From 2b77109ddb12a5ef4eb6cd1e7ae466a7ae271f09 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:19:58 +0200 Subject: [PATCH 01/13] wip --- .github/{scripts => assets}/check_no_std.sh | 0 .github/assets/hive_exclusion.json | 70 +++++++++++++++++++++ .github/{scripts => assets}/install_geth.sh | 0 .github/{scripts => assets}/label_pr.js | 0 .github/assets/parse.py | 42 +++++++++++++ .github/workflows/hive.yml | 69 ++++++++++---------- .github/workflows/integration.yml | 2 +- .github/workflows/label-pr.yml | 2 +- .github/workflows/lint.yml | 2 +- 9 files changed, 152 insertions(+), 35 deletions(-) rename .github/{scripts => assets}/check_no_std.sh (100%) create mode 100644 .github/assets/hive_exclusion.json rename .github/{scripts => assets}/install_geth.sh (100%) rename .github/{scripts => assets}/label_pr.js (100%) create mode 100644 .github/assets/parse.py diff --git a/.github/scripts/check_no_std.sh b/.github/assets/check_no_std.sh similarity index 100% rename from .github/scripts/check_no_std.sh rename to .github/assets/check_no_std.sh diff --git a/.github/assets/hive_exclusion.json b/.github/assets/hive_exclusion.json new file mode 100644 index 000000000000..991297ae03e7 --- /dev/null +++ b/.github/assets/hive_exclusion.json @@ -0,0 +1,70 @@ +{ + "rpc-compat": [ + "debug_getRawBlock/get-invalid-number (reth)", + "debug_getRawHeader/get-invalid-number (reth)", + "debug_getRawReceipts/get-invalid-number (reth)", + "debug_getRawTransaction/get-invalid-hash (reth)", + + "eth_call/call-callenv (reth)", + "eth_createAccessList/create-al-contract-eip1559 (reth)", + "eth_createAccessList/create-al-contract (reth)", + "eth_feeHistory/fee-history (reth)", + "eth_getStorageAt/get-storage-invalid-key-too-large (reth)", + "eth_getStorageAt/get-storage-invalid-key (reth)", + "eth_getTransactionReceipt/get-access-list (reth)", + "eth_getTransactionReceipt/get-blob-tx (reth)", + "eth_getTransactionReceipt/get-dynamic-fee (reth)" + ], + "engine": [ + "Blob Transaction Ordering, Multiple Clients (Cancun) (reth)", + "Inconsistent Head in ForkchoiceState (Cancun) (reth)", + "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)", + "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)", + "Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)", + "Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + "Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + "Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + "Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + + "Inconsistent Head in ForkchoiceState (Paris) (reth)", + "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)", + "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)", + "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)", + + "Withdrawals Fork On Genesis (Paris) (reth)", + "Withdrawals Fork on Block 1 (Paris) (reth)", + "Withdrawals Fork on Block 2 (Paris) (reth)", + "Withdrawals Fork on Block 3 (Paris) (reth)", + "Withdraw to a single account (Paris) (reth)", + "Withdraw to two accounts (Paris) (reth)", + "Withdraw many accounts (Paris) (reth)", + "Withdraw zero amount (Paris) (reth)", + "Empty Withdrawals (Paris) (reth)", + "Corrupted Block Hash Payload (INVALID) (Paris) (reth)", + "Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)", + "Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth)", + "Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)", + "Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth)", + "Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)", + "Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth)", + "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)", + "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)" + + ], + "sync": [ + "sync reth -> reth" + ] +} \ No newline at end of file diff --git a/.github/scripts/install_geth.sh b/.github/assets/install_geth.sh similarity index 100% rename from .github/scripts/install_geth.sh rename to .github/assets/install_geth.sh diff --git a/.github/scripts/label_pr.js b/.github/assets/label_pr.js similarity index 100% rename from .github/scripts/label_pr.js rename to .github/assets/label_pr.js diff --git a/.github/assets/parse.py b/.github/assets/parse.py new file mode 100644 index 000000000000..eece2dd5da48 --- /dev/null +++ b/.github/assets/parse.py @@ -0,0 +1,42 @@ +import json +import sys +import argparse + +# Argument parser setup +parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.") +parser.add_argument("main_json", help="Path to the main JSON file.") +parser.add_argument("--exclusion", required=True, help="Path to the exclusion JSON file.") +args = parser.parse_args() + +# Load main JSON +with open(args.main_json, 'r') as file: + main_data = json.load(file) + +# Load exclusion JSON +with open(args.exclusion, 'r') as file: + exclusion_data = json.load(file) + exclusions = exclusion_data.get(main_data['name'], []) + +# Collect unexpected failures and passes +unexpected_failures = [] +unexpected_passes = [] + +for test in main_data['testCases'].values(): + test_name = test['name'] + test_pass = test['summaryResult']['pass'] + if test_name in exclusions: + if test_pass: + unexpected_passes.append(test_name) + else: + if not test_pass: + unexpected_failures.append(test_name) + +# Check if there are any unexpected failures or passes and exit with error +if unexpected_failures or unexpected_passes: + if unexpected_failures: + print("Unexpected Failures:", unexpected_failures) + if unexpected_passes: + print("Unexpected Passes:", unexpected_passes) + sys.exit(1) + +print("All tests passed or expected failures/passes as per the exclusion list.") \ No newline at end of file diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 3340393d0873..c7ac25cc536a 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -70,8 +70,8 @@ jobs: # https://github.com/paradigmxyz/reth/issues/8579 # ethereum/rpc to be deprecated: # https://github.com/ethereum/hive/pull/1117 - # sim: [smoke/genesis, smoke/network, ethereum/sync] - sim: [smoke/genesis, smoke/network] + sim: [smoke/genesis, smoke/network, ethereum/sync] + # sim: [smoke/genesis, smoke/network] include: - sim: devp2p limit: discv4 @@ -100,8 +100,8 @@ jobs: limit: engine-exchange-capabilities # TODO: enable engine-withdrawals once resolved: # https://github.com/paradigmxyz/reth/issues/8732 - # - sim: ethereum/engine - # limit: engine-withdrawals + - sim: ethereum/engine + limit: engine-withdrawals - sim: ethereum/engine limit: engine-auth - sim: ethereum/engine @@ -109,15 +109,15 @@ jobs: # TODO: enable engine-api once resolved: # https://github.com/paradigmxyz/reth/issues/8305 # https://github.com/paradigmxyz/reth/issues/6217 - # - sim: ethereum/engine - # limit: engine-api + - sim: ethereum/engine + limit: engine-api # TODO: enable cancun once resolved: # https://github.com/paradigmxyz/reth/issues/8305 # https://github.com/paradigmxyz/reth/issues/6217 # https://github.com/paradigmxyz/reth/issues/8306 # https://github.com/paradigmxyz/reth/issues/7144 - # - sim: ethereum/engine - # limit: cancun + - sim: ethereum/engine + limit: cancun # eth_ rpc methods - sim: ethereum/rpc-compat include: @@ -142,8 +142,8 @@ jobs: # https://github.com/paradigmxyz/reth/issues/7015 # https://github.com/paradigmxyz/reth/issues/6332 # debug_ rpc methods - # - sim: ethereum/rpc-compat - # include: [debug_] + - sim: ethereum/rpc-compat + include: [debug_] # Pyspec cancun jobs - sim: pyspec include: [cancun/eip4844] @@ -209,30 +209,35 @@ jobs: - name: Run ${{ matrix.sim }} simulator run: | cd hivetests - hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth + hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth || true - - name: Create github issue if sim failed - env: - GH_TOKEN: ${{ github.token }} - if: ${{ failure() }} + # - name: Create github issue if sim failed + # env: + # GH_TOKEN: ${{ github.token }} + # if: ${{ failure() }} + # run: | + # echo "Simulator failed, creating issue" + # # Check if issue already exists + # # get all issues with the label C-hivetest, loop over each page and check if the issue already exists + + # existing_issues=$(gh api /repos/paradigmxyz/reth/issues -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -F "labels=C-hivetest" --method GET | jq '.[].title') + # if [[ $existing_issues == *"Hive Test Failure: ${{ matrix.sim }}"* ]]; then + # echo "Issue already exists" + # exit 0 + # fi + # gh api \ + # --method POST \ + # -H "Accept: application/vnd.github+json" \ + # -H "X-GitHub-Api-Version: 2022-11-28" \ + # /repos/${{ github.repository }}/issues \ + # -f title='Hive Test Failure: ${{ matrix.sim }}' \ + # -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!

The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.

[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \ + # -f "labels[]=C-hivetest" + + - name: Parse hive output run: | - echo "Simulator failed, creating issue" - # Check if issue already exists - # get all issues with the label C-hivetest, loop over each page and check if the issue already exists - - existing_issues=$(gh api /repos/paradigmxyz/reth/issues -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -F "labels=C-hivetest" --method GET | jq '.[].title') - if [[ $existing_issues == *"Hive Test Failure: ${{ matrix.sim }}"* ]]; then - echo "Issue already exists" - exit 0 - fi - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - /repos/${{ github.repository }}/issues \ - -f title='Hive Test Failure: ${{ matrix.sim }}' \ - -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!

The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.

[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \ - -f "labels[]=C-hivetest" + find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" -exec python .github/assets/parse.py {} --exclusion .github/assets/hive_exclusion.json + + - name: Print simulator output if: ${{ failure() }} run: | diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b4b494a90256..103a87706bca 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - name: Install Geth - run: .github/scripts/install_geth.sh + run: .github/assets/install_geth.sh - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 with: diff --git a/.github/workflows/label-pr.yml b/.github/workflows/label-pr.yml index 857d354a8fb8..07727173531b 100644 --- a/.github/workflows/label-pr.yml +++ b/.github/workflows/label-pr.yml @@ -19,5 +19,5 @@ jobs: uses: actions/github-script@v7 with: script: | - const label_pr = require('./.github/scripts/label_pr.js') + const label_pr = require('./.github/assets/label_pr.js') await label_pr({github, context}) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c07cee38830b..d0329aefc89f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -58,7 +58,7 @@ jobs: with: cache-on-failure: true - name: Run no_std checks - run: .github/scripts/check_no_std.sh + run: .github/assets/check_no_std.sh crate-checks: runs-on: ubuntu-latest From f3c45734bfbdb093e378e86295c08bc79cd1afe1 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:53:47 +0200 Subject: [PATCH 02/13] use xargs instead --- .github/workflows/hive.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index c7ac25cc536a..379023893849 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -236,7 +236,7 @@ jobs: - name: Parse hive output run: | - find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" -exec python .github/assets/parse.py {} --exclusion .github/assets/hive_exclusion.json + + find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/parse.py {} --exclusion .github/assets/hive_exclusion.json - name: Print simulator output if: ${{ failure() }} From 4d2147cb9142cf8726bcd1b723d5265a1a830745 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:26:56 +0200 Subject: [PATCH 03/13] checkout repo for assets --- .github/workflows/hive.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 379023893849..a1cdfe46026b 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -184,6 +184,10 @@ jobs: permissions: issues: write steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Download artifacts uses: actions/download-artifact@v4 with: @@ -233,7 +237,7 @@ jobs: # -f title='Hive Test Failure: ${{ matrix.sim }}' \ # -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!

The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.

[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \ # -f "labels[]=C-hivetest" - + - name: Parse hive output run: | find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/parse.py {} --exclusion .github/assets/hive_exclusion.json From 2f0389f58ad3a9b2ac9884c7c5405bf3007ddb27 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:00:28 +0200 Subject: [PATCH 04/13] rm expected failure for testing --- .github/assets/hive_exclusion.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/assets/hive_exclusion.json b/.github/assets/hive_exclusion.json index 991297ae03e7..bad814011b6d 100644 --- a/.github/assets/hive_exclusion.json +++ b/.github/assets/hive_exclusion.json @@ -63,8 +63,5 @@ "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)", "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)" - ], - "sync": [ - "sync reth -> reth" ] } \ No newline at end of file From 0f4e400a58489481d7a93a25db44fb08d6519921 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:26:50 +0200 Subject: [PATCH 05/13] Revert "rm expected failure for testing" This reverts commit 2f0389f58ad3a9b2ac9884c7c5405bf3007ddb27. --- .github/assets/hive_exclusion.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/assets/hive_exclusion.json b/.github/assets/hive_exclusion.json index bad814011b6d..991297ae03e7 100644 --- a/.github/assets/hive_exclusion.json +++ b/.github/assets/hive_exclusion.json @@ -63,5 +63,8 @@ "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)", "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)" + ], + "sync": [ + "sync reth -> reth" ] } \ No newline at end of file From ee9322b1d4b4043d34afe3dc2b72ea3c6b8cdf75 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:29:11 +0200 Subject: [PATCH 06/13] update --- .../assets/{hive_exclusion.json => hive/expected_failures.json} | 0 .github/assets/{ => hive}/parse.py | 2 +- .github/workflows/hive.yml | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename .github/assets/{hive_exclusion.json => hive/expected_failures.json} (100%) rename .github/assets/{ => hive}/parse.py (94%) diff --git a/.github/assets/hive_exclusion.json b/.github/assets/hive/expected_failures.json similarity index 100% rename from .github/assets/hive_exclusion.json rename to .github/assets/hive/expected_failures.json diff --git a/.github/assets/parse.py b/.github/assets/hive/parse.py similarity index 94% rename from .github/assets/parse.py rename to .github/assets/hive/parse.py index eece2dd5da48..43037a2fa260 100644 --- a/.github/assets/parse.py +++ b/.github/assets/hive/parse.py @@ -39,4 +39,4 @@ print("Unexpected Passes:", unexpected_passes) sys.exit(1) -print("All tests passed or expected failures/passes as per the exclusion list.") \ No newline at end of file +print("Success.") \ No newline at end of file diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index a1cdfe46026b..d0dc52a41ef7 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -240,7 +240,7 @@ jobs: - name: Parse hive output run: | - find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/parse.py {} --exclusion .github/assets/hive_exclusion.json + find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.json - name: Print simulator output if: ${{ failure() }} From c0d1eb0d1fda8eed0a45d50881c7a000652c8815 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:31:46 +0200 Subject: [PATCH 07/13] re-enable gh issue creation --- .github/workflows/hive.yml | 46 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index d0dc52a41ef7..463c420abe59 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -215,33 +215,33 @@ jobs: cd hivetests hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth || true - # - name: Create github issue if sim failed - # env: - # GH_TOKEN: ${{ github.token }} - # if: ${{ failure() }} - # run: | - # echo "Simulator failed, creating issue" - # # Check if issue already exists - # # get all issues with the label C-hivetest, loop over each page and check if the issue already exists - - # existing_issues=$(gh api /repos/paradigmxyz/reth/issues -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -F "labels=C-hivetest" --method GET | jq '.[].title') - # if [[ $existing_issues == *"Hive Test Failure: ${{ matrix.sim }}"* ]]; then - # echo "Issue already exists" - # exit 0 - # fi - # gh api \ - # --method POST \ - # -H "Accept: application/vnd.github+json" \ - # -H "X-GitHub-Api-Version: 2022-11-28" \ - # /repos/${{ github.repository }}/issues \ - # -f title='Hive Test Failure: ${{ matrix.sim }}' \ - # -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!

The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.

[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \ - # -f "labels[]=C-hivetest" - - name: Parse hive output run: | find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.json + - name: Create github issue if sim failed + env: + GH_TOKEN: ${{ github.token }} + if: ${{ failure() }} + run: | + echo "Simulator failed, creating issue" + # Check if issue already exists + # get all issues with the label C-hivetest, loop over each page and check if the issue already exists + + existing_issues=$(gh api /repos/paradigmxyz/reth/issues -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -F "labels=C-hivetest" --method GET | jq '.[].title') + if [[ $existing_issues == *"Hive Test Failure: ${{ matrix.sim }}"* ]]; then + echo "Issue already exists" + exit 0 + fi + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ github.repository }}/issues \ + -f title='Hive Test Failure: ${{ matrix.sim }}' \ + -f body="!!!!!!! This is an automated issue created by the hive test failure !!!!!!!

The hive test for ${{ matrix.sim }} failed. Please investigate and fix the issue.

[Link to the failed run](https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }})" \ + -f "labels[]=C-hivetest" + - name: Print simulator output if: ${{ failure() }} run: | From d3c02c560b639122a6606b363d93cdf571349dfc Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 2 Jul 2024 17:42:25 +0200 Subject: [PATCH 08/13] update suite names --- .github/assets/hive/expected_failures.json | 60 +++++++++++----------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/.github/assets/hive/expected_failures.json b/.github/assets/hive/expected_failures.json index 991297ae03e7..23c9d51305ba 100644 --- a/.github/assets/hive/expected_failures.json +++ b/.github/assets/hive/expected_failures.json @@ -15,35 +15,7 @@ "eth_getTransactionReceipt/get-blob-tx (reth)", "eth_getTransactionReceipt/get-dynamic-fee (reth)" ], - "engine": [ - "Blob Transaction Ordering, Multiple Clients (Cancun) (reth)", - "Inconsistent Head in ForkchoiceState (Cancun) (reth)", - "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)", - "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)", - "Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)", - "Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - "Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - "Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - "Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - - "Inconsistent Head in ForkchoiceState (Paris) (reth)", - "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)", - "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)", - "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)", - + "engine-withdrawals": [ "Withdrawals Fork On Genesis (Paris) (reth)", "Withdrawals Fork on Block 1 (Paris) (reth)", "Withdrawals Fork on Block 2 (Paris) (reth)", @@ -64,6 +36,36 @@ "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)" ], + "engine-api": [ + "Inconsistent Head in ForkchoiceState (Paris) (reth)", + "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)", + "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)", + "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)" + ], + "engine-cancun": [ + "Blob Transaction Ordering, Multiple Clients (Cancun) (reth)", + "Inconsistent Head in ForkchoiceState (Cancun) (reth)", + "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)", + "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)", + "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)", + "Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)", + "Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + "Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + "Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", + "Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)" + ], "sync": [ "sync reth -> reth" ] From 00b5561a23552671054715c107c7bfbb3aa83ceb Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 3 Jul 2024 16:03:55 +0200 Subject: [PATCH 09/13] use yaml instead + move issues --- .github/assets/hive/expected_failures.json | 72 -------------------- .github/assets/hive/expected_failures.yaml | 78 ++++++++++++++++++++++ .github/assets/hive/parse.py | 16 ++--- .github/workflows/hive.yml | 18 +---- 4 files changed, 87 insertions(+), 97 deletions(-) delete mode 100644 .github/assets/hive/expected_failures.json create mode 100644 .github/assets/hive/expected_failures.yaml diff --git a/.github/assets/hive/expected_failures.json b/.github/assets/hive/expected_failures.json deleted file mode 100644 index 23c9d51305ba..000000000000 --- a/.github/assets/hive/expected_failures.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "rpc-compat": [ - "debug_getRawBlock/get-invalid-number (reth)", - "debug_getRawHeader/get-invalid-number (reth)", - "debug_getRawReceipts/get-invalid-number (reth)", - "debug_getRawTransaction/get-invalid-hash (reth)", - - "eth_call/call-callenv (reth)", - "eth_createAccessList/create-al-contract-eip1559 (reth)", - "eth_createAccessList/create-al-contract (reth)", - "eth_feeHistory/fee-history (reth)", - "eth_getStorageAt/get-storage-invalid-key-too-large (reth)", - "eth_getStorageAt/get-storage-invalid-key (reth)", - "eth_getTransactionReceipt/get-access-list (reth)", - "eth_getTransactionReceipt/get-blob-tx (reth)", - "eth_getTransactionReceipt/get-dynamic-fee (reth)" - ], - "engine-withdrawals": [ - "Withdrawals Fork On Genesis (Paris) (reth)", - "Withdrawals Fork on Block 1 (Paris) (reth)", - "Withdrawals Fork on Block 2 (Paris) (reth)", - "Withdrawals Fork on Block 3 (Paris) (reth)", - "Withdraw to a single account (Paris) (reth)", - "Withdraw to two accounts (Paris) (reth)", - "Withdraw many accounts (Paris) (reth)", - "Withdraw zero amount (Paris) (reth)", - "Empty Withdrawals (Paris) (reth)", - "Corrupted Block Hash Payload (INVALID) (Paris) (reth)", - "Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)", - "Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth)", - "Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)", - "Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth)", - "Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)", - "Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth)", - "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)", - "Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)" - - ], - "engine-api": [ - "Inconsistent Head in ForkchoiceState (Paris) (reth)", - "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth)", - "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)", - "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)" - ], - "engine-cancun": [ - "Blob Transaction Ordering, Multiple Clients (Cancun) (reth)", - "Inconsistent Head in ForkchoiceState (Cancun) (reth)", - "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth)", - "Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - "Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)", - "Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)", - "Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)", - "Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - "Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - "Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)", - "Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)" - ], - "sync": [ - "sync reth -> reth" - ] -} \ No newline at end of file diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml new file mode 100644 index 000000000000..0e67b646471e --- /dev/null +++ b/.github/assets/hive/expected_failures.yaml @@ -0,0 +1,78 @@ +# https://github.com/paradigmxyz/reth/issues/7015 +# https://github.com/paradigmxyz/reth/issues/6332 +rpc-compat: + - debug_getRawBlock/get-invalid-number (reth) + - debug_getRawHeader/get-invalid-number (reth) + - debug_getRawReceipts/get-invalid-number (reth) + - debug_getRawTransaction/get-invalid-hash (reth) + + - eth_call/call-callenv (reth) + - eth_createAccessList/create-al-contract-eip1559 (reth) + - eth_createAccessList/create-al-contract (reth) + - eth_feeHistory/fee-history (reth) + - eth_getStorageAt/get-storage-invalid-key-too-large (reth) + - eth_getStorageAt/get-storage-invalid-key (reth) + - eth_getTransactionReceipt/get-access-list (reth) + - eth_getTransactionReceipt/get-blob-tx (reth) + - eth_getTransactionReceipt/get-dynamic-fee (reth) + +# https://github.com/paradigmxyz/reth/issues/8732 +engine-withdrawals: + - Withdrawals Fork On Genesis (Paris) (reth) + - Withdrawals Fork on Block 1 (Paris) (reth) + - Withdrawals Fork on Block 2 (Paris) (reth) + - Withdrawals Fork on Block 3 (Paris) (reth) + - Withdraw to a single account (Paris) (reth) + - Withdraw to two accounts (Paris) (reth) + - Withdraw many accounts (Paris) (reth) + - Withdraw zero amount (Paris) (reth) + - Empty Withdrawals (Paris) (reth) + - Corrupted Block Hash Payload (INVALID) (Paris) (reth) + - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth) + - Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth) + - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth) + - Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth) + +# https://github.com/paradigmxyz/reth/issues/8305 +# https://github.com/paradigmxyz/reth/issues/6217 +engine-api: + - Inconsistent Head in ForkchoiceState (Paris) (reth) + - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Paris) (reth) + - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth) + - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + +# https://github.com/paradigmxyz/reth/issues/8305 +# https://github.com/paradigmxyz/reth/issues/6217 +# https://github.com/paradigmxyz/reth/issues/8306 +# https://github.com/paradigmxyz/reth/issues/7144 +engine-cancun: + - Blob Transaction Ordering, Multiple Clients (Cancun) (reth) + - Inconsistent Head in ForkchoiceState (Cancun) (reth) + - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=True, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, StateRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, PrevRandao, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=True, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, StateRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth) + - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth) + - Invalid NewPayload, ParentBeaconBlockRoot, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + +# https://github.com/paradigmxyz/reth/issues/8579 +sync: + - sync reth -> reth \ No newline at end of file diff --git a/.github/assets/hive/parse.py b/.github/assets/hive/parse.py index 43037a2fa260..6011dbd5b6bd 100644 --- a/.github/assets/hive/parse.py +++ b/.github/assets/hive/parse.py @@ -1,20 +1,20 @@ -import json +import yaml import sys import argparse # Argument parser setup parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.") -parser.add_argument("main_json", help="Path to the main JSON file.") -parser.add_argument("--exclusion", required=True, help="Path to the exclusion JSON file.") +parser.add_argument("main_yaml", help="Path to the main YAML file.") +parser.add_argument("--exclusion", required=True, help="Path to the exclusion YAML file.") args = parser.parse_args() -# Load main JSON -with open(args.main_json, 'r') as file: - main_data = json.load(file) +# Load main YAML +with open(args.main_yaml, 'r') as file: + main_data = yaml.safe_load(file) -# Load exclusion JSON +# Load exclusion YAML with open(args.exclusion, 'r') as file: - exclusion_data = json.load(file) + exclusion_data = yaml.safe_load(file) exclusions = exclusion_data.get(main_data['name'], []) # Collect unexpected failures and passes diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 463c420abe59..d96e6b786910 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -66,12 +66,9 @@ jobs: strategy: fail-fast: false matrix: - # TODO: enable etherem/sync once resolved: - # https://github.com/paradigmxyz/reth/issues/8579 # ethereum/rpc to be deprecated: # https://github.com/ethereum/hive/pull/1117 sim: [smoke/genesis, smoke/network, ethereum/sync] - # sim: [smoke/genesis, smoke/network] include: - sim: devp2p limit: discv4 @@ -98,24 +95,14 @@ jobs: - TestBlobViolations - sim: ethereum/engine limit: engine-exchange-capabilities - # TODO: enable engine-withdrawals once resolved: - # https://github.com/paradigmxyz/reth/issues/8732 - sim: ethereum/engine limit: engine-withdrawals - sim: ethereum/engine limit: engine-auth - sim: ethereum/engine limit: engine-transition - # TODO: enable engine-api once resolved: - # https://github.com/paradigmxyz/reth/issues/8305 - # https://github.com/paradigmxyz/reth/issues/6217 - sim: ethereum/engine limit: engine-api - # TODO: enable cancun once resolved: - # https://github.com/paradigmxyz/reth/issues/8305 - # https://github.com/paradigmxyz/reth/issues/6217 - # https://github.com/paradigmxyz/reth/issues/8306 - # https://github.com/paradigmxyz/reth/issues/7144 - sim: ethereum/engine limit: cancun # eth_ rpc methods @@ -138,9 +125,6 @@ jobs: - eth_getTransactionReceipt - eth_sendRawTransaction - eth_syncing - # TODO: enable debug_ rpc-compat once resolved: - # https://github.com/paradigmxyz/reth/issues/7015 - # https://github.com/paradigmxyz/reth/issues/6332 # debug_ rpc methods - sim: ethereum/rpc-compat include: [debug_] @@ -217,7 +201,7 @@ jobs: - name: Parse hive output run: | - find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.json + find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml - name: Create github issue if sim failed env: From 19feadf3e2c6225d8865d74c707a8a2a1b7dd461 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 3 Jul 2024 16:06:08 +0200 Subject: [PATCH 10/13] hive uses json, exclusion list yaml --- .github/assets/hive/parse.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/assets/hive/parse.py b/.github/assets/hive/parse.py index 6011dbd5b6bd..ee75fdf55317 100644 --- a/.github/assets/hive/parse.py +++ b/.github/assets/hive/parse.py @@ -1,27 +1,28 @@ +import json import yaml import sys import argparse # Argument parser setup parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.") -parser.add_argument("main_yaml", help="Path to the main YAML file.") +parser.add_argument("report_json", help="Path to the hive report JSON file.") parser.add_argument("--exclusion", required=True, help="Path to the exclusion YAML file.") args = parser.parse_args() -# Load main YAML -with open(args.main_yaml, 'r') as file: - main_data = yaml.safe_load(file) +# Load hive JSON +with open(args.report_json, 'r') as file: + report = json.load(file) # Load exclusion YAML with open(args.exclusion, 'r') as file: exclusion_data = yaml.safe_load(file) - exclusions = exclusion_data.get(main_data['name'], []) + exclusions = exclusion_data.get(report['name'], []) # Collect unexpected failures and passes unexpected_failures = [] unexpected_passes = [] -for test in main_data['testCases'].values(): +for test in report['testCases'].values(): test_name = test['name'] test_pass = test['summaryResult']['pass'] if test_name in exclusions: From cfe9eb3c6626183aa0b8a51f9b327256ffc3d35c Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 3 Jul 2024 19:27:59 +0200 Subject: [PATCH 11/13] add more expected failures (hive related) --- .github/assets/hive/expected_failures.yaml | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml index 0e67b646471e..ba29ca5e7097 100644 --- a/.github/assets/hive/expected_failures.yaml +++ b/.github/assets/hive/expected_failures.yaml @@ -51,6 +51,27 @@ engine-api: - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + # Hive issue + # https://github.com/ethereum/hive/issues/1135 + - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + # https://github.com/paradigmxyz/reth/issues/8305 # https://github.com/paradigmxyz/reth/issues/6217 # https://github.com/paradigmxyz/reth/issues/8306 @@ -73,6 +94,27 @@ engine-cancun: - Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + # Hive issue + # https://github.com/ethereum/hive/issues/1135 + - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, ReceiptsRoot, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, GasUsed, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Incomplete Transactions, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Gas, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction GasPrice, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Cancun) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Value, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Cancun) (reth) + # https://github.com/paradigmxyz/reth/issues/8579 sync: - sync reth -> reth \ No newline at end of file From f93c740f6c8c6e63f3b618fbc87cede98a999924 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:24:59 +0200 Subject: [PATCH 12/13] add time out failures to hive --- .github/assets/hive/expected_failures.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml index ba29ca5e7097..831cb966fb64 100644 --- a/.github/assets/hive/expected_failures.yaml +++ b/.github/assets/hive/expected_failures.yaml @@ -36,6 +36,14 @@ engine-withdrawals: - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth) - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth) - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth) + + # https://github.com/paradigmxyz/reth/issues/8304#issuecomment-2208515839 + - Sync after 2 blocks - Withdrawals on Block 1 - Single Withdrawal Account - No Transactions (Paris) (reth) + - Sync after 2 blocks - Withdrawals on Block 1 - Single Withdrawal Account (Paris) (reth) + - Sync after 2 blocks - Withdrawals on Genesis - Single Withdrawal Account (Paris) (reth) + - Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts - No Transactions (Paris) (reth) + - Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth) + - Sync after 128 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth) # https://github.com/paradigmxyz/reth/issues/8305 # https://github.com/paradigmxyz/reth/issues/6217 From 882315943e1a2c735b6f5fdeed66f0e149d1092f Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:26:35 +0200 Subject: [PATCH 13/13] add parallelism 2 --- .github/workflows/hive.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index e859ec2866b8..d18ffd65ff91 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -203,7 +203,7 @@ jobs: - name: Run ${{ matrix.sim }} simulator run: | cd hivetests - hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --client reth || true + hive --sim "${{ matrix.sim }}$" --sim.limit "${{matrix.limit}}/${{join(matrix.include, '|')}}" --sim.parallelism 2 --client reth || true - name: Parse hive output run: |