Skip to content

Commit

Permalink
Merge pull request #667 from oasisprotocol/pro-wh/feature/edenfast
Browse files Browse the repository at this point in the history
tests: e2e_regression add edenfast suite
  • Loading branch information
pro-wh authored May 6, 2024
2 parents c9c7bf8 + e077ba2 commit d67678b
Show file tree
Hide file tree
Showing 28 changed files with 148 additions and 41 deletions.
1 change: 1 addition & 0 deletions .changelog/667.internal.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
tests: e2e_regression add edenfast suite
43 changes: 9 additions & 34 deletions .github/workflows/ci-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,13 @@ jobs:
/tmp/*.log
if: success() || failure() # but not if job is manually cancelled

test-e2e-regression-eden:
test-e2e-regression:
strategy:
matrix:
suite:
- damask
- eden
- edenfast
runs-on: ubuntu-20.04
steps:
- name: Checkout code
Expand All @@ -156,41 +162,10 @@ jobs:
make postgres
- name: Run e2e regression tests
run : |
make E2E_REGRESSION_SUITES=eden test-e2e-regression
make E2E_REGRESSION_SUITES=${{ matrix.suite }} test-e2e-regression
- uses: actions/upload-artifact@v3
with:
name: Actual nexus api responses
path: |
tests/e2e_regression/eden/actual
if: failure() # but not if success or job is manually cancelled

test-e2e-regression-damask:
runs-on: ubuntu-20.04
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
lfs: true
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.22.x
- name: Autogenerate Go code
run: |
go install github.com/deepmap/oapi-codegen/cmd/[email protected]
make codegen-go
- name: Build Go
run: |
make nexus
- name: Start db
run: |
make postgres
- name: Run e2e regression tests
run : |
make E2E_REGRESSION_SUITES=damask test-e2e-regression
- uses: actions/upload-artifact@v3
with:
name: Actual nexus api responses
path: |
tests/e2e_regression/damask/actual
tests/e2e_regression/${{ matrix.suite }}/actual
if: failure() # but not if success or job is manually cancelled
8 changes: 2 additions & 6 deletions .github/workflows/prepare-release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,7 @@ jobs:
name: test-e2e-localnet
runs-on: ubuntu-20.04
steps: [{name: "fake success", run: "exit 0"}]
test-e2e-regression-damask:
name: test-e2e-regression-damask
runs-on: ubuntu-20.04
steps: [{name: "fake success", run: "exit 0"}]
test-e2e-regression-eden:
name: test-e2e-regression-eden
test-e2e-regression:
name: test-e2e-regression
runs-on: ubuntu-20.04
steps: [{name: "fake success", run: "exit 0"}]
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ test-e2e:

# To run specific suites, do e.g.
# make E2E_REGRESSION_SUITES='suite1 suite2' test-e2e-regression
E2E_REGRESSION_SUITES := eden damask
E2E_REGRESSION_SUITES := eden damask edenfast

ensure-consistent-config-for-e2e-regression:
for suite in $(E2E_REGRESSION_SUITES); do ./tests/e2e_regression/ensure_consistent_config.sh $$suite; done
Expand Down
6 changes: 6 additions & 0 deletions tests/e2e_regression/accept.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ if [[ -z "$suite" || ! -e "$TEST_DIR/e2e_config_1.yml" ]]; then
exit 1
fi

[[ -L "$TEST_DIR/expected" ]] && {
echo >&2 "$suite/expected is a symbolic link to $(readlink "$TEST_DIR/expected")."
echo >&2 "Use $0 on that suite instead."
exit 1
}

[[ -d "$TEST_DIR/actual" ]] || {
echo "Note: No actual outputs found for suite $suite. Nothing to accept."
exit 0
Expand Down
2 changes: 2 additions & 0 deletions tests/e2e_regression/eden/e2e_config_1.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
# but low-effort way to ensure that all non-block analyzers perform RPCs at a deterministic
# height and therefore always hit the RPC cache. (Most EVM-related analyzers make RPCs at the
# maximum height that's been reached by block analyzers by that point.)
#
# Keep most changes in sync with the edenfast suite.

analysis:
source:
Expand Down
60 changes: 60 additions & 0 deletions tests/e2e_regression/edenfast/e2e_config_1.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Config for phase 1 of 2 of e2e regression tests.
# See also ensure_consistent_config.sh
#
# Phase 1 runs all block-based analyzers; phase 2 runs all others. This is a slightly hacky
# but low-effort way to ensure that all non-block analyzers perform RPCs at a deterministic
# height and therefore always hit the RPC cache. (Most EVM-related analyzers make RPCs at the
# maximum height that's been reached by block analyzers by that point.)
#
# The edenfast suite is the same as the eden suite, but with fast sync on the
# first half of the blocks. Keep most changes in sync with the eden suite.

analysis:
source:
cache: { cache_dir: tests/e2e_regression/edenfast/rpc-cache }
chain_name: mainnet
nodes:
# Node will be accessed only when the filling up the cache.
# In CI, the cache is expected to have all the node responses as the node is not reachable.
eden: { default: { rpc: grpc.oasis.dev:443 } }
ipfs:
gateway: https://ipfs.io
fast_startup: true
analyzers:
consensus:
from: 16_817_956 # Eden genesis
to: 16_818_955 # 1000 blocks; compromise between fast and comprehensive
batch_size: 50
fast_sync:
to: 16_818_455 # Halfway
parallelism: 10
emerald:
#from: 7_875_130 # round at Eden genesis, plus 1 (so that in finalizeFastSync we don't need Damask)
#to: 7_876_129 # 1000 blocks; compromise between fast and comprehensive
from: 8_059_340 # An NFT was created at 8059343; ERC20s were created at 7984382, 7984824, 8075695, 8166988
to: 8_060_339 # 1000 blocks; compromise between fast and comprehensive
batch_size: 50
fast_sync:
to: 8_059_839 # Halfway
parallelism: 10
# We do not test Sapphire; the node only makes recent rounds queryable, so it's awkward to test reproducibly.
# Also, sapphire does not use much code that is not covered by emerald.
storage:
backend: postgres
endpoint: postgresql://rwuser:[email protected]:5432/indexer?sslmode=disable
DANGER__WIPE_STORAGE_ON_STARTUP: true
migrations: file://storage/migrations

server:
chain_name: mainnet
endpoint: localhost:8008
storage:
endpoint: postgresql://rwuser:[email protected]:5432/indexer?sslmode=disable
backend: postgres

log:
level: debug
format: json

metrics:
pull_endpoint: localhost:8009
38 changes: 38 additions & 0 deletions tests/e2e_regression/edenfast/e2e_config_2.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# Config for phase 2 of e2e regression tests. See e2e_config_1.yml for more info.

analysis:
source:
cache: { cache_dir: tests/e2e_regression/edenfast/rpc-cache }
chain_name: mainnet
nodes:
eden: { default: { rpc: grpc.oasis.dev:443 } }
ipfs:
gateway: https://ipfs.io
fast_startup: true
helpers:
caching_proxies:
- { target: "https://sourcify.dev/server", host_addr: "localhost:9191" }
analyzers:
evm_tokens_emerald: { stop_if_queue_empty_for: 1s }
evm_nfts_emerald: { stop_if_queue_empty_for: 1s }
evm_token_balances_emerald: { stop_if_queue_empty_for: 1s }
evm_contract_code_emerald: { stop_if_queue_empty_for: 1s }
evm_abi_emerald: { stop_if_queue_empty_for: 10s } # Give evm_contract_verifier time to fetch ABIs first. The 10s has been enough in practice, but might need to be tuned in the future, espcially if the caching proxy has an empty cache.
evm_contract_verifier_emerald: { stop_if_queue_empty_for: 1s, sourcify_server_url: http://localhost:9191 }
# Some non-block analyzers are not tested in e2e regressions.
# They are largely not worth the trouble as they do not interact with rest of the system much.
# metadata_registry: {} # Awkward to inject mock registry responses.
# node_stats: {} # Awkward to inject mock node response using the current paradigm (= response caching).
# aggregate_stats: {} # Awkward to make stop after a single run.
storage:
backend: postgres
endpoint: postgresql://rwuser:[email protected]:5432/indexer?sslmode=disable
# DANGER__WIPE_STORAGE_ON_STARTUP: true
migrations: file://storage/migrations

log:
level: info
format: json

metrics:
pull_endpoint: localhost:8009
1 change: 1 addition & 0 deletions tests/e2e_regression/edenfast/expected
3 changes: 3 additions & 0 deletions tests/e2e_regression/edenfast/rpc-cache/consensus/00000-1.psg
Git LFS file not shown
Binary file not shown.
Binary file not shown.
Binary file not shown.
3 changes: 3 additions & 0 deletions tests/e2e_regression/edenfast/rpc-cache/consensus/main.pix
Git LFS file not shown
Git LFS file not shown
3 changes: 3 additions & 0 deletions tests/e2e_regression/edenfast/rpc-cache/emerald/00000-1.psg
Git LFS file not shown
Binary file not shown.
Binary file not shown.
Binary file not shown.
3 changes: 3 additions & 0 deletions tests/e2e_regression/edenfast/rpc-cache/emerald/main.pix
Git LFS file not shown
3 changes: 3 additions & 0 deletions tests/e2e_regression/edenfast/rpc-cache/emerald/overflow.pix
Git LFS file not shown
Git LFS file not shown
Binary file not shown.
Binary file not shown.
Binary file not shown.
Git LFS file not shown
Git LFS file not shown
1 change: 1 addition & 0 deletions tests/e2e_regression/edenfast/test_cases.sh

0 comments on commit d67678b

Please sign in to comment.