From bf6c2f01cdfda1279060552ad5e297c8a83afbf6 Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Thu, 7 Nov 2024 08:43:41 -0800 Subject: [PATCH 1/4] test: add e2e consensus test --- .github/workflows/e2e.yml | 5 +++++ Makefile | 8 +++++++- contrib/localnet/docker-compose.yml | 3 ++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4aadf4900e..43a8e174ce 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -111,6 +111,7 @@ jobs: outputs: DEFAULT_TESTS: ${{ steps.matrix-conditionals.outputs.DEFAULT_TESTS }} UPGRADE_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_TESTS }} + CONSENSUS_TESTS: ${{ steps.matrix-conditionals.outputs.CONSENSUS_TESTS }} UPGRADE_LIGHT_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS }} UPGRADE_IMPORT_MAINNET_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_IMPORT_MAINNET_TESTS }} ADMIN_TESTS: ${{ steps.matrix-conditionals.outputs.ADMIN_TESTS }} @@ -138,6 +139,7 @@ jobs: console.log("labels:", labels); core.setOutput('DEFAULT_TESTS', true); core.setOutput('UPGRADE_TESTS', labels.includes('UPGRADE_TESTS')); + core.setOutput('CONSENSUS_TESTS', !labels.includes('CONSENSUS_BREAKING_ACK')); core.setOutput('UPGRADE_LIGHT_TESTS', labels.includes('UPGRADE_LIGHT_TESTS')); core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', labels.includes('UPGRADE_IMPORT_MAINNET_TESTS')); core.setOutput('ADMIN_TESTS', labels.includes('ADMIN_TESTS')); @@ -206,6 +208,9 @@ jobs: - make-target: "start-e2e-test" runs-on: ubuntu-20.04 run: ${{ needs.matrix-conditionals.outputs.DEFAULT_TESTS == 'true' }} + - make-target: "start-e2e-consensus-test" + runs-on: ubuntu-20.04 + run: ${{ needs.matrix-conditionals.outputs.CONSENSUS_TESTS == 'true' }} - make-target: "start-upgrade-test" runs-on: ubuntu-20.04 run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TESTS == 'true' }} diff --git a/Makefile b/Makefile index 38cd6d3be7..ebeacdb26e 100644 --- a/Makefile +++ b/Makefile @@ -268,7 +268,7 @@ solana: start-e2e-test: e2e-images @echo "--> Starting e2e test" - cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d start-e2e-admin-test: e2e-images @echo "--> Starting e2e admin test" @@ -286,6 +286,12 @@ start-e2e-import-mainnet-test: e2e-images export ZETACORED_START_PERIOD=15m && \ cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER_COMPOSE) up -d +start-e2e-consensus-test: e2e-images + @echo "--> Starting e2e consensus test" + export ZETACORE1_IMAGE=ghcr.io/zeta-chain/zetanode:develop && \ + export ZETACORE1_PLATFORM=linux/amd64 && \ + cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d + start-stress-test: e2e-images @echo "--> Starting stress test" cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile stress up -d diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index 4f36583d91..670befa7b6 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -67,7 +67,8 @@ services: - ~/.zetacored/genesis_data:/root/genesis_data zetacore1: - image: zetanode:latest + image: ${ZETACORE1_IMAGE-zetanode:latest} + platform: ${ZETACORE1_PLATFORM-} container_name: zetacore1 hostname: zetacore1 networks: From f30e24e0392abd8d64e9bfae515467214ee750a5 Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Thu, 7 Nov 2024 09:19:07 -0800 Subject: [PATCH 2/4] move to merge queue --- .github/workflows/e2e.yml | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 43a8e174ce..ce395f73c6 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -129,17 +129,22 @@ jobs: uses: actions/github-script@v7 with: script: | - if (context.eventName === 'pull_request') { + const getPrLabels = async (pull_number) => { const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number, - }); - const labels = pr.labels.map(label => label.name); - console.log("labels:", labels); + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pull_number, + }); + const labels = pr.labels.map(label => label.name); + console.log(`labels for ${pull_number}:`, labels); + return labels; + } + + if (context.eventName === 'pull_request') { + const labels = await getPrLabels(context.payload.pull_request.number); core.setOutput('DEFAULT_TESTS', true); core.setOutput('UPGRADE_TESTS', labels.includes('UPGRADE_TESTS')); - core.setOutput('CONSENSUS_TESTS', !labels.includes('CONSENSUS_BREAKING_ACK')); + core.setOutput('CONSENSUS_TESTS', labels.includes('CONSENSUS_TESTS')); core.setOutput('UPGRADE_LIGHT_TESTS', labels.includes('UPGRADE_LIGHT_TESTS')); core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', labels.includes('UPGRADE_IMPORT_MAINNET_TESTS')); core.setOutput('ADMIN_TESTS', labels.includes('ADMIN_TESTS')); @@ -152,8 +157,20 @@ jobs: core.setOutput('V2_MIGRATION_TESTS', labels.includes('V2_MIGRATION_TESTS')); // for v2 tests, TODO: remove this once we fully migrate to v2 (https://github.com/zeta-chain/node/issues/2627) core.setOutput('ENABLE_MONITORING', labels.includes('ENABLE_MONITORING')); } else if (context.eventName === 'merge_group') { + // default mergequeue tests core.setOutput('DEFAULT_TESTS', true); core.setOutput('UPGRADE_LIGHT_TESTS', true); + + // conditional tests based on PR labels + const commit_message = context.payload.merge_group.head_commit.message; + const pr_match = commit_message.split('\n')[0].match(/\(#(\d+)\)$/); + if (!pr_match) { + console.error("unable to extract PR number from mergequeue commit message"); + return; + } + const pr_number = pr_match[1]; + const pr_labels = getPrLabels(pr_number); + core.setOutput('CONSENSUS_TESTS', !labels.includes('CONSENSUS_BREAKING_ACK')); } else if (context.eventName === 'push' && context.ref === 'refs/heads/develop') { core.setOutput('DEFAULT_TESTS', true); } else if (context.eventName === 'push' && context.ref.startsWith('refs/heads/release/')) { From ea592ebf666acc10702c2e5aa9c8c7a17736253f Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Thu, 7 Nov 2024 09:31:45 -0800 Subject: [PATCH 3/4] fix --- .github/workflows/e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ce395f73c6..cebedde240 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -169,8 +169,8 @@ jobs: return; } const pr_number = pr_match[1]; - const pr_labels = getPrLabels(pr_number); - core.setOutput('CONSENSUS_TESTS', !labels.includes('CONSENSUS_BREAKING_ACK')); + const pr_labels = await getPrLabels(pr_number); + core.setOutput('CONSENSUS_TESTS', !pr_labels.includes('CONSENSUS_BREAKING_ACK')); } else if (context.eventName === 'push' && context.ref === 'refs/heads/develop') { core.setOutput('DEFAULT_TESTS', true); } else if (context.eventName === 'push' && context.ref.startsWith('refs/heads/release/')) { From be4ad2ad6e896bf36eb322f8fb7b5d46f692196a Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Thu, 7 Nov 2024 11:26:03 -0800 Subject: [PATCH 4/4] add e2e block production monitor --- cmd/zetae2e/local/local.go | 4 ++ cmd/zetae2e/local/monitor_block_production.go | 55 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 cmd/zetae2e/local/monitor_block_production.go diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 81779faef9..e7b1ee3e29 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -190,6 +190,10 @@ func localE2ETest(cmd *cobra.Command, _ []string) { noError(deployerRunner.FundEmissionsPool()) } + // monitor block production to ensure we fail fast if there are consensus failures + // this is not run in an errgroup since only returning an error will not exit immedately + go monitorBlockProductionExit(ctx, conf) + // wait for keygen to be completed // if setup is skipped, we assume that the keygen is already completed if !skipSetup { diff --git a/cmd/zetae2e/local/monitor_block_production.go b/cmd/zetae2e/local/monitor_block_production.go new file mode 100644 index 0000000000..89de03012e --- /dev/null +++ b/cmd/zetae2e/local/monitor_block_production.go @@ -0,0 +1,55 @@ +package local + +import ( + "context" + "fmt" + "os" + "time" + + rpchttp "github.com/cometbft/cometbft/rpc/client/http" + cometbft_types "github.com/cometbft/cometbft/types" + + "github.com/zeta-chain/node/e2e/config" +) + +// monitorBlockProductionExit calls monitorBlockProduction and exits upon any error +func monitorBlockProductionExit(ctx context.Context, conf config.Config) { + err := monitorBlockProduction(ctx, conf) + if err != nil { + fmt.Printf("❌ block monitor: %v\n", err) + os.Exit(2) + } +} + +// monitorBlockProduction subscribes to new block events to monitor if blocks are being produced +// at least every four seconds +func monitorBlockProduction(ctx context.Context, conf config.Config) error { + rpcClient, err := rpchttp.New(conf.RPCs.ZetaCoreRPC, "/websocket") + if err != nil { + return fmt.Errorf("new zetacore rpc: %w", err) + } + + err = rpcClient.WSEvents.Start() + if err != nil { + return fmt.Errorf("start ws events: %w", err) + } + blockEventChan, err := rpcClient.WSEvents.Subscribe(ctx, "", "tm.event='NewBlock'") + if err != nil { + return fmt.Errorf("subscribe: %w", err) + } + latestNewBlockEvent := cometbft_types.EventDataNewBlock{} + for { + select { + case event := <-blockEventChan: + newBlockEvent, ok := event.Data.(cometbft_types.EventDataNewBlock) + if !ok { + return fmt.Errorf("expecting new block event, got %T", event.Data) + } + latestNewBlockEvent = newBlockEvent + case <-time.After(4 * time.Second): + return fmt.Errorf("timed out waiting for new block (last block %d)", latestNewBlockEvent.Block.Height) + case <-ctx.Done(): + return nil + } + } +}