Enable Parallel Transfer Test Suite

.github/workflows/e2e-test-workflow-call.yml

@@ -6,6 +6,11 @@ on:
         required: false
         type: string
         default: '' # empty string means run all tests
+      temp-run-full-suite:
+        description: 'This flag exists to run a hard coded set of tests and will be phased out'
+        required: false
+        type: boolean
+        default: false
       test:
         description: 'test name to run as standalone'
         required: false
@@ -253,3 +258,49 @@ jobs:
           name: '${{ matrix.entrypoint }}-${{ matrix.test }}'
           path: e2e/diagnostics
           retention-days: 5
+
+  e2e-test-suites:
+    # temporary flag. eventually this field will not exist and this will be the default.
+    if: ${{ inputs.temp-run-full-suite }}
+    runs-on: ubuntu-latest
+    needs:
+      - build-test-matrix
+      - docker-build
+      - docker-build-wasm
+    env:
+      CHAIN_IMAGE: '${{ inputs.chain-image }}'
+      CHAIN_A_TAG: '${{ inputs.chain-a-tag }}'
+      CHAIN_B_TAG: '${{ inputs.chain-b-tag }}'
+      RELAYER_IMAGE: '${{ inputs.relayer-image }}'
+      RELAYER_TAG: '${{ inputs.relayer-tag }}'
+      RELAYER_ID: '${{ inputs.relayer-type }}'
+      CHAIN_BINARY: '${{ inputs.chain-binary }}'
+      CHAIN_UPGRADE_TAG: '${{ inputs.chain-upgrade-tag }}'
+      CHAIN_UPGRADE_PLAN: '${{ inputs.upgrade-plan-name }}'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+            # for now we explicitly specify this test suite.
+          - entrypoint: TestTransferTestSuite
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: cosmos/ibc-go
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+          cache-dependency-path: 'e2e/go.sum'
+      - name: Run e2e Test
+        id: e2e_test
+        run: |
+          cd e2e
+          make e2e-suite entrypoint=${{ matrix.entrypoint }}
+      - name: Upload Diagnostics
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() && inputs.upload-logs }}
+        continue-on-error: true
+        with:
+          name: '${{ matrix.entrypoint }}-${{ matrix.test }}'
+          path: e2e/diagnostics
+          retention-days: 5

.github/workflows/e2e.yaml

@@ -63,4 +63,6 @@ jobs:
       chain-b-tag: '${{ needs.determine-image-tag.outputs.simd-tag }}'
       chain-binary: 'simd'
       # on regular PRs we won't run upgrade tests.
-      test-exclusions: 'TestUpgradeTestSuite,TestGrandpaTestSuite,TestIBCWasmUpgradeTestSuite'
+      # NOTE: we are exluding TestTransferTestSuite as we run this full suite instead of each individual test.
+      test-exclusions: 'TestUpgradeTestSuite,TestGrandpaTestSuite,TestIBCWasmUpgradeTestSuite,TestTransferTestSuite'
+      temp-run-full-suite: true

e2e/Makefile

@@ -14,7 +14,7 @@ e2e-test: init cleanup-ibc-test-containers
 	./scripts/run-e2e.sh $(test) $(entrypoint)
 
 e2e-suite: init cleanup-ibc-test-containers
-	RUN_SUITE="true" ./scripts/run-e2e.sh $(test) $(entrypoint)
+	RUN_SUITE="true" ./scripts/run-e2e.sh "" $(entrypoint)
 
 compatibility-tests:
 	./scripts/run-compatibility-tests.sh $(release_branch)

e2e/scripts/run-e2e.sh

@@ -2,8 +2,8 @@
 
 set -eo pipefail
 
-TEST="${1}"
-ENTRY_POINT="${2:-}"
+TEST="${1:-}"
+export ENTRY_POINT="${2:-}"
 
 function _verify_jq() {
       if ! command -v jq > /dev/null ; then
@@ -19,7 +19,7 @@ function _verify_fzf() {
       fi
 }
 
-function _verify_dependencies() {
+function _verify_test_dependencies() {
     if [ -z "${TEST}" ]; then
         # fzf is only required if we are not explicitly specifying a test.
         _verify_fzf
@@ -28,6 +28,14 @@ function _verify_dependencies() {
     _verify_jq
 }
 
+function _verify_suite_dependencies() {
+    if [ -z "${ENTRY_POINT}" ]; then
+        # fzf is only required if we are not explicitly specifying an entrypoint.
+        _verify_fzf
+    fi
+    # jq is always required to determine the entrypoint of the test.
+    _verify_jq
+}
 
 # _select_test_config lets you dynamically select a test config for the specific test.
 function _select_test_config() {
@@ -51,7 +59,7 @@ function _get_test(){
 
 # run_test runs a single E2E test.
 function run_test() {
-  # if the dev configs directory is present, enable fzf completion to select a test config file to use.
+  _verify_test_dependencies
 
   # if test is set, that is used directly, otherwise the test can be interactively provided if fzf is installed.
   TEST="$(_get_test ${TEST})"
@@ -78,23 +86,23 @@ function run_test() {
 
 # run_suite runs a full E2E test suite.
 function run_suite() {
+  _verify_suite_dependencies
   # if jq is installed, we can automatically determine the test entrypoint.
-  if command -v jq > /dev/null; then
+  if [ -z "${ENTRY_POINT}" ]; then
      cd ..
      ENTRY_POINT="$(go run -mod=readonly cmd/build_test_matrix/main.go | jq  -r '.include[] | .entrypoint' | uniq | fzf)"
      cd - > /dev/null
   fi
 
   # find the name of the file that has this test in it.
-  test_file="$(grep --recursive --files-with-matches './tests' -e "${ENTRY_POINT}")"
+  test_file="$(grep --recursive --files-with-matches './tests' -e "${ENTRY_POINT}(")"
   test_dir="$(dirname $test_file)"
 
-  # TODO: add the -p flag to run tests in parallel
-  go test -v "${test_dir}" --run ${ENTRY_POINT} -timeout 30m
+  go test -v "${test_dir}" --run ^${ENTRY_POINT}$ -timeout 30m -p 10
 }
 
-_verify_dependencies
 
+# if the dev configs directory is present, enable fzf completion to select a test config file to use.
 if [[ -d "dev-configs"  ]]; then
   export E2E_CONFIG_PATH="$(pwd)/dev-configs/$(_select_test_config)"
   echo "Using configuration file at ${E2E_CONFIG_PATH}"

e2e/tests/core/02-client/client_test.go

@@ -65,6 +65,9 @@ func (s *ClientTestSuite) TestScheduleIBCUpgrade_Succeeds() {
 	t := s.T()
 	ctx := context.TODO()
 
+	testName := t.Name()
+	s.SetupDefaultPath(testName)
+
 	chainA, chainB := s.GetChains()
 	chainAWallet := s.CreateUserOnChainA(ctx, testvalues.StartingTokenAmount)
 
@@ -169,14 +172,15 @@ func (s *ClientTestSuite) TestClientUpdateProposal_Succeeds() {
 		badTrustingPeriod = time.Second * 10
 	)
 
-	relayer := s.GetRelayer()
+	testName := t.Name()
+	relayer := s.SetupDefaultPath(testName)
 
 	t.Run("create substitute client with correct trusting period", func(t *testing.T) {
 		// TODO: update when client identifier created is accessible
 		// currently assumes first client is 07-tendermint-0
 		substituteClientID = clienttypes.FormatClientIdentifier(ibcexported.Tendermint, 0)
 
-		pathName = s.GetPaths()[0]
+		pathName = s.GetPaths(testName)[0]
 	})
 
 	chainA, chainB := s.GetChains()
@@ -249,14 +253,15 @@ func (s *ClientTestSuite) TestRecoverClient_Succeeds() {
 		badTrustingPeriod = time.Second * 10
 	)
 
-	relayer := s.GetRelayer()
+	testName := t.Name()
+	relayer := s.SetupDefaultPath(testName)
 
 	t.Run("create substitute client with correct trusting period", func(t *testing.T) {
 		// TODO: update when client identifier created is accessible
 		// currently assumes first client is 07-tendermint-0
 		substituteClientID = clienttypes.FormatClientIdentifier(ibcexported.Tendermint, 0)
 
-		pathName = s.GetPaths()[0]
+		pathName = s.GetPaths(testName)[0]
 	})
 
 	chainA, chainB := s.GetChains()
@@ -334,13 +339,15 @@ func (s *ClientTestSuite) TestClient_Update_Misbehaviour() {
 		err             error
 	)
 
-	relayer := s.GetRelayer()
+	testName := t.Name()
+	relayer := s.SetupDefaultPath(testName)
+
 	chainA, chainB := s.GetChains()
 
 	s.Require().NoError(test.WaitForBlocks(ctx, 10, chainA, chainB))
 
 	t.Run("update clients", func(t *testing.T) {
-		err := relayer.UpdateClients(ctx, s.GetRelayerExecReporter(), s.GetPaths()[0])
+		err := relayer.UpdateClients(ctx, s.GetRelayerExecReporter(), s.GetPaths(testName)[0])
 		s.Require().NoError(err)
 
 		clientState, err = query.ClientState(ctx, chainA, ibctesting.FirstClientID)
@@ -356,7 +363,7 @@ func (s *ClientTestSuite) TestClient_Update_Misbehaviour() {
 	})
 
 	t.Run("update clients", func(t *testing.T) {
-		err := relayer.UpdateClients(ctx, s.GetRelayerExecReporter(), s.GetPaths()[0])
+		err := relayer.UpdateClients(ctx, s.GetRelayerExecReporter(), s.GetPaths(testName)[0])
 		s.Require().NoError(err)
 
 		clientState, err = query.ClientState(ctx, chainA, ibctesting.FirstClientID)
@@ -438,6 +445,9 @@ func (s *ClientTestSuite) TestAllowedClientsParam() {
 	t := s.T()
 	ctx := context.TODO()
 
+	testName := t.Name()
+	s.SetupDefaultPath(testName)
+
 	chainA, chainB := s.GetChains()
 	chainAVersion := chainA.Config().Images[0].Version
 	chainAWallet := s.CreateUserOnChainA(ctx, testvalues.StartingTokenAmount)

e2e/tests/core/03-connection/connection_test.go

@@ -33,8 +33,8 @@ type ConnectionTestSuite struct {
 	testsuite.E2ETestSuite
 }
 
-func (s *ConnectionTestSuite) SetupTest() {
-	s.SetupPaths(ibc.DefaultClientOpts(), s.TransferChannelOptions())
+func (s *ConnectionTestSuite) SetupConnectionTestPath(testName string) (ibc.Relayer, ibc.ChannelOutput) {
+	return s.SetupPaths(ibc.DefaultClientOpts(), s.TransferChannelOptions(), testName), s.GetChainAChannelForTest(testName)
 }
 
 // QueryMaxExpectedTimePerBlockParam queries the on-chain max expected time per block param for 03-connection
@@ -65,7 +65,8 @@ func (s *ConnectionTestSuite) TestMaxExpectedTimePerBlockParam() {
 	t := s.T()
 	ctx := context.TODO()
 
-	relayer, channelA := s.GetRelayer(), s.GetChainAChannel()
+	testName := t.Name()
+	relayer, channelA := s.SetupConnectionTestPath(testName)
 
 	chainA, chainB := s.GetChains()
 	chainAVersion := chainA.Config().Images[0].Version
@@ -126,7 +127,7 @@ func (s *ConnectionTestSuite) TestMaxExpectedTimePerBlockParam() {
 		})
 
 		t.Run("start relayer", func(t *testing.T) {
-			s.StartRelayer(relayer)
+			s.StartRelayer(relayer, testName)
 		})
 
 		t.Run("packets are relayed", func(t *testing.T) {

e2e/tests/interchain_accounts/base_test.go

@@ -62,9 +62,9 @@ func (s *InterchainAccountsTestSuite) testMsgSendTxSuccessfulTransfer(order chan
 	t := s.T()
 	ctx := context.TODO()
 
-	// setup relayers and connection-0 between two chains
-	// channel-0 is a transfer channel but it will not be used in this test case
-	relayer := s.GetRelayer()
+	testName := t.Name()
+	relayer := s.SetupDefaultPath(testName)
+
 	chainA, chainB := s.GetChains()
 
 	// setup 2 accounts: controller account on chain A, a second chain B account.
@@ -84,7 +84,7 @@ func (s *InterchainAccountsTestSuite) testMsgSendTxSuccessfulTransfer(order chan
 	})
 
 	t.Run("start relayer", func(t *testing.T) {
-		s.StartRelayer(relayer)
+		s.StartRelayer(relayer, testName)
 	})
 
 	t.Run("verify interchain account", func(t *testing.T) {
@@ -161,7 +161,9 @@ func (s *InterchainAccountsTestSuite) TestMsgSendTx_FailedTransfer_InsufficientF
 	t := s.T()
 	ctx := context.TODO()
 
-	relayer := s.GetRelayer()
+	testName := t.Name()
+	relayer := s.SetupDefaultPath(testName)
+
 	chainA, chainB := s.GetChains()
 
 	// setup 2 accounts: controller account on chain A, a second chain B account.
@@ -181,7 +183,7 @@ func (s *InterchainAccountsTestSuite) TestMsgSendTx_FailedTransfer_InsufficientF
 	})
 
 	t.Run("start relayer", func(t *testing.T) {
-		s.StartRelayer(relayer)
+		s.StartRelayer(relayer, testName)
 	})
 
 	t.Run("verify interchain account", func(t *testing.T) {
@@ -249,7 +251,9 @@ func (s *InterchainAccountsTestSuite) TestMsgSendTx_SuccessfulTransfer_AfterReop
 	t := s.T()
 	ctx := context.TODO()
 
-	relayer := s.GetRelayer()
+	testName := t.Name()
+	relayer := s.SetupDefaultPath(testName)
+
 	chainA, chainB := s.GetChains()
 
 	// setup 2 accounts: controller account on chain A, a second chain B account.
@@ -277,7 +281,7 @@ func (s *InterchainAccountsTestSuite) TestMsgSendTx_SuccessfulTransfer_AfterReop
 	})
 
 	t.Run("start relayer", func(t *testing.T) {
-		s.StartRelayer(relayer)
+		s.StartRelayer(relayer, testName)
 	})
 
 	t.Run("verify interchain account", func(t *testing.T) {
@@ -342,7 +346,7 @@ func (s *InterchainAccountsTestSuite) TestMsgSendTx_SuccessfulTransfer_AfterReop
 	})
 
 	t.Run("start relayer", func(t *testing.T) {
-		s.StartRelayer(relayer)
+		s.StartRelayer(relayer, testName)
 	})
 
 	t.Run("verify channel is closed due to timeout on ordered channel", func(t *testing.T) {
@@ -436,9 +440,9 @@ func (s *InterchainAccountsTestSuite) testMsgSendTxSuccessfulGovProposal(order c
 	t := s.T()
 	ctx := context.TODO()
 
-	// setup relayers and connection-0 between two chains
-	// channel-0 is a transfer channel but it will not be used in this test case
-	relayer := s.GetRelayer()
+	testName := t.Name()
+	relayer := s.SetupDefaultPath(testName)
+
 	chainA, chainB := s.GetChains()
 
 	// setup 2 accounts: controller account on chain A, a second chain B account.
@@ -457,7 +461,7 @@ func (s *InterchainAccountsTestSuite) testMsgSendTxSuccessfulGovProposal(order c
 	})
 
 	t.Run("start relayer", func(t *testing.T) {
-		s.StartRelayer(relayer)
+		s.StartRelayer(relayer, testName)
 	})
 
 	t.Run("verify interchain account", func(t *testing.T) {

e2e/tests/interchain_accounts/gov_test.go

@@ -40,7 +40,9 @@ func (s *InterchainAccountsGovTestSuite) TestInterchainAccountsGovIntegration()
 	t := s.T()
 	ctx := context.TODO()
 
-	relayer := s.GetRelayer()
+	testName := t.Name()
+	relayer := s.SetupDefaultPath(testName)
+
 	chainA, chainB := s.GetChains()
 	controllerAccount := s.CreateUserOnChainA(ctx, testvalues.StartingTokenAmount)
 
@@ -58,7 +60,7 @@ func (s *InterchainAccountsGovTestSuite) TestInterchainAccountsGovIntegration()
 	})
 
 	t.Run("start relayer", func(t *testing.T) {
-		s.StartRelayer(relayer)
+		s.StartRelayer(relayer, testName)
 	})
 
 	s.Require().NoError(test.WaitForBlocks(ctx, 10, chainA, chainB))

e2e/tests/interchain_accounts/groups_test.go

@@ -86,7 +86,9 @@ func (s *InterchainAccountsGroupsTestSuite) TestInterchainAccountsGroupsIntegrat
 		err               error
 	)
 
-	relayer := s.GetRelayer()
+	testName := t.Name()
+	relayer := s.SetupDefaultPath(testName)
+
 	chainA, chainB := s.GetChains()
 
 	chainAWallet := s.CreateUserOnChainA(ctx, testvalues.StartingTokenAmount)
@@ -135,7 +137,7 @@ func (s *InterchainAccountsGroupsTestSuite) TestInterchainAccountsGroupsIntegrat
 	})
 
 	t.Run("start relayer", func(t *testing.T) {
-		s.StartRelayer(relayer)
+		s.StartRelayer(relayer, testName)
 	})
 
 	t.Run("verify interchain account registration success", func(t *testing.T) {