e2e: Cleanup test labeling

Previously test labels were applied via free-form strings and didn't provide a way to select only a single chain. Now there are constants defined for each chain to ensure consistent labeling and there is a separate constant for each chain. Tests also previously had a test-specific label applied which can be betteraccomplished with the `--focus-file` argument so long as major tests exist in separate files.
ava-labs · Sep 5, 2023 · fd8f589 · fd8f589
1 parent c3e9c83
commit fd8f589
Show file tree

Hide file tree

Showing 12 changed files with 63 additions and 68 deletions.
diff --git a/.github/workflows/test.e2e.persistent.yml b/.github/workflows/test.e2e.persistent.yml
@@ -28,7 +28,7 @@ jobs:
         run: ./scripts/build.sh -r
       - name: Run e2e tests with persistent network
         shell: bash
-        run: E2E_SERIAL=1 ./scripts/tests.e2e.persistent.sh ./build/avalanchego
+        run: E2E_SERIAL=1 ./scripts/tests.e2e.persistent.sh
       - name: Upload testnet network dir
         uses: actions/upload-artifact@v3
         if: always()

diff --git a/.github/workflows/test.e2e.yml b/.github/workflows/test.e2e.yml
@@ -28,7 +28,7 @@ jobs:
         run: ./scripts/build.sh -r
       - name: Run e2e tests
         shell: bash
-        run: E2E_SERIAL=1 ./scripts/tests.e2e.sh ./build/avalanchego
+        run: E2E_SERIAL=1 ./scripts/tests.e2e.sh
       - name: Upload testnet network dir
         uses: actions/upload-artifact@v3
         if: always()

diff --git a/scripts/tests.e2e.persistent.sh b/scripts/tests.e2e.persistent.sh
@@ -9,16 +9,17 @@ set -euo pipefail
 
 # e.g.,
 # ./scripts/build.sh
-# ./scripts/tests.e2e.persistent_network.sh ./build/avalanchego
+# ./scripts/tests.e2e.persistent.sh --ginkgo.label-filter=x               # All arguments are supplied to ginkgo
+# E2E_SERIAL=1 ./scripts/tests.e2e.sh                                     # Run tests serially
+# AVALANCHEGO_PATH=./build/avalanchego ./scripts/tests.e2e.persistent.sh  # Customization of avalanchego path
 if ! [[ "$0" =~ scripts/tests.e2e.persistent.sh ]]; then
   echo "must be run from repository root"
   exit 255
 fi
 
-AVALANCHEGO_PATH="${1-${AVALANCHEGO_PATH:-}}"
+AVALANCHEGO_PATH="${AVALANCHEGO_PATH:-./build/avalanchego}"
 if [[ -z "${AVALANCHEGO_PATH}" ]]; then
-  echo "Missing AVALANCHEGO_PATH argument!"
-  echo "Usage: ${0} [AVALANCHEGO_PATH]" >>/dev/stderr
+  echo "Empty AVALANCHEGO_PATH env var!"
   exit 255
 fi
 # Ensure an absolute path to avoid dependency on the working directory
@@ -55,6 +56,10 @@ else
 fi
 
 print_separator
-# Setting E2E_USE_PERSISTENT_NETWORK configures tests.e2e.sh to use
-# the persistent network identified by TESTNETCTL_NETWORK_DIR.
-E2E_USE_PERSISTENT_NETWORK=1 ./scripts/tests.e2e.sh
+# - Setting E2E_USE_PERSISTENT_NETWORK configures tests.e2e.sh to use
+#   the persistent network identified by TESTNETCTL_NETWORK_DIR.
+# - Only a single test is required to validate that a persistent
+#   network can be used by an e2e test suite run. Executing more tests
+#   would be duplicative of the testing performed against an ephemeral
+#   test network.
+E2E_USE_PERSISTENT_NETWORK=1 ./scripts/tests.e2e.sh --ginkgo.focus-file=permissionless_subnets.go
diff --git a/scripts/tests.e2e.sh b/scripts/tests.e2e.sh
@@ -3,9 +3,10 @@
 set -euo pipefail
 
 # e.g.,
-# ./scripts/build.sh
-# ./scripts/tests.e2e.sh ./build/avalanchego
-# E2E_SERIAL=1 ./scripts/tests.e2e.sh ./build/avalanchego
+# ./scripts/tests.e2e.sh
+# ./scripts/tests.e2e.sh --ginkgo.label-filter=x               # All arguments are supplied to ginkgo
+# E2E_SERIAL=1 ./scripts/tests.e2e.sh                          # Run tests serially
+# AVALANCHEGO_PATH=./build/avalanchego ./scripts/tests.e2e.sh  # Customization of avalanchego path
 if ! [[ "$0" =~ scripts/tests.e2e.sh ]]; then
   echo "must be run from repository root"
   exit 255
@@ -32,10 +33,9 @@ if [[ -n "${E2E_USE_PERSISTENT_NETWORK}" && -n "${TESTNETCTL_NETWORK_DIR}" ]]; t
   echo "running e2e tests against a persistent network configured at ${TESTNETCTL_NETWORK_DIR}"
   E2E_ARGS="--use-persistent-network"
 else
-  AVALANCHEGO_PATH="${1-${AVALANCHEGO_PATH:-}}"
+  AVALANCHEGO_PATH="${AVALANCHEGO_PATH:-./build/avalanchego}"
   if [[ -z "${AVALANCHEGO_PATH}" ]]; then
-    echo "Missing AVALANCHEGO_PATH argument!"
-    echo "Usage: ${0} [AVALANCHEGO_PATH]" >>/dev/stderr
+    echo "Empty AVALANCHEGO_PATH env var!"
     exit 255
   fi
   echo "running e2e tests against an ephemeral local cluster deployed with ${AVALANCHEGO_PATH}"
@@ -65,12 +65,4 @@ fi
 
 #################################
 # - Execute in random order to identify unwanted dependency
-ginkgo ${GINKGO_ARGS} -v --randomize-all ./tests/e2e/e2e.test -- ${E2E_ARGS} \
-&& EXIT_CODE=$? || EXIT_CODE=$?
-
-if [[ ${EXIT_CODE} -gt 0 ]]; then
-  echo "FAILURE with exit code ${EXIT_CODE}"
-  exit ${EXIT_CODE}
-else
-  echo "ALL SUCCESS!"
-fi
+ginkgo -p -v --randomize-all ./tests/e2e/e2e.test -- ${E2E_ARGS} "${@}"
diff --git a/tests/e2e/README.md b/tests/e2e/README.md
@@ -16,6 +16,25 @@ ACK_GINKGO_RC=true ginkgo build ./tests/e2e
 
 See [`tests.e2e.sh`](../../scripts/tests.e2e.sh) for an example.
 
+### Filtering test execution with labels
+
+In cases where a change can be verified against only a subset of
+tests, it is possible to filter the tests that will be executed by the
+declarative labels that have been applied to them. Available labels
+are defined as constants in [`describe.go`](./describe.go) with names
+of the form `*Label`. The following example runs only those tests that
+primarily target the X-Chain:
+
+
+```bash
+./tests/e2e/e2e.test \
+--avalanchego-path=./build/avalanchego \
+--ginkgo.label-filter=x
+```
+
+The ginkgo docs provide further detail on [how to compose label
+queries](https://onsi.github.io/ginkgo/#spec-labels).
+
 ## Adding tests
 
 Define any flags/configurations in [`e2e.go`](./e2e.go).
@@ -67,7 +86,7 @@ Configure testnetctl to target this network by default with one of the following
 # Start a new test run using the persistent network
 ginkgo -v ./tests/e2e -- \
     --avalanchego-path=/path/to/avalanchego \
-    --ginkgo-focus-file=[name of file containing test] \
+    --ginkgo.focus-file=[name of file containing test] \
     --use-persistent-network \
     --network-dir=/path/to/network
 

diff --git a/tests/e2e/banff/suites.go b/tests/e2e/banff/suites.go
@@ -21,12 +21,6 @@ import (
 
 var _ = ginkgo.Describe("[Banff]", func() {
 	ginkgo.It("can send custom assets X->P and P->X",
-		// use this for filtering tests by labels
-		// ref. https://onsi.github.io/ginkgo/#spec-labels
-		ginkgo.Label(
-			"xp",
-			"banff",
-		),
 		func() {
 			keychain := e2e.Env.NewKeychain(1)
 			wallet := e2e.Env.NewWallet(keychain)

diff --git a/tests/e2e/describe.go b/tests/e2e/describe.go
@@ -7,26 +7,35 @@ import (
 	ginkgo "github.com/onsi/ginkgo/v2"
 )
 
+const (
+	// Labels for test filtering
+	// For usage in ginkgo invocation, see: https://onsi.github.io/ginkgo/#spec-labels
+	XChainLabel     = "x"
+	PChainLabel     = "p"
+	CChainLabel     = "c"
+	UsesCChainLabel = "uses-c"
+)
+
 // DescribeXChain annotates the tests for X-Chain.
-// Can run with any type of cluster (e.g., local, fuji, mainnet).
-func DescribeXChain(text string, body func()) bool {
-	return ginkgo.Describe("[X-Chain] "+text, body)
+func DescribeXChain(text string, args ...interface{}) bool {
+	args = append(args, ginkgo.Label(XChainLabel))
+	return ginkgo.Describe("[X-Chain] "+text, args...)
 }
 
 // DescribeXChainSerial annotates serial tests for X-Chain.
-// Can run with any type of cluster (e.g., local, fuji, mainnet).
-func DescribeXChainSerial(text string, body func()) bool {
-	return ginkgo.Describe("[X-Chain] "+text, ginkgo.Serial, body)
+func DescribeXChainSerial(text string, args ...interface{}) bool {
+	args = append(args, ginkgo.Serial)
+	return DescribeXChain(text, args...)
 }
 
 // DescribePChain annotates the tests for P-Chain.
-// Can run with any type of cluster (e.g., local, fuji, mainnet).
-func DescribePChain(text string, body func()) bool {
-	return ginkgo.Describe("[P-Chain] "+text, body)
+func DescribePChain(text string, args ...interface{}) bool {
+	args = append(args, ginkgo.Label(PChainLabel))
+	return ginkgo.Describe("[P-Chain] "+text, args...)
 }
 
 // DescribeCChain annotates the tests for C-Chain.
-// Can run with any type of cluster (e.g., local, fuji, mainnet).
-func DescribeCChain(text string, body func()) bool {
-	return ginkgo.Describe("[C-Chain] "+text, body)
+func DescribeCChain(text string, args ...interface{}) bool {
+	args = append(args, ginkgo.Label(CChainLabel))
+	return ginkgo.Describe("[C-Chain] "+text, args...)
 }
diff --git a/tests/e2e/p/permissionless_subnets.go b/tests/e2e/p/permissionless_subnets.go
@@ -28,12 +28,6 @@ import (
 
 var _ = e2e.DescribePChain("[Permissionless Subnets]", func() {
 	ginkgo.It("subnets operations",
-		// use this for filtering tests by labels
-		// ref. https://onsi.github.io/ginkgo/#spec-labels
-		ginkgo.Label(
-			"xp",
-			"permissionless-subnets",
-		),
 		func() {
 			keychain := e2e.Env.NewKeychain(1)
 			baseWallet := e2e.Env.NewWallet(keychain)

diff --git a/tests/e2e/p/workflow.go b/tests/e2e/p/workflow.go
@@ -34,13 +34,6 @@ import (
 
 var _ = e2e.DescribePChain("[Workflow]", func() {
 	ginkgo.It("P-chain main operations",
-		// use this for filtering tests by labels
-		// ref. https://onsi.github.io/ginkgo/#spec-labels
-		ginkgo.Label(
-			"xp",
-			"workflow",
-		),
-		ginkgo.FlakeAttempts(2),
 		func() {
 			nodeURI := e2e.Env.GetRandomNodeURI()
 			keychain := e2e.Env.NewKeychain(2)

diff --git a/tests/e2e/static-handlers/suites.go b/tests/e2e/static-handlers/suites.go
@@ -27,11 +27,6 @@ import (
 
 var _ = ginkgo.Describe("[StaticHandlers]", func() {
 	ginkgo.It("can make calls to avm static api",
-		// use this for filtering tests by labels
-		// ref. https://onsi.github.io/ginkgo/#spec-labels
-		ginkgo.Label(
-			"static-handlers",
-		),
 		func() {
 			addrMap := map[string]string{}
 			for _, addrStr := range []string{

diff --git a/tests/e2e/x/interchain_workflow.go b/tests/e2e/x/interchain_workflow.go
@@ -23,7 +23,7 @@ import (
 	"github.com/ava-labs/avalanchego/wallet/subnet/primary/common"
 )
 
-var _ = e2e.DescribeXChain("[Interchain Workflow]", func() {
+var _ = e2e.DescribeXChain("[Interchain Workflow]", ginkgo.Label(e2e.UsesCChainLabel), func() {
 	require := require.New(ginkgo.GinkgoT())
 
 	const transferAmount = 10 * units.Avax

diff --git a/tests/e2e/x/transfer/virtuous.go b/tests/e2e/x/transfer/virtuous.go
@@ -37,12 +37,6 @@ const (
 // cannot reliably be run in parallel.
 var _ = e2e.DescribeXChainSerial("[Virtuous Transfer Tx AVAX]", func() {
 	ginkgo.It("can issue a virtuous transfer tx for AVAX asset",
-		// use this for filtering tests by labels
-		// ref. https://onsi.github.io/ginkgo/#spec-labels
-		ginkgo.Label(
-			"x",
-			"virtuous-transfer-tx-avax",
-		),
 		func() {
 			rpcEps := e2e.Env.URIs