[Draft] Group docker simple flow tests to suite

cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml

@@ -224,8 +224,9 @@ phases:
         -i ${INTEGRATION_TEST_INSTANCE_PROFILE}
         -m ${INTEGRATION_TEST_MAX_EC2_COUNT}
         -p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
-        -r ${TESTS}
+        -r 'Test'
         -v 4
+        --select ${TESTS}
         --skip ${SKIPPED_TESTS}
         --bundles-override=${BUNDLES_OVERRIDE}
         --cleanup-resources=true

cmd/integration_test/cmd/run.go

@@ -20,6 +20,7 @@ const (
 	regexFlagName              = "regex"
 	maxInstancesFlagName       = "max-instances"
 	maxConcurrentTestsFlagName = "max-concurrent-tests"
+	selectFlagName             = "select"
 	skipFlagName               = "skip"
 	bundlesOverrideFlagName    = "bundles-override"
 	cleanupResourcesFlagName   = "cleanup-resources"
@@ -64,6 +65,7 @@ func init() {
 	runE2ECmd.Flags().StringP(regexFlagName, "r", "", "Run only those tests and examples matching the regular expression. Equivalent to go test -run")
 	runE2ECmd.Flags().IntP(maxInstancesFlagName, "m", 1, "Run tests in parallel on same instance within the max EC2 instance count")
 	runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "p", 1, "Maximum number of parallel tests that can be run at a time")
+	runE2ECmd.Flags().StringSlice(selectFlagName, nil, "List of tests to select")
 	runE2ECmd.Flags().StringSlice(skipFlagName, nil, "List of tests to skip")
 	runE2ECmd.Flags().Bool(bundlesOverrideFlagName, false, "Flag to indicate if the tests should run with a bundles override")
 	runE2ECmd.Flags().Bool(cleanupResourcesFlagName, false, "Flag to indicate if test resources should be cleaned up automatically as tests complete")
@@ -86,6 +88,7 @@ func runE2E(ctx context.Context) error {
 	testRegex := viper.GetString(regexFlagName)
 	maxInstances := viper.GetInt(maxInstancesFlagName)
 	maxConcurrentTests := viper.GetInt(maxConcurrentTestsFlagName)
+	testsToSelect := viper.GetStringSlice(selectFlagName)
 	testsToSkip := viper.GetStringSlice(skipFlagName)
 	bundlesOverride := viper.GetBool(bundlesOverrideFlagName)
 	cleanupResources := viper.GetBool(cleanupResourcesFlagName)
@@ -100,6 +103,7 @@ func runE2E(ctx context.Context) error {
 		StorageBucket:          storageBucket,
 		JobId:                  jobId,
 		Regex:                  testRegex,
+		TestsToSelect:          testsToSelect,
 		TestsToSkip:            testsToSkip,
 		BundlesOverride:        bundlesOverride,
 		CleanupResources:       cleanupResources,

internal/test/e2e/artifacts.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"github.com/go-logr/logr"
 
@@ -58,7 +59,8 @@ func (e *E2ESession) uploadJUnitReportFromInstance(testName string) {
 func (e *E2ESession) downloadJUnitReportToLocalDisk(testName, destinationFolder string) {
 	junitFile := "junit-testing.xml"
 	key := filepath.Join(e.generatedArtifactsPath(), testName, junitFile)
-	dst := filepath.Join(destinationFolder, fmt.Sprintf("junit-testing-%s.xml", testName))
+	// Subtest contains '/' in the name.
+	dst := filepath.Join(destinationFolder, fmt.Sprintf("junit-testing-%s.xml", strings.Replace(testName, "/", "-", -1)))
 
 	e.logger.V(1).Info("Downloading JUnit report to disk", "dst", dst)
 	if err := s3.DownloadToDisk(e.session, key, e.storageBucket, dst); err != nil {

internal/test/e2e/list.go

@@ -2,16 +2,21 @@ package e2e
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"fmt"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/aws/eks-anywhere/pkg/executables"
 	"github.com/aws/eks-anywhere/pkg/types"
+	e2etest "github.com/aws/eks-anywhere/test/e2e"
 )
 
-func listTests(regex string, testsToSkip []string) (tests, skippedTests []string, err error) {
+// regex will filter out the tests that match the regex, but it does't see subtests.
+// TestsToSelect and testsToSkip will filter out all test cases including subtests.
+func listTests(regex string, testsToSelect []string, testsToSkip []string) (tests, skippedTests []string, err error) {
 	e := executables.NewExecutable(filepath.Join("bin", e2eBinary))
 	ctx := context.Background()
 	testResponse, err := e.Execute(ctx, "-test.list", regex)
@@ -21,16 +26,26 @@ func listTests(regex string, testsToSkip []string) (tests, skippedTests []string
 
 	skipLookup := types.SliceToLookup(testsToSkip)
 	scanner := bufio.NewScanner(&testResponse)
-	for scanner.Scan() {
-		line := scanner.Text()
-		if skipLookup.IsPresent(line) {
-			skippedTests = append(skippedTests, line)
+	rawTestList := processTestListOutput(testResponse)
+
+	for _, t := range rawTestList {
+		selected := len(testsToSelect) == 0
+		for _, s := range testsToSelect {
+			re := regexp.MustCompile(s)
+			if re.MatchString(t) {
+				selected = true
+				break
+			}
+		}
+		if !selected {
 			continue
 		}
-
-		if strings.HasPrefix(line, "Test") {
-			tests = append(tests, line)
+		if skipLookup.IsPresent(t) {
+			skippedTests = append(skippedTests, t)
+			continue
 		}
+
+		tests = append(tests, t)
 	}
 
 	if err := scanner.Err(); err != nil {
@@ -39,3 +54,31 @@ func listTests(regex string, testsToSkip []string) (tests, skippedTests []string
 
 	return tests, skippedTests, nil
 }
+
+// parse the output of -test.list, and add subtests.
+func processTestListOutput(b bytes.Buffer) (tests []string) {
+	s := bufio.NewScanner(&b)
+	for s.Scan() {
+		line := s.Text()
+		if !strings.HasPrefix(line, "Test") {
+			continue
+		}
+
+		if !strings.HasSuffix(line, "Suite") {
+			tests = append(tests, line)
+			continue
+		}
+
+		if strings.HasSuffix(line, "Suite") {
+			for k, s := range e2etest.Suites {
+				if strings.HasSuffix(line, k) {
+					for _, st := range s {
+						tests = append(tests, line+"/"+st.GetName())
+					}
+					break
+				}
+			}
+		}
+	}
+	return tests
+}

internal/test/e2e/run.go

@@ -45,6 +45,7 @@ type ParallelRunConf struct {
 	StorageBucket          string
 	JobId                  string
 	Regex                  string
+	TestsToSelect          []string
 	TestsToSkip            []string
 	BundlesOverride        bool
 	CleanupResources       bool
@@ -65,7 +66,7 @@ type (
 
 // RunTestsInParallel Run Tests in parallel by spawning multiple admin machines.
 func RunTestsInParallel(conf ParallelRunConf) error {
-	testsList, skippedTests, err := listTests(conf.Regex, conf.TestsToSkip)
+	testsList, skippedTests, err := listTests(conf.Regex, conf.TestsToSelect, conf.TestsToSkip)
 	if err != nil {
 		return err
 	}
@@ -282,15 +283,16 @@ func RunTests(conf instanceRunConf, inventoryCatalogue map[string]*hardwareCatal
 }
 
 func (e *E2ESession) runTests(regex string) (testCommandResult *testCommandResult, err error) {
-	e.logger.V(1).Info("Running e2e tests", "regex", regex)
 	command := "GOVERSION=go1.16.6 gotestsum --junitfile=junit-testing.xml --raw-command --format=standard-verbose --hide-summary=all --ignore-non-json-output-lines -- test2json -t -p e2e ./bin/e2e.test -test.v"
 
 	if regex != "" {
-		command = fmt.Sprintf("%s -test.run \"^(%s)$\" -test.timeout %s", command, regex, e2eTimeout)
+		command = fmt.Sprintf("%s -test.run \"^%s$\" -test.timeout %s", command, regex, e2eTimeout)
 	}
 
 	command = e.commandWithEnvVars(command)
 
+	e.logger.V(1).Info("Running e2e tests", "regex", regex, "command", command)
+
 	opt := ssm.WithOutputToCloudwatch()
 
 	testCommandResult, err = ssm.RunCommand(

test/e2e/docker_test.go

@@ -621,52 +621,6 @@ func TestDockerKubernetes130RegistryMirrorInsecureSkipVerify(t *testing.T) {
 	runRegistryMirrorConfigFlow(test)
 }
 
-// Simple flow
-func TestDockerKubernetes126SimpleFlow(t *testing.T) {
-	test := framework.NewClusterE2ETest(
-		t,
-		framework.NewDocker(t),
-		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube126)),
-	)
-	runSimpleFlow(test)
-}
-
-func TestDockerKubernetes127SimpleFlow(t *testing.T) {
-	test := framework.NewClusterE2ETest(
-		t,
-		framework.NewDocker(t),
-		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)),
-	)
-	runSimpleFlow(test)
-}
-
-func TestDockerKubernetes128SimpleFlow(t *testing.T) {
-	test := framework.NewClusterE2ETest(
-		t,
-		framework.NewDocker(t),
-		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)),
-	)
-	runSimpleFlow(test)
-}
-
-func TestDockerKubernetes129SimpleFlow(t *testing.T) {
-	test := framework.NewClusterE2ETest(
-		t,
-		framework.NewDocker(t),
-		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
-	)
-	runSimpleFlow(test)
-}
-
-func TestDockerKubernetes130SimpleFlow(t *testing.T) {
-	test := framework.NewClusterE2ETest(
-		t,
-		framework.NewDocker(t),
-		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)),
-	)
-	runSimpleFlow(test)
-}
-
 // Stacked etcd
 func TestDockerKubernetesStackedEtcd(t *testing.T) {
 	test := framework.NewClusterE2ETest(t,
@@ -1480,3 +1434,16 @@ func TestDockerKubernetes130KubeletConfigurationSimpleFlow(t *testing.T) {
 	)
 	runKubeletConfigurationFlow(test)
 }
+
+func TestDockerKubernetesSimpleFlowSuite(t *testing.T) {
+	for _, ts := range Suites[simpleFlowSubtest.GetSuiteSuffix()] {
+		t.Run(ts.GetName(), func(t *testing.T) {
+			test := framework.NewClusterE2ETest(
+				t,
+				framework.NewDocker(t),
+				framework.WithClusterFiller(api.WithKubernetesVersion(ts.(*SimpleFlowSubtest).KubeVersion)),
+			)
+			runSimpleFlow(test)
+		})
+	}
+}

test/e2e/test_suite.go

@@ -0,0 +1,48 @@
+package e2e
+
+import (
+	"strings"
+
+	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+)
+
+var kubeVersions = []v1alpha1.KubernetesVersion{v1alpha1.Kube126, v1alpha1.Kube127, v1alpha1.Kube128, v1alpha1.Kube129, v1alpha1.Kube130}
+
+// Subtest is an interface to represent a test case.
+type Subtest interface {
+	// Return the name of the subtest case.
+	GetName() string
+	// Return the name suffix of the parent test, which includes this subtest. It should be "*Suite".
+	GetSuiteSuffix() string
+}
+
+// Suites contain all test suites. The key is the suite suffix.
+var Suites = map[string][]Subtest{
+	simpleFlowSubtest.GetSuiteSuffix(): {},
+}
+
+// SimpleFlowSubtest is a struct to represent a simple flow test.
+type SimpleFlowSubtest struct {
+	KubeVersion v1alpha1.KubernetesVersion
+}
+
+// GetName returns the subtest name.
+func (st *SimpleFlowSubtest) GetName() string {
+	return strings.ReplaceAll(string(st.KubeVersion), ".", "")
+}
+
+// GetSuiteSuffix returns the Suite suffix.
+func (st *SimpleFlowSubtest) GetSuiteSuffix() string {
+	return "SimpleFlowSuite"
+}
+
+// Make sure the SimpleFlowSubtest implements the Subtest interface.
+var simpleFlowSubtest Subtest = (*SimpleFlowSubtest)(nil)
+
+// Init SimpleFlowSuite.
+func init() {
+	s := simpleFlowSubtest.GetSuiteSuffix()
+	for _, k := range kubeVersions {
+		Suites[s] = append(Suites[s], &SimpleFlowSubtest{KubeVersion: k})
+	}
+}