diff --git a/cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml index 46776a199820..8a25c5827321 100644 --- a/cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/quick-test-eks-a-cli.yml @@ -224,8 +224,9 @@ phases: -i ${INTEGRATION_TEST_INSTANCE_PROFILE} -m ${INTEGRATION_TEST_MAX_EC2_COUNT} -p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT} - -r ${TESTS} + -r 'Test' -v 4 + --select ${TESTS} --skip ${SKIPPED_TESTS} --bundles-override=${BUNDLES_OVERRIDE} --cleanup-resources=true diff --git a/cmd/integration_test/cmd/run.go b/cmd/integration_test/cmd/run.go index fbb4580cc77f..867ad5ce76c4 100644 --- a/cmd/integration_test/cmd/run.go +++ b/cmd/integration_test/cmd/run.go @@ -20,6 +20,7 @@ const ( regexFlagName = "regex" maxInstancesFlagName = "max-instances" maxConcurrentTestsFlagName = "max-concurrent-tests" + selectFlagName = "select" skipFlagName = "skip" bundlesOverrideFlagName = "bundles-override" cleanupResourcesFlagName = "cleanup-resources" @@ -64,6 +65,7 @@ func init() { runE2ECmd.Flags().StringP(regexFlagName, "r", "", "Run only those tests and examples matching the regular expression. Equivalent to go test -run") runE2ECmd.Flags().IntP(maxInstancesFlagName, "m", 1, "Run tests in parallel on same instance within the max EC2 instance count") runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "p", 1, "Maximum number of parallel tests that can be run at a time") + runE2ECmd.Flags().StringSlice(selectFlagName, nil, "List of tests to select") runE2ECmd.Flags().StringSlice(skipFlagName, nil, "List of tests to skip") runE2ECmd.Flags().Bool(bundlesOverrideFlagName, false, "Flag to indicate if the tests should run with a bundles override") runE2ECmd.Flags().Bool(cleanupResourcesFlagName, false, "Flag to indicate if test resources should be cleaned up automatically as tests complete") @@ -86,6 +88,7 @@ func runE2E(ctx context.Context) error { testRegex := viper.GetString(regexFlagName) maxInstances := viper.GetInt(maxInstancesFlagName) maxConcurrentTests := viper.GetInt(maxConcurrentTestsFlagName) + testsToSelect := viper.GetStringSlice(selectFlagName) testsToSkip := viper.GetStringSlice(skipFlagName) bundlesOverride := viper.GetBool(bundlesOverrideFlagName) cleanupResources := viper.GetBool(cleanupResourcesFlagName) @@ -100,6 +103,7 @@ func runE2E(ctx context.Context) error { StorageBucket: storageBucket, JobId: jobId, Regex: testRegex, + TestsToSelect: testsToSelect, TestsToSkip: testsToSkip, BundlesOverride: bundlesOverride, CleanupResources: cleanupResources, diff --git a/internal/test/e2e/artifacts.go b/internal/test/e2e/artifacts.go index 44accbe034d5..b6e511658603 100644 --- a/internal/test/e2e/artifacts.go +++ b/internal/test/e2e/artifacts.go @@ -3,6 +3,7 @@ package e2e import ( "fmt" "path/filepath" + "strings" "github.com/go-logr/logr" @@ -58,7 +59,8 @@ func (e *E2ESession) uploadJUnitReportFromInstance(testName string) { func (e *E2ESession) downloadJUnitReportToLocalDisk(testName, destinationFolder string) { junitFile := "junit-testing.xml" key := filepath.Join(e.generatedArtifactsPath(), testName, junitFile) - dst := filepath.Join(destinationFolder, fmt.Sprintf("junit-testing-%s.xml", testName)) + // Subtest contains '/' in the name. + dst := filepath.Join(destinationFolder, fmt.Sprintf("junit-testing-%s.xml", strings.Replace(testName, "/", "-", -1))) e.logger.V(1).Info("Downloading JUnit report to disk", "dst", dst) if err := s3.DownloadToDisk(e.session, key, e.storageBucket, dst); err != nil { diff --git a/internal/test/e2e/list.go b/internal/test/e2e/list.go index a209c80ce9c3..8ad48f710f40 100644 --- a/internal/test/e2e/list.go +++ b/internal/test/e2e/list.go @@ -2,16 +2,21 @@ package e2e import ( "bufio" + "bytes" "context" "fmt" "path/filepath" + "regexp" "strings" "github.com/aws/eks-anywhere/pkg/executables" "github.com/aws/eks-anywhere/pkg/types" + e2etest "github.com/aws/eks-anywhere/test/e2e" ) -func listTests(regex string, testsToSkip []string) (tests, skippedTests []string, err error) { +// regex will filter out the tests that match the regex, but it does't see subtests. +// TestsToSelect and testsToSkip will filter out all test cases including subtests. +func listTests(regex string, testsToSelect []string, testsToSkip []string) (tests, skippedTests []string, err error) { e := executables.NewExecutable(filepath.Join("bin", e2eBinary)) ctx := context.Background() testResponse, err := e.Execute(ctx, "-test.list", regex) @@ -21,16 +26,26 @@ func listTests(regex string, testsToSkip []string) (tests, skippedTests []string skipLookup := types.SliceToLookup(testsToSkip) scanner := bufio.NewScanner(&testResponse) - for scanner.Scan() { - line := scanner.Text() - if skipLookup.IsPresent(line) { - skippedTests = append(skippedTests, line) + rawTestList := processTestListOutput(testResponse) + + for _, t := range rawTestList { + selected := len(testsToSelect) == 0 + for _, s := range testsToSelect { + re := regexp.MustCompile(s) + if re.MatchString(t) { + selected = true + break + } + } + if !selected { continue } - - if strings.HasPrefix(line, "Test") { - tests = append(tests, line) + if skipLookup.IsPresent(t) { + skippedTests = append(skippedTests, t) + continue } + + tests = append(tests, t) } if err := scanner.Err(); err != nil { @@ -39,3 +54,31 @@ func listTests(regex string, testsToSkip []string) (tests, skippedTests []string return tests, skippedTests, nil } + +// parse the output of -test.list, and add subtests. +func processTestListOutput(b bytes.Buffer) (tests []string) { + s := bufio.NewScanner(&b) + for s.Scan() { + line := s.Text() + if !strings.HasPrefix(line, "Test") { + continue + } + + if !strings.HasSuffix(line, "Suite") { + tests = append(tests, line) + continue + } + + if strings.HasSuffix(line, "Suite") { + for k, s := range e2etest.Suites { + if strings.HasSuffix(line, k) { + for _, st := range s { + tests = append(tests, line+"/"+st.GetName()) + } + break + } + } + } + } + return tests +} diff --git a/internal/test/e2e/run.go b/internal/test/e2e/run.go index 1f7a550e9987..da0904b661bd 100644 --- a/internal/test/e2e/run.go +++ b/internal/test/e2e/run.go @@ -45,6 +45,7 @@ type ParallelRunConf struct { StorageBucket string JobId string Regex string + TestsToSelect []string TestsToSkip []string BundlesOverride bool CleanupResources bool @@ -65,7 +66,7 @@ type ( // RunTestsInParallel Run Tests in parallel by spawning multiple admin machines. func RunTestsInParallel(conf ParallelRunConf) error { - testsList, skippedTests, err := listTests(conf.Regex, conf.TestsToSkip) + testsList, skippedTests, err := listTests(conf.Regex, conf.TestsToSelect, conf.TestsToSkip) if err != nil { return err } @@ -282,15 +283,16 @@ func RunTests(conf instanceRunConf, inventoryCatalogue map[string]*hardwareCatal } func (e *E2ESession) runTests(regex string) (testCommandResult *testCommandResult, err error) { - e.logger.V(1).Info("Running e2e tests", "regex", regex) command := "GOVERSION=go1.16.6 gotestsum --junitfile=junit-testing.xml --raw-command --format=standard-verbose --hide-summary=all --ignore-non-json-output-lines -- test2json -t -p e2e ./bin/e2e.test -test.v" if regex != "" { - command = fmt.Sprintf("%s -test.run \"^(%s)$\" -test.timeout %s", command, regex, e2eTimeout) + command = fmt.Sprintf("%s -test.run \"^%s$\" -test.timeout %s", command, regex, e2eTimeout) } command = e.commandWithEnvVars(command) + e.logger.V(1).Info("Running e2e tests", "regex", regex, "command", command) + opt := ssm.WithOutputToCloudwatch() testCommandResult, err = ssm.RunCommand( diff --git a/test/e2e/docker_test.go b/test/e2e/docker_test.go index 1557f73b5d57..1e95f1514de5 100644 --- a/test/e2e/docker_test.go +++ b/test/e2e/docker_test.go @@ -621,52 +621,6 @@ func TestDockerKubernetes130RegistryMirrorInsecureSkipVerify(t *testing.T) { runRegistryMirrorConfigFlow(test) } -// Simple flow -func TestDockerKubernetes126SimpleFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewDocker(t), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube126)), - ) - runSimpleFlow(test) -} - -func TestDockerKubernetes127SimpleFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewDocker(t), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube127)), - ) - runSimpleFlow(test) -} - -func TestDockerKubernetes128SimpleFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewDocker(t), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube128)), - ) - runSimpleFlow(test) -} - -func TestDockerKubernetes129SimpleFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewDocker(t), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)), - ) - runSimpleFlow(test) -} - -func TestDockerKubernetes130SimpleFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewDocker(t), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube130)), - ) - runSimpleFlow(test) -} - // Stacked etcd func TestDockerKubernetesStackedEtcd(t *testing.T) { test := framework.NewClusterE2ETest(t, @@ -1480,3 +1434,16 @@ func TestDockerKubernetes130KubeletConfigurationSimpleFlow(t *testing.T) { ) runKubeletConfigurationFlow(test) } + +func TestDockerKubernetesSimpleFlowSuite(t *testing.T) { + for _, ts := range Suites[simpleFlowSubtest.GetSuiteSuffix()] { + t.Run(ts.GetName(), func(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewDocker(t), + framework.WithClusterFiller(api.WithKubernetesVersion(ts.(*SimpleFlowSubtest).KubeVersion)), + ) + runSimpleFlow(test) + }) + } +} diff --git a/test/e2e/test_suite.go b/test/e2e/test_suite.go new file mode 100644 index 000000000000..55cbf6aff835 --- /dev/null +++ b/test/e2e/test_suite.go @@ -0,0 +1,48 @@ +package e2e + +import ( + "strings" + + "github.com/aws/eks-anywhere/pkg/api/v1alpha1" +) + +var kubeVersions = []v1alpha1.KubernetesVersion{v1alpha1.Kube126, v1alpha1.Kube127, v1alpha1.Kube128, v1alpha1.Kube129, v1alpha1.Kube130} + +// Subtest is an interface to represent a test case. +type Subtest interface { + // Return the name of the subtest case. + GetName() string + // Return the name suffix of the parent test, which includes this subtest. It should be "*Suite". + GetSuiteSuffix() string +} + +// Suites contain all test suites. The key is the suite suffix. +var Suites = map[string][]Subtest{ + simpleFlowSubtest.GetSuiteSuffix(): {}, +} + +// SimpleFlowSubtest is a struct to represent a simple flow test. +type SimpleFlowSubtest struct { + KubeVersion v1alpha1.KubernetesVersion +} + +// GetName returns the subtest name. +func (st *SimpleFlowSubtest) GetName() string { + return strings.ReplaceAll(string(st.KubeVersion), ".", "") +} + +// GetSuiteSuffix returns the Suite suffix. +func (st *SimpleFlowSubtest) GetSuiteSuffix() string { + return "SimpleFlowSuite" +} + +// Make sure the SimpleFlowSubtest implements the Subtest interface. +var simpleFlowSubtest Subtest = (*SimpleFlowSubtest)(nil) + +// Init SimpleFlowSuite. +func init() { + s := simpleFlowSubtest.GetSuiteSuffix() + for _, k := range kubeVersions { + Suites[s] = append(Suites[s], &SimpleFlowSubtest{KubeVersion: k}) + } +}