Fix e2e test grid by using ginkgo labels (#3451)

* Fix e2e gha num chunks Signed-off-by: Per Goncalves da Silva <[email protected]> * Refactor e2e split tool for labels Signed-off-by: Per Goncalves da Silva <[email protected]> * Add ginkgo labels to e2e tests Signed-off-by: Per Goncalves da Silva <[email protected]> --------- Signed-off-by: Per Goncalves da Silva <[email protected]> Co-authored-by: Per Goncalves da Silva <[email protected]>
operator-framework · Nov 26, 2024 · 77ce6ff · 77ce6ff
1 parent 92011d9
commit 77ce6ff
Show file tree

Hide file tree

Showing 29 changed files with 143 additions and 204 deletions.
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
@@ -92,12 +92,12 @@ jobs:
 
       # run non-flakes if matrix-id is not 'flakes'
       - name: Run e2e tests
-        # calculate the number of chunks as the number of parallel jobs minus 1 (flakes job)
+        # calculate the number of chunks as the number of parallel jobs
         # use the split tool to split the test suite into chunks and run the chunk corresponding to the matrix-id
         # focus on those tests and skip tests marked as FLAKE
         run: |
-          E2E_TEST_NUM_CHUNKS=$(( ${{ strategy.job-total }} - 1 )) \
-          GINKGO_OPTS="${GINKGO_OPTS} -focus '$(go run ./test/e2e/split/... -chunks $E2E_TEST_NUM_CHUNKS -print-chunk $E2E_TEST_CHUNK ./test/e2e)' -skip '\[FLAKE\]'" \
+          E2E_TEST_NUM_CHUNKS=$(( ${{ strategy.job-total }} )) \
+          GINKGO_OPTS="${GINKGO_OPTS} -label-filter '$(go run ./test/e2e/split/... -chunks $E2E_TEST_NUM_CHUNKS -print-chunk $E2E_TEST_CHUNK ./test/e2e)' -skip '\[FLAKE\]'" \
           make e2e;
 
       # archive test results

diff --git a/test/e2e/bundle_e2e_test.go b/test/e2e/bundle_e2e_test.go
@@ -26,7 +26,7 @@ import (
 //go:embed testdata/vpa/crd.yaml
 var vpaCRDRaw []byte
 
-var _ = Describe("Installing bundles with new object types", func() {
+var _ = Describe("Installing bundles with new object types", Label("ObjectTypes"), func() {
 	var (
 		kubeClient         operatorclient.ClientInterface
 		operatorClient     versioned.Interface

diff --git a/test/e2e/catalog_e2e_test.go b/test/e2e/catalog_e2e_test.go
@@ -43,7 +43,7 @@ const (
 	badCSVDir             = "bad-csv"
 )
 
-var _ = Describe("Starting CatalogSource e2e tests", func() {
+var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), func() {
 	var (
 		generatedNamespace  corev1.Namespace
 		c                   operatorclient.ClientInterface

diff --git a/test/e2e/catalog_exclusion_test.go b/test/e2e/catalog_exclusion_test.go
@@ -19,7 +19,7 @@ import (
 
 const magicCatalogDir = "magiccatalog"
 
-var _ = Describe("Global Catalog Exclusion", func() {
+var _ = Describe("Global Catalog Exclusion", Label("CatalogExclusion"), func() {
 	var (
 		generatedNamespace  corev1.Namespace
 		determinedE2eClient *util.DeterminedE2EClient

diff --git a/test/e2e/catsrc_pod_config_e2e_test.go b/test/e2e/catsrc_pod_config_e2e_test.go
@@ -18,7 +18,7 @@ const catalogSourceLabel = "olm.catalogSource"
 
 var _ = By
 
-var _ = Describe("CatalogSource Grpc Pod Config", func() {
+var _ = Describe("CatalogSource Grpc Pod Config", Label("CatalogSourcePodConfig"), func() {
 
 	var (
 		generatedNamespace corev1.Namespace

diff --git a/test/e2e/crd_e2e_test.go b/test/e2e/crd_e2e_test.go
@@ -22,7 +22,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var _ = Describe("CRD Versions", func() {
+var _ = Describe("CRD Versions", Label("CRDs"), func() {
 	var (
 		generatedNamespace corev1.Namespace
 		c                  operatorclient.ClientInterface

diff --git a/test/e2e/csv_e2e_test.go b/test/e2e/csv_e2e_test.go
@@ -34,7 +34,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("ClusterServiceVersion", func() {
+var _ = Describe("ClusterServiceVersion", Label("ClusterServiceVersion"), func() {
 	var (
 		generatedNamespace corev1.Namespace
 		c                  operatorclient.ClientInterface

diff --git a/test/e2e/deprecated_e2e_test.go b/test/e2e/deprecated_e2e_test.go
@@ -18,7 +18,7 @@ import (
 
 var missingAPI = `{"apiVersion":"verticalpodautoscalers.autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"name":"my.thing","namespace":"foo"}}`
 
-var _ = Describe("Not found APIs", func() {
+var _ = Describe("Not found APIs", Label("APIDeprecation"), func() {
 	var generatedNamespace corev1.Namespace
 
 	BeforeEach(func() {

diff --git a/test/e2e/disabling_copied_csv_e2e_test.go b/test/e2e/disabling_copied_csv_e2e_test.go
@@ -26,7 +26,7 @@ const (
 	protectedCopiedCSVNamespacesRuntimeFlag = "--protectedCopiedCSVNamespaces"
 )
 
-var _ = Describe("Disabling copied CSVs", func() {
+var _ = Describe("Disabling copied CSVs", Label("DisablingCopiedCSVs"), func() {
 	var (
 		generatedNamespace              corev1.Namespace
 		csv                             operatorsv1alpha1.ClusterServiceVersion

diff --git a/test/e2e/dynamic_resource_e2e_test.go b/test/e2e/dynamic_resource_e2e_test.go
@@ -20,7 +20,7 @@ import (
 // This test was disabled because both of its tests are currently being skipped
 // We need to understand why and whether this test is even needed:
 // https://github.com/operator-framework/operator-lifecycle-manager/issues/3402
-var _ = XDescribe("Subscriptions create required objects from Catalogs", func() {
+var _ = XDescribe("Subscriptions create required objects from Catalogs", Label("DynamicResource"), func() {
 	var (
 		crc                versioned.Interface
 		generatedNamespace corev1.Namespace

diff --git a/test/e2e/fail_forward_e2e_test.go b/test/e2e/fail_forward_e2e_test.go
@@ -123,7 +123,7 @@ func updateCatalogSource(namespace, name string, packages ...string) (func(), er
 	return deployCatalogSource(namespace, name, packages...)
 }
 
-var _ = Describe("Fail Forward Upgrades", func() {
+var _ = Describe("Fail Forward Upgrades", Label("FailForward"), func() {
 
 	var (
 		generatedNamespace corev1.Namespace

diff --git a/test/e2e/gc_e2e_test.go b/test/e2e/gc_e2e_test.go
@@ -23,7 +23,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Garbage collection for dependent resources", func() {
+var _ = Describe("Garbage collection for dependent resources", Label("GarbageCollection"), func() {
 	var (
 		kubeClient         operatorclient.ClientInterface
 		operatorClient     versioned.Interface

diff --git a/test/e2e/installplan_e2e_test.go b/test/e2e/installplan_e2e_test.go
@@ -50,7 +50,7 @@ const (
 	deprecatedCRDDir = "deprecated-crd"
 )
 
-var _ = Describe("Install Plan", func() {
+var _ = Describe("Install Plan", Label("InstallPlan"), func() {
 	var (
 		c                  operatorclient.ClientInterface
 		crc                versioned.Interface

diff --git a/test/e2e/magic_catalog_test.go b/test/e2e/magic_catalog_test.go
@@ -13,7 +13,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-var _ = Describe("MagicCatalog", func() {
+var _ = Describe("MagicCatalog", Label("MagicCatalog"), func() {
 	var (
 		generatedNamespace corev1.Namespace
 		c                  client.Client

diff --git a/test/e2e/metrics_e2e_test.go b/test/e2e/metrics_e2e_test.go
@@ -27,7 +27,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Metrics are generated for OLM managed resources", func() {
+var _ = Describe("Metrics are generated for OLM managed resources", Label("Metrics"), func() {
 	var (
 		c                  operatorclient.ClientInterface
 		crc                versioned.Interface

diff --git a/test/e2e/operator_condition_e2e_test.go b/test/e2e/operator_condition_e2e_test.go
@@ -19,7 +19,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Operator Condition", func() {
+var _ = Describe("Operator Condition", Label("OperatorCondition"), func() {
 	var (
 		generatedNamespace corev1.Namespace
 	)

diff --git a/test/e2e/operator_groups_e2e_test.go b/test/e2e/operator_groups_e2e_test.go
@@ -35,7 +35,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Operator Group", func() {
+var _ = Describe("Operator Group", Label("OperatorGroup"), func() {
 	var (
 		c                  operatorclient.ClientInterface
 		crc                versioned.Interface

diff --git a/test/e2e/operator_test.go b/test/e2e/operator_test.go
@@ -31,7 +31,7 @@ import (
 )
 
 // Describes test specs for the Operator resource.
-var _ = Describe("Operator API", func() {
+var _ = Describe("Operator API", Label("Operator"), func() {
 	var (
 		clientCtx       context.Context
 		scheme          *runtime.Scheme

diff --git a/test/e2e/packagemanifest_e2e_test.go b/test/e2e/packagemanifest_e2e_test.go
@@ -23,7 +23,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Package Manifest API lists available Operators from Catalog Sources", func() {
+var _ = Describe("Package Manifest API lists available Operators from Catalog Sources", Label("PackageManifest"), func() {
 	var (
 		crc                versioned.Interface
 		pmc                pmversioned.Interface

diff --git a/test/e2e/resource_manager_test.go b/test/e2e/resource_manager_test.go
@@ -14,7 +14,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("ResourceManager", func() {
+var _ = Describe("ResourceManager", Label("ResourceManager"), func() {
 	var generatedNamespace corev1.Namespace
 
 	BeforeEach(func() {

diff --git a/test/e2e/scoped_client_test.go b/test/e2e/scoped_client_test.go
@@ -18,7 +18,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Scoped Client bound to a service account can be used to make API calls", func() {
+var _ = Describe("Scoped Client bound to a service account can be used to make API calls", Label("ScopedClient"), func() {
 	// TestScopedClient ensures that we can create a scoped client bound to a
 	// service account and then we can use the scoped client to make API calls.
 	var (

diff --git a/test/e2e/split/integration_test.sh b/test/e2e/split/integration_test.sh
@@ -5,8 +5,8 @@ function get_total_specs() {
 }
 
 unfocused_specs=$(get_total_specs)
-regexp=$(go run ./test/e2e/split/... -chunks 1 -print-chunk 0 ./test/e2e)
-focused_specs=$(get_total_specs -focus "$regexp")
+label_filter=$(go run ./test/e2e/split/... -chunks 1 -print-chunk 0 ./test/e2e)
+focused_specs=$(get_total_specs -label-filter "$label_filter")
 
 if ! [ $unfocused_specs -eq $focused_specs ]; then
   echo "expected number of unfocused specs $unfocused_specs to equal focus specs $focused_specs"