Add additional support to performance suite for SNO/CRC (#1048)

* Add additional support to performance suite for SNO/CRC

performance: skip if test pod are unschedulable

* Modify operator

tests/accesscontrol/tests/access_control_pod_host_path.go

@@ -1,6 +1,8 @@
 package accesscontrol
 
 import (
+	"strings"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
@@ -45,6 +47,9 @@ var _ = Describe("Access-control pod-host-path, ", func() {
 
 		By("Create deployment")
 		err = globalhelper.CreateAndWaitUntilDeploymentIsReady(dep, tsparams.Timeout)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Assert deployment does not have hostPath set")
@@ -77,6 +82,9 @@ var _ = Describe("Access-control pod-host-path, ", func() {
 
 		By("Create deployment")
 		err = globalhelper.CreateAndWaitUntilDeploymentIsReady(dep, tsparams.Timeout)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Assert deployment has hostPath set")

tests/globalhelper/catalogsources.go

@@ -82,6 +82,7 @@ func CreateCommunityOperatorsCatalogSource() error {
 	communityOperatorIndex415 := "registry.redhat.io/redhat/community-operator-index:v4.15"
 	communityOperatorIndex416 := "registry.redhat.io/redhat/community-operator-index:v4.16"
 	communityOperatorIndex417 := "registry.redhat.io/redhat/community-operator-index:v4.17"
+	communityOperatorIndex418 := "registry.redhat.io/redhat/community-operator-index:v4.18"
 
 	// determine which index to use based on ocp version
 	ocpVersion, err := GetClusterVersion()
@@ -106,6 +107,8 @@ func CreateCommunityOperatorsCatalogSource() error {
 		return createCatalogSource("community-operators", communityOperatorIndex416)
 	case "4.17":
 		return createCatalogSource("community-operators", communityOperatorIndex417)
+	case "4.18":
+		return createCatalogSource("community-operators", communityOperatorIndex418)
 	default:
 		return fmt.Errorf("unsupported ocp version %s", ocpVersion)
 	}

tests/globalhelper/deployment.go

@@ -48,6 +48,46 @@ func createAndWaitUntilDeploymentIsReady(client *egiClients.Settings, deployment
 		return fmt.Errorf("failed to create deployment %q (ns %s): %w", deployment.Name, deployment.Namespace, err)
 	}
 
+	// Check if all pods in the deployment are schedulable
+	deploymentUnschedulable := false
+
+	Eventually(func() bool {
+		testDeployment, err := client.Deployments(deployment.Namespace).Get(
+			context.TODO(), deployment.Name, metav1.GetOptions{})
+
+		if err != nil {
+			glog.V(5).Info(fmt.Sprintf(
+				"deployment %s is not running, retry in 5 seconds", testDeployment.Name))
+
+			return false
+		}
+
+		// If it is running, we can break the loop
+		if testDeployment.Status.ReadyReplicas == *testDeployment.Spec.Replicas {
+			glog.V(5).Info(fmt.Sprintf("deployment %s is running", testDeployment.Name))
+			glog.V(5).Info(fmt.Sprintf("deployment %s status: %v", testDeployment.Name, testDeployment.Status))
+
+			return true
+		}
+
+		// print the conditions
+		fmt.Printf("Deployment %s conditions: %v\n", testDeployment.Name, testDeployment.Status.Conditions)
+
+		for _, condition := range testDeployment.Status.Conditions {
+			if condition.Type == appsv1.DeploymentReplicaFailure && condition.Status == corev1.ConditionTrue {
+				deploymentUnschedulable = true
+
+				break
+			}
+		}
+
+		return deploymentUnschedulable
+	}, timeout, 5*time.Second).Should(Equal(true), "Deployment is not running")
+
+	if deploymentUnschedulable {
+		return fmt.Errorf("deployment %s is not schedulable", runningDeployment.Name)
+	}
+
 	Eventually(func() bool {
 		status, err := IsDeploymentReady(client, runningDeployment.Namespace, runningDeployment.Name)
 		if err != nil {

tests/globalhelper/pod.go

@@ -75,6 +75,45 @@ func CreateAndWaitUntilPodIsReady(pod *corev1.Pod, timeout time.Duration) error
 		return fmt.Errorf("failed to create pod %q (ns %s): %w", pod.Name, pod.Namespace, err)
 	}
 
+	// Pod isn't running, we need to wait for it to be scheduled
+	// Loop through pod conditions to check if pod is schedulable
+	// If it is not schedulable, we return an error
+	podUnschedulable := false
+
+	Eventually(func() bool {
+		runningPod, err := GetAPIClient().Pods(pod.Namespace).Get(
+			context.TODO(), pod.Name, metav1.GetOptions{})
+
+		if err != nil {
+			glog.V(5).Info(fmt.Sprintf(
+				"Pod %s is not running, retry in %d seconds", createdPod.Name, retryInterval))
+
+			return false
+		}
+
+		// If it is running, we can break the loop
+		if runningPod.Status.Phase == corev1.PodRunning {
+			return true
+		}
+
+		// print the conditions
+		fmt.Printf("Pod %s conditions: %v\n", runningPod.Name, runningPod.Status.Conditions)
+
+		for _, condition := range runningPod.Status.Conditions {
+			if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && condition.Reason == "Unschedulable" {
+				podUnschedulable = true
+
+				break
+			}
+		}
+
+		return podUnschedulable
+	}, timeout, retryInterval*time.Second).Should(Equal(true), "Pod is not running")
+
+	if podUnschedulable {
+		return fmt.Errorf("pod %s is not schedulable", createdPod.Name)
+	}
+
 	Eventually(func() bool {
 		status, err := isPodReady(createdPod.Namespace, createdPod.Name)
 		if err != nil {

tests/globalhelper/runhelper.go

@@ -109,11 +109,6 @@ func launchTestsViaImage(testCaseName string, tcNameForReport string, reportDir
 	// print the command
 	glog.V(5).Info(fmt.Sprintf("Running command: %s %s", containerEngine, strings.Join(certsuiteCmdArgs, " ")))
 
-	// fmt.Printf("Running command: %s %s", containerEngine, strings.Join(certsuiteCmdArgs, " "))
-
-	// fmt.Println("Sleeping for 5 minutes")
-	// time.Sleep(5 * time.Minute)
-
 	cmd := exec.Command(containerEngine, certsuiteCmdArgs...)
 
 	debugCertsuite, err := GetConfiguration().DebugCertsuite()

tests/operator/parameters/parameters.go

@@ -33,21 +33,23 @@ var (
 		testPodLabelPrefixName: testPodLabelValue,
 		"app":                  "test",
 	}
-	CertsuiteTargetOperatorLabels = fmt.Sprintf("%s: %s", "redhat-best-practices-for-k8s.com/operator", "target")
-	CertsuiteTargetCrdFilters     = []string{"charts.operatorhub.io"}
-	OperatorGroupName             = "operator-test-operator-group"
-	OperatorLabel                 = map[string]string{"redhat-best-practices-for-k8s.com/operator": "target"}
-	CertifiedOperatorGroup        = "certified-operators"
-	RedhatOperatorGroup           = "redhat-operators"
-	CommunityOperatorGroup        = "community-operators"
-	OperatorSourceNamespace       = "openshift-marketplace"
-	OperatorPrefixCloudbees       = "cloudbees-ci"
-	OperatorPrefixAnchore         = "anchore-engine"
-	OperatorPrefixQuay            = "quay-operator"
-	OperatorPrefixKiali           = "kiali-operator"
-	OperatorPrefixOpenvino        = "openvino-operator"
-	CertifiedOperatorPrefixNginx  = "nginx-ingress-operator"
-	SubscriptionNameOpenvino      = "ovms-operator-subscription"
+	CertsuiteTargetOperatorLabels             = fmt.Sprintf("%s: %s", "redhat-best-practices-for-k8s.com/operator", "target")
+	CertsuiteTargetCrdFilters                 = []string{"nginxingresses.charts.nginx.org"}
+	OperatorGroupName                         = "operator-test-operator-group"
+	OperatorLabel                             = map[string]string{"redhat-best-practices-for-k8s.com/operator": "target"}
+	CertifiedOperatorGroup                    = "certified-operators"
+	RedhatOperatorGroup                       = "redhat-operators"
+	CommunityOperatorGroup                    = "community-operators"
+	OperatorSourceNamespace                   = "openshift-marketplace"
+	OperatorPrefixCloudbees                   = "cloudbees-ci"
+	OperatorPrefixAnchore                     = "anchore-engine"
+	OperatorPrefixQuay                        = "quay-operator"
+	OperatorPrefixKiali                       = "kiali-operator"
+	OperatorPrefixOpenvino                    = "openvino-operator"
+	CertifiedOperatorPrefixNginx              = "nginx-ingress-operator"
+	SubscriptionNameOpenvino                  = "ovms-operator-subscription"
+	UncertifiedOperatorPrefixCockroach        = "cockroachdb"
+	CertifiedOperatorPrefixCockroachCertified = "cockroach-operator"
 )
 
 const (

tests/operator/tests/operator_bundle_image.go

@@ -2,6 +2,7 @@ package operator
 
 import (
 	"fmt"
+	"runtime"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -40,6 +41,11 @@ var _ = Describe("Operator bundle count,", Serial, func() {
 	})
 
 	It("CatalogSource with less than 1000 bundle images", func() {
+		// Skip if the host is not x86 based
+		if runtime.GOARCH != "amd64" {
+			Skip("This test is only supported on x86 based hosts")
+		}
+
 		By("Create custom-operator catalog source")
 		err := globalhelper.DeployCustomOperatorSource("quay.io/deliedit/test:catalog-index-test")
 		Expect(err).ToNot(HaveOccurred())

tests/operator/tests/operator_crd_openapi_schema.go

@@ -1,6 +1,8 @@
 package operator
 
 import (
+	"fmt"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
@@ -34,44 +36,56 @@ var _ = Describe("Operator crd-openapi-schema", func() {
 		By("Deploy operator group")
 		err = tshelper.DeployTestOperatorGroup(randomNamespace, false)
 		Expect(err).ToNot(HaveOccurred(), "Error deploying operator group")
+	})
+
+	AfterEach(func() {
+		globalhelper.AfterEachCleanupWithRandomNamespace(randomNamespace,
+			randomReportDir, randomCertsuiteConfigDir, tsparams.Timeout)
+	})
 
-		By("Deploy openvino operator for testing")
+	It("operator crd is defined with openapi schema", func() {
+		By("Query the packagemanifest for the default channel")
+		channel, err := globalhelper.QueryPackageManifestForDefaultChannel(
+			tsparams.CertifiedOperatorPrefixNginx,
+			randomNamespace,
+		)
+		Expect(err).ToNot(HaveOccurred(), "Error querying package manifest for nginx-ingress-operator")
+
+		By("Query the packagemanifest for the " + tsparams.CertifiedOperatorPrefixNginx)
+		version, err := globalhelper.QueryPackageManifestForVersion(tsparams.CertifiedOperatorPrefixNginx, randomNamespace, channel)
+		Expect(err).ToNot(HaveOccurred(), "Error querying package manifest for nginx-ingress-operator")
+
+		By(fmt.Sprintf("Deploy nginx-ingress-operator%s for testing", "."+version))
+		// nginx-ingress-operator: in certified-operators group and version is certified
 		err = tshelper.DeployOperatorSubscription(
-			"ovms-operator",
-			"ovms-operator",
-			"alpha",
+			tsparams.CertifiedOperatorPrefixNginx,
+			tsparams.CertifiedOperatorPrefixNginx,
+			channel,
 			randomNamespace,
 			tsparams.CertifiedOperatorGroup,
 			tsparams.OperatorSourceNamespace,
-			"",
+			tsparams.CertifiedOperatorPrefixNginx+".v"+version,
 			v1alpha1.ApprovalAutomatic,
 		)
 		Expect(err).ToNot(HaveOccurred(), ErrorDeployOperatorStr+
-			tsparams.OperatorPrefixOpenvino)
+			tsparams.CertifiedOperatorPrefixNginx)
 
-		err = tshelper.WaitUntilOperatorIsReady(tsparams.OperatorPrefixOpenvino,
+		err = waitUntilOperatorIsReady(tsparams.CertifiedOperatorPrefixNginx,
 			randomNamespace)
-		Expect(err).ToNot(HaveOccurred(), "Operator "+tsparams.OperatorPrefixOpenvino+
+		Expect(err).ToNot(HaveOccurred(), "Operator "+tsparams.CertifiedOperatorPrefixNginx+".v"+version+
 			" is not ready")
-	})
 
-	AfterEach(func() {
-		globalhelper.AfterEachCleanupWithRandomNamespace(randomNamespace,
-			randomReportDir, randomCertsuiteConfigDir, tsparams.Timeout)
-	})
-
-	It("operator crd is defined with openapi schema", func() {
 		By("Label operator")
 		Eventually(func() error {
 			return tshelper.AddLabelToInstalledCSV(
-				tsparams.OperatorPrefixOpenvino,
+				tsparams.CertifiedOperatorPrefixNginx,
 				randomNamespace,
 				tsparams.OperatorLabel)
 		}, tsparams.TimeoutLabelCsv, tsparams.PollingInterval).Should(Not(HaveOccurred()),
-			ErrorLabelingOperatorStr+tsparams.OperatorPrefixOpenvino)
+			ErrorLabelingOperatorStr+tsparams.CertifiedOperatorPrefixNginx)
 
 		By("Start test")
-		err := globalhelper.LaunchTests(
+		err = globalhelper.LaunchTests(
 			tsparams.CertsuiteOperatorCrdOpenAPISchema,
 			globalhelper.ConvertSpecNameToFileName(CurrentSpecReport().FullText()),
 			randomReportDir,

tests/operator/tests/operator_crd_versioning.go

@@ -1,6 +1,8 @@
 package operator
 
 import (
+	"fmt"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
@@ -34,44 +36,56 @@ var _ = Describe("Operator crd-versioning,", func() {
 		By("Deploy operator group")
 		err = tshelper.DeployTestOperatorGroup(randomNamespace, false)
 		Expect(err).ToNot(HaveOccurred(), "Error deploying operator group")
+	})
+
+	AfterEach(func() {
+		globalhelper.AfterEachCleanupWithRandomNamespace(randomNamespace,
+			randomReportDir, randomCertsuiteConfigDir, tsparams.Timeout)
+	})
 
-		By("Deploy openvino operator for testing")
+	It("operator crd has valid versioning", func() {
+		By("Query the packagemanifest for the default channel")
+		channel, err := globalhelper.QueryPackageManifestForDefaultChannel(
+			tsparams.CertifiedOperatorPrefixNginx,
+			randomNamespace,
+		)
+		Expect(err).ToNot(HaveOccurred(), "Error querying package manifest for nginx-ingress-operator")
+
+		By("Query the packagemanifest for the " + tsparams.CertifiedOperatorPrefixNginx)
+		version, err := globalhelper.QueryPackageManifestForVersion(tsparams.CertifiedOperatorPrefixNginx, randomNamespace, channel)
+		Expect(err).ToNot(HaveOccurred(), "Error querying package manifest for nginx-ingress-operator")
+
+		By(fmt.Sprintf("Deploy nginx-ingress-operator%s for testing", "."+version))
+		// nginx-ingress-operator: in certified-operators group and version is certified
 		err = tshelper.DeployOperatorSubscription(
-			"ovms-operator",
-			"ovms-operator",
-			"alpha",
+			tsparams.CertifiedOperatorPrefixNginx,
+			tsparams.CertifiedOperatorPrefixNginx,
+			channel,
 			randomNamespace,
 			tsparams.CertifiedOperatorGroup,
 			tsparams.OperatorSourceNamespace,
-			"",
+			tsparams.CertifiedOperatorPrefixNginx+".v"+version,
 			v1alpha1.ApprovalAutomatic,
 		)
 		Expect(err).ToNot(HaveOccurred(), ErrorDeployOperatorStr+
-			tsparams.OperatorPrefixOpenvino)
+			tsparams.CertifiedOperatorPrefixNginx)
 
-		err = tshelper.WaitUntilOperatorIsReady(tsparams.OperatorPrefixOpenvino,
+		err = waitUntilOperatorIsReady(tsparams.CertifiedOperatorPrefixNginx,
 			randomNamespace)
-		Expect(err).ToNot(HaveOccurred(), "Operator "+tsparams.OperatorPrefixOpenvino+
+		Expect(err).ToNot(HaveOccurred(), "Operator "+tsparams.CertifiedOperatorPrefixNginx+".v"+version+
 			" is not ready")
-	})
 
-	AfterEach(func() {
-		globalhelper.AfterEachCleanupWithRandomNamespace(randomNamespace,
-			randomReportDir, randomCertsuiteConfigDir, tsparams.Timeout)
-	})
-
-	It("operator crd has valid versioning", func() {
 		By("Label operator")
 		Eventually(func() error {
 			return tshelper.AddLabelToInstalledCSV(
-				tsparams.OperatorPrefixOpenvino,
+				tsparams.CertifiedOperatorPrefixNginx,
 				randomNamespace,
 				tsparams.OperatorLabel)
 		}, tsparams.TimeoutLabelCsv, tsparams.PollingInterval).Should(Not(HaveOccurred()),
-			ErrorLabelingOperatorStr+tsparams.OperatorPrefixOpenvino)
+			ErrorLabelingOperatorStr+tsparams.CertifiedOperatorPrefixNginx)
 
 		By("Start test")
-		err := globalhelper.LaunchTests(
+		err = globalhelper.LaunchTests(
 			tsparams.CertsuiteOperatorCrdVersioning,
 			globalhelper.ConvertSpecNameToFileName(CurrentSpecReport().FullText()),
 			randomReportDir,

tests/operator/tests/operator_multiple_installed.go

@@ -2,6 +2,7 @@ package operator
 
 import (
 	"fmt"
+	"runtime"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -141,6 +142,11 @@ var _ = Describe("Operator multiple installed,", Serial, func() {
 		// This means we will have access to a "new" and "old" channel for the nginx-ingress-operator.
 		// We will deploy the "new" channel in the first namespace and the "old" channel in the second namespace.
 
+		// Skip if the host is not x86 based
+		if runtime.GOARCH != "amd64" {
+			Skip("This test is only supported on x86 based hosts")
+		}
+
 		By("Create custom-operator catalog source")
 		err := globalhelper.DeployCustomOperatorSource("quay.io/deliedit/test:catalog-index-test")
 		Expect(err).ToNot(HaveOccurred())