Add additional support to performance suite for SNO/CRC

performance: skip if test pod are unschedulable

tests/accesscontrol/tests/access_control_pod_host_path.go

@@ -1,6 +1,8 @@
 package accesscontrol
 
 import (
+	"strings"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
@@ -45,6 +47,9 @@ var _ = Describe("Access-control pod-host-path, ", func() {
 
 		By("Create deployment")
 		err = globalhelper.CreateAndWaitUntilDeploymentIsReady(dep, tsparams.Timeout)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Assert deployment does not have hostPath set")
@@ -77,6 +82,9 @@ var _ = Describe("Access-control pod-host-path, ", func() {
 
 		By("Create deployment")
 		err = globalhelper.CreateAndWaitUntilDeploymentIsReady(dep, tsparams.Timeout)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Assert deployment has hostPath set")

tests/globalhelper/deployment.go

@@ -48,6 +48,46 @@ func createAndWaitUntilDeploymentIsReady(client *egiClients.Settings, deployment
 		return fmt.Errorf("failed to create deployment %q (ns %s): %w", deployment.Name, deployment.Namespace, err)
 	}
 
+	// Check if all pods in the deployment are schedulable
+	deploymentUnschedulable := false
+
+	Eventually(func() bool {
+		testDeployment, err := client.Deployments(deployment.Namespace).Get(
+			context.TODO(), deployment.Name, metav1.GetOptions{})
+
+		if err != nil {
+			glog.V(5).Info(fmt.Sprintf(
+				"deployment %s is not running, retry in 5 seconds", testDeployment.Name))
+
+			return false
+		}
+
+		// If it is running, we can break the loop
+		if testDeployment.Status.ReadyReplicas == *testDeployment.Spec.Replicas {
+			glog.V(5).Info(fmt.Sprintf("deployment %s is running", testDeployment.Name))
+			glog.V(5).Info(fmt.Sprintf("deployment %s status: %v", testDeployment.Name, testDeployment.Status))
+
+			return true
+		}
+
+		// print the conditions
+		fmt.Printf("Deployment %s conditions: %v\n", testDeployment.Name, testDeployment.Status.Conditions)
+
+		for _, condition := range testDeployment.Status.Conditions {
+			if condition.Type == appsv1.DeploymentReplicaFailure && condition.Status == corev1.ConditionTrue {
+				deploymentUnschedulable = true
+
+				break
+			}
+		}
+
+		return deploymentUnschedulable
+	}, timeout, 5*time.Second).Should(Equal(true), "Deployment is not running")
+
+	if deploymentUnschedulable {
+		return fmt.Errorf("deployment %s is not schedulable", runningDeployment.Name)
+	}
+
 	Eventually(func() bool {
 		status, err := IsDeploymentReady(client, runningDeployment.Namespace, runningDeployment.Name)
 		if err != nil {

tests/globalhelper/pod.go

@@ -75,6 +75,45 @@ func CreateAndWaitUntilPodIsReady(pod *corev1.Pod, timeout time.Duration) error
 		return fmt.Errorf("failed to create pod %q (ns %s): %w", pod.Name, pod.Namespace, err)
 	}
 
+	// Pod isn't running, we need to wait for it to be scheduled
+	// Loop through pod conditions to check if pod is schedulable
+	// If it is not schedulable, we return an error
+	podUnschedulable := false
+
+	Eventually(func() bool {
+		runningPod, err := GetAPIClient().Pods(pod.Namespace).Get(
+			context.TODO(), pod.Name, metav1.GetOptions{})
+
+		if err != nil {
+			glog.V(5).Info(fmt.Sprintf(
+				"Pod %s is not running, retry in %d seconds", createdPod.Name, retryInterval))
+
+			return false
+		}
+
+		// If it is running, we can break the loop
+		if runningPod.Status.Phase == corev1.PodRunning {
+			return true
+		}
+
+		// print the conditions
+		fmt.Printf("Pod %s conditions: %v\n", runningPod.Name, runningPod.Status.Conditions)
+
+		for _, condition := range runningPod.Status.Conditions {
+			if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && condition.Reason == "Unschedulable" {
+				podUnschedulable = true
+
+				break
+			}
+		}
+
+		return podUnschedulable
+	}, timeout, retryInterval*time.Second).Should(Equal(true), "Pod is not running")
+
+	if podUnschedulable {
+		return fmt.Errorf("pod %s is not schedulable", createdPod.Name)
+	}
+
 	Eventually(func() bool {
 		status, err := isPodReady(createdPod.Namespace, createdPod.Name)
 		if err != nil {

tests/globalhelper/runhelper.go

@@ -109,11 +109,6 @@ func launchTestsViaImage(testCaseName string, tcNameForReport string, reportDir
 	// print the command
 	glog.V(5).Info(fmt.Sprintf("Running command: %s %s", containerEngine, strings.Join(certsuiteCmdArgs, " ")))
 
-	// fmt.Printf("Running command: %s %s", containerEngine, strings.Join(certsuiteCmdArgs, " "))
-
-	// fmt.Println("Sleeping for 5 minutes")
-	// time.Sleep(5 * time.Minute)
-
 	cmd := exec.Command(containerEngine, certsuiteCmdArgs...)
 
 	debugCertsuite, err := GetConfiguration().DebugCertsuite()

tests/operator/tests/operator_bundle_image.go

@@ -2,6 +2,7 @@ package operator
 
 import (
 	"fmt"
+	"runtime"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -40,6 +41,11 @@ var _ = Describe("Operator bundle count,", Serial, func() {
 	})
 
 	It("CatalogSource with less than 1000 bundle images", func() {
+		// Skip if the host is not x86 based
+		if runtime.GOARCH != "amd64" {
+			Skip("This test is only supported on x86 based hosts")
+		}
+
 		By("Create custom-operator catalog source")
 		err := globalhelper.DeployCustomOperatorSource("quay.io/deliedit/test:catalog-index-test")
 		Expect(err).ToNot(HaveOccurred())

tests/operator/tests/operator_crd_versioning.go

@@ -1,6 +1,8 @@
 package operator
 
 import (
+	"time"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
@@ -60,7 +62,7 @@ var _ = Describe("Operator crd-versioning,", func() {
 			randomReportDir, randomCertsuiteConfigDir, tsparams.Timeout)
 	})
 
-	It("operator crd has valid versioning", func() {
+	FIt("operator crd has valid versioning", func() {
 		By("Label operator")
 		Eventually(func() error {
 			return tshelper.AddLabelToInstalledCSV(
@@ -70,6 +72,8 @@ var _ = Describe("Operator crd-versioning,", func() {
 		}, tsparams.TimeoutLabelCsv, tsparams.PollingInterval).Should(Not(HaveOccurred()),
 			ErrorLabelingOperatorStr+tsparams.OperatorPrefixOpenvino)
 
+		time.Sleep(5 * time.Minute)
+
 		By("Start test")
 		err := globalhelper.LaunchTests(
 			tsparams.CertsuiteOperatorCrdVersioning,

tests/operator/tests/operator_multiple_installed.go

@@ -2,6 +2,7 @@ package operator
 
 import (
 	"fmt"
+	"runtime"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -141,6 +142,11 @@ var _ = Describe("Operator multiple installed,", Serial, func() {
 		// This means we will have access to a "new" and "old" channel for the nginx-ingress-operator.
 		// We will deploy the "new" channel in the first namespace and the "old" channel in the second namespace.
 
+		// Skip if the host is not x86 based
+		if runtime.GOARCH != "amd64" {
+			Skip("This test is only supported on x86 based hosts")
+		}
+
 		By("Create custom-operator catalog source")
 		err := globalhelper.DeployCustomOperatorSource("quay.io/deliedit/test:catalog-index-test")
 		Expect(err).ToNot(HaveOccurred())

tests/performance/tests/exclusive_cpu_pools.go

@@ -2,6 +2,7 @@ package tests
 
 import (
 	"runtime"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -56,6 +57,9 @@ var _ = Describe("performance-exclusive-cpu-pool", func() {
 			globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels)
 
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Start exclusive-cpu-pool test")
@@ -70,14 +74,16 @@ var _ = Describe("performance-exclusive-cpu-pool", func() {
 	})
 
 	It("One pod with one exclusive container, and one shared container", func() {
-
 		By("Define pod")
 		testPod := tshelper.DefineExclusivePod(tsparams.TestPodName, randomNamespace,
 			globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels)
 
 		tshelper.RedefinePodWithSharedContainer(testPod, 0)
 
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Start exclusive-cpu-pool test")
@@ -92,14 +98,16 @@ var _ = Describe("performance-exclusive-cpu-pool", func() {
 	})
 
 	It("One pod with only shared containers", func() {
-
 		By("Define pod")
 		testPod := tshelper.DefineExclusivePod(tsparams.TestPodName, randomNamespace,
 			globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels)
 
 		pod.RedefineWithCPUResources(testPod, "0.75", "0.5")
 
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Start exclusive-cpu-pool test")

tests/performance/tests/rt_app_no_exec_probes.go

@@ -1,6 +1,8 @@
 package tests
 
 import (
+	"strings"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/redhat-best-practices-for-k8s/certsuite-qe/tests/globalhelper"
@@ -46,6 +48,9 @@ var _ = Describe("performance-rt-apps-no-exec-probes", func() {
 			tsparams.RtImageName, tsparams.CertsuiteTargetPodLabels)
 
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, 2*tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		command := "chrt -f -p 50 1" // To change the scheduling policy of the container start process to FIFO scheduling
@@ -71,6 +76,9 @@ var _ = Describe("performance-rt-apps-no-exec-probes", func() {
 		pod.RedefineWithLivenessProbe(testPod)
 
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, 2*tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		command := "chrt -f -p 50 1" // To change the scheduling policy of the container start process to FIFO scheduling
@@ -89,7 +97,6 @@ var _ = Describe("performance-rt-apps-no-exec-probes", func() {
 	})
 
 	It("One non-Rt exclusive pod with no exec probes", func() {
-
 		By("Define pod")
 		testPod := pod.DefinePod(tsparams.TestPodName, randomNamespace,
 			globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels)
@@ -99,6 +106,9 @@ var _ = Describe("performance-rt-apps-no-exec-probes", func() {
 
 		By("Create and wait until pod is ready")
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Assert pod has modified resources")

tests/performance/tests/rt_exclusive_cpu_pool_scheduling_policy.go

@@ -1,6 +1,8 @@
 package tests
 
 import (
+	"strings"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/redhat-best-practices-for-k8s/certsuite-qe/tests/globalhelper"
@@ -45,6 +47,9 @@ var _ = Describe("performance-exclusive-cpu-pool-rt-scheduling-policy", func() {
 			tsparams.RtImageName, tsparams.CertsuiteTargetPodLabels)
 
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Start exclusive-cpu-pool-rt-scheduling-policy test")
@@ -67,6 +72,9 @@ var _ = Describe("performance-exclusive-cpu-pool-rt-scheduling-policy", func() {
 			tsparams.RtImageName, tsparams.CertsuiteTargetPodLabels)
 
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Change to rt scheduling policy")
@@ -94,6 +102,9 @@ var _ = Describe("performance-exclusive-cpu-pool-rt-scheduling-policy", func() {
 			tsparams.RtImageName, tsparams.CertsuiteTargetPodLabels)
 
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Change to rt scheduling policy")
@@ -120,6 +131,9 @@ var _ = Describe("performance-exclusive-cpu-pool-rt-scheduling-policy", func() {
 			globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels)
 
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Start exclusive-cpu-pool-rt-scheduling-policy test")

tests/performance/tests/rt_isolated_cpu_pool.go

@@ -1,6 +1,8 @@
 package tests
 
 import (
+	"strings"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/redhat-best-practices-for-k8s/certsuite-qe/tests/globalhelper"
@@ -61,6 +63,9 @@ var _ = Describe("performance-isolated-cpu-pool-rt-scheduling-policy", Serial, f
 		Expect(err).To(BeNil())
 
 		err = globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Change to rt scheduling policy")
@@ -101,7 +106,10 @@ var _ = Describe("performance-isolated-cpu-pool-rt-scheduling-policy", Serial, f
 		Expect(err).To(BeNil())
 
 		err = globalhelper.CreateAndWaitUntilPodIsReady(testPod, 2*tsparams.WaitingTime)
-		Expect(err).NotTo(HaveOccurred())
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
+		Expect(err).ToNot(HaveOccurred())
 
 		By("Start isolated-cpu-pool-rt-scheduling-policy test")
 		err = globalhelper.LaunchTests(
@@ -122,6 +130,9 @@ var _ = Describe("performance-isolated-cpu-pool-rt-scheduling-policy", Serial, f
 			globalhelper.GetConfiguration().General.TestImage, tsparams.CertsuiteTargetPodLabels)
 
 		err := globalhelper.CreateAndWaitUntilPodIsReady(testPod, tsparams.WaitingTime)
+		if err != nil && strings.Contains(err.Error(), "not schedulable") {
+			Skip("This test cannot run because the pod is not schedulable due to insufficient resources")
+		}
 		Expect(err).ToNot(HaveOccurred())
 
 		By("Start isolated-cpu-pool-rt-scheduling-policy test")