operator: image bundle < 1000 references test

.github/workflows/pre-main.yaml

@@ -250,6 +250,9 @@ jobs:
       - name: Check the smoke test results against the expected results template
         run: ./certsuite check results --log-file="certsuite-out/certsuite.log"
 
+      - name: Print the certsuite.log
+        run: cat certsuite-out/certsuite.log
+
       - name: 'Test: Run preflight specific test suite'
         run: ./certsuite run --label-filter=preflight --log-level="${SMOKE_TESTS_LOG_LEVEL}"
 
@@ -373,6 +376,9 @@ jobs:
       - name: Build the Certsuite tool
         run: make build-certsuite-tool
 
+      - name: Print the certsuite.log
+        run: cat "${CERTSUITE_OUTPUT_DIR}"/certsuite.log
+
       - name: Check the smoke test results against the expected results template
         run: ./certsuite check results --log-file="${CERTSUITE_OUTPUT_DIR}"/certsuite.log
 

CATALOG.md

@@ -7,7 +7,7 @@ Depending on the workload type, not all tests are required to pass to satisfy be
 
 ## Test cases summary
 
-### Total test cases: 116
+### Total test cases: 117
 
 ### Total suites: 10
 
@@ -19,7 +19,7 @@ Depending on the workload type, not all tests are required to pass to satisfy be
 |manageability|2|
 |networking|12|
 |observability|5|
-|operator|10|
+|operator|11|
 |performance|6|
 |platform-alteration|13|
 |preflight|17|
@@ -36,11 +36,11 @@ Depending on the workload type, not all tests are required to pass to satisfy be
 |---|---|
 |8|1|
 
-### Non-Telco specific tests only: 68
+### Non-Telco specific tests only: 69
 
 |Mandatory|Optional|
 |---|---|
-|43|25|
+|44|25|
 
 ### Telco specific tests only: 27
 
@@ -1186,6 +1186,22 @@ Tags|telco,observability
 
 ### operator
 
+#### operator-bundle-count
+
+Property|Description
+---|---
+Unique ID|operator-bundle-count
+Description|Tests operator bundle count is less than 1000
+Suggested Remediation|Ensure that the Operator has a valid bundle count less than 1000.
+Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements
+Exception Process|No exceptions
+Tags|common,operator
+|**Scenario**|**Optional/Mandatory**|
+|Extended|Mandatory|
+|Far-Edge|Mandatory|
+|Non-Telco|Mandatory|
+|Telco|Mandatory|
+
 #### operator-crd-openapi-schema
 
 Property|Description

expected_results.yaml

@@ -59,6 +59,7 @@ testCases:
     - operator-single-crd-owner
     - operator-pods-no-hugepages
     - operator-multiple-same-operators
+    - operator-bundle-count
     - performance-exclusive-cpu-pool
     - performance-max-resources-exec-probes
     - performance-shared-cpu-pool-non-rt-scheduling-policy       # hazelcast pod meets requirements

pkg/autodiscover/autodiscover.go

@@ -158,6 +158,7 @@ func DoAutoDiscover(config *configuration.TestConfiguration) DiscoveredTestData
 	}
 	data.AllInstallPlans = getAllInstallPlans(oc.OlmClient)
 	data.AllCatalogSources = getAllCatalogSources(oc.OlmClient)
+
 	data.Namespaces = namespacesListToStringList(config.TargetNameSpaces)
 	data.Pods, data.AllPods = findPodsByLabels(oc.K8sClient.CoreV1(), podsUnderTestLabelsObjects, data.Namespaces)
 	data.AbnormalEvents = findAbnormalEvents(oc.K8sClient.CoreV1(), data.Namespaces)

pkg/provider/catalogsources.go

@@ -0,0 +1,81 @@
+package provider
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	olmv1Alpha "github.com/operator-framework/api/pkg/operators/v1alpha1"
+	"github.com/redhat-best-practices-for-k8s/certsuite/internal/clientsholder"
+	"github.com/redhat-best-practices-for-k8s/certsuite/pkg/stringhelper"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type CatalogSource struct {
+	*olmv1Alpha.CatalogSource
+}
+
+func NewCatalogSource(cs *olmv1Alpha.CatalogSource) *CatalogSource {
+	return &CatalogSource{
+		CatalogSource: cs,
+	}
+}
+
+func (cs *CatalogSource) GetBundleCount(env *TestEnvironment) (int, error) {
+	const (
+		grpCurlVersion  = "1.8.5"
+		grpCurlFileName = "grpcurl_" + grpCurlVersion + "_linux_x86_64.tar.gz"
+	)
+
+	// List of images that are allowlisted to be skipped.
+	allowlistedBundleImages := []string{
+		"registry.redhat.io/redhat/",
+		// TODO: Add more images to the allowlist if needed
+	}
+
+	o := clientsholder.GetClientsHolder()
+
+	// The index image needs to be queried to get the bundle count.
+	// We accomplish this by exec'ing into the running pod.
+
+	// Get all pods in the cluster wide
+	allPods, err := o.K8sClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return -1, err
+	}
+
+	// Look through all of the pods to find the find the pod that is running the index image
+	// If the pod is running an image that is allowlisted, then we can skip it
+
+	// First find the pod that is running the index image
+	for p := range allPods.Items {
+		for c := range allPods.Items[p].Spec.Containers {
+			if allPods.Items[p].Spec.Containers[c].Image != cs.Spec.Image ||
+				stringhelper.StringInSlice(allowlistedBundleImages,
+					allPods.Items[p].Spec.Containers[c].Image, true) {
+				continue
+			}
+
+			// Found the pod that is running the index image
+			// Now exec into the pod and run the command to get the bundle count
+
+			ctx := clientsholder.NewContext(allPods.Items[p].Namespace,
+				allPods.Items[p].Name, allPods.Items[p].Spec.Containers[c].Name)
+
+			grpCurlURL := "https://github.com/fullstorydev/grpcurl/releases/download/v" + grpCurlVersion + "/" + grpCurlFileName
+
+			createBundlesCommand := "curl -s -L0 " + grpCurlURL + " -o " + grpCurlFileName + "; tar -xf " + grpCurlFileName +
+				"; " + "./grpcurl -plaintext localhost:50051 api.Registry.ListBundles > bundles.txt; cat bundles.txt | grep bundlePath | wc -l"
+
+			// exec into the pod and run the commands
+			cmdValue, errStr, err := o.ExecCommandContainer(ctx, createBundlesCommand)
+			if err != nil || errStr != "" {
+				return -1, err
+			}
+
+			return strconv.Atoi(strings.TrimSpace(cmdValue))
+		}
+	}
+
+	return -1, nil
+}

pkg/provider/catalogsources_test.go

@@ -0,0 +1 @@
+package provider

pkg/provider/containers_test.go

@@ -207,7 +207,7 @@ func TestIsTagEmpty(t *testing.T) {
 	}
 }
 
-func TestIsreadOnlyRootFilessystem(t *testing.T) {
+func TestIsreadOnlyRootFilesystem(t *testing.T) {
 	trueVal := true
 	falseVal := false
 	testCases := []struct {

pkg/provider/provider.go

@@ -33,6 +33,7 @@ import (
 	"github.com/redhat-best-practices-for-k8s/certsuite/internal/log"
 	"github.com/redhat-best-practices-for-k8s/certsuite/pkg/autodiscover"
 	"github.com/redhat-best-practices-for-k8s/certsuite/pkg/configuration"
+	"github.com/redhat-best-practices-for-k8s/certsuite/pkg/stringhelper"
 	k8sPrivilegedDs "github.com/redhat-best-practices-for-k8s/privileged-daemonset"
 	plibRuntime "github.com/redhat-openshift-ecosystem/openshift-preflight/certification"
 	"helm.sh/helm/v3/pkg/release"
@@ -112,10 +113,11 @@ type TestEnvironment struct { // rename this with testTarget
 	ResourceQuotas         []corev1.ResourceQuota
 	PodDisruptionBudgets   []policyv1.PodDisruptionBudget
 	NetworkPolicies        []networkingv1.NetworkPolicy
-	AllInstallPlans        []*olmv1Alpha.InstallPlan   `json:"AllInstallPlans"`
-	AllSubscriptions       []olmv1Alpha.Subscription   `json:"AllSubscriptions"`
-	AllCatalogSources      []*olmv1Alpha.CatalogSource `json:"-"`
-	OperatorGroups         []*olmv1.OperatorGroup      `json:"OperatorGroups"`
+	AllInstallPlans        []*olmv1Alpha.InstallPlan `json:"AllInstallPlans"`
+	AllSubscriptions       []olmv1Alpha.Subscription `json:"AllSubscriptions"`
+	CatalogSources         []*CatalogSource          `json:"-"` // CatalogSources from targetNamespaces
+	AllCatalogSources      []*CatalogSource          `json:"-"` // All CatalogSources from the entire cluster
+	OperatorGroups         []*olmv1.OperatorGroup    `json:"OperatorGroups"`
 	IstioServiceMeshFound  bool
 	ValidProtocolNames     []string
 	DaemonsetFailedToSpawn bool
@@ -209,7 +211,7 @@ func deployDaemonSet(namespace string) error {
 	return nil
 }
 
-func buildTestEnvironment() { //nolint:funlen
+func buildTestEnvironment() { //nolint:funlen,gocyclo
 	start := time.Now()
 	env = TestEnvironment{}
 
@@ -239,7 +241,6 @@ func buildTestEnvironment() { //nolint:funlen
 		log.Fatal("Cannot get OperatorGroups: %v", err)
 	}
 	env.AllSubscriptions = data.AllSubscriptions
-	env.AllCatalogSources = data.AllCatalogSources
 	env.AllOperators = createOperators(data.AllCsvs, data.AllSubscriptions, data.AllInstallPlans, data.AllCatalogSources, false, true)
 	env.AllOperatorsSummary = getSummaryAllOperators(env.AllOperators)
 	env.AllCrds = data.AllCrds
@@ -251,6 +252,20 @@ func buildTestEnvironment() { //nolint:funlen
 		aEvent := NewEvent(&data.AbnormalEvents[i])
 		env.AbnormalEvents = append(env.AbnormalEvents, &aEvent)
 	}
+
+	// CatalogSources
+	// Store every catalog source in the cluster to AllCatalogSources.
+	// Store only the catalog sources in the target namespaces to CatalogSources.
+	for _, cs := range data.AllCatalogSources {
+		if stringhelper.StringInSlice(env.Namespaces, cs.Namespace, false) {
+			env.CatalogSources = append(env.CatalogSources, NewCatalogSource(cs))
+		}
+		env.AllCatalogSources = append(env.AllCatalogSources, NewCatalogSource(cs))
+	}
+
+	log.Info("Found %d catalog sources in the target namespaces", len(env.CatalogSources))
+	log.Info("Found %d catalog sources in the entire cluster", len(env.AllCatalogSources))
+
 	// Service accounts
 	env.ServiceAccounts = data.ServiceAccounts
 	env.AllServiceAccounts = data.AllServiceAccounts

pkg/testhelper/testhelper.go

@@ -309,6 +309,16 @@ func NewOperatorReportObject(aNamespace, aOperatorName, aReason string, isCompli
 	return out
 }
 
+// NewServiceReportObject creates a new ReportObject for a service.
+// It takes the namespace, service name, reason, and compliance status as input parameters.
+// It returns the created ReportObject.
+func NewCatalogSourceReportObject(aNamespace, aCatalogSourceName, aReason string, isCompliant bool) (out *ReportObject) {
+	out = NewReportObject(aReason, OperatorType, isCompliant)
+	out.AddField(Namespace, aNamespace)
+	out.AddField(Name, aCatalogSourceName)
+	return out
+}
+
 // NewDeploymentReportObject creates a new ReportObject for a deployment.
 // It takes the namespace, deployment name, reason, and compliance status as input parameters.
 // It returns a pointer to the created ReportObject.
@@ -638,6 +648,15 @@ func GetNoHugepagesPodsSkipFn(env *provider.TestEnvironment) func() (bool, strin
 	}
 }
 
+func GetNoCatalogSourcesSkipFn(env *provider.TestEnvironment) func() (bool, string) {
+	return func() (bool, string) {
+		if len(env.CatalogSources) == 0 {
+			return true, "no catalog sources found"
+		}
+		return false, ""
+	}
+}
+
 func GetNoOperatorsSkipFn(env *provider.TestEnvironment) func() (bool, string) {
 	return func() (bool, string) {
 		if len(env.Operators) == 0 {

tests/identifiers/doclinks.go

@@ -116,6 +116,7 @@ const (
 	TestOperatorAutomountTokensDocLink                  = DocOperatorRequirement
 	TestOperatorReadOnlyFilesystemDocLink               = DocOperatorRequirement
 	TestOperatorPodsNoHugepagesDocLink                  = DocOperatorRequirement
+	TestOperatorBundleCountIdentifierDocLink            = DocOperatorRequirement
 	TestOperatorOlmSkipRangeDocLink                     = DocOperatorRequirement
 	TestMultipleSameOperatorsIdentifierDocLink          = DocOperatorRequirement
 

tests/identifiers/identifiers.go

@@ -134,6 +134,7 @@ var (
 	TestOperatorSingleCrdOwnerIdentifier              claim.Identifier
 	TestOperatorPodsNoHugepages                       claim.Identifier
 	TestMultipleSameOperatorsIdentifier               claim.Identifier
+	TestOperatorBundleCountIdentifier                 claim.Identifier
 	TestPodNodeSelectorAndAffinityBestPractices       claim.Identifier
 	TestPodHighAvailabilityBestPractices              claim.Identifier
 	TestPodClusterRoleBindingsBestPracticesIdentifier claim.Identifier
@@ -1068,6 +1069,22 @@ that Node's kernel may not have the same hacks.'`,
 		},
 		TagCommon)
 
+	TestOperatorBundleCountIdentifier = AddCatalogEntry(
+		"bundle-count",
+		common.OperatorTestKey,
+		`Tests operator bundle count is less than 1000`,
+		OperatorBundleCountRemediation,
+		NoExceptions,
+		TestOperatorBundleCountIdentifierDocLink,
+		false,
+		map[string]string{
+			FarEdge:  Mandatory,
+			Telco:    Mandatory,
+			NonTelco: Mandatory,
+			Extended: Mandatory,
+		},
+		TagCommon)
+
 	TestMultipleSameOperatorsIdentifier = AddCatalogEntry(
 		"multiple-same-operators",
 		common.OperatorTestKey,

tests/identifiers/remediation.go

@@ -101,6 +101,8 @@ const (
 
 	OperatorPodsNoHugepagesRemediation = `Ensure that the pods are not using hugepages`
 
+	OperatorBundleCountRemediation = `Ensure that the Operator has a valid bundle count less than 1000.`
+
 	MultipleSameOperatorsRemediation = `Ensure that only one Operator of the same type is installed in the cluster.`
 
 	PodNodeSelectorAndAffinityBestPracticesRemediation = `In most cases, Pod's should not specify their host Nodes through nodeSelector or nodeAffinity. However, there are cases in which workloads require specialized hardware specific to a particular class of Node.`

tests/operator/suite.go

@@ -17,6 +17,7 @@
 package operator
 
 import (
+	"strconv"
 	"strings"
 
 	"github.com/redhat-best-practices-for-k8s/certsuite/tests/common"
@@ -115,6 +116,13 @@ func LoadChecks() {
 			testMultipleSameOperators(c, &env)
 			return nil
 		}))
+
+	checksGroup.Add(checksdb.NewCheck(identifiers.GetTestIDAndLabels(identifiers.TestOperatorBundleCountIdentifier)).
+		WithSkipCheckFn(testhelper.GetNoCatalogSourcesSkipFn(&env)).
+		WithCheckFn(func(c *checksdb.Check) error {
+			testOperatorBundleCount(c, &env)
+			return nil
+		}))
 }
 
 // This function check if the Operator CRD version follows K8s versioning
@@ -428,3 +436,41 @@ func testMultipleSameOperators(check *checksdb.Check, env *provider.TestEnvironm
 
 	check.SetResult(compliantObjects, nonCompliantObjects)
 }
+
+func testOperatorBundleCount(check *checksdb.Check, env *provider.TestEnvironment) {
+	var compliantObjects []*testhelper.ReportObject
+	var nonCompliantObjects []*testhelper.ReportObject
+
+	const (
+		bundleCountLimit = 1000
+	)
+
+	// Convert the bundle count limit to a string for logging purposes.
+	bundleCountLimitStr := strconv.Itoa(bundleCountLimit)
+
+	// Ensure the operator bundle has less than 1000 referenced images.
+	for _, catalogSource := range env.CatalogSources {
+		bundleCount, err := catalogSource.GetBundleCount(env)
+		if err != nil {
+			check.LogError("Error getting bundle count for operator %q: %v", catalogSource.Name, err)
+			nonCompliantObjects = append(nonCompliantObjects, testhelper.NewCatalogSourceReportObject(catalogSource.Namespace, catalogSource.Name, "Error getting bundle count", false))
+			continue
+		}
+
+		if bundleCount == -1 {
+			check.LogError("Error getting bundle count for operator %q", catalogSource.Name)
+			nonCompliantObjects = append(nonCompliantObjects, testhelper.NewCatalogSourceReportObject(catalogSource.Namespace, catalogSource.Name, "Error getting bundle count", false))
+			continue
+		}
+
+		if bundleCount > bundleCountLimit {
+			check.LogError("Operator %q has more than "+bundleCountLimitStr+" ("+strconv.Itoa(bundleCount)+") referenced images", catalogSource.Name)
+			nonCompliantObjects = append(nonCompliantObjects, testhelper.NewCatalogSourceReportObject(catalogSource.Namespace, catalogSource.Name, "Operator has more than "+bundleCountLimitStr+" referenced images", false))
+		} else {
+			check.LogInfo("Operator %q has less than "+bundleCountLimitStr+" ("+strconv.Itoa(bundleCount)+") referenced images", catalogSource.Name)
+			compliantObjects = append(compliantObjects, testhelper.NewCatalogSourceReportObject(catalogSource.Namespace, catalogSource.Name, "Operator has less than "+bundleCountLimitStr+" referenced images", true))
+		}
+	}
+
+	check.SetResult(compliantObjects, nonCompliantObjects)
+}