From 7e57fdec32535ef50c8ea63ff226bc3ebb48d20a Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Tue, 5 Nov 2024 10:48:07 -0600 Subject: [PATCH] operator: image bundle < 1000 references test --- .github/workflows/pre-main.yaml | 6 +++ CATALOG.md | 24 +++++++-- expected_results.yaml | 1 + pkg/autodiscover/autodiscover.go | 1 + pkg/provider/catalogsources.go | 81 +++++++++++++++++++++++++++++ pkg/provider/catalogsources_test.go | 1 + pkg/provider/containers_test.go | 2 +- pkg/provider/provider.go | 27 +++++++--- pkg/testhelper/testhelper.go | 19 +++++++ tests/identifiers/doclinks.go | 1 + tests/identifiers/identifiers.go | 17 ++++++ tests/identifiers/remediation.go | 2 + tests/operator/suite.go | 46 ++++++++++++++++ 13 files changed, 217 insertions(+), 11 deletions(-) create mode 100644 pkg/provider/catalogsources.go create mode 100644 pkg/provider/catalogsources_test.go diff --git a/.github/workflows/pre-main.yaml b/.github/workflows/pre-main.yaml index 889b226ab..d9493856b 100644 --- a/.github/workflows/pre-main.yaml +++ b/.github/workflows/pre-main.yaml @@ -250,6 +250,9 @@ jobs: - name: Check the smoke test results against the expected results template run: ./certsuite check results --log-file="certsuite-out/certsuite.log" + - name: Print the certsuite.log + run: cat certsuite-out/certsuite.log + - name: 'Test: Run preflight specific test suite' run: ./certsuite run --label-filter=preflight --log-level="${SMOKE_TESTS_LOG_LEVEL}" @@ -373,6 +376,9 @@ jobs: - name: Build the Certsuite tool run: make build-certsuite-tool + - name: Print the certsuite.log + run: cat "${CERTSUITE_OUTPUT_DIR}"/certsuite.log + - name: Check the smoke test results against the expected results template run: ./certsuite check results --log-file="${CERTSUITE_OUTPUT_DIR}"/certsuite.log diff --git a/CATALOG.md b/CATALOG.md index c28fda8d8..85060a006 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -7,7 +7,7 @@ Depending on the workload type, not all tests are required to pass to satisfy be ## Test cases summary -### Total test cases: 116 +### Total test cases: 117 ### Total suites: 10 @@ -19,7 +19,7 @@ Depending on the workload type, not all tests are required to pass to satisfy be |manageability|2| |networking|12| |observability|5| -|operator|10| +|operator|11| |performance|6| |platform-alteration|13| |preflight|17| @@ -36,11 +36,11 @@ Depending on the workload type, not all tests are required to pass to satisfy be |---|---| |8|1| -### Non-Telco specific tests only: 68 +### Non-Telco specific tests only: 69 |Mandatory|Optional| |---|---| -|43|25| +|44|25| ### Telco specific tests only: 27 @@ -1186,6 +1186,22 @@ Tags|telco,observability ### operator +#### operator-bundle-count + +Property|Description +---|--- +Unique ID|operator-bundle-count +Description|Tests operator bundle count is less than 1000 +Suggested Remediation|Ensure that the Operator has a valid bundle count less than 1000. +Best Practice Reference|https://redhat-best-practices-for-k8s.github.io/guide/#redhat-best-practices-for-k8s-cnf-operator-requirements +Exception Process|No exceptions +Tags|common,operator +|**Scenario**|**Optional/Mandatory**| +|Extended|Mandatory| +|Far-Edge|Mandatory| +|Non-Telco|Mandatory| +|Telco|Mandatory| + #### operator-crd-openapi-schema Property|Description diff --git a/expected_results.yaml b/expected_results.yaml index 9fa8b0f23..b1bc6bb07 100644 --- a/expected_results.yaml +++ b/expected_results.yaml @@ -59,6 +59,7 @@ testCases: - operator-single-crd-owner - operator-pods-no-hugepages - operator-multiple-same-operators + - operator-bundle-count - performance-exclusive-cpu-pool - performance-max-resources-exec-probes - performance-shared-cpu-pool-non-rt-scheduling-policy # hazelcast pod meets requirements diff --git a/pkg/autodiscover/autodiscover.go b/pkg/autodiscover/autodiscover.go index d0086ca46..2285791e8 100644 --- a/pkg/autodiscover/autodiscover.go +++ b/pkg/autodiscover/autodiscover.go @@ -158,6 +158,7 @@ func DoAutoDiscover(config *configuration.TestConfiguration) DiscoveredTestData } data.AllInstallPlans = getAllInstallPlans(oc.OlmClient) data.AllCatalogSources = getAllCatalogSources(oc.OlmClient) + data.Namespaces = namespacesListToStringList(config.TargetNameSpaces) data.Pods, data.AllPods = findPodsByLabels(oc.K8sClient.CoreV1(), podsUnderTestLabelsObjects, data.Namespaces) data.AbnormalEvents = findAbnormalEvents(oc.K8sClient.CoreV1(), data.Namespaces) diff --git a/pkg/provider/catalogsources.go b/pkg/provider/catalogsources.go new file mode 100644 index 000000000..f8e205f31 --- /dev/null +++ b/pkg/provider/catalogsources.go @@ -0,0 +1,81 @@ +package provider + +import ( + "context" + "strconv" + "strings" + + olmv1Alpha "github.com/operator-framework/api/pkg/operators/v1alpha1" + "github.com/redhat-best-practices-for-k8s/certsuite/internal/clientsholder" + "github.com/redhat-best-practices-for-k8s/certsuite/pkg/stringhelper" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type CatalogSource struct { + *olmv1Alpha.CatalogSource +} + +func NewCatalogSource(cs *olmv1Alpha.CatalogSource) *CatalogSource { + return &CatalogSource{ + CatalogSource: cs, + } +} + +func (cs *CatalogSource) GetBundleCount(env *TestEnvironment) (int, error) { + const ( + grpCurlVersion = "1.8.5" + grpCurlFileName = "grpcurl_" + grpCurlVersion + "_linux_x86_64.tar.gz" + ) + + // List of images that are allowlisted to be skipped. + allowlistedBundleImages := []string{ + "registry.redhat.io/redhat/", + // TODO: Add more images to the allowlist if needed + } + + o := clientsholder.GetClientsHolder() + + // The index image needs to be queried to get the bundle count. + // We accomplish this by exec'ing into the running pod. + + // Get all pods in the cluster wide + allPods, err := o.K8sClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return -1, err + } + + // Look through all of the pods to find the find the pod that is running the index image + // If the pod is running an image that is allowlisted, then we can skip it + + // First find the pod that is running the index image + for p := range allPods.Items { + for c := range allPods.Items[p].Spec.Containers { + if allPods.Items[p].Spec.Containers[c].Image != cs.Spec.Image || + stringhelper.StringInSlice(allowlistedBundleImages, + allPods.Items[p].Spec.Containers[c].Image, true) { + continue + } + + // Found the pod that is running the index image + // Now exec into the pod and run the command to get the bundle count + + ctx := clientsholder.NewContext(allPods.Items[p].Namespace, + allPods.Items[p].Name, allPods.Items[p].Spec.Containers[c].Name) + + grpCurlURL := "https://github.com/fullstorydev/grpcurl/releases/download/v" + grpCurlVersion + "/" + grpCurlFileName + + createBundlesCommand := "curl -s -L0 " + grpCurlURL + " -o " + grpCurlFileName + "; tar -xf " + grpCurlFileName + + "; " + "./grpcurl -plaintext localhost:50051 api.Registry.ListBundles > bundles.txt; cat bundles.txt | grep bundlePath | wc -l" + + // exec into the pod and run the commands + cmdValue, errStr, err := o.ExecCommandContainer(ctx, createBundlesCommand) + if err != nil || errStr != "" { + return -1, err + } + + return strconv.Atoi(strings.TrimSpace(cmdValue)) + } + } + + return -1, nil +} diff --git a/pkg/provider/catalogsources_test.go b/pkg/provider/catalogsources_test.go new file mode 100644 index 000000000..4f504f668 --- /dev/null +++ b/pkg/provider/catalogsources_test.go @@ -0,0 +1 @@ +package provider diff --git a/pkg/provider/containers_test.go b/pkg/provider/containers_test.go index afaf07db5..4a9aa3bb5 100644 --- a/pkg/provider/containers_test.go +++ b/pkg/provider/containers_test.go @@ -207,7 +207,7 @@ func TestIsTagEmpty(t *testing.T) { } } -func TestIsreadOnlyRootFilessystem(t *testing.T) { +func TestIsreadOnlyRootFilesystem(t *testing.T) { trueVal := true falseVal := false testCases := []struct { diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 7945c33fa..a0b32ce39 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -33,6 +33,7 @@ import ( "github.com/redhat-best-practices-for-k8s/certsuite/internal/log" "github.com/redhat-best-practices-for-k8s/certsuite/pkg/autodiscover" "github.com/redhat-best-practices-for-k8s/certsuite/pkg/configuration" + "github.com/redhat-best-practices-for-k8s/certsuite/pkg/stringhelper" k8sPrivilegedDs "github.com/redhat-best-practices-for-k8s/privileged-daemonset" plibRuntime "github.com/redhat-openshift-ecosystem/openshift-preflight/certification" "helm.sh/helm/v3/pkg/release" @@ -112,10 +113,11 @@ type TestEnvironment struct { // rename this with testTarget ResourceQuotas []corev1.ResourceQuota PodDisruptionBudgets []policyv1.PodDisruptionBudget NetworkPolicies []networkingv1.NetworkPolicy - AllInstallPlans []*olmv1Alpha.InstallPlan `json:"AllInstallPlans"` - AllSubscriptions []olmv1Alpha.Subscription `json:"AllSubscriptions"` - AllCatalogSources []*olmv1Alpha.CatalogSource `json:"-"` - OperatorGroups []*olmv1.OperatorGroup `json:"OperatorGroups"` + AllInstallPlans []*olmv1Alpha.InstallPlan `json:"AllInstallPlans"` + AllSubscriptions []olmv1Alpha.Subscription `json:"AllSubscriptions"` + CatalogSources []*CatalogSource `json:"-"` // CatalogSources from targetNamespaces + AllCatalogSources []*CatalogSource `json:"-"` // All CatalogSources from the entire cluster + OperatorGroups []*olmv1.OperatorGroup `json:"OperatorGroups"` IstioServiceMeshFound bool ValidProtocolNames []string DaemonsetFailedToSpawn bool @@ -209,7 +211,7 @@ func deployDaemonSet(namespace string) error { return nil } -func buildTestEnvironment() { //nolint:funlen +func buildTestEnvironment() { //nolint:funlen,gocyclo start := time.Now() env = TestEnvironment{} @@ -239,7 +241,6 @@ func buildTestEnvironment() { //nolint:funlen log.Fatal("Cannot get OperatorGroups: %v", err) } env.AllSubscriptions = data.AllSubscriptions - env.AllCatalogSources = data.AllCatalogSources env.AllOperators = createOperators(data.AllCsvs, data.AllSubscriptions, data.AllInstallPlans, data.AllCatalogSources, false, true) env.AllOperatorsSummary = getSummaryAllOperators(env.AllOperators) env.AllCrds = data.AllCrds @@ -251,6 +252,20 @@ func buildTestEnvironment() { //nolint:funlen aEvent := NewEvent(&data.AbnormalEvents[i]) env.AbnormalEvents = append(env.AbnormalEvents, &aEvent) } + + // CatalogSources + // Store every catalog source in the cluster to AllCatalogSources. + // Store only the catalog sources in the target namespaces to CatalogSources. + for _, cs := range data.AllCatalogSources { + if stringhelper.StringInSlice(env.Namespaces, cs.Namespace, false) { + env.CatalogSources = append(env.CatalogSources, NewCatalogSource(cs)) + } + env.AllCatalogSources = append(env.AllCatalogSources, NewCatalogSource(cs)) + } + + log.Info("Found %d catalog sources in the target namespaces", len(env.CatalogSources)) + log.Info("Found %d catalog sources in the entire cluster", len(env.AllCatalogSources)) + // Service accounts env.ServiceAccounts = data.ServiceAccounts env.AllServiceAccounts = data.AllServiceAccounts diff --git a/pkg/testhelper/testhelper.go b/pkg/testhelper/testhelper.go index 41b94c25d..2c19b8df4 100644 --- a/pkg/testhelper/testhelper.go +++ b/pkg/testhelper/testhelper.go @@ -309,6 +309,16 @@ func NewOperatorReportObject(aNamespace, aOperatorName, aReason string, isCompli return out } +// NewServiceReportObject creates a new ReportObject for a service. +// It takes the namespace, service name, reason, and compliance status as input parameters. +// It returns the created ReportObject. +func NewCatalogSourceReportObject(aNamespace, aCatalogSourceName, aReason string, isCompliant bool) (out *ReportObject) { + out = NewReportObject(aReason, OperatorType, isCompliant) + out.AddField(Namespace, aNamespace) + out.AddField(Name, aCatalogSourceName) + return out +} + // NewDeploymentReportObject creates a new ReportObject for a deployment. // It takes the namespace, deployment name, reason, and compliance status as input parameters. // It returns a pointer to the created ReportObject. @@ -638,6 +648,15 @@ func GetNoHugepagesPodsSkipFn(env *provider.TestEnvironment) func() (bool, strin } } +func GetNoCatalogSourcesSkipFn(env *provider.TestEnvironment) func() (bool, string) { + return func() (bool, string) { + if len(env.CatalogSources) == 0 { + return true, "no catalog sources found" + } + return false, "" + } +} + func GetNoOperatorsSkipFn(env *provider.TestEnvironment) func() (bool, string) { return func() (bool, string) { if len(env.Operators) == 0 { diff --git a/tests/identifiers/doclinks.go b/tests/identifiers/doclinks.go index 9fa0d3d17..a16774e37 100644 --- a/tests/identifiers/doclinks.go +++ b/tests/identifiers/doclinks.go @@ -116,6 +116,7 @@ const ( TestOperatorAutomountTokensDocLink = DocOperatorRequirement TestOperatorReadOnlyFilesystemDocLink = DocOperatorRequirement TestOperatorPodsNoHugepagesDocLink = DocOperatorRequirement + TestOperatorBundleCountIdentifierDocLink = DocOperatorRequirement TestOperatorOlmSkipRangeDocLink = DocOperatorRequirement TestMultipleSameOperatorsIdentifierDocLink = DocOperatorRequirement diff --git a/tests/identifiers/identifiers.go b/tests/identifiers/identifiers.go index ec505d4e2..77e659b50 100644 --- a/tests/identifiers/identifiers.go +++ b/tests/identifiers/identifiers.go @@ -134,6 +134,7 @@ var ( TestOperatorSingleCrdOwnerIdentifier claim.Identifier TestOperatorPodsNoHugepages claim.Identifier TestMultipleSameOperatorsIdentifier claim.Identifier + TestOperatorBundleCountIdentifier claim.Identifier TestPodNodeSelectorAndAffinityBestPractices claim.Identifier TestPodHighAvailabilityBestPractices claim.Identifier TestPodClusterRoleBindingsBestPracticesIdentifier claim.Identifier @@ -1068,6 +1069,22 @@ that Node's kernel may not have the same hacks.'`, }, TagCommon) + TestOperatorBundleCountIdentifier = AddCatalogEntry( + "bundle-count", + common.OperatorTestKey, + `Tests operator bundle count is less than 1000`, + OperatorBundleCountRemediation, + NoExceptions, + TestOperatorBundleCountIdentifierDocLink, + false, + map[string]string{ + FarEdge: Mandatory, + Telco: Mandatory, + NonTelco: Mandatory, + Extended: Mandatory, + }, + TagCommon) + TestMultipleSameOperatorsIdentifier = AddCatalogEntry( "multiple-same-operators", common.OperatorTestKey, diff --git a/tests/identifiers/remediation.go b/tests/identifiers/remediation.go index 510594360..80e04f26d 100644 --- a/tests/identifiers/remediation.go +++ b/tests/identifiers/remediation.go @@ -101,6 +101,8 @@ const ( OperatorPodsNoHugepagesRemediation = `Ensure that the pods are not using hugepages` + OperatorBundleCountRemediation = `Ensure that the Operator has a valid bundle count less than 1000.` + MultipleSameOperatorsRemediation = `Ensure that only one Operator of the same type is installed in the cluster.` PodNodeSelectorAndAffinityBestPracticesRemediation = `In most cases, Pod's should not specify their host Nodes through nodeSelector or nodeAffinity. However, there are cases in which workloads require specialized hardware specific to a particular class of Node.` diff --git a/tests/operator/suite.go b/tests/operator/suite.go index ed44397ce..f2f0f78a0 100644 --- a/tests/operator/suite.go +++ b/tests/operator/suite.go @@ -17,6 +17,7 @@ package operator import ( + "strconv" "strings" "github.com/redhat-best-practices-for-k8s/certsuite/tests/common" @@ -115,6 +116,13 @@ func LoadChecks() { testMultipleSameOperators(c, &env) return nil })) + + checksGroup.Add(checksdb.NewCheck(identifiers.GetTestIDAndLabels(identifiers.TestOperatorBundleCountIdentifier)). + WithSkipCheckFn(testhelper.GetNoCatalogSourcesSkipFn(&env)). + WithCheckFn(func(c *checksdb.Check) error { + testOperatorBundleCount(c, &env) + return nil + })) } // This function check if the Operator CRD version follows K8s versioning @@ -428,3 +436,41 @@ func testMultipleSameOperators(check *checksdb.Check, env *provider.TestEnvironm check.SetResult(compliantObjects, nonCompliantObjects) } + +func testOperatorBundleCount(check *checksdb.Check, env *provider.TestEnvironment) { + var compliantObjects []*testhelper.ReportObject + var nonCompliantObjects []*testhelper.ReportObject + + const ( + bundleCountLimit = 1000 + ) + + // Convert the bundle count limit to a string for logging purposes. + bundleCountLimitStr := strconv.Itoa(bundleCountLimit) + + // Ensure the operator bundle has less than 1000 referenced images. + for _, catalogSource := range env.CatalogSources { + bundleCount, err := catalogSource.GetBundleCount(env) + if err != nil { + check.LogError("Error getting bundle count for operator %q: %v", catalogSource.Name, err) + nonCompliantObjects = append(nonCompliantObjects, testhelper.NewCatalogSourceReportObject(catalogSource.Namespace, catalogSource.Name, "Error getting bundle count", false)) + continue + } + + if bundleCount == -1 { + check.LogError("Error getting bundle count for operator %q", catalogSource.Name) + nonCompliantObjects = append(nonCompliantObjects, testhelper.NewCatalogSourceReportObject(catalogSource.Namespace, catalogSource.Name, "Error getting bundle count", false)) + continue + } + + if bundleCount > bundleCountLimit { + check.LogError("Operator %q has more than "+bundleCountLimitStr+" ("+strconv.Itoa(bundleCount)+") referenced images", catalogSource.Name) + nonCompliantObjects = append(nonCompliantObjects, testhelper.NewCatalogSourceReportObject(catalogSource.Namespace, catalogSource.Name, "Operator has more than "+bundleCountLimitStr+" referenced images", false)) + } else { + check.LogInfo("Operator %q has less than "+bundleCountLimitStr+" ("+strconv.Itoa(bundleCount)+") referenced images", catalogSource.Name) + compliantObjects = append(compliantObjects, testhelper.NewCatalogSourceReportObject(catalogSource.Namespace, catalogSource.Name, "Operator has less than "+bundleCountLimitStr+" referenced images", true)) + } + } + + check.SetResult(compliantObjects, nonCompliantObjects) +}