Library: add ability to get a test list

certification/results.go

@@ -12,6 +12,10 @@ type openshiftClusterVersion = runtime.OpenshiftClusterVersion
 type Result struct {
 	check.Check
 	ElapsedTime time.Duration
+	// Err contains the error a check itself throws if it failed to run.
+	// If populated, the expectation is that this Result is in the
+	// Results{}.Errors slice.
+	Err error
 }
 
 type Results struct {

container/check_container.go

@@ -38,11 +38,42 @@ func NewCheck(image string, opts ...Option) *containerCheck {
 // pyxis information is provided. Calls should add a relevant ArtifactWriter to the context if they
 // wish to work with artifact files written by checks.
 func (c *containerCheck) Run(ctx context.Context) (certification.Results, error) {
+	err := c.resolve(ctx)
+	if err != nil {
+		return certification.Results{}, err
+	}
+
+	cfg := runtime.Config{
+		Image:              c.image,
+		DockerConfig:       c.dockerconfigjson,
+		Scratch:            c.policy == policy.PolicyScratch,
+		Bundle:             false,
+		Insecure:           c.insecure,
+		Platform:           c.platform,
+		ManifestListDigest: c.manifestListDigest,
+	}
+	eng, err := engine.New(ctx, c.checks, nil, cfg)
+	if err != nil {
+		return certification.Results{}, err
+	}
+
+	if err := eng.ExecuteChecks(ctx); err != nil {
+		return certification.Results{}, err
+	}
+
+	return eng.Results(ctx), nil
+}
+
+func (c *containerCheck) resolve(ctx context.Context) error {
+	if c.resolved {
+		return nil
+	}
+
 	if c.image == "" {
-		return certification.Results{}, preflighterr.ErrImageEmpty
+		return preflighterr.ErrImageEmpty
 	}
 
-	pol := policy.PolicyContainer
+	c.policy = policy.PolicyContainer
 
 	// If we have enough Pyxis information, resolve the policy.
 	if c.hasPyxisData() {
@@ -55,44 +86,30 @@ func (c *containerCheck) Run(ctx context.Context) (certification.Results, error)
 
 		override, err := lib.GetContainerPolicyExceptions(ctx, p)
 		if err != nil {
-			return certification.Results{}, fmt.Errorf("%w: %s", preflighterr.ErrCannotResolvePolicyException, err)
+			return fmt.Errorf("%w: %s", preflighterr.ErrCannotResolvePolicyException, err)
 		}
 
-		pol = override
+		c.policy = override
 	}
 
-	checks, err := engine.InitializeContainerChecks(ctx, pol, engine.ContainerCheckConfig{
+	newChecks, err := engine.InitializeContainerChecks(ctx, c.policy, engine.ContainerCheckConfig{
 		DockerConfig:           c.dockerconfigjson,
 		PyxisAPIToken:          c.pyxisToken,
 		CertificationProjectID: c.certificationProjectID,
 	})
 	if err != nil {
-		return certification.Results{}, fmt.Errorf("%w: %s", preflighterr.ErrCannotInitializeChecks, err)
+		return fmt.Errorf("%w: %s", preflighterr.ErrCannotInitializeChecks, err)
 	}
+	c.checks = newChecks
+	c.resolved = true
 
-	cfg := runtime.Config{
-		Image:              c.image,
-		DockerConfig:       c.dockerconfigjson,
-		Scratch:            pol == policy.PolicyScratch,
-		Bundle:             false,
-		Insecure:           c.insecure,
-		Platform:           c.platform,
-		ManifestListDigest: c.manifestListDigest,
-	}
-	eng, err := engine.New(ctx, checks, nil, cfg)
-	if err != nil {
-		return certification.Results{}, err
-	}
-
-	if err != nil {
-		return certification.Results{}, err
-	}
-
-	if err := eng.ExecuteChecks(ctx); err != nil {
-		return certification.Results{}, err
-	}
+	return nil
+}
 
-	return eng.Results(ctx), nil
+// List the available container checks. Policy exceptions will be resolved if the proper
+// pyxis information is provided.
+func (c *containerCheck) List(ctx context.Context) (policy.Policy, []check.Check, error) {
+	return c.policy, c.checks, c.resolve(ctx)
 }
 
 // hasPyxisData returns true of the values necessary to make a pyxis
@@ -170,4 +187,7 @@ type containerCheck struct {
 	platform               string
 	insecure               bool
 	manifestListDigest     string
+	checks                 []check.Check
+	resolved               bool
+	policy                 policy.Policy
 }

container/check_container_test.go

@@ -64,6 +64,23 @@ var _ = Describe("Container Check Execution", func() {
 			chk = NewCheck(goodImage)
 		})
 
+		It("Should resolve checks without issue", func() {
+			ctx := context.TODO()
+			err := chk.resolve(ctx)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(chk.policy).To(Equal("container"))
+			Expect(chk.resolved).To(Equal(true))
+			Expect(len(chk.checks)).To(Equal(8))
+		})
+
+		It("Should list checks without issue", func() {
+			ctx := context.TODO()
+			policy, checks, err := chk.List(ctx)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(policy).To(Equal("container"))
+			Expect(len(checks)).To(Equal(8))
+		})
+
 		It("Should run without issue", func() {
 			ctx := context.TODO()
 			results, err := chk.Run(ctx)

internal/engine/engine.go

@@ -231,7 +231,7 @@ func (c *craneEngine) ExecuteChecks(ctx context.Context) error {
 
 		if err != nil {
 			logger.WithValues("result", "ERROR", "err", err.Error()).Info("check completed", "check", executedCheck.Name())
-			c.results.Errors = appendUnlessOptional(c.results.Errors, certification.Result{Check: executedCheck, ElapsedTime: checkElapsedTime})
+			c.results.Errors = appendUnlessOptional(c.results.Errors, certification.Result{Check: executedCheck, ElapsedTime: checkElapsedTime, Err: err})
 			continue
 		}
 

operator/check_operator.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/redhat-openshift-ecosystem/openshift-preflight/certification"
 	preflighterr "github.com/redhat-openshift-ecosystem/openshift-preflight/errors"
+	"github.com/redhat-openshift-ecosystem/openshift-preflight/internal/check"
 	"github.com/redhat-openshift-ecosystem/openshift-preflight/internal/engine"
 	"github.com/redhat-openshift-ecosystem/openshift-preflight/internal/policy"
 	"github.com/redhat-openshift-ecosystem/openshift-preflight/internal/runtime"
@@ -35,29 +36,9 @@ func NewCheck(image, indeximage string, kubeconfig []byte, opts ...Option) *oper
 
 // Run executes the check and returns the results.
 func (c operatorCheck) Run(ctx context.Context) (certification.Results, error) {
-	switch {
-	case c.image == "":
-		return certification.Results{}, preflighterr.ErrImageEmpty
-	case c.kubeconfig == nil:
-		return certification.Results{}, preflighterr.ErrKubeconfigEmpty
-	case c.indeximage == "":
-		return certification.Results{}, preflighterr.ErrIndexImageEmpty
-	}
-
-	pol := policy.PolicyOperator
-
-	checks, err := engine.InitializeOperatorChecks(ctx, pol, engine.OperatorCheckConfig{
-		ScorecardImage:          c.scorecardImage,
-		ScorecardWaitTime:       c.scorecardWaitTime,
-		ScorecardNamespace:      c.scorecardNamespace,
-		ScorecardServiceAccount: c.scorecardServiceAccount,
-		IndexImage:              c.indeximage,
-		DockerConfig:            c.dockerConfigFilePath,
-		Channel:                 c.operatorChannel,
-		Kubeconfig:              c.kubeconfig,
-	})
+	err := c.resolve(ctx)
 	if err != nil {
-		return certification.Results{}, fmt.Errorf("%w: %s", preflighterr.ErrCannotInitializeChecks, err)
+		return certification.Results{}, err
 	}
 
 	cfg := runtime.Config{
@@ -68,7 +49,7 @@ func (c operatorCheck) Run(ctx context.Context) (certification.Results, error) {
 		Insecure:     c.insecure,
 		Platform:     goruntime.GOARCH,
 	}
-	eng, err := engine.New(ctx, checks, c.kubeconfig, cfg)
+	eng, err := engine.New(ctx, c.checks, c.kubeconfig, cfg)
 	if err != nil {
 		return certification.Results{}, err
 	}
@@ -84,11 +65,46 @@ func (c operatorCheck) Run(ctx context.Context) (certification.Results, error) {
 		return certification.Results{}, err
 	}
 
+	return eng.Results(ctx), nil
+}
+
+func (c *operatorCheck) resolve(ctx context.Context) error {
+	if c.resolved {
+		return nil
+	}
+
+	switch {
+	case c.image == "":
+		return preflighterr.ErrImageEmpty
+	case c.kubeconfig == nil:
+		return preflighterr.ErrKubeconfigEmpty
+	case c.indeximage == "":
+		return preflighterr.ErrIndexImageEmpty
+	}
+
+	c.policy = policy.PolicyOperator
+	newChecks, err := engine.InitializeOperatorChecks(ctx, c.policy, engine.OperatorCheckConfig{
+		ScorecardImage:          c.scorecardImage,
+		ScorecardWaitTime:       c.scorecardWaitTime,
+		ScorecardNamespace:      c.scorecardNamespace,
+		ScorecardServiceAccount: c.scorecardServiceAccount,
+		IndexImage:              c.indeximage,
+		DockerConfig:            c.dockerConfigFilePath,
+		Channel:                 c.operatorChannel,
+		Kubeconfig:              c.kubeconfig,
+	})
 	if err != nil {
-		return certification.Results{}, err
+		return fmt.Errorf("%w: %s", preflighterr.ErrCannotInitializeChecks, err)
 	}
+	c.checks = newChecks
+	c.resolved = true
 
-	return eng.Results(ctx), nil
+	return nil
+}
+
+// List the available operator checks.
+func (c operatorCheck) List(ctx context.Context) (policy.Policy, []check.Check, error) {
+	return c.policy, c.checks, c.resolve(ctx)
 }
 
 // WithScorecardNamespace configures the namespace value to use for OperatorSDK Scorecard checks.
@@ -161,4 +177,7 @@ type operatorCheck struct {
 	operatorChannel         string
 	dockerConfigFilePath    string
 	insecure                bool
+	checks                  []check.Check
+	resolved                bool
+	policy                  policy.Policy
 }

operator/check_operator_test.go

@@ -49,6 +49,29 @@ var _ = Describe("Operator Check initialization", func() {
 
 var _ = Describe("Operator Check Execution", func() {
 	// NOTE: There's no unit test for running the operator check because it requires a cluster.
+	When("testing against a known-good image", func() {
+		var chk *operatorCheck
+		BeforeEach(func() {
+			chk = NewCheck("Image", "IndexImage", []byte("Kubeconfig"))
+		})
+
+		It("Should resolve checks without issue", func() {
+			ctx := context.TODO()
+			err := chk.resolve(ctx)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(chk.policy).To(Equal("operator"))
+			Expect(chk.resolved).To(Equal(true))
+			Expect(len(chk.checks)).To(Equal(9))
+		})
+
+		It("Should list checks without issue", func() {
+			ctx := context.TODO()
+			policy, checks, err := chk.List(ctx)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(policy).To(Equal("operator"))
+			Expect(len(checks)).To(Equal(9))
+		})
+	})
 
 	When("Calling the check", func() {
 		It("should fail if you passed an empty image", func() {