scorecard: exit 1 when tests fail, update docs about stages/parallelism

cmd/operator-sdk/alpha/scorecard/cmd.go

@@ -183,7 +183,25 @@ func (c *scorecardCmd) run() (err error) {
 		}
 	}
 
-	return c.printOutput(scorecardTests)
+	if err := c.printOutput(scorecardTests); err != nil {
+		return err
+	}
+
+	if hasFailingTest(scorecardTests) {
+		os.Exit(1)
+	}
+	return nil
+}
+
+func hasFailingTest(list v1alpha3.TestList) bool {
+	for _, t := range list.Items {
+		for _, r := range t.Status.Results {
+			if r.State != v1alpha3.PassState {
+				return true
+			}
+		}
+	}
+	return false
 }
 
 func (c *scorecardCmd) validate(args []string) error {

internal/registry/labels.go

@@ -26,6 +26,9 @@ import (
 	"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
 	registrybundle "github.com/operator-framework/operator-registry/pkg/lib/bundle"
 	log "github.com/sirupsen/logrus"
+
+	// TODO: replace `gopkg.in/yaml.v3` with `sigs.k8s.io/yaml` once operator-registry has `json` tags in the
+	// annotations struct.
 	yaml "gopkg.in/yaml.v3"
 )
 

internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml

@@ -1,20 +1,16 @@
 stages:
-  - tests:
-    - name: "customtest1"
-      image: quay.io/username/custom-scorecard-tests:dev
-      entrypoint: 
-      - custom-scorecard-tests
-      - customtest1
-      labels:
-        suite: custom
-        test: customtest1
-      description: an ISV custom test
-    - name: "customtest2"
-      entrypoint: 
-      - custom-scorecard-tests
-      - customtest2
-      image: quay.io/username/custom-scorecard-tests:dev
-      labels:
-        suite: custom
-        test: customtest2
-      description: an ISV custom test
+- tests:
+  - image: quay.io/username/custom-scorecard-tests:dev
+    entrypoint: 
+    - custom-scorecard-tests
+    - customtest1
+    labels:
+      suite: custom
+      test: customtest1
+  - image: quay.io/username/custom-scorecard-tests:dev
+    entrypoint: 
+    - custom-scorecard-tests
+    - customtest2
+    labels:
+      suite: custom
+      test: customtest2

internal/scorecard/alpha/formatting.go

@@ -39,20 +39,20 @@ func (r PodTestRunner) getTestStatus(ctx context.Context, p *v1.Pod) (output *v1
 	return output
 }
 
-// ListTests lists the scorecard tests as configured that would be
+// List lists the scorecard tests as configured that would be
 // run based on user selection
 func (o Scorecard) List() v1alpha3.TestList {
 	output := v1alpha3.NewTestList()
 	for _, stage := range o.Config.Stages {
 		tests := o.selectTests(stage)
 		for _, test := range tests {
-			output.Items = append(output.Items, v1alpha3.Test{
-				Spec: v1alpha3.TestSpec{
-					Image:      test.Image,
-					Entrypoint: test.Entrypoint,
-					Labels:     test.Labels,
-				},
-			})
+			item := v1alpha3.NewTest()
+			item.Spec = v1alpha3.TestSpec{
+				Image:      test.Image,
+				Entrypoint: test.Entrypoint,
+				Labels:     test.Labels,
+			}
+			output.Items = append(output.Items, item)
 		}
 	}
 

internal/scorecard/alpha/labels_test.go

@@ -67,64 +67,50 @@ func TestEmptySelector(t *testing.T) {
 
 const testConfig = `stages:
 - tests:
-  - name: "customtest1"
-    image: quay.io/someuser/customtest1:v0.0.1
+  - image: quay.io/someuser/customtest1:v0.0.1
     entrypoint: 
     - custom-test
     labels:
       suite: custom
       test: customtest1
-    description: an ISV custom test that does...
-  - name: "customtest2"
-    image: quay.io/someuser/customtest2:v0.0.1
+  - image: quay.io/someuser/customtest2:v0.0.1
     entrypoint: 
     - custom-test
     labels:
       suite: custom
       test: customtest2
-    description: an ISV custom test that does...
-  - name: "basic-check-spec"
-    image: quay.io/redhat/basictests:v0.0.1
+  - image: quay.io/redhat/basictests:v0.0.1
     entrypoint: 
     - scorecard-test
     - basic-check-spec
     labels:
       suite: basic
       test: basic-check-spec-test
-    description: check the spec test
-  - name: "basic-check-status"
-    image: quay.io/redhat/basictests:v0.0.1
+  - image: quay.io/redhat/basictests:v0.0.1
     entrypoint: 
     - scorecard-test
     - basic-check-status
     labels:
       suite: basic
       test: basic-check-status-test
-    description: check the status test
-  - name: "olm-bundle-validation"
-    image: quay.io/redhat/olmtests:v0.0.1
+  - image: quay.io/redhat/olmtests:v0.0.1
     entrypoint: 
     - scorecard-test
     - olm-bundle-validation
     labels:
       suite: olm
       test: olm-bundle-validation-test
-    description: validate the bundle test
-  - name: "olm-crds-have-validation"
-    image: quay.io/redhat/olmtests:v0.0.1
+  - image: quay.io/redhat/olmtests:v0.0.1
     entrypoint: 
     - scorecard-test
     - olm-crds-have-validation
     labels:
       suite: olm
       test: olm-crds-have-validation-test
-    description: CRDs have validation
-  - name: "kuttl-tests"
-    image: quay.io/redhat/kuttltests:v0.0.1
+  - image: quay.io/redhat/kuttltests:v0.0.1
     labels:
       suite: kuttl
     entrypoint:
     - kuttl-test
     - olm-status-descriptors
-    description: Kuttl tests
 `

internal/scorecard/alpha/run_test.go

@@ -109,10 +109,10 @@ func TestRunParallelPass(t *testing.T) {
 
 	tests, err := scorecard.Run(ctx)
 	if err != nil {
-		t.Fatalf("expected no error, got error: %v", err)
+		t.Fatalf("Expected no error, got error: %v", err)
 	}
 	if len(tests.Items) != 2 {
-		t.Fatalf("expected 2 tests, got %d", len(tests.Items))
+		t.Fatalf("Expected 2 tests, got %d", len(tests.Items))
 	}
 	for _, test := range tests.Items {
 		expectPass(t, test)
@@ -127,10 +127,10 @@ func TestRunSequentialPass(t *testing.T) {
 
 	tests, err := scorecard.Run(ctx)
 	if err != nil {
-		t.Fatalf("expected no error, got error: %v", err)
+		t.Fatalf("Expected no error, got error: %v", err)
 	}
 	if len(tests.Items) != 2 {
-		t.Fatalf("expected 2 tests, got %d", len(tests.Items))
+		t.Fatalf("Expected 2 tests, got %d", len(tests.Items))
 	}
 	for _, test := range tests.Items {
 		expectPass(t, test)
@@ -146,10 +146,10 @@ func TestRunSequentialFail(t *testing.T) {
 
 	tests, err := scorecard.Run(ctx)
 	if err != nil {
-		t.Fatalf("expected no error, got error: %v", err)
+		t.Fatalf("Expected no error, got error: %v", err)
 	}
 	if len(tests.Items) != 2 {
-		t.Fatalf("expected 2 tests, got %d", len(tests.Items))
+		t.Fatalf("Expected 2 tests, got %d", len(tests.Items))
 	}
 
 	expectPass(t, tests.Items[0])
@@ -184,28 +184,28 @@ func getFakeScorecard(parallel bool) Scorecard {
 
 func expectPass(t *testing.T, test v1alpha3.Test) {
 	if len(test.Status.Results) != 1 {
-		t.Fatalf("expected 1 results, got %d", len(test.Status.Results))
+		t.Fatalf("Expected 1 results, got %d", len(test.Status.Results))
 	}
 	for _, r := range test.Status.Results {
 		if len(r.Errors) > 0 {
-			t.Fatalf("expected no errors, got %v", r.Errors)
+			t.Fatalf("Expected no errors, got %v", r.Errors)
 		}
 		if r.State != v1alpha3.PassState {
-			t.Fatalf("expected result state %q, got %q", v1alpha3.PassState, r.State)
+			t.Fatalf("Expected result state %q, got %q", v1alpha3.PassState, r.State)
 		}
 	}
 }
 
 func expectDeadlineExceeded(t *testing.T, test v1alpha3.Test) {
 	if len(test.Status.Results) != 1 {
-		t.Fatalf("expected 1 results, got %d", len(test.Status.Results))
+		t.Fatalf("Expected 1 results, got %d", len(test.Status.Results))
 	}
 	for _, r := range test.Status.Results {
 		if len(r.Errors) != 1 || r.Errors[0] != context.DeadlineExceeded.Error() {
-			t.Fatalf("expected error %q error, got %v", context.DeadlineExceeded, r.Errors)
+			t.Fatalf("Expected error %q error, got %v", context.DeadlineExceeded, r.Errors)
 		}
 		if r.State != v1alpha3.FailState {
-			t.Fatalf("expected result state %q, got %q", v1alpha3.FailState, r.State)
+			t.Fatalf("Expected result state %q, got %q", v1alpha3.FailState, r.State)
 		}
 	}
 }

internal/scorecard/alpha/scorecard.go

@@ -94,7 +94,7 @@ func (o Scorecard) runTest(ctx context.Context, test Test) v1alpha3.Test {
 	return out
 }
 
-// RunTests executes the scorecard tests as configured
+// Run executes the scorecard tests as configured
 func (o Scorecard) Run(ctx context.Context) (v1alpha3.TestList, error) {
 	testOutput := v1alpha3.NewTestList()
 	if err := o.TestRunner.Initialize(ctx); err != nil {

internal/scorecard/alpha/tar.go

@@ -194,7 +194,7 @@ func newTarDirHeader(path string) *tar.Header {
 		Typeflag: tar.TypeDir,
 		Name:     filepath.Clean(path) + "/",
 		ModTime:  time.Now(),
-		Mode:     0700,
+		Mode:     0755,
 		Uid:      os.Getuid(),
 		Gid:      os.Getgid(),
 	}

internal/scorecard/alpha/testdata/bundle/tests/scorecard/config.yaml

@@ -8,44 +8,38 @@ stages:
     labels:
       suite: basic
       test: basic-check-spec-test
-    description: check the spec test
   - image: quay.io/operator-framework/scorecard-test:dev
     entrypoint:
     - scorecard-test
     - olm-bundle-validation
     labels:
       suite: olm
       test: olm-bundle-validation-test
-    description: validate the bundle test
   - image: quay.io/operator-framework/scorecard-test:dev
     entrypoint:
     - scorecard-test
     - olm-crds-have-validation
     labels:
       suite: olm
       test: olm-crds-have-validation-test
-    description: CRDs have validation
   - image: quay.io/operator-framework/scorecard-test:dev
     entrypoint:
     - scorecard-test
     - olm-crds-have-resources
     labels:
       suite: olm
       test: olm-crds-have-resources-test
-    description: CRDs have resources
   - image: quay.io/operator-framework/scorecard-test:dev
     entrypoint:
     - scorecard-test
     - olm-spec-descriptors
     labels:
       suite: olm
       test: olm-spec-descriptors-test
-    description: OLM Spec Descriptors
   - image: quay.io/operator-framework/scorecard-test:dev
     entrypoint:
     - scorecard-test
     - olm-status-descriptors
     labels:
       suite: olm
       test: olm-status-descriptors-test
-    description: OLM Status Descriptors

pkg/apis/scorecard/v1alpha3/formatter.go

@@ -62,7 +62,7 @@ func (s Test) MarshalText() (string, error) {
 		sb.WriteString("Results: \n")
 		for _, result := range s.Status.Results {
 			if len(result.Name) > 0 {
-				sb.WriteString(fmt.Sprintf("\tName: %s", result.Name))
+				sb.WriteString(fmt.Sprintf("\tName: %s\n", result.Name))
 			}
 			sb.WriteString("\tState: ")
 			if result.State == PassState {

website/content/en/docs/scorecard/custom-tests.md

@@ -86,17 +86,16 @@ func CustomTest1(bundle registry.Bundle) scapiv1alpha2.ScorecardTestResult {
 The [configuration file][config_yaml] includes test definitions and metadata to run the test. For the example `CustomTest1` function, the following fields should be specified in `config.yaml`.
 
 ```yaml
-tests:
-- name: "customtest1"
-  image: quay.io/username/custom-scorecard-tests:dev
-  entrypoint: 
-  - custom-scorecard-tests
-  - customtest1
-  labels:
-    suite: custom
-    test: customtest1
-  description: an ISV custom test
-  ```
+stages:
+- tests:
+  - image: quay.io/username/custom-scorecard-tests:dev
+    entrypoint: 
+    - custom-scorecard-tests
+    - customtest1
+    labels:
+      suite: custom
+      test: customtest1
+ ```
 
 The important fields to note here are:
 1. `image` - name and tag of the test image which was specified in the Makefile.

website/content/en/docs/scorecard/kuttl-tests.md

@@ -48,13 +48,12 @@ cases under the scorecard/kuttl directory within the bundle contents.
 In the scorecard configuration file, you might have the following
 definition of what the selector `suite=kuttlsuite` will translate to:
 ```yaml
-tests:
-- name: "kuttltest1"
-  image: quay.io/operator-framework/scorecard-test-kuttl:dev
-  labels:
-    suite: kuttlsuite
-    test: kuttltest1
-  description: an ISV custom test that does...
+stages:
+- tests:
+  - image: quay.io/operator-framework/scorecard-test-kuttl:dev
+    labels:
+      suite: kuttlsuite
+      test: kuttltest1
 ```
 
 This test configuration will execute the scorecard-test-kuttl