From dc3d55e279c6c93f670e2cdc88dafd4e5d1587ed Mon Sep 17 00:00:00 2001 From: Joe Lanford Date: Tue, 14 Jul 2020 14:14:45 -0400 Subject: [PATCH] scorecard: exit 1 when tests fail, update docs about stages/parallelism --- cmd/operator-sdk/alpha/scorecard/cmd.go | 20 ++- hack/boilerplate.go.txt | 13 -- internal/registry/labels.go | 3 + .../bundle/tests/scorecard/config.yaml | 34 ++--- internal/scorecard/alpha/formatting.go | 16 +- internal/scorecard/alpha/labels_test.go | 28 +--- internal/scorecard/alpha/run_test.go | 24 +-- internal/scorecard/alpha/scorecard.go | 2 +- internal/scorecard/alpha/tar.go | 2 +- .../bundle/tests/scorecard/config.yaml | 6 - pkg/apis/scorecard/v1alpha3/formatter.go | 2 +- .../content/en/docs/scorecard/custom-tests.md | 21 ++- .../content/en/docs/scorecard/kuttl-tests.md | 13 +- .../en/docs/scorecard/scorecard-alpha.md | 144 +++++++++++------- 14 files changed, 173 insertions(+), 155 deletions(-) delete mode 100644 hack/boilerplate.go.txt diff --git a/cmd/operator-sdk/alpha/scorecard/cmd.go b/cmd/operator-sdk/alpha/scorecard/cmd.go index fd8a7f36468..ae0f18d00f9 100644 --- a/cmd/operator-sdk/alpha/scorecard/cmd.go +++ b/cmd/operator-sdk/alpha/scorecard/cmd.go @@ -183,7 +183,25 @@ func (c *scorecardCmd) run() (err error) { } } - return c.printOutput(scorecardTests) + if err := c.printOutput(scorecardTests); err != nil { + return err + } + + if hasFailingTest(scorecardTests) { + os.Exit(1) + } + return nil +} + +func hasFailingTest(list v1alpha3.TestList) bool { + for _, t := range list.Items { + for _, r := range t.Status.Results { + if r.State != v1alpha3.PassState { + return true + } + } + } + return false } func (c *scorecardCmd) validate(args []string) error { diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt deleted file mode 100644 index b9791e49042..00000000000 --- a/hack/boilerplate.go.txt +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2020 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. diff --git a/internal/registry/labels.go b/internal/registry/labels.go index 91bfeb28e62..b51ec231567 100644 --- a/internal/registry/labels.go +++ b/internal/registry/labels.go @@ -26,6 +26,9 @@ import ( "github.com/operator-framework/operator-registry/pkg/image/containerdregistry" registrybundle "github.com/operator-framework/operator-registry/pkg/lib/bundle" log "github.com/sirupsen/logrus" + + // TODO: replace `gopkg.in/yaml.v3` with `sigs.k8s.io/yaml` once operator-registry has `json` tags in the + // annotations struct. yaml "gopkg.in/yaml.v3" ) diff --git a/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml b/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml index f4ef3600439..62f1a50b470 100644 --- a/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml @@ -1,20 +1,16 @@ stages: - - tests: - - name: "customtest1" - image: quay.io/username/custom-scorecard-tests:dev - entrypoint: - - custom-scorecard-tests - - customtest1 - labels: - suite: custom - test: customtest1 - description: an ISV custom test - - name: "customtest2" - entrypoint: - - custom-scorecard-tests - - customtest2 - image: quay.io/username/custom-scorecard-tests:dev - labels: - suite: custom - test: customtest2 - description: an ISV custom test +- tests: + - image: quay.io/username/custom-scorecard-tests:dev + entrypoint: + - custom-scorecard-tests + - customtest1 + labels: + suite: custom + test: customtest1 + - image: quay.io/username/custom-scorecard-tests:dev + entrypoint: + - custom-scorecard-tests + - customtest2 + labels: + suite: custom + test: customtest2 diff --git a/internal/scorecard/alpha/formatting.go b/internal/scorecard/alpha/formatting.go index 9d057b3abe1..89345541742 100644 --- a/internal/scorecard/alpha/formatting.go +++ b/internal/scorecard/alpha/formatting.go @@ -39,20 +39,20 @@ func (r PodTestRunner) getTestStatus(ctx context.Context, p *v1.Pod) (output *v1 return output } -// ListTests lists the scorecard tests as configured that would be +// List lists the scorecard tests as configured that would be // run based on user selection func (o Scorecard) List() v1alpha3.TestList { output := v1alpha3.NewTestList() for _, stage := range o.Config.Stages { tests := o.selectTests(stage) for _, test := range tests { - output.Items = append(output.Items, v1alpha3.Test{ - Spec: v1alpha3.TestSpec{ - Image: test.Image, - Entrypoint: test.Entrypoint, - Labels: test.Labels, - }, - }) + item := v1alpha3.NewTest() + item.Spec = v1alpha3.TestSpec{ + Image: test.Image, + Entrypoint: test.Entrypoint, + Labels: test.Labels, + } + output.Items = append(output.Items, item) } } diff --git a/internal/scorecard/alpha/labels_test.go b/internal/scorecard/alpha/labels_test.go index 2fffd3c4ca6..8bd4d2d6055 100644 --- a/internal/scorecard/alpha/labels_test.go +++ b/internal/scorecard/alpha/labels_test.go @@ -67,64 +67,50 @@ func TestEmptySelector(t *testing.T) { const testConfig = `stages: - tests: - - name: "customtest1" - image: quay.io/someuser/customtest1:v0.0.1 + - image: quay.io/someuser/customtest1:v0.0.1 entrypoint: - custom-test labels: suite: custom test: customtest1 - description: an ISV custom test that does... - - name: "customtest2" - image: quay.io/someuser/customtest2:v0.0.1 + - image: quay.io/someuser/customtest2:v0.0.1 entrypoint: - custom-test labels: suite: custom test: customtest2 - description: an ISV custom test that does... - - name: "basic-check-spec" - image: quay.io/redhat/basictests:v0.0.1 + - image: quay.io/redhat/basictests:v0.0.1 entrypoint: - scorecard-test - basic-check-spec labels: suite: basic test: basic-check-spec-test - description: check the spec test - - name: "basic-check-status" - image: quay.io/redhat/basictests:v0.0.1 + - image: quay.io/redhat/basictests:v0.0.1 entrypoint: - scorecard-test - basic-check-status labels: suite: basic test: basic-check-status-test - description: check the status test - - name: "olm-bundle-validation" - image: quay.io/redhat/olmtests:v0.0.1 + - image: quay.io/redhat/olmtests:v0.0.1 entrypoint: - scorecard-test - olm-bundle-validation labels: suite: olm test: olm-bundle-validation-test - description: validate the bundle test - - name: "olm-crds-have-validation" - image: quay.io/redhat/olmtests:v0.0.1 + - image: quay.io/redhat/olmtests:v0.0.1 entrypoint: - scorecard-test - olm-crds-have-validation labels: suite: olm test: olm-crds-have-validation-test - description: CRDs have validation - - name: "kuttl-tests" - image: quay.io/redhat/kuttltests:v0.0.1 + - image: quay.io/redhat/kuttltests:v0.0.1 labels: suite: kuttl entrypoint: - kuttl-test - olm-status-descriptors - description: Kuttl tests ` diff --git a/internal/scorecard/alpha/run_test.go b/internal/scorecard/alpha/run_test.go index b840f600d35..2add2b63ba0 100644 --- a/internal/scorecard/alpha/run_test.go +++ b/internal/scorecard/alpha/run_test.go @@ -109,10 +109,10 @@ func TestRunParallelPass(t *testing.T) { tests, err := scorecard.Run(ctx) if err != nil { - t.Fatalf("expected no error, got error: %v", err) + t.Fatalf("Expected no error, got error: %v", err) } if len(tests.Items) != 2 { - t.Fatalf("expected 2 tests, got %d", len(tests.Items)) + t.Fatalf("Expected 2 tests, got %d", len(tests.Items)) } for _, test := range tests.Items { expectPass(t, test) @@ -127,10 +127,10 @@ func TestRunSequentialPass(t *testing.T) { tests, err := scorecard.Run(ctx) if err != nil { - t.Fatalf("expected no error, got error: %v", err) + t.Fatalf("Expected no error, got error: %v", err) } if len(tests.Items) != 2 { - t.Fatalf("expected 2 tests, got %d", len(tests.Items)) + t.Fatalf("Expected 2 tests, got %d", len(tests.Items)) } for _, test := range tests.Items { expectPass(t, test) @@ -146,10 +146,10 @@ func TestRunSequentialFail(t *testing.T) { tests, err := scorecard.Run(ctx) if err != nil { - t.Fatalf("expected no error, got error: %v", err) + t.Fatalf("Expected no error, got error: %v", err) } if len(tests.Items) != 2 { - t.Fatalf("expected 2 tests, got %d", len(tests.Items)) + t.Fatalf("Expected 2 tests, got %d", len(tests.Items)) } expectPass(t, tests.Items[0]) @@ -184,28 +184,28 @@ func getFakeScorecard(parallel bool) Scorecard { func expectPass(t *testing.T, test v1alpha3.Test) { if len(test.Status.Results) != 1 { - t.Fatalf("expected 1 results, got %d", len(test.Status.Results)) + t.Fatalf("Expected 1 results, got %d", len(test.Status.Results)) } for _, r := range test.Status.Results { if len(r.Errors) > 0 { - t.Fatalf("expected no errors, got %v", r.Errors) + t.Fatalf("Expected no errors, got %v", r.Errors) } if r.State != v1alpha3.PassState { - t.Fatalf("expected result state %q, got %q", v1alpha3.PassState, r.State) + t.Fatalf("Expected result state %q, got %q", v1alpha3.PassState, r.State) } } } func expectDeadlineExceeded(t *testing.T, test v1alpha3.Test) { if len(test.Status.Results) != 1 { - t.Fatalf("expected 1 results, got %d", len(test.Status.Results)) + t.Fatalf("Expected 1 results, got %d", len(test.Status.Results)) } for _, r := range test.Status.Results { if len(r.Errors) != 1 || r.Errors[0] != context.DeadlineExceeded.Error() { - t.Fatalf("expected error %q error, got %v", context.DeadlineExceeded, r.Errors) + t.Fatalf("Expected error %q error, got %v", context.DeadlineExceeded, r.Errors) } if r.State != v1alpha3.FailState { - t.Fatalf("expected result state %q, got %q", v1alpha3.FailState, r.State) + t.Fatalf("Expected result state %q, got %q", v1alpha3.FailState, r.State) } } } diff --git a/internal/scorecard/alpha/scorecard.go b/internal/scorecard/alpha/scorecard.go index b00410714a5..5de11acc5e7 100644 --- a/internal/scorecard/alpha/scorecard.go +++ b/internal/scorecard/alpha/scorecard.go @@ -94,7 +94,7 @@ func (o Scorecard) runTest(ctx context.Context, test Test) v1alpha3.Test { return out } -// RunTests executes the scorecard tests as configured +// Run executes the scorecard tests as configured func (o Scorecard) Run(ctx context.Context) (v1alpha3.TestList, error) { testOutput := v1alpha3.NewTestList() if err := o.TestRunner.Initialize(ctx); err != nil { diff --git a/internal/scorecard/alpha/tar.go b/internal/scorecard/alpha/tar.go index fbd836dcd19..2e7dcae60a0 100644 --- a/internal/scorecard/alpha/tar.go +++ b/internal/scorecard/alpha/tar.go @@ -194,7 +194,7 @@ func newTarDirHeader(path string) *tar.Header { Typeflag: tar.TypeDir, Name: filepath.Clean(path) + "/", ModTime: time.Now(), - Mode: 0700, + Mode: 0755, Uid: os.Getuid(), Gid: os.Getgid(), } diff --git a/internal/scorecard/alpha/testdata/bundle/tests/scorecard/config.yaml b/internal/scorecard/alpha/testdata/bundle/tests/scorecard/config.yaml index e76b3c77a78..81562b8c44e 100644 --- a/internal/scorecard/alpha/testdata/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/alpha/testdata/bundle/tests/scorecard/config.yaml @@ -8,7 +8,6 @@ stages: labels: suite: basic test: basic-check-spec-test - description: check the spec test - image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test @@ -16,7 +15,6 @@ stages: labels: suite: olm test: olm-bundle-validation-test - description: validate the bundle test - image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test @@ -24,7 +22,6 @@ stages: labels: suite: olm test: olm-crds-have-validation-test - description: CRDs have validation - image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test @@ -32,7 +29,6 @@ stages: labels: suite: olm test: olm-crds-have-resources-test - description: CRDs have resources - image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test @@ -40,7 +36,6 @@ stages: labels: suite: olm test: olm-spec-descriptors-test - description: OLM Spec Descriptors - image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test @@ -48,4 +43,3 @@ stages: labels: suite: olm test: olm-status-descriptors-test - description: OLM Status Descriptors diff --git a/pkg/apis/scorecard/v1alpha3/formatter.go b/pkg/apis/scorecard/v1alpha3/formatter.go index ea7fad9273d..2f0f76a4124 100644 --- a/pkg/apis/scorecard/v1alpha3/formatter.go +++ b/pkg/apis/scorecard/v1alpha3/formatter.go @@ -62,7 +62,7 @@ func (s Test) MarshalText() (string, error) { sb.WriteString("Results: \n") for _, result := range s.Status.Results { if len(result.Name) > 0 { - sb.WriteString(fmt.Sprintf("\tName: %s", result.Name)) + sb.WriteString(fmt.Sprintf("\tName: %s\n", result.Name)) } sb.WriteString("\tState: ") if result.State == PassState { diff --git a/website/content/en/docs/scorecard/custom-tests.md b/website/content/en/docs/scorecard/custom-tests.md index 40b54253e7f..36a169e2be3 100644 --- a/website/content/en/docs/scorecard/custom-tests.md +++ b/website/content/en/docs/scorecard/custom-tests.md @@ -86,17 +86,16 @@ func CustomTest1(bundle registry.Bundle) scapiv1alpha2.ScorecardTestResult { The [configuration file][config_yaml] includes test definitions and metadata to run the test. For the example `CustomTest1` function, the following fields should be specified in `config.yaml`. ```yaml -tests: -- name: "customtest1" - image: quay.io/username/custom-scorecard-tests:dev - entrypoint: - - custom-scorecard-tests - - customtest1 - labels: - suite: custom - test: customtest1 - description: an ISV custom test - ``` +stages: +- tests: + - image: quay.io/username/custom-scorecard-tests:dev + entrypoint: + - custom-scorecard-tests + - customtest1 + labels: + suite: custom + test: customtest1 + ``` The important fields to note here are: 1. `image` - name and tag of the test image which was specified in the Makefile. diff --git a/website/content/en/docs/scorecard/kuttl-tests.md b/website/content/en/docs/scorecard/kuttl-tests.md index 207473b3ece..c685b36a7bf 100644 --- a/website/content/en/docs/scorecard/kuttl-tests.md +++ b/website/content/en/docs/scorecard/kuttl-tests.md @@ -48,13 +48,12 @@ cases under the scorecard/kuttl directory within the bundle contents. In the scorecard configuration file, you might have the following definition of what the selector `suite=kuttlsuite` will translate to: ```yaml -tests: -- name: "kuttltest1" - image: quay.io/operator-framework/scorecard-test-kuttl:dev - labels: - suite: kuttlsuite - test: kuttltest1 - description: an ISV custom test that does... +stages: +- tests: + - image: quay.io/operator-framework/scorecard-test-kuttl:dev + labels: + suite: kuttlsuite + test: kuttltest1 ``` This test configuration will execute the scorecard-test-kuttl diff --git a/website/content/en/docs/scorecard/scorecard-alpha.md b/website/content/en/docs/scorecard/scorecard-alpha.md index 365f7db8c51..7e4d72f80ac 100644 --- a/website/content/en/docs/scorecard/scorecard-alpha.md +++ b/website/content/en/docs/scorecard/scorecard-alpha.md @@ -13,16 +13,16 @@ Tests are implemented within test images that are configured and constructed to be executed by scorecard. Scorecard assumes it is being executed with access to a configured -Kube cluster. Each test is executed within a Pod by scorecard, +Kubernetes cluster. Each test is executed within a Pod by scorecard, from which pod logs are aggregated and test results sent to the console. -Scorecard has built-in basic and OLM tests but also provides a +Scorecard has built-in basic and OLM tests, and it also provides a means to execute custom test definitions. ## Requirements The scorecard tests make no assumptions as to the state of the -operator being tested. Creating operators and custom resources +operator being tested. Creating operators and custom resources for an operator are left outside the scope of the scorecard itself. Scorecard tests can however create whatever resources they @@ -37,7 +37,7 @@ of the configuration file format. Unless you are executing custom tests, you can just copy the provided example configuration file into your project. 2. Place the scorecard configuration file within your project -bundle directory a the following location `tests/scorecard/config.yaml`. +bundle directory at the following location `tests/scorecard/config.yaml`. You can override the default location of the configuration file by specifying the `--config` flag. 3. Execute the [`scorecard` command][cli-scorecard]. See the @@ -60,43 +60,41 @@ and used for running the scorecard pre-defined tests that ship with the SDK. A sample of the scorecard configuration file may look as follows: ```yaml -tests: -- name: "basic-check-spec" - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - basic-check-spec - labels: - suite: basic - test: basic-check-spec-test - description: check the spec test -- name: "olm-bundle-validation" - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-bundle-validation - labels: - suite: olm - test: olm-bundle-validation-test - description: validate the bundle test +stages: +- tests:` + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - basic-check-spec + labels: + suite: basic + test: basic-check-spec-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-bundle-validation + labels: + suite: olm + test: olm-bundle-validation-test ``` -The configuration file defines each test that scorecard can execute. The -following fields of the scorecard configuration file define the test -as follows: +The configuration file defines the tests that scorecard execute. Tests are +grouped into stages for fine-grained control of [parallelism](#parallelism). +The following fields of the scorecard configuration file define the test as +follows: -| Config Field | Description | -| -------- | -------- | -| image | the test container image name that implements a test -| entrypoint | the command and arguments that are invoked in the test image to execute a test -| labels | scorecard-defined or custom labels that [select](#selecting-tests) which tests to run +| Config Field | Description +| ------------ | ----------- +| image | the test container image name that implements a test +| entrypoint | the command and arguments that are invoked in the test image to execute a test +| labels | scorecard-defined or custom labels that [select](#selecting-tests) which tests to run ### Command Args The scorecard command has the following syntax: ``` -operator-sdk alpha scorecard [bundle path] | [bundle image name] [flags] +operator-sdk alpha scorecard [flags] ``` The scorecard requires a positional argument that holds either the @@ -104,6 +102,24 @@ on-disk path to your operator bundle or the name of a bundle image. For further information about the flags see the [CLI documentation][cli-scorecard]. +## Parallelism + +The configuration file allows operator developers to define separate stages for +their tests. Stages run sequentially in the order they are defined in the +configuration file. A stage contains a list of tests and a configurable +`parallel` setting. + +By default (or when a `stage` explicitly sets `parallel` to `false`), tests in +a stage are run sequentially in the order they are defined in the configuration +file. Running tests one at a time is helpful to guarantee that no two tests +interact and conflict with each other. + +However, if tests are designed to be fully isolated, they can be parallelized. +To run a set of isolated tests in parallel, include them in the same stage and +set `parallel` to `true`. All tests in a parallel stage are executed +simultaneously, and scorecard waits for all of them to finish before proceding +to the next stage. This can make your tests run much faster. + ## Selecting Tests Tests are selected by setting the `--selector` CLI flag to @@ -160,20 +176,34 @@ See an example of the JSON format produced by a scorecard test: ```json { - "spec": { - "image": "" - }, - "status": { - "results": [ - { - "name": "olm-bundle-validation", - "log": "time=\"2020-06-10T19:02:49Z\" level=debug msg=\"Found manifests directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=debug msg=\"Found metadata directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level -=debug msg=\"Getting mediaType info from manifests directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=info msg=\"Found annotations file\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=info msg=\"Could not find optio -nal dependencies file\" name=bundle-test\n", - "state": "pass" + "apiVersion": "scorecard.operatorframework.io/v1alpha3", + "kind": "TestList", + "items": [ + { + "kind": "Test", + "apiVersion": "scorecard.operatorframework.io/v1alpha3", + "spec": { + "image": "quay.io/operator-framework/scorecard-test:dev", + "entrypoint": [ + "scorecard-test", + "olm-bundle-validation" + ], + "labels": { + "suite": "olm", + "test": "olm-bundle-validation-test" + } + }, + "status": { + "results": [ + { + "name": "olm-bundle-validation", + "log": "time=\"2020-06-10T19:02:49Z\" level=debug msg=\"Found manifests directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=debug msg=\"Found metadata directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=debug msg=\"Getting mediaType info from manifests directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=info msg=\"Found annotations file\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=info msg=\"Could not find optional dependencies file\" name=bundle-test\n", + "state": "pass" + } + ] } - ] - } + } + ] } ``` @@ -182,14 +212,20 @@ nal dependencies file\" name=bundle-test\n", See an example of the text format produced by a scorecard test: ``` - Labels: - olm-bundle-validation : pass - Log: - time="2020-06-10T19:00:43Z" level=debug msg="Found manifests directory" name=bundle-test - time="2020-06-10T19:00:43Z" level=debug msg="Found metadata directory" name=bundle-test - time="2020-06-10T19:00:43Z" level=debug msg="Getting mediaType info from manifests directory" name=bundle-test - time="2020-06-10T19:00:43Z" level=info msg="Found annotations file" name=bundle-test - time="2020-06-10T19:00:43Z" level=info msg="Could not find optional dependencies file" name=bundle-test +-------------------------------------------------------------------------------- +Image: quay.io/operator-framework/scorecard-test:dev +Entrypoint: [scorecard-test olm-bundle-validation] +Labels: + "suite":"olm" + "test":"olm-bundle-validation-test" +Results: + State: pass + Log: + time="2020-06-10T19:00:43Z" level=debug msg="Found manifests directory" name=bundle-test + time="2020-06-10T19:00:43Z" level=debug msg="Found metadata directory" name=bundle-test + time="2020-06-10T19:00:43Z" level=debug msg="Getting mediaType info from manifests directory" name=bundle-test + time="2020-06-10T19:00:43Z" level=info msg="Found annotations file" name=bundle-test + time="2020-06-10T19:00:43Z" level=info msg="Could not find optional dependencies file" name=bundle-test ``` **NOTE** The output format spec matches the [`Test`](https://godoc.org/github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3#Test) type layout. @@ -208,7 +244,7 @@ Scorecard will execute custom tests if they follow these mandated conventions: * tests accept an entrypoint which include a command and arguments * tests produce v1alpha3 scorecard output in JSON format with no extraneous logging in the test output * tests can obtain the bundle contents at a shared mount point of /bundle - * tests can access the Kube API using an in-cluster client connection + * tests can access the Kubernetes API using an in-cluster client connection