From b743d511934839b2159ca2f6cf87f3b440cd10fc Mon Sep 17 00:00:00 2001 From: Binbin Li Date: Thu, 1 Aug 2024 14:46:09 +0000 Subject: [PATCH] feat: fill ErrorReason and Remediation during verifierReport generation --- pkg/executor/core/executor.go | 12 +- pkg/executor/types/types.go | 1 + .../configpolicy/configpolicy.go | 9 +- pkg/verifier/cosign/cosign.go | 6 +- plugins/verifier/sbom/sbom.go | 80 ++++--- plugins/verifier/sbom/sbom_test.go | 184 ++++++++++++++- .../schemavalidator/schema_validator.go | 34 ++- .../schemavalidator/schema_validator_test.go | 170 ++++++++++++++ .../vulnerability_report.go | 222 +++++++++++------- .../vulnerability_report_test.go | 91 ++++--- 10 files changed, 645 insertions(+), 164 deletions(-) create mode 100644 plugins/verifier/schemavalidator/schema_validator_test.go diff --git a/pkg/executor/core/executor.go b/pkg/executor/core/executor.go index 11231f740..abe1f7153 100644 --- a/pkg/executor/core/executor.go +++ b/pkg/executor/core/executor.go @@ -176,13 +176,17 @@ func (executor Executor) verifyReferenceForJSONPolicy(ctx context.Context, subje verifierStartTime := time.Now() verifyResult, err := verifier.Verify(ctx, subjectRef, referenceDesc, referrerStore) if err != nil { + verifierErr := errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace) verifyResult = vr.VerifierResult{ IsSuccess: false, Name: verifier.Name(), // Deprecating Name in v2, switch to VerifierName instead. Type: verifier.Type(), // Deprecating Type in v2, switch to VerifierType instead. VerifierName: verifier.Name(), VerifierType: verifier.Type(), - Message: errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace).Error()} + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), + } } if len(verifier.GetNestedReferences()) > 0 { @@ -228,13 +232,17 @@ func (executor Executor) verifyReferenceForRegoPolicy(ctx context.Context, subje verifierStartTime := time.Now() verifierResult, err := verifier.Verify(errCtx, subjectRef, referenceDesc, referrerStore) if err != nil { + verifierErr := errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace) verifierReport = vt.VerifierResult{ IsSuccess: false, Name: verifier.Name(), // Deprecating Name in v2, switch to VerifierName instead. Type: verifier.Type(), // Deprecating Type in v2, switch to VerifierType instead. VerifierName: verifier.Name(), VerifierType: verifier.Type(), - Message: errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace).Error()} + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), + } } else { verifierReport = vt.NewVerifierResult(verifierResult) } diff --git a/pkg/executor/types/types.go b/pkg/executor/types/types.go index ccf78c337..30bac9a1b 100644 --- a/pkg/executor/types/types.go +++ b/pkg/executor/types/types.go @@ -29,6 +29,7 @@ type VerifyResult struct { // NestedVerifierReport describes the results of verifying an artifact and its // nested artifacts by available verifiers. +// Note: NestedVerifierReport is used for verification results in v1. type NestedVerifierReport struct { Subject string `json:"subject"` ReferenceDigest string `json:"referenceDigest"` diff --git a/pkg/policyprovider/configpolicy/configpolicy.go b/pkg/policyprovider/configpolicy/configpolicy.go index a0475bbc2..56dfb4c39 100644 --- a/pkg/policyprovider/configpolicy/configpolicy.go +++ b/pkg/policyprovider/configpolicy/configpolicy.go @@ -97,10 +97,13 @@ func (enforcer PolicyEnforcer) ContinueVerifyOnFailure(_ context.Context, _ comm // ErrorToVerifyResult converts an error to a properly formatted verify result func (enforcer PolicyEnforcer) ErrorToVerifyResult(_ context.Context, subjectRefString string, verifyError error) types.VerifyResult { + verifierErr := re.ErrorCodeVerifyReferenceFailure.WithDetail(fmt.Sprintf("failed to verify artifact: %s", subjectRefString)).WithError(verifyError) errorReport := verifier.VerifierResult{ - Subject: subjectRefString, - IsSuccess: false, - Message: fmt.Sprintf("verification failed: %v", verifyError), + Subject: subjectRefString, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), } var reports []interface{} reports = append(reports, errorReport) diff --git a/pkg/verifier/cosign/cosign.go b/pkg/verifier/cosign/cosign.go index 8f272ddb7..ca79d6302 100644 --- a/pkg/verifier/cosign/cosign.go +++ b/pkg/verifier/cosign/cosign.go @@ -485,14 +485,16 @@ func staticLayerOpts(desc imgspec.Descriptor) ([]static.Option, error) { // ErrorToVerifyResult returns a verifier result with the error message and isSuccess set to false func errorToVerifyResult(name string, verifierType string, err error) verifier.VerifierResult { + verifierErr := re.ErrorCodeVerifyReferenceFailure.WithDetail("cosign verification failed").WithError(err) return verifier.VerifierResult{ IsSuccess: false, Name: name, // Deprecating Name in v2, switch to VerifierName instead. Type: verifierType, // Deprecating Type in v2, switch to VerifierType instead. VerifierName: name, VerifierType: verifierType, - Message: "cosign verification failed", - ErrorReason: err.Error(), + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), } } diff --git a/plugins/verifier/sbom/sbom.go b/plugins/verifier/sbom/sbom.go index cd8448da3..31c98a566 100644 --- a/plugins/verifier/sbom/sbom.go +++ b/plugins/verifier/sbom/sbom.go @@ -28,6 +28,7 @@ import ( "github.com/ratify-project/ratify/plugins/verifier/sbom/utils" // This import is required to utilize the oras built-in referrer store + re "github.com/ratify-project/ratify/errors" _ "github.com/ratify-project/ratify/pkg/referrerstore/oras" "github.com/ratify-project/ratify/pkg/verifier" "github.com/ratify-project/ratify/pkg/verifier/plugin/skel" @@ -82,19 +83,28 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe ctx := context.Background() referenceManifest, err := referrerStore.GetReferenceManifest(ctx, subjectReference, referenceDescriptor) if err != nil { + storeErr := re.ErrorCodeGetReferenceManifestFailure.WithDetail(fmt.Sprintf("Failed to fetch reference manifest for subject: %s reference descriptor: %v", subjectReference, referenceDescriptor.Descriptor)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Error fetching reference manifest for subject: %s reference descriptor: %v, err: %v", subjectReference, referenceDescriptor.Descriptor, err), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: storeErr.GetFullDetails(), + ErrorReason: storeErr.GetRootCause(), + Remediation: storeErr.GetRootRemediation(), }, nil } if len(referenceManifest.Blobs) == 0 { return &verifier.VerifierResult{ - Name: input.Name, - IsSuccess: false, - Message: fmt.Sprintf("SBOM validation failed: no layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: "SBOM validation failed", + ErrorReason: fmt.Sprintf("No layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), }, nil } @@ -103,11 +113,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe refBlob, err := referrerStore.GetBlobContent(ctx, subjectReference, blobDesc.Digest) if err != nil { + storeErr := re.ErrorCodeGetBlobContentFailure.WithDetail(fmt.Sprintf("Failed to fetch blob for subject: %s digest: %s", subjectReference, blobDesc.Digest)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Error fetching blob for subject: %s digest: %s, err: %v", subjectReference, blobDesc.Digest, err), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: storeErr.GetFullDetails(), + ErrorReason: storeErr.GetRootCause(), + Remediation: storeErr.GetRootRemediation(), }, nil } @@ -116,19 +131,24 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe return processSpdxJSONMediaType(input.Name, verifierType, refBlob, input.DisallowedLicenses, input.DisallowedPackages), nil default: return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Unsupported artifactType: %s", artifactType), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: "Failed to process SBOM blobs.", + ErrorReason: fmt.Sprintf("Unsupported artifactType: %s", artifactType), }, nil } } return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "SBOM verification success. No license or package violation found.", + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: true, + Message: "SBOM verification success. No license or package violation found.", }, nil } @@ -178,10 +198,12 @@ func processSpdxJSONMediaType(name string, verifierType string, refBlob []byte, if len(licenseViolation) != 0 || len(packageViolation) != 0 { return &verifier.VerifierResult{ - Name: name, - IsSuccess: false, - Extensions: extensionData, - Message: "SBOM validation failed. Please review extensions data for license and package violation found.", + Name: name, + IsSuccess: false, + Extensions: extensionData, + Message: "SBOM validation failed.", + ErrorReason: "License or package violation found.", + Remediation: "Please review extensions data for license and package violation found.", } } } @@ -196,11 +218,15 @@ func processSpdxJSONMediaType(name string, verifierType string, refBlob []byte, Message: "SBOM verification success. No license or package violation found.", } } + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("failed to verify artifact: %s", name)).WithError(err) return &verifier.VerifierResult{ - Name: name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("SBOM failed to parse: %v", err), + Name: name, + Type: verifierType, + VerifierName: name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), } } diff --git a/plugins/verifier/sbom/sbom_test.go b/plugins/verifier/sbom/sbom_test.go index 8dc5a4c3f..8609fc500 100644 --- a/plugins/verifier/sbom/sbom_test.go +++ b/plugins/verifier/sbom/sbom_test.go @@ -15,14 +15,24 @@ limitations under the License. package main import ( + "errors" + "fmt" "os" "path/filepath" "strings" "testing" + "github.com/opencontainers/go-digest" + oci "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/ratify-project/ratify/pkg/common" + "github.com/ratify-project/ratify/pkg/ocispecs" + "github.com/ratify-project/ratify/pkg/referrerstore/mocks" + "github.com/ratify-project/ratify/pkg/verifier/plugin/skel" "github.com/ratify-project/ratify/plugins/verifier/sbom/utils" ) +const mediaType string = "application/vnd.syft+json" + func TestProcessSPDXJsonMediaType(t *testing.T) { b, err := os.ReadFile(filepath.Join("testdata", "bom.json")) if err != nil { @@ -41,8 +51,178 @@ func TestProcessInvalidSPDXJsonMediaType(t *testing.T) { } report := processSpdxJSONMediaType("test", "", b, nil, nil) - if !strings.Contains(report.Message, "SBOM failed to parse") { - t.Fatalf("expected to have an error processing spdx json file: %s", filepath.Join("testdata", "bom.json")) + if !strings.Contains(report.Message, "failed to verify artifact") { + t.Fatalf("report message: %s does not contain expected error message", report.Message) + } + if report.ErrorReason != "JSON document does not contain spdxVersion field" { + t.Fatalf("expected error reason: %s, got: %s", "JSON document does not contain spdxVersion field", report.ErrorReason) + } +} + +func TestVerifyReference(t *testing.T) { + manifestDigest := digest.FromString("test_manifest_digest") + manifestDigest2 := digest.FromString("test_manifest_digest_2") + blobDigest := digest.FromString("test_blob_digest") + blobDigest2 := digest.FromString("test_blob_digest_2") + type args struct { + stdinData string + referenceManifest ocispecs.ReferenceManifest + blobContent string + refDesc ocispecs.ReferenceDescriptor + } + type want struct { + message string + errorReason string + err error + } + + tests := []struct { + name string + args args + want want + }{ + { + name: "invalid stdin data", + args: args{}, + want: want{ + err: errors.New("failed to parse stdin for the input: unexpected end of JSON input"), + }, + }, + { + name: "failed to get reference manifest", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest2, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: "Failed to fetch reference manifest for subject: test_subject reference descriptor: { sha256:b55e209647d87fcd95a94c59ff4d342e42bf10f02a7c10b5192131f8d959ff5a 0 [] map[] [] }", + errorReason: "manifest not found", + }, + }, + { + name: "empty blobs", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: "SBOM validation failed", + errorReason: fmt.Sprintf("No layers found in manifest for referrer %s@%s", "test_subject_path", manifestDigest.String()), + }, + }, + { + name: "get blob content error", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest2, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: fmt.Sprintf("Failed to fetch blob for subject: test_subject digest: %s", blobDigest2.String()), + errorReason: "blob not found", + }, + }, + { + name: "unsupported artifactType", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: "Failed to process SBOM blobs.", + errorReason: "Unsupported artifactType: application/vnd.syft+json", + }, + }, + { + name: "process spdx json mediaType error", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: SpdxJSONMediaType, + }, + }, + want: want{ + message: "failed to verify artifact: sbom", + errorReason: "unexpected end of JSON input", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmdArgs := &skel.CmdArgs{ + Version: "1.0.0", + Subject: "test_subject", + StdinData: []byte(tt.args.stdinData), + } + testStore := &mocks.MemoryTestStore{ + Manifests: map[digest.Digest]ocispecs.ReferenceManifest{manifestDigest: tt.args.referenceManifest}, + Blobs: map[digest.Digest][]byte{blobDigest: []byte(tt.args.blobContent)}, + } + subjectRef := common.Reference{ + Path: "test_subject_path", + Original: "test_subject", + } + verifierResult, err := VerifyReference(cmdArgs, subjectRef, tt.args.refDesc, testStore) + if err != nil && err.Error() != tt.want.err.Error() { + t.Fatalf("verifyReference() error = %v, wantErr %v", err, tt.want.err) + } + if verifierResult != nil { + if verifierResult.Message != tt.want.message { + t.Fatalf("verifyReference() verifier report message = %s, want = %s", verifierResult.Message, tt.want.message) + } + if verifierResult.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyReference() verifier report error reason = %s, want = %s", verifierResult.ErrorReason, tt.want.errorReason) + } + } + }) } } diff --git a/plugins/verifier/schemavalidator/schema_validator.go b/plugins/verifier/schemavalidator/schema_validator.go index 2af66396e..78b5562ff 100644 --- a/plugins/verifier/schemavalidator/schema_validator.go +++ b/plugins/verifier/schemavalidator/schema_validator.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" + re "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/common" "github.com/ratify-project/ratify/pkg/ocispecs" "github.com/ratify-project/ratify/pkg/referrerstore" @@ -72,10 +73,12 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe if len(referenceManifest.Blobs) == 0 { return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("schema validation failed: no blobs found for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("No blobs found for referrer %s@%s.", subjectReference.Path, referenceDescriptor.Digest.String()), }, nil } @@ -87,20 +90,27 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe err = processMediaType(schemaMap, blobDesc.MediaType, refBlob) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("schema validation failed for digest:[%s], media type:[%s].", blobDesc.Digest, blobDesc.MediaType)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("schema validation failed for digest:[%s],media type:[%s],parse errors:[%v]", blobDesc.Digest, blobDesc.MediaType, err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), }, nil } } return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "schema validation passed for configured media types", + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: true, + Message: "schema validation passed for configured media types", }, nil } diff --git a/plugins/verifier/schemavalidator/schema_validator_test.go b/plugins/verifier/schemavalidator/schema_validator_test.go new file mode 100644 index 000000000..d0f28f60c --- /dev/null +++ b/plugins/verifier/schemavalidator/schema_validator_test.go @@ -0,0 +1,170 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "testing" + + "github.com/opencontainers/go-digest" + oci "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/ratify-project/ratify/pkg/common" + "github.com/ratify-project/ratify/pkg/ocispecs" + "github.com/ratify-project/ratify/pkg/referrerstore/mocks" + "github.com/ratify-project/ratify/pkg/verifier/plugin/skel" +) + +const mediaType string = "application/schema+json" + +func TestVerifyReference(t *testing.T) { + manifestDigest := digest.FromString("test_manifest_digest") + manifestDigest2 := digest.FromString("test_manifest_digest_2") + blobDigest := digest.FromString("test_blob_digest") + blobDigest2 := digest.FromString("test_blob_digest_2") + type args struct { + stdinData string + referenceManifest ocispecs.ReferenceManifest + blobContent string + refDesc ocispecs.ReferenceDescriptor + } + type want struct { + message string + errorReason string + err error + } + tests := []struct { + name string + args args + want want + }{ + { + name: "invalid stdin data", + args: args{}, + want: want{ + err: errors.New("failed to parse stdin for the input: unexpected end of JSON input"), + }, + }, + { + name: "failed to get reference manifest", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest2, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + err: errors.New("error fetching reference manifest for subject: test_subject reference descriptor: { sha256:b55e209647d87fcd95a94c59ff4d342e42bf10f02a7c10b5192131f8d959ff5a 0 [] map[] [] }"), + }, + }, + { + name: "empty blobs", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: fmt.Sprintf("No blobs found for referrer %s@%s.", "test_subject_path", manifestDigest.String()), + }, + }, + { + name: "get blob content error", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest2, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + err: fmt.Errorf("error fetching blob for subject:[%s] digest:[%s]", "test_subject", blobDigest2.String()), + }, + }, + { + name: "process mediaType error", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: fmt.Sprintf("schema validation failed for digest:[%s], media type:[%s].", blobDigest.String(), mediaType), + errorReason: "media type not configured for plugin:[application/schema+json]", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmdArgs := &skel.CmdArgs{ + Version: "1.0.0", + Subject: "test_subject", + StdinData: []byte(tt.args.stdinData), + } + testStore := &mocks.MemoryTestStore{ + Manifests: map[digest.Digest]ocispecs.ReferenceManifest{manifestDigest: tt.args.referenceManifest}, + Blobs: map[digest.Digest][]byte{blobDigest: []byte(tt.args.blobContent)}, + } + subjectRef := common.Reference{ + Path: "test_subject_path", + Original: "test_subject", + } + verifierResult, err := VerifyReference(cmdArgs, subjectRef, tt.args.refDesc, testStore) + if err != nil && err.Error() != tt.want.err.Error() { + t.Fatalf("verifyReference() error = %v, wantErr %v", err, tt.want.err) + } + if verifierResult != nil { + if verifierResult.Message != tt.want.message { + t.Fatalf("verifyReference() verifier report message = %s, want = %s", verifierResult.Message, tt.want.message) + } + if verifierResult.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyReference() verifier report error reason = %s, want = %s", verifierResult.ErrorReason, tt.want.errorReason) + } + } + }) + } +} diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report.go b/plugins/verifier/vulnerabilityreport/vulnerability_report.go index 37089ccf4..fda7aa8e8 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report.go @@ -26,6 +26,7 @@ import ( imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/owenrumney/go-sarif/v2/sarif" + re "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/common" "github.com/ratify-project/ratify/pkg/ocispecs" "github.com/ratify-project/ratify/pkg/referrerstore" @@ -93,11 +94,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe } createdTime, err := extractCreationTimestamp(input.CreatedAnnotationName, referenceDescriptor) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to create timestamp annotation.").WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error extracting create timestamp annotation:[%v]", err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), }, nil } @@ -105,11 +111,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe if input.MaximumAge != "" { ok, err := validateMaximumAge(input.MaximumAge, createdTime) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to validate maximum age.").WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error validating maximum age:[%v]", err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -117,10 +128,12 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe } if !ok { return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: report is older than maximum age:[%s]", input.MaximumAge), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("Report is older than maximum age:[%s].", input.MaximumAge), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -132,11 +145,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe referenceManifest, err := referrerStore.GetReferenceManifest(ctx, subjectReference, referenceDescriptor) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Failed to fetch reference manifest for subject: %s, reference descriptor: %v.", subjectReference, referenceDescriptor)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error fetching reference manifest for subject: %s reference descriptor: %v: [%v]", subjectReference, referenceDescriptor.Descriptor, err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -145,10 +163,12 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe if len(referenceManifest.Blobs) == 0 { return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: no layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("No layers found in manifest for referrer %s@%s.", subjectReference.Path, referenceDescriptor.Digest.String()), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -158,11 +178,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe blobDesc := referenceManifest.Blobs[0] refBlob, err := referrerStore.GetBlobContent(ctx, subjectReference, blobDesc.Digest) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Failed to fetch blob for subject:[%s] digest:[%s].", subjectReference, blobDesc.Digest)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error fetching blob for subject:[%s] digest:[%s]: [%v]", subjectReference, blobDesc.Digest, err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -172,10 +197,12 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe // skip all validation if passthrough is enabled if input.Passthrough { return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "Validation skipped. passthrough enabled", + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: true, + Message: "Validation skipped. passthrough enabled", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, "passthrough": true, @@ -186,11 +213,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe // validate json schema if err := verifyJSONSchema(referenceDescriptor.ArtifactType, refBlob, input.SchemaURL); err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Schema validation failed for digest:[%s],artifact type:[%s].", blobDesc.Digest, referenceDescriptor.ArtifactType)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: schema validation failed for digest:[%s],artifact type:[%s],parse errors:[%v]", blobDesc.Digest, referenceDescriptor.ArtifactType, err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -202,10 +234,12 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe } return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: true, + Message: "Validation succeeded", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -234,11 +268,16 @@ func verifyJSONSchema(artifactType string, refBlob []byte, schemaURL string) err func processSarifReport(input *PluginConfig, verifierName string, verifierType string, blob []byte, createdTime time.Time) (*verifier.VerifierResult, error) { sarifReport, err := sarif.FromBytes(blob) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to parse sarif report.").WithError(err) return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error parsing sarif report:[%v]", err.Error()), + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -247,10 +286,12 @@ func processSarifReport(input *PluginConfig, verifierName string, verifierType s // verify that there is at least one run in the report if len(sarifReport.Runs) < 1 { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: "Validation failed: no runs found in sarif report", + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: "No runs found in sarif report.", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -277,10 +318,12 @@ func processSarifReport(input *PluginConfig, verifierName string, verifierType s } return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: true, + Message: "Validation succeeded", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, "scanner": scannerName, @@ -302,10 +345,12 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st for _, result := range sarifReport.Runs[0].Results { if result.RuleID == nil || *result.RuleID == "" { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", result), + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("Rule id not found for result:[%v].", result), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -327,24 +372,28 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st if len(denylistViolations) > 0 { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, Extensions: map[string]interface{}{ "scanner": scannerName, "denylistCVEs": denylistCVEs, "cveViolations": denylistViolations, CreatedAnnotation: createdTime, }, - Message: "Validation failed: found denied CVEs. See extensions field for details.", + Message: "Found denied CVEs. See extensions field for details.", }, nil } return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: true, + Message: "Validation succeeded", Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -364,10 +413,12 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne for _, result := range sarifReport.Runs[0].Results { if result.RuleID == nil || *result.RuleID == "" { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", result), + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("Rule id not found for result:[%v].", result), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -377,10 +428,12 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne rule, ok := ruleMap[*result.RuleID] if !ok { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: rule not found for result:[%v]", result), + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("Rule not found for result:[%v].", result), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -389,11 +442,16 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne } severity, err := extractSeverity(scannerName, *rule) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to extract severity.").WithError(err) return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error extracting severity:[%v]", err.Error()), + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -410,23 +468,27 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne // if there are violating rules, return them as custom extension field if len(violatingRules) > 0 { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, Extensions: map[string]interface{}{ "scanner": scannerName, "disallowedSeverities": disallowedSeverities, "severityViolations": violatingRules, CreatedAnnotation: createdTime, }, - Message: "Validation failed: found disallowed severities. See extensions field for details.", + Message: "Found disallowed severities. See extensions field for details.", }, nil } return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: true, + Message: "Validation succeeded", Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go index e6341bfe1..84728f529 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go @@ -76,8 +76,9 @@ func TestVerifyReference(t *testing.T) { blobContent string } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -103,7 +104,8 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: error extracting create timestamp annotation:[%s]", "no annotations found for descriptor:[{{ sha256:b2f67b016d3c646f025099b363b4f83a56a44d067a846be74e8866342c56f216 0 [] map[] [] } application/sarif+json}]"), + message: "Failed to create timestamp annotation.", + errorReason: "no annotations found for descriptor:[{{ sha256:b2f67b016d3c646f025099b363b4f83a56a44d067a846be74e8866342c56f216 0 [] map[] [] } application/sarif+json}]", }, }, { @@ -118,7 +120,8 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: error validating maximum age:[%s]", "error parsing maximum age:[1d]"), + message: "Failed to validate maximum age.", + errorReason: "error parsing maximum age:[1d]", }, }, { @@ -133,7 +136,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: report is older than maximum age:[%s]", "24h"), + message: "Report is older than maximum age:[24h].", }, }, { @@ -148,7 +151,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: no layers found in manifest for referrer %s@%s", "test_subject_path", manifestDigest.String()), + message: fmt.Sprintf("No layers found in manifest for referrer %s@%s.", "test_subject_path", manifestDigest.String()), }, }, { @@ -189,7 +192,8 @@ func TestVerifyReference(t *testing.T) { blobContent: "{}", }, want: want{ - message: fmt.Sprintf("Validation failed: schema validation failed for digest:[%s],artifact type:[%s],parse errors:[%v]", blobDigest, SarifArtifactType, "version is required: runs is required: "), + message: fmt.Sprintf("Schema validation failed for digest:[%s],artifact type:[%s].", blobDigest, SarifArtifactType), + errorReason: "version is required: runs is required: ", }, }, { @@ -238,12 +242,15 @@ func TestVerifyReference(t *testing.T) { } verifierResult, err := VerifyReference(&cmdArgs, subjectRef, refDesc, testStore) if err != nil && err.Error() != tt.want.err.Error() { - t.Errorf("verifyReference() error = %v, wantErr %v", err, tt.want.err) - return + t.Fatalf("verifyReference() error = %v, wantErr %v", err, tt.want.err) } - if verifierResult != nil && verifierResult.Message != tt.want.message { - t.Errorf("verifyReference() verifier report message = %s, want %s", verifierResult.Message, tt.want.message) - return + if verifierResult != nil { + if verifierResult.Message != tt.want.message { + t.Fatalf("verifyReference() verifier report message = %s, want = %s", verifierResult.Message, tt.want.message) + } + if verifierResult.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyReference() verifier report error reason = %s, want = %s", verifierResult.ErrorReason, tt.want.errorReason) + } } }) } @@ -313,8 +320,9 @@ func TestProcessSarifReport(t *testing.T) { blobContent string } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -328,8 +336,9 @@ func TestProcessSarifReport(t *testing.T) { blobContent: "invalid", }, want: want{ - message: fmt.Sprintf("Validation failed: error parsing sarif report:[%s]", "invalid character 'i' looking for beginning of value"), - err: nil, + message: "Failed to parse sarif report.", + errorReason: "invalid character 'i' looking for beginning of value", + err: nil, }, }, { @@ -343,7 +352,7 @@ func TestProcessSarifReport(t *testing.T) { }`, }, want: want{ - message: "Validation failed: no runs found in sarif report", + message: "No runs found in sarif report.", err: nil, }, }, @@ -357,7 +366,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "Validation failed: found denied CVEs. See extensions field for details.", + message: "Found denied CVEs. See extensions field for details.", err: nil, }, }, @@ -374,7 +383,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "Validation failed: found disallowed severities. See extensions field for details.", + message: "Found disallowed severities. See extensions field for details.", err: nil, }, }, @@ -407,6 +416,9 @@ func TestProcessSarifReport(t *testing.T) { t.Errorf("processSarifReport() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) return } + if verifierReport.ErrorReason != tt.want.errorReason { + t.Errorf("processSarifReport() verifier report error reason = %s, want %s", verifierReport.ErrorReason, tt.want.errorReason) + } }) } } @@ -419,8 +431,9 @@ func TestVerifyDenyListCVEs(t *testing.T) { sarifReport sarif.Report } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -452,7 +465,7 @@ func TestVerifyDenyListCVEs(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", &sarif.Result{}), + message: fmt.Sprintf("Rule id not found for result:[%v].", &sarif.Result{}), err: nil, }, }, @@ -483,7 +496,7 @@ func TestVerifyDenyListCVEs(t *testing.T) { }, }, want: want{ - message: "Validation failed: found denied CVEs. See extensions field for details.", + message: "Found denied CVEs. See extensions field for details.", err: nil, }, }, @@ -523,12 +536,13 @@ func TestVerifyDenyListCVEs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { verifierReport, err := verifyDenyListCVEs("test_verifier", "", TrivyScannerName, &tests[i].args.sarifReport, tt.args.denyListCVEs, time.Now()) if err != nil && err.Error() != tt.want.err.Error() { - t.Errorf("verifyDenyListCVEs() error = %v, wantErr %v", err, tt.want.err) - return + t.Fatalf("verifyDenyListCVEs() error = %v, wantErr %v", err, tt.want.err) } if verifierReport.Message != tt.want.message { - t.Errorf("verifyDenyListCVEs() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) - return + t.Fatalf("verifyDenyListCVEs() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) + } + if verifierReport.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyDenyListCVEs() verifier report error reaon = %s, want = %s", verifierReport.ErrorReason, tt.want.errorReason) } }) } @@ -545,8 +559,9 @@ func TestVerifyDisallowedSeverities(t *testing.T) { sarifReport sarif.Report } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -582,7 +597,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", &sarif.Result{}), + message: fmt.Sprintf("Rule id not found for result:[%v].", &sarif.Result{}), err: nil, }, }, @@ -617,7 +632,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: rule not found for result:[%v]", &sarif.Result{RuleID: &invalidRuleID}), + message: fmt.Sprintf("Rule not found for result:[%v].", &sarif.Result{RuleID: &invalidRuleID}), err: nil, }, }, @@ -652,8 +667,9 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: error extracting severity:[severity not found in help text:[%s]]", invalidSeverityText), - err: nil, + message: "Failed to extract severity.", + errorReason: fmt.Sprintf("severity not found in help text:[%s]", invalidSeverityText), + err: nil, }, }, { @@ -687,7 +703,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: "Validation failed: found disallowed severities. See extensions field for details.", + message: "Found disallowed severities. See extensions field for details.", err: nil, }, }, @@ -731,13 +747,16 @@ func TestVerifyDisallowedSeverities(t *testing.T) { t.Run(tt.name, func(t *testing.T) { verifierReport, err := verifyDisallowedSeverities("test_verifier", "", TrivyScannerName, &tests[i].args.sarifReport, tt.args.disallowedSeverities, time.Now()) if err != nil && err.Error() != tt.want.err.Error() { - t.Errorf("verifyDisallowedSeverities() error = %v, wantErr %v", err, tt.want.err) + t.Fatalf("verifyDisallowedSeverities() error = %v, wantErr %v", err, tt.want.err) return } if verifierReport.Message != tt.want.message { - t.Errorf("verifyDisallowedSeverities() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) + t.Fatalf("verifyDisallowedSeverities() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) return } + if verifierReport.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyDisalowedServerities() verifier report error reason = %s, want = %s", verifierReport.ErrorReason, tt.want.errorReason) + } }) } }