From 57d97fd1416d8fbf41f0401128d67f6a844916fd Mon Sep 17 00:00:00 2001 From: Binbin Li Date: Wed, 21 Aug 2024 10:21:21 +0800 Subject: [PATCH] feat: fill ErrorReason and Remediation during verifierReport generation (#1682) --- pkg/executor/core/executor.go | 18 +- pkg/executor/types/types.go | 1 + .../configpolicy/configpolicy.go | 7 +- pkg/verifier/api.go | 17 - pkg/verifier/cosign/cosign.go | 55 +-- pkg/verifier/cosign/cosign_test.go | 3 + pkg/verifier/notation/notation.go | 10 +- pkg/verifier/result.go | 62 +++ pkg/verifier/result_test.go | 83 ++++ pkg/verifier/types/types.go | 40 +- pkg/verifier/types/types_test.go | 83 ++++ plugins/verifier/sbom/sbom.go | 83 ++-- plugins/verifier/sbom/sbom_test.go | 184 ++++++++- .../schemavalidator/schema_validator.go | 28 +- .../schemavalidator/schema_validator_test.go | 170 +++++++++ .../vulnerability_report.go | 360 ++++++++++-------- .../vulnerability_report_test.go | 91 +++-- 17 files changed, 954 insertions(+), 341 deletions(-) create mode 100644 pkg/verifier/result.go create mode 100644 pkg/verifier/result_test.go create mode 100644 pkg/verifier/types/types_test.go create mode 100644 plugins/verifier/schemavalidator/schema_validator_test.go diff --git a/pkg/executor/core/executor.go b/pkg/executor/core/executor.go index 2cc90b0280..e5d8ec2993 100644 --- a/pkg/executor/core/executor.go +++ b/pkg/executor/core/executor.go @@ -176,13 +176,8 @@ func (executor Executor) verifyReferenceForJSONPolicy(ctx context.Context, subje verifierStartTime := time.Now() verifyResult, err := verifier.Verify(ctx, subjectRef, referenceDesc, referrerStore) if err != nil { - verifyResult = vr.VerifierResult{ - IsSuccess: false, - Name: verifier.Name(), - Type: verifier.Type(), - VerifierName: verifier.Name(), - VerifierType: verifier.Type(), - Message: errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace).Error()} + verifierErr := errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace) + verifyResult = vr.NewVerifierResult("", verifier.Name(), verifier.Type(), "", false, &verifierErr, nil) } if len(verifier.GetNestedReferences()) > 0 { @@ -229,13 +224,8 @@ func (executor Executor) verifyReferenceForRegoPolicy(ctx context.Context, subje verifierStartTime := time.Now() verifierResult, err := verifier.Verify(errCtx, subjectRef, referenceDesc, referrerStore) if err != nil { - verifierReport = vt.VerifierResult{ - IsSuccess: false, - Name: verifier.Name(), // Deprecating Name in next release, reference to VerifierName instead. - Type: verifier.Type(), // Deprecating Type in next release, reference to VerifierType instead. - VerifierName: verifier.Name(), - VerifierType: verifier.Type(), - Message: errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace).Error()} + verifierErr := errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace) + verifierReport = vt.CreateVerifierResult(verifier.Name(), verifier.Type(), "", false, &verifierErr) } else { verifierReport = vt.NewVerifierResult(verifierResult) } diff --git a/pkg/executor/types/types.go b/pkg/executor/types/types.go index ccf78c3372..30bac9a1be 100644 --- a/pkg/executor/types/types.go +++ b/pkg/executor/types/types.go @@ -29,6 +29,7 @@ type VerifyResult struct { // NestedVerifierReport describes the results of verifying an artifact and its // nested artifacts by available verifiers. +// Note: NestedVerifierReport is used for verification results in v1. type NestedVerifierReport struct { Subject string `json:"subject"` ReferenceDigest string `json:"referenceDigest"` diff --git a/pkg/policyprovider/configpolicy/configpolicy.go b/pkg/policyprovider/configpolicy/configpolicy.go index a0475bbc26..2ab93ca02e 100644 --- a/pkg/policyprovider/configpolicy/configpolicy.go +++ b/pkg/policyprovider/configpolicy/configpolicy.go @@ -97,11 +97,8 @@ func (enforcer PolicyEnforcer) ContinueVerifyOnFailure(_ context.Context, _ comm // ErrorToVerifyResult converts an error to a properly formatted verify result func (enforcer PolicyEnforcer) ErrorToVerifyResult(_ context.Context, subjectRefString string, verifyError error) types.VerifyResult { - errorReport := verifier.VerifierResult{ - Subject: subjectRefString, - IsSuccess: false, - Message: fmt.Sprintf("verification failed: %v", verifyError), - } + verifierErr := re.ErrorCodeVerifyReferenceFailure.WithDetail(fmt.Sprintf("failed to verify artifact: %s", subjectRefString)).WithError(verifyError) + errorReport := verifier.NewVerifierResult(subjectRefString, "", "", "", false, &verifierErr, nil) var reports []interface{} reports = append(reports, errorReport) return types.VerifyResult{IsSuccess: false, VerifierReports: reports} diff --git a/pkg/verifier/api.go b/pkg/verifier/api.go index 8f9695515e..d24246664a 100644 --- a/pkg/verifier/api.go +++ b/pkg/verifier/api.go @@ -23,23 +23,6 @@ import ( "github.com/ratify-project/ratify/pkg/referrerstore" ) -// VerifierResult describes the result of verifying a reference manifest for a subject -type VerifierResult struct { //nolint:revive // ignore linter to have unique type name - Subject string `json:"subject,omitempty"` - IsSuccess bool `json:"isSuccess"` - Name string `json:"name,omitempty"` - VerifierName string `json:"verifierName,omitempty"` - Type string `json:"type,omitempty"` - VerifierType string `json:"verifierType,omitempty"` - Message string `json:"message,omitempty"` - ErrorReason string `json:"errorReason,omitempty"` - Extensions interface{} `json:"extensions,omitempty"` - NestedResults []VerifierResult `json:"nestedResults,omitempty"` - ArtifactType string `json:"artifactType,omitempty"` - ReferenceDigest string `json:"referenceDigest,omitempty"` - Remediation string `json:"remediation,omitempty"` -} - // ReferenceVerifier is an interface that defines methods to verify a reference // for a subject by a verifier. type ReferenceVerifier interface { diff --git a/pkg/verifier/cosign/cosign.go b/pkg/verifier/cosign/cosign.go index a5a3bc4c41..fb3efd7140 100644 --- a/pkg/verifier/cosign/cosign.go +++ b/pkg/verifier/cosign/cosign.go @@ -284,15 +284,15 @@ func (v *cosignVerifier) verifyInternal(ctx context.Context, subjectReference co } if hasValidSignature { - return verifier.VerifierResult{ - Name: v.name, - Type: v.verifierType, - VerifierName: v.name, - VerifierType: v.verifierType, - IsSuccess: true, - Message: "Verification success. Valid signatures found. Please refer to extensions field for verifications performed.", - Extensions: Extension{SignatureExtension: sigExtensions, TrustPolicy: trustPolicy.GetName()}, - }, nil + return verifier.NewVerifierResult( + "", + v.name, + v.verifierType, + "Verification success. Valid signatures found. Please refer to extensions field for verifications performed.", + true, + nil, + Extension{SignatureExtension: sigExtensions, TrustPolicy: trustPolicy.GetName()}, + ), nil } errorResult := errorToVerifyResult(v.name, v.verifierType, fmt.Errorf("no valid signatures found")) @@ -397,15 +397,15 @@ func (v *cosignVerifier) verifyLegacy(ctx context.Context, subjectReference comm } if len(signatures) > 0 { - return verifier.VerifierResult{ - Name: v.name, - Type: v.verifierType, - VerifierName: v.name, - VerifierType: v.verifierType, - IsSuccess: true, - Message: "Verification success. Valid signatures found", - Extensions: LegacyExtension{SignatureExtension: sigExtensions}, - }, nil + return verifier.NewVerifierResult( + "", + v.name, + v.verifierType, + "Verification success. Valid signatures found", + true, + nil, + LegacyExtension{SignatureExtension: sigExtensions}, + ), nil } errorResult := errorToVerifyResult(v.name, v.verifierType, fmt.Errorf("no valid signatures found")) @@ -485,15 +485,16 @@ func staticLayerOpts(desc imgspec.Descriptor) ([]static.Option, error) { // ErrorToVerifyResult returns a verifier result with the error message and isSuccess set to false func errorToVerifyResult(name string, verifierType string, err error) verifier.VerifierResult { - return verifier.VerifierResult{ - IsSuccess: false, - Name: name, - Type: verifierType, - VerifierName: name, - VerifierType: verifierType, - Message: "Verification failed", - ErrorReason: err.Error(), - } + verifierErr := re.ErrorCodeVerifyReferenceFailure.WithDetail("Verification failed").WithError(err) + return verifier.NewVerifierResult( + "", + name, + verifierType, + "", + false, + &verifierErr, + nil, + ) } // decodeASN1Signature decodes the ASN.1 signature to raw signature bytes diff --git a/pkg/verifier/cosign/cosign_test.go b/pkg/verifier/cosign/cosign_test.go index fde624a3e9..b4e368617b 100644 --- a/pkg/verifier/cosign/cosign_test.go +++ b/pkg/verifier/cosign/cosign_test.go @@ -410,6 +410,9 @@ func TestErrorToVerifyResult(t *testing.T) { if verifierResult.Message != "Verification failed" { t.Errorf("errorToVerifyResult() = %v, want %v", verifierResult.Message, "Verification failed") } + if verifierResult.ErrorReason != "test error" { + t.Errorf("errorToVerifyResult() = %v, want %v", verifierResult.ErrorReason, "test error") + } } // TestDecodeASN1Signature tests the decodeASN1Signature function diff --git a/pkg/verifier/notation/notation.go b/pkg/verifier/notation/notation.go index 1a43493470..ef412b9f0a 100644 --- a/pkg/verifier/notation/notation.go +++ b/pkg/verifier/notation/notation.go @@ -174,15 +174,7 @@ func (v *notationPluginVerifier) Verify(ctx context.Context, extensions["SN"] = cert.Subject.String() } - return verifier.VerifierResult{ - Name: v.name, - Type: v.verifierType, - VerifierName: v.name, - VerifierType: v.verifierType, - IsSuccess: true, - Message: "Signature verification success", - Extensions: extensions, - }, nil + return verifier.NewVerifierResult("", v.name, v.verifierType, "Signature verification success", true, nil, extensions), nil } func getVerifierService(conf *NotationPluginVerifierConfig, pluginDirectory string) (notation.Verifier, error) { diff --git a/pkg/verifier/result.go b/pkg/verifier/result.go new file mode 100644 index 0000000000..dff28e9e8d --- /dev/null +++ b/pkg/verifier/result.go @@ -0,0 +1,62 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package verifier + +import "github.com/ratify-project/ratify/errors" + +// VerifierResult describes the result of verifying a reference manifest for a subject. +// Note: This struct is used to represent the result of verification in v0. +type VerifierResult struct { //nolint:revive // ignore linter to have unique type name + Subject string `json:"subject,omitempty"` + IsSuccess bool `json:"isSuccess"` + // Name will be deprecated in v2, tracking issue: https://github.com/ratify-project/ratify/issues/1707 + Name string `json:"name,omitempty"` + VerifierName string `json:"verifierName,omitempty"` + // Type will be deprecated in v2, tracking issue: https://github.com/ratify-project/ratify/issues/1707 + Type string `json:"type,omitempty"` + VerifierType string `json:"verifierType,omitempty"` + ReferenceDigest string `json:"referenceDigest,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` + Message string `json:"message,omitempty"` + ErrorReason string `json:"errorReason,omitempty"` + Remediation string `json:"remediation,omitempty"` + Extensions interface{} `json:"extensions,omitempty"` + NestedResults []VerifierResult `json:"nestedResults,omitempty"` +} + +// NewVerifierResult creates a new VerifierResult object with the given parameters. +func NewVerifierResult(subject, verifierName, verifierType, message string, isSuccess bool, err *errors.Error, extensions interface{}) VerifierResult { + var errorReason, remediation string + if err != nil { + if err.GetDetail() != "" { + message = err.GetDetail() + } + errorReason = err.GetErrorReason() + remediation = err.GetRemediation() + } + return VerifierResult{ + Subject: subject, + IsSuccess: isSuccess, + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + Message: message, + ErrorReason: errorReason, + Remediation: remediation, + Extensions: extensions, + } +} diff --git a/pkg/verifier/result_test.go b/pkg/verifier/result_test.go new file mode 100644 index 0000000000..64efd2c52d --- /dev/null +++ b/pkg/verifier/result_test.go @@ -0,0 +1,83 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package verifier + +import ( + "fmt" + "testing" + + "github.com/ratify-project/ratify/errors" +) + +const ( + testMsg1 = "test message 1" + testMsg2 = "test message 2" + testErrReason = "test error reason" + testRemediation = "test remediation" +) + +func TestNewVerifierResult(t *testing.T) { + tests := []struct { + name string + message string + err errors.Error + expectedMsg string + expectedErrReason string + expectedRemediation string + }{ + { + name: "nil error", + message: testMsg1, + err: errors.Error{}, + expectedMsg: testMsg1, + }, + { + name: "error without detail", + message: testMsg1, + err: errors.ErrorCodeUnknown.WithError(fmt.Errorf(testErrReason)).WithRemediation(testRemediation), + expectedMsg: testMsg1, + expectedErrReason: testErrReason, + expectedRemediation: testRemediation, + }, + { + name: "error with detail", + message: testMsg1, + err: errors.ErrorCodeUnknown.WithError(fmt.Errorf(testErrReason)).WithRemediation(testRemediation).WithDetail(testMsg2), + expectedMsg: testMsg2, + expectedErrReason: testErrReason, + expectedRemediation: testRemediation, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := &tt.err + if tt.err == (errors.Error{}) { + err = nil + } + + result := NewVerifierResult("", "", "", tt.message, false, err, nil) + if result.Message != tt.expectedMsg { + t.Errorf("expected message %s, got %s", tt.expectedMsg, result.Message) + } + if result.ErrorReason != tt.expectedErrReason { + t.Errorf("expected error reason %s, got %s", tt.expectedErrReason, result.ErrorReason) + } + if result.Remediation != tt.expectedRemediation { + t.Errorf("expected remediation %s, got %s", tt.expectedRemediation, result.Remediation) + } + }) + } +} diff --git a/pkg/verifier/types/types.go b/pkg/verifier/types/types.go index 2eed0a669e..d21e9b0989 100644 --- a/pkg/verifier/types/types.go +++ b/pkg/verifier/types/types.go @@ -19,6 +19,7 @@ import ( "encoding/json" "io" + "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/verifier" ) @@ -47,11 +48,14 @@ const ( // VerifierResult describes the verification result returned from the verifier plugin type VerifierResult struct { - IsSuccess bool `json:"isSuccess"` - Message string `json:"message"` - ErrorReason string `json:"errorReason,omitempty"` - Name string `json:"name"` - VerifierName string `json:"verifierName,omitempty"` + IsSuccess bool `json:"isSuccess"` + Message string `json:"message"` + ErrorReason string `json:"errorReason,omitempty"` + Remediation string `json:"remediation,omitempty"` + // Name will be deprecated in v2, tracking issue: https://github.com/ratify-project/ratify/issues/1707 + Name string `json:"name"` + VerifierName string `json:"verifierName,omitempty"` + // Type will be deprecated in v2, tracking issue: https://github.com/ratify-project/ratify/issues/1707 Type string `json:"type,omitempty"` VerifierType string `json:"verifierType,omitempty"` Extensions interface{} `json:"extensions"` @@ -79,6 +83,30 @@ func WriteVerifyResultResult(result *verifier.VerifierResult, w io.Writer) error return json.NewEncoder(w).Encode(result) } +// CreateVerifierResult creates a new verifier result object from given input. +func CreateVerifierResult(verifierName, verifierType, message string, isSuccess bool, err *errors.Error) VerifierResult { + var errorReason string + var remediation string + if err != nil { + if err.GetDetail() != "" { + message = err.GetDetail() + } + errorReason = err.GetErrorReason() + remediation = err.GetRemediation() + } + + return VerifierResult{ + IsSuccess: isSuccess, + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + Message: message, + ErrorReason: errorReason, + Remediation: remediation, + } +} + // NewVerifierResult creates a new verifier result object from the given // verifier.VerifierResult. func NewVerifierResult(result verifier.VerifierResult) VerifierResult { @@ -90,5 +118,7 @@ func NewVerifierResult(result verifier.VerifierResult) VerifierResult { VerifierName: result.VerifierName, VerifierType: result.VerifierType, Extensions: result.Extensions, + ErrorReason: result.ErrorReason, + Remediation: result.Remediation, } } diff --git a/pkg/verifier/types/types_test.go b/pkg/verifier/types/types_test.go new file mode 100644 index 0000000000..ce1cd39f65 --- /dev/null +++ b/pkg/verifier/types/types_test.go @@ -0,0 +1,83 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "fmt" + "testing" + + "github.com/ratify-project/ratify/errors" +) + +const ( + testMsg1 = "test message 1" + testMsg2 = "test message 2" + testErrReason = "test error reason" + testRemediation = "test remediation" +) + +func TestCreateVerifierResult(t *testing.T) { + tests := []struct { + name string + message string + err errors.Error + expectedMsg string + expectedErrReason string + expectedRemediation string + }{ + { + name: "nil error", + message: testMsg1, + err: errors.Error{}, + expectedMsg: testMsg1, + }, + { + name: "error without detail", + message: testMsg1, + err: errors.ErrorCodeUnknown.WithError(fmt.Errorf(testErrReason)).WithRemediation(testRemediation), + expectedMsg: testMsg1, + expectedErrReason: testErrReason, + expectedRemediation: testRemediation, + }, + { + name: "error with detail", + message: testMsg1, + err: errors.ErrorCodeUnknown.WithError(fmt.Errorf(testErrReason)).WithRemediation(testRemediation).WithDetail(testMsg2), + expectedMsg: testMsg2, + expectedErrReason: testErrReason, + expectedRemediation: testRemediation, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := &tt.err + if tt.err == (errors.Error{}) { + err = nil + } + + result := CreateVerifierResult("", "", tt.message, false, err) + if result.Message != tt.expectedMsg { + t.Errorf("expected message %s, got %s", tt.expectedMsg, result.Message) + } + if result.ErrorReason != tt.expectedErrReason { + t.Errorf("expected error reason %s, got %s", tt.expectedErrReason, result.ErrorReason) + } + if result.Remediation != tt.expectedRemediation { + t.Errorf("expected remediation %s, got %s", tt.expectedRemediation, result.Remediation) + } + }) + } +} diff --git a/plugins/verifier/sbom/sbom.go b/plugins/verifier/sbom/sbom.go index cd8448da3b..70a18ad2ac 100644 --- a/plugins/verifier/sbom/sbom.go +++ b/plugins/verifier/sbom/sbom.go @@ -22,12 +22,14 @@ import ( "fmt" "strings" + "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/common" "github.com/ratify-project/ratify/pkg/ocispecs" "github.com/ratify-project/ratify/pkg/referrerstore" "github.com/ratify-project/ratify/plugins/verifier/sbom/utils" // This import is required to utilize the oras built-in referrer store + re "github.com/ratify-project/ratify/errors" _ "github.com/ratify-project/ratify/pkg/referrerstore/oras" "github.com/ratify-project/ratify/pkg/verifier" "github.com/ratify-project/ratify/pkg/verifier/plugin/skel" @@ -82,20 +84,15 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe ctx := context.Background() referenceManifest, err := referrerStore.GetReferenceManifest(ctx, subjectReference, referenceDescriptor) if err != nil { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Error fetching reference manifest for subject: %s reference descriptor: %v, err: %v", subjectReference, referenceDescriptor.Descriptor, err), - }, nil + storeErr := re.ErrorCodeGetReferenceManifestFailure.WithDetail(fmt.Sprintf("Failed to fetch reference manifest for subject: %s reference descriptor: %v", subjectReference, referenceDescriptor.Descriptor)).WithError(err) + result := verifier.NewVerifierResult("", input.Name, verifierType, "", false, &storeErr, nil) + return &result, nil } if len(referenceManifest.Blobs) == 0 { - return &verifier.VerifierResult{ - Name: input.Name, - IsSuccess: false, - Message: fmt.Sprintf("SBOM validation failed: no layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), - }, nil + noBlobErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("No layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String())) + result := verifier.NewVerifierResult("", input.Name, verifierType, "SBOM validation failed", false, &noBlobErr, nil) + return &result, nil } artifactType := referenceDescriptor.ArtifactType @@ -103,33 +100,22 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe refBlob, err := referrerStore.GetBlobContent(ctx, subjectReference, blobDesc.Digest) if err != nil { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Error fetching blob for subject: %s digest: %s, err: %v", subjectReference, blobDesc.Digest, err), - }, nil + storeErr := re.ErrorCodeGetBlobContentFailure.WithDetail(fmt.Sprintf("Failed to fetch blob for subject: %s digest: %s", subjectReference, blobDesc.Digest)).WithError(err) + result := verifier.NewVerifierResult("", input.Name, verifierType, "", false, &storeErr, nil) + return &result, nil } switch artifactType { case SpdxJSONMediaType: return processSpdxJSONMediaType(input.Name, verifierType, refBlob, input.DisallowedLicenses, input.DisallowedPackages), nil default: - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Unsupported artifactType: %s", artifactType), - }, nil + storeErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Unsupported artifactType: %s", artifactType)) + result := verifier.NewVerifierResult("", input.Name, verifierType, "Failed to process SBOM blobs.", false, &storeErr, nil) + return &result, nil } } - - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "SBOM verification success. No license or package violation found.", - }, nil + result := verifier.NewVerifierResult("", input.Name, verifierType, "SBOM verification success. No license or package violation found.", true, nil, nil) + return &result, nil } // getViolations returns the package and license violations based on the deny list @@ -177,31 +163,26 @@ func processSpdxJSONMediaType(name string, verifierType string, refBlob []byte, } if len(licenseViolation) != 0 || len(packageViolation) != 0 { - return &verifier.VerifierResult{ - Name: name, - IsSuccess: false, - Extensions: extensionData, - Message: "SBOM validation failed. Please review extensions data for license and package violation found.", - } + sbomErr := errors.ErrorCodeVerifyPluginFailure.WithDetail("License or package violation found.").WithRemediation("Please review extensions data for license and package violation found.") + result := verifier.NewVerifierResult("", name, verifierType, "SBOM validation failed", false, &sbomErr, extensionData) + return &result } } - return &verifier.VerifierResult{ - Name: name, - Type: verifierType, - IsSuccess: true, - Extensions: map[string]interface{}{ - CreationInfo: spdxDoc.CreationInfo, - }, - Message: "SBOM verification success. No license or package violation found.", - } - } - return &verifier.VerifierResult{ - Name: name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("SBOM failed to parse: %v", err), + result := verifier.NewVerifierResult( + "", + name, + verifierType, + "SBOM verification success. No license or package violation found.", + true, + nil, + map[string]interface{}{CreationInfo: spdxDoc.CreationInfo}, + ) + return &result } + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("failed to verify artifact: %s", name)).WithError(err) + result := verifier.NewVerifierResult("", name, verifierType, "", false, &verifierErr, nil) + return &result } // iterate through all package info and check against the deny list diff --git a/plugins/verifier/sbom/sbom_test.go b/plugins/verifier/sbom/sbom_test.go index 8dc5a4c3f5..8609fc500e 100644 --- a/plugins/verifier/sbom/sbom_test.go +++ b/plugins/verifier/sbom/sbom_test.go @@ -15,14 +15,24 @@ limitations under the License. package main import ( + "errors" + "fmt" "os" "path/filepath" "strings" "testing" + "github.com/opencontainers/go-digest" + oci "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/ratify-project/ratify/pkg/common" + "github.com/ratify-project/ratify/pkg/ocispecs" + "github.com/ratify-project/ratify/pkg/referrerstore/mocks" + "github.com/ratify-project/ratify/pkg/verifier/plugin/skel" "github.com/ratify-project/ratify/plugins/verifier/sbom/utils" ) +const mediaType string = "application/vnd.syft+json" + func TestProcessSPDXJsonMediaType(t *testing.T) { b, err := os.ReadFile(filepath.Join("testdata", "bom.json")) if err != nil { @@ -41,8 +51,178 @@ func TestProcessInvalidSPDXJsonMediaType(t *testing.T) { } report := processSpdxJSONMediaType("test", "", b, nil, nil) - if !strings.Contains(report.Message, "SBOM failed to parse") { - t.Fatalf("expected to have an error processing spdx json file: %s", filepath.Join("testdata", "bom.json")) + if !strings.Contains(report.Message, "failed to verify artifact") { + t.Fatalf("report message: %s does not contain expected error message", report.Message) + } + if report.ErrorReason != "JSON document does not contain spdxVersion field" { + t.Fatalf("expected error reason: %s, got: %s", "JSON document does not contain spdxVersion field", report.ErrorReason) + } +} + +func TestVerifyReference(t *testing.T) { + manifestDigest := digest.FromString("test_manifest_digest") + manifestDigest2 := digest.FromString("test_manifest_digest_2") + blobDigest := digest.FromString("test_blob_digest") + blobDigest2 := digest.FromString("test_blob_digest_2") + type args struct { + stdinData string + referenceManifest ocispecs.ReferenceManifest + blobContent string + refDesc ocispecs.ReferenceDescriptor + } + type want struct { + message string + errorReason string + err error + } + + tests := []struct { + name string + args args + want want + }{ + { + name: "invalid stdin data", + args: args{}, + want: want{ + err: errors.New("failed to parse stdin for the input: unexpected end of JSON input"), + }, + }, + { + name: "failed to get reference manifest", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest2, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: "Failed to fetch reference manifest for subject: test_subject reference descriptor: { sha256:b55e209647d87fcd95a94c59ff4d342e42bf10f02a7c10b5192131f8d959ff5a 0 [] map[] [] }", + errorReason: "manifest not found", + }, + }, + { + name: "empty blobs", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: "SBOM validation failed", + errorReason: fmt.Sprintf("No layers found in manifest for referrer %s@%s", "test_subject_path", manifestDigest.String()), + }, + }, + { + name: "get blob content error", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest2, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: fmt.Sprintf("Failed to fetch blob for subject: test_subject digest: %s", blobDigest2.String()), + errorReason: "blob not found", + }, + }, + { + name: "unsupported artifactType", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: "Failed to process SBOM blobs.", + errorReason: "Unsupported artifactType: application/vnd.syft+json", + }, + }, + { + name: "process spdx json mediaType error", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: SpdxJSONMediaType, + }, + }, + want: want{ + message: "failed to verify artifact: sbom", + errorReason: "unexpected end of JSON input", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmdArgs := &skel.CmdArgs{ + Version: "1.0.0", + Subject: "test_subject", + StdinData: []byte(tt.args.stdinData), + } + testStore := &mocks.MemoryTestStore{ + Manifests: map[digest.Digest]ocispecs.ReferenceManifest{manifestDigest: tt.args.referenceManifest}, + Blobs: map[digest.Digest][]byte{blobDigest: []byte(tt.args.blobContent)}, + } + subjectRef := common.Reference{ + Path: "test_subject_path", + Original: "test_subject", + } + verifierResult, err := VerifyReference(cmdArgs, subjectRef, tt.args.refDesc, testStore) + if err != nil && err.Error() != tt.want.err.Error() { + t.Fatalf("verifyReference() error = %v, wantErr %v", err, tt.want.err) + } + if verifierResult != nil { + if verifierResult.Message != tt.want.message { + t.Fatalf("verifyReference() verifier report message = %s, want = %s", verifierResult.Message, tt.want.message) + } + if verifierResult.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyReference() verifier report error reason = %s, want = %s", verifierResult.ErrorReason, tt.want.errorReason) + } + } + }) } } diff --git a/plugins/verifier/schemavalidator/schema_validator.go b/plugins/verifier/schemavalidator/schema_validator.go index 2af66396ea..d318862227 100644 --- a/plugins/verifier/schemavalidator/schema_validator.go +++ b/plugins/verifier/schemavalidator/schema_validator.go @@ -20,6 +20,8 @@ import ( "encoding/json" "fmt" + "github.com/ratify-project/ratify/errors" + re "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/common" "github.com/ratify-project/ratify/pkg/ocispecs" "github.com/ratify-project/ratify/pkg/referrerstore" @@ -71,12 +73,9 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe } if len(referenceManifest.Blobs) == 0 { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("schema validation failed: no blobs found for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), - }, nil + noBlobErr := errors.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("No blobs found for referrer %s@%s.", subjectReference.Path, referenceDescriptor.Digest.String())) + result := verifier.NewVerifierResult("", input.Name, verifierType, "", false, &noBlobErr, nil) + return &result, nil } for _, blobDesc := range referenceManifest.Blobs { @@ -87,21 +86,14 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe err = processMediaType(schemaMap, blobDesc.MediaType, refBlob) if err != nil { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("schema validation failed for digest:[%s],media type:[%s],parse errors:[%v]", blobDesc.Digest, blobDesc.MediaType, err.Error()), - }, nil + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("schema validation failed for digest:[%s], media type:[%s].", blobDesc.Digest, blobDesc.MediaType)).WithError(err) + result := verifier.NewVerifierResult("", input.Name, verifierType, "", false, &verifierErr, nil) + return &result, nil } } - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "schema validation passed for configured media types", - }, nil + result := verifier.NewVerifierResult("", input.Name, verifierType, "schema validation passed for configured media types", true, nil, nil) + return &result, nil } func processMediaType(schemaMap map[string]string, mediaType string, refBlob []byte) error { diff --git a/plugins/verifier/schemavalidator/schema_validator_test.go b/plugins/verifier/schemavalidator/schema_validator_test.go new file mode 100644 index 0000000000..3448ec8977 --- /dev/null +++ b/plugins/verifier/schemavalidator/schema_validator_test.go @@ -0,0 +1,170 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "testing" + + "github.com/opencontainers/go-digest" + oci "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/ratify-project/ratify/pkg/common" + "github.com/ratify-project/ratify/pkg/ocispecs" + "github.com/ratify-project/ratify/pkg/referrerstore/mocks" + "github.com/ratify-project/ratify/pkg/verifier/plugin/skel" +) + +const mediaType string = "application/schema+json" + +func TestVerifyReference(t *testing.T) { + manifestDigest := digest.FromString("test_manifest_digest") + manifestDigest2 := digest.FromString("test_manifest_digest_2") + blobDigest := digest.FromString("test_blob_digest") + blobDigest2 := digest.FromString("test_blob_digest_2") + type args struct { + stdinData string + referenceManifest ocispecs.ReferenceManifest + blobContent string + refDesc ocispecs.ReferenceDescriptor + } + type want struct { + message string + errorReason string + err error + } + tests := []struct { + name string + args args + want want + }{ + { + name: "invalid stdin data", + args: args{}, + want: want{ + err: errors.New("failed to parse stdin for the input: unexpected end of JSON input"), + }, + }, + { + name: "failed to get reference manifest", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest2, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + err: errors.New("error fetching reference manifest for subject: test_subject reference descriptor: { sha256:b55e209647d87fcd95a94c59ff4d342e42bf10f02a7c10b5192131f8d959ff5a 0 [] map[] [] }"), + }, + }, + { + name: "empty blobs", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + errorReason: fmt.Sprintf("No blobs found for referrer %s@%s.", "test_subject_path", manifestDigest.String()), + }, + }, + { + name: "get blob content error", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest2, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + err: fmt.Errorf("error fetching blob for subject:[%s] digest:[%s]", "test_subject", blobDigest2.String()), + }, + }, + { + name: "process mediaType error", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: fmt.Sprintf("schema validation failed for digest:[%s], media type:[%s].", blobDigest.String(), mediaType), + errorReason: "media type not configured for plugin:[application/schema+json]", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmdArgs := &skel.CmdArgs{ + Version: "1.0.0", + Subject: "test_subject", + StdinData: []byte(tt.args.stdinData), + } + testStore := &mocks.MemoryTestStore{ + Manifests: map[digest.Digest]ocispecs.ReferenceManifest{manifestDigest: tt.args.referenceManifest}, + Blobs: map[digest.Digest][]byte{blobDigest: []byte(tt.args.blobContent)}, + } + subjectRef := common.Reference{ + Path: "test_subject_path", + Original: "test_subject", + } + verifierResult, err := VerifyReference(cmdArgs, subjectRef, tt.args.refDesc, testStore) + if err != nil && err.Error() != tt.want.err.Error() { + t.Fatalf("verifyReference() error = %v, wantErr %v", err, tt.want.err) + } + if verifierResult != nil { + if verifierResult.Message != tt.want.message { + t.Fatalf("verifyReference() verifier report message = %s, want = %s", verifierResult.Message, tt.want.message) + } + if verifierResult.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyReference() verifier report error reason = %s, want = %s", verifierResult.ErrorReason, tt.want.errorReason) + } + } + }) + } +} diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report.go b/plugins/verifier/vulnerabilityreport/vulnerability_report.go index 37089ccf4a..ec4004f8f1 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report.go @@ -26,6 +26,8 @@ import ( imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/owenrumney/go-sarif/v2/sarif" + "github.com/ratify-project/ratify/errors" + re "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/common" "github.com/ratify-project/ratify/pkg/ocispecs" "github.com/ratify-project/ratify/pkg/referrerstore" @@ -93,38 +95,39 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe } createdTime, err := extractCreationTimestamp(input.CreatedAnnotationName, referenceDescriptor) if err != nil { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error extracting create timestamp annotation:[%v]", err.Error()), - }, nil + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to create timestamp annotation.").WithError(err) + result := verifier.NewVerifierResult("", input.Name, verifierType, "", false, &verifierErr, nil) + return &result, nil } // check report is newer than allowed maximum age if input.MaximumAge != "" { ok, err := validateMaximumAge(input.MaximumAge, createdTime) if err != nil { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error validating maximum age:[%v]", err.Error()), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to validate maximum age.").WithError(err) + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } if !ok { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: report is older than maximum age:[%s]", input.MaximumAge), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Report is older than maximum age:[%s].", input.MaximumAge)) + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } } @@ -132,84 +135,96 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe referenceManifest, err := referrerStore.GetReferenceManifest(ctx, subjectReference, referenceDescriptor) if err != nil { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error fetching reference manifest for subject: %s reference descriptor: %v: [%v]", subjectReference, referenceDescriptor.Descriptor, err.Error()), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Failed to fetch reference manifest for subject: %s, reference descriptor: %v.", subjectReference, referenceDescriptor)).WithError(err) + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } if len(referenceManifest.Blobs) == 0 { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: no layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + noBlobErr := errors.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("No layers found in manifest for referrer %s@%s.", subjectReference.Path, referenceDescriptor.Digest.String())) + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &noBlobErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } blobDesc := referenceManifest.Blobs[0] refBlob, err := referrerStore.GetBlobContent(ctx, subjectReference, blobDesc.Digest) if err != nil { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error fetching blob for subject:[%s] digest:[%s]: [%v]", subjectReference, blobDesc.Digest, err.Error()), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Failed to fetch blob for subject:[%s] digest:[%s].", subjectReference, blobDesc.Digest)).WithError(err) + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } // skip all validation if passthrough is enabled if input.Passthrough { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "Validation skipped. passthrough enabled", - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "Validation skipped. passthrough enabled", + true, + nil, + map[string]interface{}{ CreatedAnnotation: createdTime, "passthrough": true, "report": string(refBlob), }, - }, nil + ) + return &result, nil } // validate json schema if err := verifyJSONSchema(referenceDescriptor.ArtifactType, refBlob, input.SchemaURL); err != nil { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: schema validation failed for digest:[%s],artifact type:[%s],parse errors:[%v]", blobDesc.Digest, referenceDescriptor.ArtifactType, err.Error()), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Schema validation failed for digest:[%s],artifact type:[%s].", blobDesc.Digest, referenceDescriptor.ArtifactType)).WithError(err) + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } if referenceDescriptor.ArtifactType == SarifArtifactType { return processSarifReport(input, input.Name, verifierType, refBlob, createdTime) } - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "Validation succeeded", + true, + nil, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } // verifyJSONSchema validates the json schema of the report @@ -234,27 +249,30 @@ func verifyJSONSchema(artifactType string, refBlob []byte, schemaURL string) err func processSarifReport(input *PluginConfig, verifierName string, verifierType string, blob []byte, createdTime time.Time) (*verifier.VerifierResult, error) { sarifReport, err := sarif.FromBytes(blob) if err != nil { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error parsing sarif report:[%v]", err.Error()), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to parse sarif report.").WithError(err) + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } // verify that there is at least one run in the report if len(sarifReport.Runs) < 1 { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: "Validation failed: no runs found in sarif report", - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "No runs found in sarif report.", + false, + nil, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } scannerName := strings.ToLower(sarifReport.Runs[0].Tool.Driver.Name) if len(input.DenylistCVEs) > 0 { @@ -276,16 +294,19 @@ func processSarifReport(input *PluginConfig, verifierName string, verifierType s } } - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "Validation succeeded", + true, + nil, + map[string]interface{}{ CreatedAnnotation: createdTime, "scanner": scannerName, }, - }, nil + ) + return &result, nil } // verifyDenyListCVEs verifies that the report does not contain any deny-listed CVEs @@ -301,16 +322,19 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st // iterate over the results and check which cves are deny-listed for _, result := range sarifReport.Runs[0].Results { if result.RuleID == nil || *result.RuleID == "" { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", result), - Extensions: map[string]interface{}{ + verifierResult := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + fmt.Sprintf("Rule id not found for result:[%v].", result), + false, + nil, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &verifierResult, nil } ruleIDLower := strings.ToLower(*result.RuleID) if _, ok := denylistCVESet[ruleIDLower]; ok { @@ -326,30 +350,36 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st } if len(denylistViolations) > 0 { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "Found denied CVEs. See extensions field for details.", + false, + nil, + map[string]interface{}{ "scanner": scannerName, "denylistCVEs": denylistCVEs, "cveViolations": denylistViolations, CreatedAnnotation: createdTime, }, - Message: "Validation failed: found denied CVEs. See extensions field for details.", - }, nil + ) + return &result, nil } - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "Validation succeeded", + true, + nil, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &result, nil } // verifyDisallowedSeverities verifies that the report does not contain any disallowed severity levels @@ -363,42 +393,52 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne // iterate over the results and check if the severity is disallowed for _, result := range sarifReport.Runs[0].Results { if result.RuleID == nil || *result.RuleID == "" { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", result), - Extensions: map[string]interface{}{ + verifierResult := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + fmt.Sprintf("Rule id not found for result:[%v].", result), + false, + nil, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &verifierResult, nil } rule, ok := ruleMap[*result.RuleID] if !ok { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: rule not found for result:[%v]", result), - Extensions: map[string]interface{}{ + verifierResult := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + fmt.Sprintf("Rule not found for result:[%v].", result), + false, + nil, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &verifierResult, nil } severity, err := extractSeverity(scannerName, *rule) if err != nil { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error extracting severity:[%v]", err.Error()), - Extensions: map[string]interface{}{ + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to extract severity.").WithError(err) + verifierResult := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &verifierResult, nil } // check if the severity is disallowed and add it to the map of violating CVE IDs for _, disallowed := range disallowedSeverities { @@ -409,29 +449,35 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne } // if there are violating rules, return them as custom extension field if len(violatingRules) > 0 { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "Found disallowed severities. See extensions field for details.", + false, + nil, + map[string]interface{}{ "scanner": scannerName, "disallowedSeverities": disallowedSeverities, "severityViolations": violatingRules, CreatedAnnotation: createdTime, }, - Message: "Validation failed: found disallowed severities. See extensions field for details.", - }, nil + ) + return &result, nil } - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "Validation succeeded", + true, + nil, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &result, nil } // extractSeverity extracts the severity from the rule help text using regex diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go index e6341bfe10..0823d2ab9f 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go @@ -76,8 +76,9 @@ func TestVerifyReference(t *testing.T) { blobContent string } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -103,7 +104,8 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: error extracting create timestamp annotation:[%s]", "no annotations found for descriptor:[{{ sha256:b2f67b016d3c646f025099b363b4f83a56a44d067a846be74e8866342c56f216 0 [] map[] [] } application/sarif+json}]"), + message: "Failed to create timestamp annotation.", + errorReason: "no annotations found for descriptor:[{{ sha256:b2f67b016d3c646f025099b363b4f83a56a44d067a846be74e8866342c56f216 0 [] map[] [] } application/sarif+json}]", }, }, { @@ -118,7 +120,8 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: error validating maximum age:[%s]", "error parsing maximum age:[1d]"), + message: "Failed to validate maximum age.", + errorReason: "error parsing maximum age:[1d]", }, }, { @@ -133,7 +136,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: report is older than maximum age:[%s]", "24h"), + errorReason: "Report is older than maximum age:[24h].", }, }, { @@ -148,7 +151,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: no layers found in manifest for referrer %s@%s", "test_subject_path", manifestDigest.String()), + errorReason: fmt.Sprintf("No layers found in manifest for referrer %s@%s.", "test_subject_path", manifestDigest.String()), }, }, { @@ -189,7 +192,8 @@ func TestVerifyReference(t *testing.T) { blobContent: "{}", }, want: want{ - message: fmt.Sprintf("Validation failed: schema validation failed for digest:[%s],artifact type:[%s],parse errors:[%v]", blobDigest, SarifArtifactType, "version is required: runs is required: "), + message: fmt.Sprintf("Schema validation failed for digest:[%s],artifact type:[%s].", blobDigest, SarifArtifactType), + errorReason: "version is required: runs is required: ", }, }, { @@ -238,12 +242,15 @@ func TestVerifyReference(t *testing.T) { } verifierResult, err := VerifyReference(&cmdArgs, subjectRef, refDesc, testStore) if err != nil && err.Error() != tt.want.err.Error() { - t.Errorf("verifyReference() error = %v, wantErr %v", err, tt.want.err) - return + t.Fatalf("verifyReference() error = %v, wantErr %v", err, tt.want.err) } - if verifierResult != nil && verifierResult.Message != tt.want.message { - t.Errorf("verifyReference() verifier report message = %s, want %s", verifierResult.Message, tt.want.message) - return + if verifierResult != nil { + if verifierResult.Message != tt.want.message { + t.Fatalf("verifyReference() verifier report message = %s, want = %s", verifierResult.Message, tt.want.message) + } + if verifierResult.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyReference() verifier report error reason = %s, want = %s", verifierResult.ErrorReason, tt.want.errorReason) + } } }) } @@ -313,8 +320,9 @@ func TestProcessSarifReport(t *testing.T) { blobContent string } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -328,8 +336,9 @@ func TestProcessSarifReport(t *testing.T) { blobContent: "invalid", }, want: want{ - message: fmt.Sprintf("Validation failed: error parsing sarif report:[%s]", "invalid character 'i' looking for beginning of value"), - err: nil, + message: "Failed to parse sarif report.", + errorReason: "invalid character 'i' looking for beginning of value", + err: nil, }, }, { @@ -343,7 +352,7 @@ func TestProcessSarifReport(t *testing.T) { }`, }, want: want{ - message: "Validation failed: no runs found in sarif report", + message: "No runs found in sarif report.", err: nil, }, }, @@ -357,7 +366,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "Validation failed: found denied CVEs. See extensions field for details.", + message: "Found denied CVEs. See extensions field for details.", err: nil, }, }, @@ -374,7 +383,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "Validation failed: found disallowed severities. See extensions field for details.", + message: "Found disallowed severities. See extensions field for details.", err: nil, }, }, @@ -407,6 +416,9 @@ func TestProcessSarifReport(t *testing.T) { t.Errorf("processSarifReport() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) return } + if verifierReport.ErrorReason != tt.want.errorReason { + t.Errorf("processSarifReport() verifier report error reason = %s, want %s", verifierReport.ErrorReason, tt.want.errorReason) + } }) } } @@ -419,8 +431,9 @@ func TestVerifyDenyListCVEs(t *testing.T) { sarifReport sarif.Report } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -452,7 +465,7 @@ func TestVerifyDenyListCVEs(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", &sarif.Result{}), + message: fmt.Sprintf("Rule id not found for result:[%v].", &sarif.Result{}), err: nil, }, }, @@ -483,7 +496,7 @@ func TestVerifyDenyListCVEs(t *testing.T) { }, }, want: want{ - message: "Validation failed: found denied CVEs. See extensions field for details.", + message: "Found denied CVEs. See extensions field for details.", err: nil, }, }, @@ -523,12 +536,13 @@ func TestVerifyDenyListCVEs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { verifierReport, err := verifyDenyListCVEs("test_verifier", "", TrivyScannerName, &tests[i].args.sarifReport, tt.args.denyListCVEs, time.Now()) if err != nil && err.Error() != tt.want.err.Error() { - t.Errorf("verifyDenyListCVEs() error = %v, wantErr %v", err, tt.want.err) - return + t.Fatalf("verifyDenyListCVEs() error = %v, wantErr %v", err, tt.want.err) } if verifierReport.Message != tt.want.message { - t.Errorf("verifyDenyListCVEs() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) - return + t.Fatalf("verifyDenyListCVEs() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) + } + if verifierReport.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyDenyListCVEs() verifier report error reaon = %s, want = %s", verifierReport.ErrorReason, tt.want.errorReason) } }) } @@ -545,8 +559,9 @@ func TestVerifyDisallowedSeverities(t *testing.T) { sarifReport sarif.Report } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -582,7 +597,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", &sarif.Result{}), + message: fmt.Sprintf("Rule id not found for result:[%v].", &sarif.Result{}), err: nil, }, }, @@ -617,7 +632,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: rule not found for result:[%v]", &sarif.Result{RuleID: &invalidRuleID}), + message: fmt.Sprintf("Rule not found for result:[%v].", &sarif.Result{RuleID: &invalidRuleID}), err: nil, }, }, @@ -652,8 +667,9 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: error extracting severity:[severity not found in help text:[%s]]", invalidSeverityText), - err: nil, + message: "Failed to extract severity.", + errorReason: fmt.Sprintf("severity not found in help text:[%s]", invalidSeverityText), + err: nil, }, }, { @@ -687,7 +703,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: "Validation failed: found disallowed severities. See extensions field for details.", + message: "Found disallowed severities. See extensions field for details.", err: nil, }, }, @@ -731,13 +747,16 @@ func TestVerifyDisallowedSeverities(t *testing.T) { t.Run(tt.name, func(t *testing.T) { verifierReport, err := verifyDisallowedSeverities("test_verifier", "", TrivyScannerName, &tests[i].args.sarifReport, tt.args.disallowedSeverities, time.Now()) if err != nil && err.Error() != tt.want.err.Error() { - t.Errorf("verifyDisallowedSeverities() error = %v, wantErr %v", err, tt.want.err) + t.Fatalf("verifyDisallowedSeverities() error = %v, wantErr %v", err, tt.want.err) return } if verifierReport.Message != tt.want.message { - t.Errorf("verifyDisallowedSeverities() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) + t.Fatalf("verifyDisallowedSeverities() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) return } + if verifierReport.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyDisalowedServerities() verifier report error reason = %s, want = %s", verifierReport.ErrorReason, tt.want.errorReason) + } }) } }