From 2bc5b6c7d869b32c48fcaa5ff3dda9a9ff645dd0 Mon Sep 17 00:00:00 2001 From: Binbin Li Date: Thu, 25 Jul 2024 15:47:01 +0000 Subject: [PATCH 1/5] feat: add verifierName, verifierType and errorReason fields to verifierReport --- httpserver/types.go | 6 ++++-- pkg/executor/core/executor.go | 20 ++++++++++------- pkg/verifier/api.go | 3 +++ pkg/verifier/cosign/cosign.go | 35 ++++++++++++++++++------------ pkg/verifier/notation/notation.go | 12 ++++++----- pkg/verifier/types/types.go | 36 +++++++++++++++++++------------ 6 files changed, 69 insertions(+), 43 deletions(-) diff --git a/httpserver/types.go b/httpserver/types.go index f307bc55d..8e3655cb5 100644 --- a/httpserver/types.go +++ b/httpserver/types.go @@ -22,9 +22,11 @@ import ( const ( VerificationResultVersion = "0.1.0" + ResultVersion0_2_0 = "0.2.0" // Starting from this version, the verification result can be // evaluated by Ratify embedded OPA engine. ResultVersionSupportingRego = "1.0.0" + ResultVersion1_1_0 = "1.1.0" ) type VerificationResponse struct { @@ -34,9 +36,9 @@ type VerificationResponse struct { } func fromVerifyResult(res types.VerifyResult, policyType string) VerificationResponse { - version := VerificationResultVersion + version := ResultVersion0_2_0 if policyType == pt.RegoPolicy { - version = ResultVersionSupportingRego + version = ResultVersion1_1_0 } return VerificationResponse{ Version: version, diff --git a/pkg/executor/core/executor.go b/pkg/executor/core/executor.go index ef9d27210..4e37bd48d 100644 --- a/pkg/executor/core/executor.go +++ b/pkg/executor/core/executor.go @@ -178,10 +178,12 @@ func (executor Executor) verifyReferenceForJSONPolicy(ctx context.Context, subje verifyResult.Subject = subjectRef.String() if err != nil { verifyResult = vr.VerifierResult{ - IsSuccess: false, - Name: verifier.Name(), - Type: verifier.Type(), - Message: errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace).Error()} + IsSuccess: false, + Name: verifier.Name(), + Type: verifier.Type(), + VerifierName: verifier.Name(), + VerifierType: verifier.Type(), + Message: errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace).Error()} } if len(verifier.GetNestedReferences()) > 0 { @@ -227,10 +229,12 @@ func (executor Executor) verifyReferenceForRegoPolicy(ctx context.Context, subje verifierResult, err := verifier.Verify(errCtx, subjectRef, referenceDesc, referrerStore) if err != nil { verifierReport = vt.VerifierResult{ - IsSuccess: false, - Name: verifier.Name(), - Type: verifier.Type(), - Message: errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace).Error()} + IsSuccess: false, + Name: verifier.Name(), // Deprecating Name in next release, reference to VerifierName instead. + Type: verifier.Type(), // Deprecating Type in next release, reference to VerifierType instead. + VerifierName: verifier.Name(), + VerifierType: verifier.Type(), + Message: errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace).Error()} } else { verifierReport = vt.NewVerifierResult(verifierResult) } diff --git a/pkg/verifier/api.go b/pkg/verifier/api.go index dcc549662..53c14e150 100644 --- a/pkg/verifier/api.go +++ b/pkg/verifier/api.go @@ -28,8 +28,11 @@ type VerifierResult struct { //nolint:revive // ignore linter to have unique typ Subject string `json:"subject,omitempty"` IsSuccess bool `json:"isSuccess"` Name string `json:"name,omitempty"` + VerifierName string `json:"verifierName,omitempty"` Type string `json:"type,omitempty"` + VerifierType string `json:"verifierType,omitempty"` Message string `json:"message,omitempty"` + ErrorReason string `json:"errorReason,omitempty"` Extensions interface{} `json:"extensions,omitempty"` NestedResults []VerifierResult `json:"nestedResults,omitempty"` ArtifactType string `json:"artifactType,omitempty"` diff --git a/pkg/verifier/cosign/cosign.go b/pkg/verifier/cosign/cosign.go index 40a67fbfb..0cfaf7c5e 100644 --- a/pkg/verifier/cosign/cosign.go +++ b/pkg/verifier/cosign/cosign.go @@ -285,11 +285,13 @@ func (v *cosignVerifier) verifyInternal(ctx context.Context, subjectReference co if hasValidSignature { return verifier.VerifierResult{ - Name: v.name, - Type: v.verifierType, - IsSuccess: true, - Message: "cosign verification success. valid signatures found. please refer to extensions field for verifications performed.", - Extensions: Extension{SignatureExtension: sigExtensions, TrustPolicy: trustPolicy.GetName()}, + Name: v.name, + Type: v.verifierType, + VerifierName: v.name, + VerifierType: v.verifierType, + IsSuccess: true, + Message: "cosign verification success. valid signatures found. please refer to extensions field for verifications performed.", + Extensions: Extension{SignatureExtension: sigExtensions, TrustPolicy: trustPolicy.GetName()}, }, nil } @@ -396,11 +398,13 @@ func (v *cosignVerifier) verifyLegacy(ctx context.Context, subjectReference comm if len(signatures) > 0 { return verifier.VerifierResult{ - Name: v.name, - Type: v.verifierType, - IsSuccess: true, - Message: "cosign verification success. valid signatures found", - Extensions: LegacyExtension{SignatureExtension: sigExtensions}, + Name: v.name, + Type: v.verifierType, + VerifierName: v.name, + VerifierType: v.verifierType, + IsSuccess: true, + Message: "cosign verification success. valid signatures found", + Extensions: LegacyExtension{SignatureExtension: sigExtensions}, }, nil } @@ -482,10 +486,13 @@ func staticLayerOpts(desc imgspec.Descriptor) ([]static.Option, error) { // ErrorToVerifyResult returns a verifier result with the error message and isSuccess set to false func errorToVerifyResult(name string, verifierType string, err error) verifier.VerifierResult { return verifier.VerifierResult{ - IsSuccess: false, - Name: name, - Type: verifierType, - Message: errors.Wrap(err, "cosign verification failed").Error(), + IsSuccess: false, + Name: name, + Type: verifierType, + VerifierName: name, + VerifierType: verifierType, + Message: "cosign verification failed", + ErrorReason: err.Error(), } } diff --git a/pkg/verifier/notation/notation.go b/pkg/verifier/notation/notation.go index 30648e474..41f3dec53 100644 --- a/pkg/verifier/notation/notation.go +++ b/pkg/verifier/notation/notation.go @@ -175,11 +175,13 @@ func (v *notationPluginVerifier) Verify(ctx context.Context, } return verifier.VerifierResult{ - Name: v.name, - Type: v.verifierType, - IsSuccess: true, - Message: "signature verification success", - Extensions: extensions, + Name: v.name, + Type: v.verifierType, + VerifierName: v.name, + VerifierType: v.verifierType, + IsSuccess: true, + Message: "signature verification success", + Extensions: extensions, }, nil } diff --git a/pkg/verifier/types/types.go b/pkg/verifier/types/types.go index 1c12dc086..2eed0a669 100644 --- a/pkg/verifier/types/types.go +++ b/pkg/verifier/types/types.go @@ -47,11 +47,14 @@ const ( // VerifierResult describes the verification result returned from the verifier plugin type VerifierResult struct { - IsSuccess bool `json:"isSuccess"` - Message string `json:"message"` - Name string `json:"name"` - Type string `json:"type,omitempty"` - Extensions interface{} `json:"extensions"` + IsSuccess bool `json:"isSuccess"` + Message string `json:"message"` + ErrorReason string `json:"errorReason,omitempty"` + Name string `json:"name"` + VerifierName string `json:"verifierName,omitempty"` + Type string `json:"type,omitempty"` + VerifierType string `json:"verifierType,omitempty"` + Extensions interface{} `json:"extensions"` } // GetVerifierResult encodes the given JSON data into verify result object @@ -61,11 +64,13 @@ func GetVerifierResult(result []byte) (*verifier.VerifierResult, error) { return nil, err } return &verifier.VerifierResult{ - IsSuccess: vResult.IsSuccess, - Message: vResult.Message, - Name: vResult.Name, - Type: vResult.Type, - Extensions: vResult.Extensions, + IsSuccess: vResult.IsSuccess, + Message: vResult.Message, + Name: vResult.Name, + Type: vResult.Type, + VerifierName: vResult.Name, + VerifierType: vResult.Type, + Extensions: vResult.Extensions, }, nil } @@ -78,9 +83,12 @@ func WriteVerifyResultResult(result *verifier.VerifierResult, w io.Writer) error // verifier.VerifierResult. func NewVerifierResult(result verifier.VerifierResult) VerifierResult { return VerifierResult{ - IsSuccess: result.IsSuccess, - Message: result.Message, - Name: result.Name, - Extensions: result.Extensions, + IsSuccess: result.IsSuccess, + Message: result.Message, + Name: result.Name, + Type: result.Type, + VerifierName: result.VerifierName, + VerifierType: result.VerifierType, + Extensions: result.Extensions, } } From 65eb936b9308919dffb8c22aa881e11a09b9350a Mon Sep 17 00:00:00 2001 From: Binbin Li Date: Wed, 31 Jul 2024 08:27:52 +0000 Subject: [PATCH 2/5] feat: refactor error message format --- cmd/ratify/cmd/cmd_test.go | 4 +- errors/types.go | 132 ++++++++++++++++++++++--------------- errors/types_test.go | 127 ++++++++++++++++++++++++----------- test/bats/plugin-test.bats | 2 +- 4 files changed, 172 insertions(+), 93 deletions(-) diff --git a/cmd/ratify/cmd/cmd_test.go b/cmd/ratify/cmd/cmd_test.go index 49149dcf5..9016198ab 100644 --- a/cmd/ratify/cmd/cmd_test.go +++ b/cmd/ratify/cmd/cmd_test.go @@ -50,8 +50,8 @@ func TestDiscover(t *testing.T) { // TODO: make ratify cli more unit testable // unit test should not need to resolve real image - if !strings.Contains(err.Error(), "referrer store failure") { - t.Errorf("error expected") + if !strings.Contains(err.Error(), "REFERRER_STORE_FAILURE") { + t.Errorf("expected containing: %s, but got: %s", "REFERRER_STORE_FAILURE", err.Error()) } } diff --git a/errors/types.go b/errors/types.go index 3a211c80a..733de658e 100644 --- a/errors/types.go +++ b/errors/types.go @@ -60,15 +60,16 @@ type ErrorCode int // Error provides a wrapper around ErrorCode with extra Details provided. type Error struct { - OriginalError error `json:"originalError,omitempty"` - Code ErrorCode `json:"code"` - Message string `json:"message"` - Detail interface{} `json:"detail,omitempty"` - ComponentType ComponentType `json:"componentType,omitempty"` - PluginName string `json:"pluginName,omitempty"` - LinkToDoc string `json:"linkToDoc,omitempty"` - Stack string `json:"stack,omitempty"` - Description string `json:"description,omitempty"` + originalError error + code ErrorCode + message string + detail interface{} + componentType ComponentType + remediation string + pluginName string + stack string + description string + isRootError bool // isRootError is true if the originalError is either nil or not an Error type } // ErrorDescriptor provides relevant information about a given error code. @@ -152,7 +153,7 @@ func (ec ErrorCode) WithComponentType(componentType ComponentType) Error { // WithLinkToDoc returns a new Error object with attached link to the documentation. func (ec ErrorCode) WithLinkToDoc(link string) Error { - return newError(ec, ec.Message()).WithLinkToDoc(link) + return newError(ec, ec.Message()).WithRemediation(link) } func (ec ErrorCode) WithDescription() Error { @@ -165,27 +166,29 @@ func (ec ErrorCode) WithPluginName(pluginName string) Error { } // NewError returns a new Error object. -func (ec ErrorCode) NewError(componentType ComponentType, pluginName, link string, err error, detail interface{}, printStackTrace bool) Error { +func (ec ErrorCode) NewError(componentType ComponentType, pluginName, remediation string, err error, detail interface{}, printStackTrace bool) Error { stack := "" if printStackTrace { stack = getStackTrace() } return Error{ - Code: ec, - Message: ec.Message(), - OriginalError: err, - Detail: detail, - ComponentType: componentType, - PluginName: pluginName, - LinkToDoc: link, - Stack: stack, + code: ec, + message: ec.Message(), + originalError: err, + detail: detail, + componentType: componentType, + pluginName: pluginName, + remediation: remediation, + stack: stack, + isRootError: err == nil || !errors.As(err, &Error{}), } } func newError(code ErrorCode, message string) Error { return Error{ - Code: code, - Message: message, + code: code, + message: message, + isRootError: true, } } @@ -193,90 +196,113 @@ func newError(code ErrorCode, message string) Error { func (e Error) Is(target error) bool { t := &Error{} if errors.As(target, t) { - return e.Code.ErrorCode() == t.Code.ErrorCode() + return e.code.ErrorCode() == t.code.ErrorCode() } return false } // ErrorCode returns the ID/Value of this Error func (e Error) ErrorCode() ErrorCode { - return e.Code + return e.code } // Unwrap returns the original error func (e Error) Unwrap() error { - return e.OriginalError + return e.originalError } // Error returns a human readable representation of the error. +// An Error message includes the error code, detail from nested errors, root cause and remediation, all separated by ": ". func (e Error) Error() string { - var errStr string - if e.OriginalError != nil { - errStr += fmt.Sprintf("Original Error: (%s), ", e.OriginalError.Error()) + err, details := e.getRootError() + if err.detail != nil { + details = append(details, fmt.Sprintf("%s", err.detail)) } - - errStr += fmt.Sprintf("Error: %s, Code: %s", e.Message, e.Code.String()) - - if e.PluginName != "" { - errStr += fmt.Sprintf(", Plugin Name: %s", e.PluginName) + if err.originalError != nil { + details = append(details, err.originalError.Error()) } - if e.ComponentType != "" { - errStr += fmt.Sprintf(", Component Type: %s", e.ComponentType) + if err.remediation != "" { + details = append(details, err.remediation) } + return fmt.Sprintf("%s: %s", err.ErrorCode().Descriptor().Value, strings.Join(details, ": ")) +} - if e.LinkToDoc != "" { - errStr += fmt.Sprintf(", Documentation: %s", e.LinkToDoc) +// GetFullDetails returns details from all nested errors. +func (e Error) GetFullDetails() string { + err, details := e.getRootError() + if err.originalError != nil && err.detail != nil { + details = append(details, fmt.Sprintf("%s", err.detail)) } - if e.Detail != nil { - errStr += fmt.Sprintf(", Detail: %v", e.Detail) - } + return strings.Join(details, ": ") +} - if e.Description != "" { - errStr += fmt.Sprintf(", Description: %v", e.Description) +// GetRootCause returns the root cause of the error. +func (e Error) GetRootCause() string { + err, _ := e.getRootError() + if err.originalError != nil { + return err.originalError.Error() } + return fmt.Sprintf("%s", err.detail) +} - if e.Stack != "" { - errStr += fmt.Sprintf(", Stack trace: %s", e.Stack) - } +func (e Error) GetRootRemediation() string { + err, _ := e.getRootError() + return err.remediation +} - return errStr +func (e Error) getRootError() (err Error, details []string) { + err = e + for !err.isRootError { + if err.detail != nil { + details = append(details, fmt.Sprintf("%s", err.detail)) + } + var ratifyError Error + if errors.As(err.originalError, &ratifyError) { + err = ratifyError + } else { + // break is unnecessary, but added for safety + break + } + } + return err, details } // WithDetail will return a new Error, based on the current one, but with // some Detail info added func (e Error) WithDetail(detail interface{}) Error { - e.Detail = detail + e.detail = detail return e } // WithPluginName returns a new Error object with pluginName set. func (e Error) WithPluginName(pluginName string) Error { - e.PluginName = pluginName + e.pluginName = pluginName return e } // WithComponentType returns a new Error object with ComponentType set. func (e Error) WithComponentType(componentType ComponentType) Error { - e.ComponentType = componentType + e.componentType = componentType return e } // WithError returns a new Error object with original error. func (e Error) WithError(err error) Error { - e.OriginalError = err + e.originalError = err + e.isRootError = err == nil || !errors.As(err, &Error{}) return e } -// WithLinkToDoc returns a new Error object attached with link to documentation. -func (e Error) WithLinkToDoc(link string) Error { - e.LinkToDoc = link +// WithRemediation returns a new Error object attached with remediation. +func (e Error) WithRemediation(remediation string) Error { + e.remediation = remediation return e } func (e Error) WithDescription() Error { - e.Description = e.Code.Description() + e.description = e.code.Description() return e } diff --git a/errors/types_test.go b/errors/types_test.go index 5e97400d1..6a8fc2ce3 100644 --- a/errors/types_test.go +++ b/errors/types_test.go @@ -22,26 +22,31 @@ import ( ) const ( - testGroup = "test-group" - testValue = "test-value" - testMessage = "test-message" - testDescription = "test-description" - testDetail = "test-detail" - testComponentType = "test-component-type" - testLink = "test-link" - testPluginName = "test-plugin-name" - testErrorString = "Original Error: (Error: , Code: UNKNOWN), Error: test-message, Code: test-value, Plugin Name: test-plugin-name, Component Type: test-component-type, Documentation: test-link, Detail: test-detail" - nonexistentEC = 2000 + testGroup = "test-group" + testErrCode1 = "TEST_ERROR_CODE_1" + testErrCode2 = "TEST_ERROR_CODE_2" + testMessage = "test-message" + testDescription = "test-description" + testDetail1 = "test-detail-1" + testDetail2 = "test-detail-2" + testComponentType1 = "test-component-type-1" + testComponentType2 = "test-component-type-2" + testLink1 = "test-link-1" + testLink2 = "test-link-2" + testPluginName = "test-plugin-name" + nonexistentEC = 2000 ) var ( testEC = Register(testGroup, ErrorDescriptor{ - Value: testValue, + Value: testErrCode1, Message: testMessage, Description: testDescription, }) - testEC2 = Register(testGroup, ErrorDescriptor{}) + testEC2 = Register(testGroup, ErrorDescriptor{ + Value: testErrCode2, + }) ) func TestErrorCode(t *testing.T) { @@ -52,8 +57,9 @@ func TestErrorCode(t *testing.T) { } func TestError(t *testing.T) { - if testEC.Error() != testValue { - t.Fatalf("expected: %s, got: %s", testValue, testEC.Error()) + expectedStr := "test error code 1" + if testEC.Error() != expectedStr { + t.Fatalf("expected: %s, got: %s", expectedStr, testEC.Error()) } } @@ -66,7 +72,7 @@ func TestDescriptor(t *testing.T) { { name: "existing error code", ec: testEC, - expectedValue: testValue, + expectedValue: testErrCode1, }, { name: "nonexistent error code", @@ -92,9 +98,9 @@ func TestMessage(t *testing.T) { } func TestWithDetail(t *testing.T) { - err := testEC.WithDetail(testDetail) - if err.Detail != testDetail { - t.Fatalf("expected detail: %s, got: %s", testDetail, err.Detail) + err := testEC.WithDetail(testDetail1) + if err.detail != testDetail1 { + t.Fatalf("expected detail: %s, got: %s", testDetail1, err.detail) } } @@ -106,35 +112,35 @@ func TestWithError(t *testing.T) { } func TestWithComponentType(t *testing.T) { - err := testEC.WithComponentType(testComponentType) - if err.ComponentType != testComponentType { - t.Fatalf("expected component type: %s, got: %s", testComponentType, err.ComponentType) + err := testEC.WithComponentType(testComponentType1) + if err.componentType != testComponentType1 { + t.Fatalf("expected component type: %s, got: %s", testComponentType1, err.componentType) } } func TestWithLinkToDoc(t *testing.T) { - err := testEC.WithLinkToDoc(testLink) - if err.LinkToDoc != testLink { - t.Fatalf("expected link to doc: %s, got: %s", testLink, err.LinkToDoc) + err := testEC.WithLinkToDoc(testLink1) + if err.remediation != testLink1 { + t.Fatalf("expected link to doc: %s, got: %s", testLink1, err.remediation) } } func TestWithPluginName(t *testing.T) { err := testEC.WithPluginName(testPluginName) - if err.PluginName != testPluginName { - t.Fatalf("expected plugin name: %s, got: %s", testPluginName, err.PluginName) + if err.pluginName != testPluginName { + t.Fatalf("expected plugin name: %s, got: %s", testPluginName, err.pluginName) } } func TestWithDescription(t *testing.T) { err := testEC.WithDescription() - if err.Description != testDescription { - t.Fatalf("expected description: %s, got: %s", testDescription, err.Description) + if err.description != testDescription { + t.Fatalf("expected description: %s, got: %s", testDescription, err.description) } } func TestIs(t *testing.T) { - err := testEC.WithDetail(testDetail) + err := testEC.WithDetail(testDetail1) result := err.Is(err) if !result { t.Fatalf("expected true, got: %v", result) @@ -149,7 +155,7 @@ func TestIs(t *testing.T) { func TestError_ErrorCode(t *testing.T) { err := Error{ - Code: 1, + code: 1, } if err.ErrorCode() != 1 { t.Fatalf("expected 1, got: %d", err.ErrorCode()) @@ -168,17 +174,64 @@ func TestIsEmpty(t *testing.T) { } func TestError_Error(t *testing.T) { - err := testEC.WithPluginName(testPluginName).WithComponentType(testComponentType).WithLinkToDoc(testLink).WithDetail(testDetail).WithError(Error{}).WithDescription() - result := err.Error() - if !strings.HasPrefix(result, testErrorString) { - t.Fatalf("expected string starts with: %s, but got: %s", testErrorString, result) + // Nested errors. + rootErr := testEC.NewError(testComponentType1, "", testLink1, errors.New(testMessage), testDetail1, false) + err := testEC2.WithPluginName(testPluginName).WithComponentType(testComponentType2).WithRemediation(testLink2).WithDetail(testDetail2).WithError(rootErr) + + expectedErrStr := strings.Join([]string{testErrCode1, testDetail2, testDetail1, testMessage, testLink1}, ": ") + if err.Error() != expectedErrStr { + t.Fatalf("expected string: %s, but got: %s", expectedErrStr, err.Error()) + } + + // Single error. + err = testEC.WithDetail(testDetail1) + expectedErrStr = "TEST_ERROR_CODE_1: test-detail-1" + if err.Error() != expectedErrStr { + t.Fatalf("expected string: %s, but got: %s", expectedErrStr, err.Error()) + } +} + +func TestError_GetRootCause(t *testing.T) { + // rootErr contains original error. + rootErr := testEC.NewError(testComponentType1, "", testLink1, errors.New(testMessage), testDetail1, false) + err := testEC.WithPluginName(testPluginName).WithComponentType(testComponentType2).WithRemediation(testLink2).WithDetail(testDetail2).WithError(rootErr) + + if err.GetRootCause() != testMessage { + t.Fatalf("expected root cause: %v, but got: %v", err.GetRootCause(), testMessage) + } + + // rootErr does not contain original error. + rootErr = testEC.NewError(testComponentType1, "", testLink1, nil, testDetail1, false) + err = testEC.WithPluginName(testPluginName).WithComponentType(testComponentType2).WithRemediation(testLink2).WithDetail(testDetail2).WithError(rootErr) + + if err.GetRootCause() != testDetail1 { + t.Fatalf("expected root cause: %v, but got: %v", err.GetRootCause(), testDetail1) + } +} + +func TestError_GetFullDetails(t *testing.T) { + rootErr := testEC.NewError(testComponentType1, "", testLink1, errors.New(testMessage), testDetail1, false) + err := testEC.WithPluginName(testPluginName).WithComponentType(testComponentType2).WithRemediation(testLink2).WithDetail(testDetail2).WithError(rootErr) + + expectedDetails := strings.Join([]string{testDetail2, testDetail1}, ": ") + if err.GetFullDetails() != expectedDetails { + t.Fatalf("expected full details: %v, but got: %v", expectedDetails, err.GetFullDetails()) + } +} + +func TestError_GetRootRemediation(t *testing.T) { + rootErr := testEC.NewError(testComponentType1, "", testLink1, errors.New(testMessage), testDetail1, false) + err := testEC.WithPluginName(testPluginName).WithComponentType(testComponentType2).WithRemediation(testLink2).WithDetail(testDetail2).WithError(rootErr) + + if err.GetRootRemediation() != testLink1 { + t.Fatalf("expected root remediation: %v, but got: %v", err.GetRootRemediation(), testLink1) } } func TestNewError(t *testing.T) { - err := testEC.NewError(testComponentType, testPluginName, testLink, Error{}, testDetail, false) + err := testEC.NewError(testComponentType1, testPluginName, testLink1, Error{}, testDetail1, false) - if err.ComponentType != testComponentType || err.PluginName != testPluginName || err.LinkToDoc != testLink || err.Detail != testDetail { - t.Fatalf("expected component type: %s, plugin name: %s, link to doc: %s, detail: %s, but got: %s, %s, %s, %s", testComponentType, testPluginName, testLink, testDetail, err.ComponentType, err.PluginName, err.LinkToDoc, err.Detail) + if err.componentType != testComponentType1 || err.pluginName != testPluginName || err.remediation != testLink1 || err.detail != testDetail1 { + t.Fatalf("expected component type: %s, plugin name: %s, link to doc: %s, detail: %s, but got: %s, %s, %s, %s", testComponentType1, testPluginName, testLink1, testDetail1, err.componentType, err.pluginName, err.remediation, err.detail) } } diff --git a/test/bats/plugin-test.bats b/test/bats/plugin-test.bats index 3d37bbd4e..0cc9dea3a 100644 --- a/test/bats/plugin-test.bats +++ b/test/bats/plugin-test.bats @@ -312,7 +312,7 @@ RATIFY_NAMESPACE=gatekeeper-system sed 's/licensechecker/invalidlicensechecker/' ./config/samples/clustered/verifier/config_v1beta1_verifier_complete_licensechecker.yaml >invalidVerifier.yaml run kubectl apply -f invalidVerifier.yaml assert_success - run bash -c "kubectl describe verifiers.config.ratify.deislabs.io/verifier-license-checker -n ${RATIFY_NAMESPACE} | grep 'Brieferror: Original Error:'" + run bash -c "kubectl describe verifiers.config.ratify.deislabs.io/verifier-license-checker -n ${RATIFY_NAMESPACE} | grep 'Brieferror: PLUGIN_NOT_FOUND:'" assert_success # apply a valid verifier, validate status property shows success From b743d511934839b2159ca2f6cf87f3b440cd10fc Mon Sep 17 00:00:00 2001 From: Binbin Li Date: Thu, 1 Aug 2024 14:46:09 +0000 Subject: [PATCH 3/5] feat: fill ErrorReason and Remediation during verifierReport generation --- pkg/executor/core/executor.go | 12 +- pkg/executor/types/types.go | 1 + .../configpolicy/configpolicy.go | 9 +- pkg/verifier/cosign/cosign.go | 6 +- plugins/verifier/sbom/sbom.go | 80 ++++--- plugins/verifier/sbom/sbom_test.go | 184 ++++++++++++++- .../schemavalidator/schema_validator.go | 34 ++- .../schemavalidator/schema_validator_test.go | 170 ++++++++++++++ .../vulnerability_report.go | 222 +++++++++++------- .../vulnerability_report_test.go | 91 ++++--- 10 files changed, 645 insertions(+), 164 deletions(-) create mode 100644 plugins/verifier/schemavalidator/schema_validator_test.go diff --git a/pkg/executor/core/executor.go b/pkg/executor/core/executor.go index 11231f740..abe1f7153 100644 --- a/pkg/executor/core/executor.go +++ b/pkg/executor/core/executor.go @@ -176,13 +176,17 @@ func (executor Executor) verifyReferenceForJSONPolicy(ctx context.Context, subje verifierStartTime := time.Now() verifyResult, err := verifier.Verify(ctx, subjectRef, referenceDesc, referrerStore) if err != nil { + verifierErr := errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace) verifyResult = vr.VerifierResult{ IsSuccess: false, Name: verifier.Name(), // Deprecating Name in v2, switch to VerifierName instead. Type: verifier.Type(), // Deprecating Type in v2, switch to VerifierType instead. VerifierName: verifier.Name(), VerifierType: verifier.Type(), - Message: errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace).Error()} + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), + } } if len(verifier.GetNestedReferences()) > 0 { @@ -228,13 +232,17 @@ func (executor Executor) verifyReferenceForRegoPolicy(ctx context.Context, subje verifierStartTime := time.Now() verifierResult, err := verifier.Verify(errCtx, subjectRef, referenceDesc, referrerStore) if err != nil { + verifierErr := errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace) verifierReport = vt.VerifierResult{ IsSuccess: false, Name: verifier.Name(), // Deprecating Name in v2, switch to VerifierName instead. Type: verifier.Type(), // Deprecating Type in v2, switch to VerifierType instead. VerifierName: verifier.Name(), VerifierType: verifier.Type(), - Message: errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace).Error()} + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), + } } else { verifierReport = vt.NewVerifierResult(verifierResult) } diff --git a/pkg/executor/types/types.go b/pkg/executor/types/types.go index ccf78c337..30bac9a1b 100644 --- a/pkg/executor/types/types.go +++ b/pkg/executor/types/types.go @@ -29,6 +29,7 @@ type VerifyResult struct { // NestedVerifierReport describes the results of verifying an artifact and its // nested artifacts by available verifiers. +// Note: NestedVerifierReport is used for verification results in v1. type NestedVerifierReport struct { Subject string `json:"subject"` ReferenceDigest string `json:"referenceDigest"` diff --git a/pkg/policyprovider/configpolicy/configpolicy.go b/pkg/policyprovider/configpolicy/configpolicy.go index a0475bbc2..56dfb4c39 100644 --- a/pkg/policyprovider/configpolicy/configpolicy.go +++ b/pkg/policyprovider/configpolicy/configpolicy.go @@ -97,10 +97,13 @@ func (enforcer PolicyEnforcer) ContinueVerifyOnFailure(_ context.Context, _ comm // ErrorToVerifyResult converts an error to a properly formatted verify result func (enforcer PolicyEnforcer) ErrorToVerifyResult(_ context.Context, subjectRefString string, verifyError error) types.VerifyResult { + verifierErr := re.ErrorCodeVerifyReferenceFailure.WithDetail(fmt.Sprintf("failed to verify artifact: %s", subjectRefString)).WithError(verifyError) errorReport := verifier.VerifierResult{ - Subject: subjectRefString, - IsSuccess: false, - Message: fmt.Sprintf("verification failed: %v", verifyError), + Subject: subjectRefString, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), } var reports []interface{} reports = append(reports, errorReport) diff --git a/pkg/verifier/cosign/cosign.go b/pkg/verifier/cosign/cosign.go index 8f272ddb7..ca79d6302 100644 --- a/pkg/verifier/cosign/cosign.go +++ b/pkg/verifier/cosign/cosign.go @@ -485,14 +485,16 @@ func staticLayerOpts(desc imgspec.Descriptor) ([]static.Option, error) { // ErrorToVerifyResult returns a verifier result with the error message and isSuccess set to false func errorToVerifyResult(name string, verifierType string, err error) verifier.VerifierResult { + verifierErr := re.ErrorCodeVerifyReferenceFailure.WithDetail("cosign verification failed").WithError(err) return verifier.VerifierResult{ IsSuccess: false, Name: name, // Deprecating Name in v2, switch to VerifierName instead. Type: verifierType, // Deprecating Type in v2, switch to VerifierType instead. VerifierName: name, VerifierType: verifierType, - Message: "cosign verification failed", - ErrorReason: err.Error(), + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), } } diff --git a/plugins/verifier/sbom/sbom.go b/plugins/verifier/sbom/sbom.go index cd8448da3..31c98a566 100644 --- a/plugins/verifier/sbom/sbom.go +++ b/plugins/verifier/sbom/sbom.go @@ -28,6 +28,7 @@ import ( "github.com/ratify-project/ratify/plugins/verifier/sbom/utils" // This import is required to utilize the oras built-in referrer store + re "github.com/ratify-project/ratify/errors" _ "github.com/ratify-project/ratify/pkg/referrerstore/oras" "github.com/ratify-project/ratify/pkg/verifier" "github.com/ratify-project/ratify/pkg/verifier/plugin/skel" @@ -82,19 +83,28 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe ctx := context.Background() referenceManifest, err := referrerStore.GetReferenceManifest(ctx, subjectReference, referenceDescriptor) if err != nil { + storeErr := re.ErrorCodeGetReferenceManifestFailure.WithDetail(fmt.Sprintf("Failed to fetch reference manifest for subject: %s reference descriptor: %v", subjectReference, referenceDescriptor.Descriptor)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Error fetching reference manifest for subject: %s reference descriptor: %v, err: %v", subjectReference, referenceDescriptor.Descriptor, err), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: storeErr.GetFullDetails(), + ErrorReason: storeErr.GetRootCause(), + Remediation: storeErr.GetRootRemediation(), }, nil } if len(referenceManifest.Blobs) == 0 { return &verifier.VerifierResult{ - Name: input.Name, - IsSuccess: false, - Message: fmt.Sprintf("SBOM validation failed: no layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: "SBOM validation failed", + ErrorReason: fmt.Sprintf("No layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), }, nil } @@ -103,11 +113,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe refBlob, err := referrerStore.GetBlobContent(ctx, subjectReference, blobDesc.Digest) if err != nil { + storeErr := re.ErrorCodeGetBlobContentFailure.WithDetail(fmt.Sprintf("Failed to fetch blob for subject: %s digest: %s", subjectReference, blobDesc.Digest)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Error fetching blob for subject: %s digest: %s, err: %v", subjectReference, blobDesc.Digest, err), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: storeErr.GetFullDetails(), + ErrorReason: storeErr.GetRootCause(), + Remediation: storeErr.GetRootRemediation(), }, nil } @@ -116,19 +131,24 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe return processSpdxJSONMediaType(input.Name, verifierType, refBlob, input.DisallowedLicenses, input.DisallowedPackages), nil default: return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Unsupported artifactType: %s", artifactType), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: "Failed to process SBOM blobs.", + ErrorReason: fmt.Sprintf("Unsupported artifactType: %s", artifactType), }, nil } } return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "SBOM verification success. No license or package violation found.", + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: true, + Message: "SBOM verification success. No license or package violation found.", }, nil } @@ -178,10 +198,12 @@ func processSpdxJSONMediaType(name string, verifierType string, refBlob []byte, if len(licenseViolation) != 0 || len(packageViolation) != 0 { return &verifier.VerifierResult{ - Name: name, - IsSuccess: false, - Extensions: extensionData, - Message: "SBOM validation failed. Please review extensions data for license and package violation found.", + Name: name, + IsSuccess: false, + Extensions: extensionData, + Message: "SBOM validation failed.", + ErrorReason: "License or package violation found.", + Remediation: "Please review extensions data for license and package violation found.", } } } @@ -196,11 +218,15 @@ func processSpdxJSONMediaType(name string, verifierType string, refBlob []byte, Message: "SBOM verification success. No license or package violation found.", } } + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("failed to verify artifact: %s", name)).WithError(err) return &verifier.VerifierResult{ - Name: name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("SBOM failed to parse: %v", err), + Name: name, + Type: verifierType, + VerifierName: name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), } } diff --git a/plugins/verifier/sbom/sbom_test.go b/plugins/verifier/sbom/sbom_test.go index 8dc5a4c3f..8609fc500 100644 --- a/plugins/verifier/sbom/sbom_test.go +++ b/plugins/verifier/sbom/sbom_test.go @@ -15,14 +15,24 @@ limitations under the License. package main import ( + "errors" + "fmt" "os" "path/filepath" "strings" "testing" + "github.com/opencontainers/go-digest" + oci "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/ratify-project/ratify/pkg/common" + "github.com/ratify-project/ratify/pkg/ocispecs" + "github.com/ratify-project/ratify/pkg/referrerstore/mocks" + "github.com/ratify-project/ratify/pkg/verifier/plugin/skel" "github.com/ratify-project/ratify/plugins/verifier/sbom/utils" ) +const mediaType string = "application/vnd.syft+json" + func TestProcessSPDXJsonMediaType(t *testing.T) { b, err := os.ReadFile(filepath.Join("testdata", "bom.json")) if err != nil { @@ -41,8 +51,178 @@ func TestProcessInvalidSPDXJsonMediaType(t *testing.T) { } report := processSpdxJSONMediaType("test", "", b, nil, nil) - if !strings.Contains(report.Message, "SBOM failed to parse") { - t.Fatalf("expected to have an error processing spdx json file: %s", filepath.Join("testdata", "bom.json")) + if !strings.Contains(report.Message, "failed to verify artifact") { + t.Fatalf("report message: %s does not contain expected error message", report.Message) + } + if report.ErrorReason != "JSON document does not contain spdxVersion field" { + t.Fatalf("expected error reason: %s, got: %s", "JSON document does not contain spdxVersion field", report.ErrorReason) + } +} + +func TestVerifyReference(t *testing.T) { + manifestDigest := digest.FromString("test_manifest_digest") + manifestDigest2 := digest.FromString("test_manifest_digest_2") + blobDigest := digest.FromString("test_blob_digest") + blobDigest2 := digest.FromString("test_blob_digest_2") + type args struct { + stdinData string + referenceManifest ocispecs.ReferenceManifest + blobContent string + refDesc ocispecs.ReferenceDescriptor + } + type want struct { + message string + errorReason string + err error + } + + tests := []struct { + name string + args args + want want + }{ + { + name: "invalid stdin data", + args: args{}, + want: want{ + err: errors.New("failed to parse stdin for the input: unexpected end of JSON input"), + }, + }, + { + name: "failed to get reference manifest", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest2, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: "Failed to fetch reference manifest for subject: test_subject reference descriptor: { sha256:b55e209647d87fcd95a94c59ff4d342e42bf10f02a7c10b5192131f8d959ff5a 0 [] map[] [] }", + errorReason: "manifest not found", + }, + }, + { + name: "empty blobs", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: "SBOM validation failed", + errorReason: fmt.Sprintf("No layers found in manifest for referrer %s@%s", "test_subject_path", manifestDigest.String()), + }, + }, + { + name: "get blob content error", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest2, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: fmt.Sprintf("Failed to fetch blob for subject: test_subject digest: %s", blobDigest2.String()), + errorReason: "blob not found", + }, + }, + { + name: "unsupported artifactType", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: "Failed to process SBOM blobs.", + errorReason: "Unsupported artifactType: application/vnd.syft+json", + }, + }, + { + name: "process spdx json mediaType error", + args: args{ + stdinData: `{"config":{"name":"sbom","type":"sbom"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: SpdxJSONMediaType, + }, + }, + want: want{ + message: "failed to verify artifact: sbom", + errorReason: "unexpected end of JSON input", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmdArgs := &skel.CmdArgs{ + Version: "1.0.0", + Subject: "test_subject", + StdinData: []byte(tt.args.stdinData), + } + testStore := &mocks.MemoryTestStore{ + Manifests: map[digest.Digest]ocispecs.ReferenceManifest{manifestDigest: tt.args.referenceManifest}, + Blobs: map[digest.Digest][]byte{blobDigest: []byte(tt.args.blobContent)}, + } + subjectRef := common.Reference{ + Path: "test_subject_path", + Original: "test_subject", + } + verifierResult, err := VerifyReference(cmdArgs, subjectRef, tt.args.refDesc, testStore) + if err != nil && err.Error() != tt.want.err.Error() { + t.Fatalf("verifyReference() error = %v, wantErr %v", err, tt.want.err) + } + if verifierResult != nil { + if verifierResult.Message != tt.want.message { + t.Fatalf("verifyReference() verifier report message = %s, want = %s", verifierResult.Message, tt.want.message) + } + if verifierResult.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyReference() verifier report error reason = %s, want = %s", verifierResult.ErrorReason, tt.want.errorReason) + } + } + }) } } diff --git a/plugins/verifier/schemavalidator/schema_validator.go b/plugins/verifier/schemavalidator/schema_validator.go index 2af66396e..78b5562ff 100644 --- a/plugins/verifier/schemavalidator/schema_validator.go +++ b/plugins/verifier/schemavalidator/schema_validator.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" + re "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/common" "github.com/ratify-project/ratify/pkg/ocispecs" "github.com/ratify-project/ratify/pkg/referrerstore" @@ -72,10 +73,12 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe if len(referenceManifest.Blobs) == 0 { return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("schema validation failed: no blobs found for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("No blobs found for referrer %s@%s.", subjectReference.Path, referenceDescriptor.Digest.String()), }, nil } @@ -87,20 +90,27 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe err = processMediaType(schemaMap, blobDesc.MediaType, refBlob) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("schema validation failed for digest:[%s], media type:[%s].", blobDesc.Digest, blobDesc.MediaType)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("schema validation failed for digest:[%s],media type:[%s],parse errors:[%v]", blobDesc.Digest, blobDesc.MediaType, err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), }, nil } } return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "schema validation passed for configured media types", + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: true, + Message: "schema validation passed for configured media types", }, nil } diff --git a/plugins/verifier/schemavalidator/schema_validator_test.go b/plugins/verifier/schemavalidator/schema_validator_test.go new file mode 100644 index 000000000..d0f28f60c --- /dev/null +++ b/plugins/verifier/schemavalidator/schema_validator_test.go @@ -0,0 +1,170 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "testing" + + "github.com/opencontainers/go-digest" + oci "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/ratify-project/ratify/pkg/common" + "github.com/ratify-project/ratify/pkg/ocispecs" + "github.com/ratify-project/ratify/pkg/referrerstore/mocks" + "github.com/ratify-project/ratify/pkg/verifier/plugin/skel" +) + +const mediaType string = "application/schema+json" + +func TestVerifyReference(t *testing.T) { + manifestDigest := digest.FromString("test_manifest_digest") + manifestDigest2 := digest.FromString("test_manifest_digest_2") + blobDigest := digest.FromString("test_blob_digest") + blobDigest2 := digest.FromString("test_blob_digest_2") + type args struct { + stdinData string + referenceManifest ocispecs.ReferenceManifest + blobContent string + refDesc ocispecs.ReferenceDescriptor + } + type want struct { + message string + errorReason string + err error + } + tests := []struct { + name string + args args + want want + }{ + { + name: "invalid stdin data", + args: args{}, + want: want{ + err: errors.New("failed to parse stdin for the input: unexpected end of JSON input"), + }, + }, + { + name: "failed to get reference manifest", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest2, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + err: errors.New("error fetching reference manifest for subject: test_subject reference descriptor: { sha256:b55e209647d87fcd95a94c59ff4d342e42bf10f02a7c10b5192131f8d959ff5a 0 [] map[] [] }"), + }, + }, + { + name: "empty blobs", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{}, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: fmt.Sprintf("No blobs found for referrer %s@%s.", "test_subject_path", manifestDigest.String()), + }, + }, + { + name: "get blob content error", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest2, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + err: fmt.Errorf("error fetching blob for subject:[%s] digest:[%s]", "test_subject", blobDigest2.String()), + }, + }, + { + name: "process mediaType error", + args: args{ + stdinData: `{"config":{"name":"schemavalidator","type":"schemavalidator"}}`, + referenceManifest: ocispecs.ReferenceManifest{ + Blobs: []oci.Descriptor{ + { + MediaType: mediaType, + Digest: blobDigest, + }, + }, + }, + refDesc: ocispecs.ReferenceDescriptor{ + Descriptor: oci.Descriptor{ + Digest: manifestDigest, + }, + ArtifactType: mediaType, + }, + }, + want: want{ + message: fmt.Sprintf("schema validation failed for digest:[%s], media type:[%s].", blobDigest.String(), mediaType), + errorReason: "media type not configured for plugin:[application/schema+json]", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmdArgs := &skel.CmdArgs{ + Version: "1.0.0", + Subject: "test_subject", + StdinData: []byte(tt.args.stdinData), + } + testStore := &mocks.MemoryTestStore{ + Manifests: map[digest.Digest]ocispecs.ReferenceManifest{manifestDigest: tt.args.referenceManifest}, + Blobs: map[digest.Digest][]byte{blobDigest: []byte(tt.args.blobContent)}, + } + subjectRef := common.Reference{ + Path: "test_subject_path", + Original: "test_subject", + } + verifierResult, err := VerifyReference(cmdArgs, subjectRef, tt.args.refDesc, testStore) + if err != nil && err.Error() != tt.want.err.Error() { + t.Fatalf("verifyReference() error = %v, wantErr %v", err, tt.want.err) + } + if verifierResult != nil { + if verifierResult.Message != tt.want.message { + t.Fatalf("verifyReference() verifier report message = %s, want = %s", verifierResult.Message, tt.want.message) + } + if verifierResult.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyReference() verifier report error reason = %s, want = %s", verifierResult.ErrorReason, tt.want.errorReason) + } + } + }) + } +} diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report.go b/plugins/verifier/vulnerabilityreport/vulnerability_report.go index 37089ccf4..fda7aa8e8 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report.go @@ -26,6 +26,7 @@ import ( imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/owenrumney/go-sarif/v2/sarif" + re "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/common" "github.com/ratify-project/ratify/pkg/ocispecs" "github.com/ratify-project/ratify/pkg/referrerstore" @@ -93,11 +94,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe } createdTime, err := extractCreationTimestamp(input.CreatedAnnotationName, referenceDescriptor) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to create timestamp annotation.").WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error extracting create timestamp annotation:[%v]", err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), }, nil } @@ -105,11 +111,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe if input.MaximumAge != "" { ok, err := validateMaximumAge(input.MaximumAge, createdTime) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to validate maximum age.").WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error validating maximum age:[%v]", err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -117,10 +128,12 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe } if !ok { return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: report is older than maximum age:[%s]", input.MaximumAge), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("Report is older than maximum age:[%s].", input.MaximumAge), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -132,11 +145,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe referenceManifest, err := referrerStore.GetReferenceManifest(ctx, subjectReference, referenceDescriptor) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Failed to fetch reference manifest for subject: %s, reference descriptor: %v.", subjectReference, referenceDescriptor)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error fetching reference manifest for subject: %s reference descriptor: %v: [%v]", subjectReference, referenceDescriptor.Descriptor, err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -145,10 +163,12 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe if len(referenceManifest.Blobs) == 0 { return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: no layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("No layers found in manifest for referrer %s@%s.", subjectReference.Path, referenceDescriptor.Digest.String()), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -158,11 +178,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe blobDesc := referenceManifest.Blobs[0] refBlob, err := referrerStore.GetBlobContent(ctx, subjectReference, blobDesc.Digest) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Failed to fetch blob for subject:[%s] digest:[%s].", subjectReference, blobDesc.Digest)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error fetching blob for subject:[%s] digest:[%s]: [%v]", subjectReference, blobDesc.Digest, err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -172,10 +197,12 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe // skip all validation if passthrough is enabled if input.Passthrough { return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "Validation skipped. passthrough enabled", + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: true, + Message: "Validation skipped. passthrough enabled", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, "passthrough": true, @@ -186,11 +213,16 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe // validate json schema if err := verifyJSONSchema(referenceDescriptor.ArtifactType, refBlob, input.SchemaURL); err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Schema validation failed for digest:[%s],artifact type:[%s].", blobDesc.Digest, referenceDescriptor.ArtifactType)).WithError(err) return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: schema validation failed for digest:[%s],artifact type:[%s],parse errors:[%v]", blobDesc.Digest, referenceDescriptor.ArtifactType, err.Error()), + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -202,10 +234,12 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe } return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", + Name: input.Name, + Type: verifierType, + VerifierName: input.Name, + VerifierType: verifierType, + IsSuccess: true, + Message: "Validation succeeded", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -234,11 +268,16 @@ func verifyJSONSchema(artifactType string, refBlob []byte, schemaURL string) err func processSarifReport(input *PluginConfig, verifierName string, verifierType string, blob []byte, createdTime time.Time) (*verifier.VerifierResult, error) { sarifReport, err := sarif.FromBytes(blob) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to parse sarif report.").WithError(err) return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error parsing sarif report:[%v]", err.Error()), + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -247,10 +286,12 @@ func processSarifReport(input *PluginConfig, verifierName string, verifierType s // verify that there is at least one run in the report if len(sarifReport.Runs) < 1 { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: "Validation failed: no runs found in sarif report", + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: "No runs found in sarif report.", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -277,10 +318,12 @@ func processSarifReport(input *PluginConfig, verifierName string, verifierType s } return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: true, + Message: "Validation succeeded", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, "scanner": scannerName, @@ -302,10 +345,12 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st for _, result := range sarifReport.Runs[0].Results { if result.RuleID == nil || *result.RuleID == "" { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", result), + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("Rule id not found for result:[%v].", result), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -327,24 +372,28 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st if len(denylistViolations) > 0 { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, Extensions: map[string]interface{}{ "scanner": scannerName, "denylistCVEs": denylistCVEs, "cveViolations": denylistViolations, CreatedAnnotation: createdTime, }, - Message: "Validation failed: found denied CVEs. See extensions field for details.", + Message: "Found denied CVEs. See extensions field for details.", }, nil } return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: true, + Message: "Validation succeeded", Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -364,10 +413,12 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne for _, result := range sarifReport.Runs[0].Results { if result.RuleID == nil || *result.RuleID == "" { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", result), + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("Rule id not found for result:[%v].", result), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -377,10 +428,12 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne rule, ok := ruleMap[*result.RuleID] if !ok { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: rule not found for result:[%v]", result), + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: fmt.Sprintf("Rule not found for result:[%v].", result), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -389,11 +442,16 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne } severity, err := extractSeverity(scannerName, *rule) if err != nil { + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to extract severity.").WithError(err) return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Validation failed: error extracting severity:[%v]", err.Error()), + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, + Message: verifierErr.GetFullDetails(), + ErrorReason: verifierErr.GetRootCause(), + Remediation: verifierErr.GetRootRemediation(), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -410,23 +468,27 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne // if there are violating rules, return them as custom extension field if len(violatingRules) > 0 { return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: false, + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: false, Extensions: map[string]interface{}{ "scanner": scannerName, "disallowedSeverities": disallowedSeverities, "severityViolations": violatingRules, CreatedAnnotation: createdTime, }, - Message: "Validation failed: found disallowed severities. See extensions field for details.", + Message: "Found disallowed severities. See extensions field for details.", }, nil } return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - IsSuccess: true, - Message: "Validation succeeded", + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + IsSuccess: true, + Message: "Validation succeeded", Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go index e6341bfe1..84728f529 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go @@ -76,8 +76,9 @@ func TestVerifyReference(t *testing.T) { blobContent string } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -103,7 +104,8 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: error extracting create timestamp annotation:[%s]", "no annotations found for descriptor:[{{ sha256:b2f67b016d3c646f025099b363b4f83a56a44d067a846be74e8866342c56f216 0 [] map[] [] } application/sarif+json}]"), + message: "Failed to create timestamp annotation.", + errorReason: "no annotations found for descriptor:[{{ sha256:b2f67b016d3c646f025099b363b4f83a56a44d067a846be74e8866342c56f216 0 [] map[] [] } application/sarif+json}]", }, }, { @@ -118,7 +120,8 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: error validating maximum age:[%s]", "error parsing maximum age:[1d]"), + message: "Failed to validate maximum age.", + errorReason: "error parsing maximum age:[1d]", }, }, { @@ -133,7 +136,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: report is older than maximum age:[%s]", "24h"), + message: "Report is older than maximum age:[24h].", }, }, { @@ -148,7 +151,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("Validation failed: no layers found in manifest for referrer %s@%s", "test_subject_path", manifestDigest.String()), + message: fmt.Sprintf("No layers found in manifest for referrer %s@%s.", "test_subject_path", manifestDigest.String()), }, }, { @@ -189,7 +192,8 @@ func TestVerifyReference(t *testing.T) { blobContent: "{}", }, want: want{ - message: fmt.Sprintf("Validation failed: schema validation failed for digest:[%s],artifact type:[%s],parse errors:[%v]", blobDigest, SarifArtifactType, "version is required: runs is required: "), + message: fmt.Sprintf("Schema validation failed for digest:[%s],artifact type:[%s].", blobDigest, SarifArtifactType), + errorReason: "version is required: runs is required: ", }, }, { @@ -238,12 +242,15 @@ func TestVerifyReference(t *testing.T) { } verifierResult, err := VerifyReference(&cmdArgs, subjectRef, refDesc, testStore) if err != nil && err.Error() != tt.want.err.Error() { - t.Errorf("verifyReference() error = %v, wantErr %v", err, tt.want.err) - return + t.Fatalf("verifyReference() error = %v, wantErr %v", err, tt.want.err) } - if verifierResult != nil && verifierResult.Message != tt.want.message { - t.Errorf("verifyReference() verifier report message = %s, want %s", verifierResult.Message, tt.want.message) - return + if verifierResult != nil { + if verifierResult.Message != tt.want.message { + t.Fatalf("verifyReference() verifier report message = %s, want = %s", verifierResult.Message, tt.want.message) + } + if verifierResult.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyReference() verifier report error reason = %s, want = %s", verifierResult.ErrorReason, tt.want.errorReason) + } } }) } @@ -313,8 +320,9 @@ func TestProcessSarifReport(t *testing.T) { blobContent string } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -328,8 +336,9 @@ func TestProcessSarifReport(t *testing.T) { blobContent: "invalid", }, want: want{ - message: fmt.Sprintf("Validation failed: error parsing sarif report:[%s]", "invalid character 'i' looking for beginning of value"), - err: nil, + message: "Failed to parse sarif report.", + errorReason: "invalid character 'i' looking for beginning of value", + err: nil, }, }, { @@ -343,7 +352,7 @@ func TestProcessSarifReport(t *testing.T) { }`, }, want: want{ - message: "Validation failed: no runs found in sarif report", + message: "No runs found in sarif report.", err: nil, }, }, @@ -357,7 +366,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "Validation failed: found denied CVEs. See extensions field for details.", + message: "Found denied CVEs. See extensions field for details.", err: nil, }, }, @@ -374,7 +383,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "Validation failed: found disallowed severities. See extensions field for details.", + message: "Found disallowed severities. See extensions field for details.", err: nil, }, }, @@ -407,6 +416,9 @@ func TestProcessSarifReport(t *testing.T) { t.Errorf("processSarifReport() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) return } + if verifierReport.ErrorReason != tt.want.errorReason { + t.Errorf("processSarifReport() verifier report error reason = %s, want %s", verifierReport.ErrorReason, tt.want.errorReason) + } }) } } @@ -419,8 +431,9 @@ func TestVerifyDenyListCVEs(t *testing.T) { sarifReport sarif.Report } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -452,7 +465,7 @@ func TestVerifyDenyListCVEs(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", &sarif.Result{}), + message: fmt.Sprintf("Rule id not found for result:[%v].", &sarif.Result{}), err: nil, }, }, @@ -483,7 +496,7 @@ func TestVerifyDenyListCVEs(t *testing.T) { }, }, want: want{ - message: "Validation failed: found denied CVEs. See extensions field for details.", + message: "Found denied CVEs. See extensions field for details.", err: nil, }, }, @@ -523,12 +536,13 @@ func TestVerifyDenyListCVEs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { verifierReport, err := verifyDenyListCVEs("test_verifier", "", TrivyScannerName, &tests[i].args.sarifReport, tt.args.denyListCVEs, time.Now()) if err != nil && err.Error() != tt.want.err.Error() { - t.Errorf("verifyDenyListCVEs() error = %v, wantErr %v", err, tt.want.err) - return + t.Fatalf("verifyDenyListCVEs() error = %v, wantErr %v", err, tt.want.err) } if verifierReport.Message != tt.want.message { - t.Errorf("verifyDenyListCVEs() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) - return + t.Fatalf("verifyDenyListCVEs() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) + } + if verifierReport.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyDenyListCVEs() verifier report error reaon = %s, want = %s", verifierReport.ErrorReason, tt.want.errorReason) } }) } @@ -545,8 +559,9 @@ func TestVerifyDisallowedSeverities(t *testing.T) { sarifReport sarif.Report } type want struct { - message string - err error + message string + errorReason string + err error } tests := []struct { name string @@ -582,7 +597,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", &sarif.Result{}), + message: fmt.Sprintf("Rule id not found for result:[%v].", &sarif.Result{}), err: nil, }, }, @@ -617,7 +632,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: rule not found for result:[%v]", &sarif.Result{RuleID: &invalidRuleID}), + message: fmt.Sprintf("Rule not found for result:[%v].", &sarif.Result{RuleID: &invalidRuleID}), err: nil, }, }, @@ -652,8 +667,9 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("Validation failed: error extracting severity:[severity not found in help text:[%s]]", invalidSeverityText), - err: nil, + message: "Failed to extract severity.", + errorReason: fmt.Sprintf("severity not found in help text:[%s]", invalidSeverityText), + err: nil, }, }, { @@ -687,7 +703,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: "Validation failed: found disallowed severities. See extensions field for details.", + message: "Found disallowed severities. See extensions field for details.", err: nil, }, }, @@ -731,13 +747,16 @@ func TestVerifyDisallowedSeverities(t *testing.T) { t.Run(tt.name, func(t *testing.T) { verifierReport, err := verifyDisallowedSeverities("test_verifier", "", TrivyScannerName, &tests[i].args.sarifReport, tt.args.disallowedSeverities, time.Now()) if err != nil && err.Error() != tt.want.err.Error() { - t.Errorf("verifyDisallowedSeverities() error = %v, wantErr %v", err, tt.want.err) + t.Fatalf("verifyDisallowedSeverities() error = %v, wantErr %v", err, tt.want.err) return } if verifierReport.Message != tt.want.message { - t.Errorf("verifyDisallowedSeverities() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) + t.Fatalf("verifyDisallowedSeverities() verifier report message = %s, want %s", verifierReport.Message, tt.want.message) return } + if verifierReport.ErrorReason != tt.want.errorReason { + t.Fatalf("verifyDisalowedServerities() verifier report error reason = %s, want = %s", verifierReport.ErrorReason, tt.want.errorReason) + } }) } } From 04d2e611a161a15f067299fd5acd2efb8f63d635 Mon Sep 17 00:00:00 2001 From: Binbin Li Date: Tue, 6 Aug 2024 08:49:40 +0000 Subject: [PATCH 4/5] fix: remove duplicate code --- pkg/executor/core/executor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/executor/core/executor.go b/pkg/executor/core/executor.go index ebbdd6f13..c33223f54 100644 --- a/pkg/executor/core/executor.go +++ b/pkg/executor/core/executor.go @@ -196,7 +196,6 @@ func (executor Executor) verifyReferenceForJSONPolicy(ctx context.Context, subje verifyResult.Subject = subjectRef.String() verifyResult.ReferenceDigest = referenceDesc.Digest.String() verifyResult.ArtifactType = referenceDesc.ArtifactType - verifyResult.ReferenceDigest = referenceDesc.Digest.String() verifyResults = append(verifyResults, verifyResult) isSuccess = verifyResult.IsSuccess metrics.ReportVerifierDuration(ctx, time.Since(verifierStartTime).Milliseconds(), verifier.Name(), subjectRef.String(), isSuccess, err != nil) From fda9dfc90574d67907dedb2c014173e8741efe64 Mon Sep 17 00:00:00 2001 From: Binbin Li Date: Tue, 13 Aug 2024 03:00:35 +0000 Subject: [PATCH 5/5] chore: refactor verifierResult construction into method --- pkg/executor/core/executor.go | 22 +- .../configpolicy/configpolicy.go | 8 +- pkg/verifier/api.go | 20 - pkg/verifier/cosign/cosign.go | 55 ++- pkg/verifier/notation/notation.go | 10 +- pkg/verifier/result.go | 62 +++ pkg/verifier/result_test.go | 83 ++++ pkg/verifier/types/types.go | 25 ++ pkg/verifier/types/types_test.go | 83 ++++ plugins/verifier/sbom/sbom.go | 101 ++--- .../schemavalidator/schema_validator.go | 34 +- .../schemavalidator/schema_validator_test.go | 2 +- .../vulnerability_report.go | 406 +++++++++--------- .../vulnerability_report_test.go | 4 +- 14 files changed, 518 insertions(+), 397 deletions(-) create mode 100644 pkg/verifier/result.go create mode 100644 pkg/verifier/result_test.go create mode 100644 pkg/verifier/types/types_test.go diff --git a/pkg/executor/core/executor.go b/pkg/executor/core/executor.go index c33223f54..e5d8ec299 100644 --- a/pkg/executor/core/executor.go +++ b/pkg/executor/core/executor.go @@ -177,16 +177,7 @@ func (executor Executor) verifyReferenceForJSONPolicy(ctx context.Context, subje verifyResult, err := verifier.Verify(ctx, subjectRef, referenceDesc, referrerStore) if err != nil { verifierErr := errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace) - verifyResult = vr.VerifierResult{ - IsSuccess: false, - Name: verifier.Name(), // Deprecating Name in v2, switch to VerifierName instead. - Type: verifier.Type(), // Deprecating Type in v2, switch to VerifierType instead. - VerifierName: verifier.Name(), - VerifierType: verifier.Type(), - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - } + verifyResult = vr.NewVerifierResult("", verifier.Name(), verifier.Type(), "", false, &verifierErr, nil) } if len(verifier.GetNestedReferences()) > 0 { @@ -234,16 +225,7 @@ func (executor Executor) verifyReferenceForRegoPolicy(ctx context.Context, subje verifierResult, err := verifier.Verify(errCtx, subjectRef, referenceDesc, referrerStore) if err != nil { verifierErr := errors.ErrorCodeVerifyReferenceFailure.NewError(errors.Verifier, verifier.Name(), errors.EmptyLink, err, nil, errors.HideStackTrace) - verifierReport = vt.VerifierResult{ - IsSuccess: false, - Name: verifier.Name(), // Deprecating Name in v2, switch to VerifierName instead. - Type: verifier.Type(), // Deprecating Type in v2, switch to VerifierType instead. - VerifierName: verifier.Name(), - VerifierType: verifier.Type(), - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - } + verifierReport = vt.CreateVerifierResult(verifier.Name(), verifier.Type(), "", false, &verifierErr) } else { verifierReport = vt.NewVerifierResult(verifierResult) } diff --git a/pkg/policyprovider/configpolicy/configpolicy.go b/pkg/policyprovider/configpolicy/configpolicy.go index 812dc21ca..2ab93ca02 100644 --- a/pkg/policyprovider/configpolicy/configpolicy.go +++ b/pkg/policyprovider/configpolicy/configpolicy.go @@ -98,13 +98,7 @@ func (enforcer PolicyEnforcer) ContinueVerifyOnFailure(_ context.Context, _ comm // ErrorToVerifyResult converts an error to a properly formatted verify result func (enforcer PolicyEnforcer) ErrorToVerifyResult(_ context.Context, subjectRefString string, verifyError error) types.VerifyResult { verifierErr := re.ErrorCodeVerifyReferenceFailure.WithDetail(fmt.Sprintf("failed to verify artifact: %s", subjectRefString)).WithError(verifyError) - errorReport := verifier.VerifierResult{ - Subject: subjectRefString, - IsSuccess: false, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - } + errorReport := verifier.NewVerifierResult(subjectRefString, "", "", "", false, &verifierErr, nil) var reports []interface{} reports = append(reports, errorReport) return types.VerifyResult{IsSuccess: false, VerifierReports: reports} diff --git a/pkg/verifier/api.go b/pkg/verifier/api.go index ac166b698..d24246664 100644 --- a/pkg/verifier/api.go +++ b/pkg/verifier/api.go @@ -23,26 +23,6 @@ import ( "github.com/ratify-project/ratify/pkg/referrerstore" ) -// VerifierResult describes the result of verifying a reference manifest for a subject. -// Note: This struct is used to represent the result of verification in v0. -type VerifierResult struct { //nolint:revive // ignore linter to have unique type name - Subject string `json:"subject,omitempty"` - IsSuccess bool `json:"isSuccess"` - // Name will be deprecated in v2, switch to VerifierName instead. - Name string `json:"name,omitempty"` - VerifierName string `json:"verifierName,omitempty"` - // Type will be deprecated in v2, switch to VerifierType instead. - Type string `json:"type,omitempty"` - VerifierType string `json:"verifierType,omitempty"` - ReferenceDigest string `json:"referenceDigest,omitempty"` - ArtifactType string `json:"artifactType,omitempty"` - Message string `json:"message,omitempty"` - ErrorReason string `json:"errorReason,omitempty"` - Remediation string `json:"remediation,omitempty"` - Extensions interface{} `json:"extensions,omitempty"` - NestedResults []VerifierResult `json:"nestedResults,omitempty"` -} - // ReferenceVerifier is an interface that defines methods to verify a reference // for a subject by a verifier. type ReferenceVerifier interface { diff --git a/pkg/verifier/cosign/cosign.go b/pkg/verifier/cosign/cosign.go index dfc3622ed..fb3efd714 100644 --- a/pkg/verifier/cosign/cosign.go +++ b/pkg/verifier/cosign/cosign.go @@ -284,15 +284,15 @@ func (v *cosignVerifier) verifyInternal(ctx context.Context, subjectReference co } if hasValidSignature { - return verifier.VerifierResult{ - Name: v.name, // Deprecating Name in v2, switch to VerifierName instead. - Type: v.verifierType, // Deprecating Type in v2, switch to VerifierType instead. - VerifierName: v.name, - VerifierType: v.verifierType, - IsSuccess: true, - Message: "Verification success. Valid signatures found. Please refer to extensions field for verifications performed.", - Extensions: Extension{SignatureExtension: sigExtensions, TrustPolicy: trustPolicy.GetName()}, - }, nil + return verifier.NewVerifierResult( + "", + v.name, + v.verifierType, + "Verification success. Valid signatures found. Please refer to extensions field for verifications performed.", + true, + nil, + Extension{SignatureExtension: sigExtensions, TrustPolicy: trustPolicy.GetName()}, + ), nil } errorResult := errorToVerifyResult(v.name, v.verifierType, fmt.Errorf("no valid signatures found")) @@ -397,15 +397,15 @@ func (v *cosignVerifier) verifyLegacy(ctx context.Context, subjectReference comm } if len(signatures) > 0 { - return verifier.VerifierResult{ - Name: v.name, // Deprecating Name in v2, switch to VerifierName instead. - Type: v.verifierType, // Deprecating Type in v2, switch to VerifierType instead. - VerifierName: v.name, - VerifierType: v.verifierType, - IsSuccess: true, - Message: "Verification success. Valid signatures found", - Extensions: LegacyExtension{SignatureExtension: sigExtensions}, - }, nil + return verifier.NewVerifierResult( + "", + v.name, + v.verifierType, + "Verification success. Valid signatures found", + true, + nil, + LegacyExtension{SignatureExtension: sigExtensions}, + ), nil } errorResult := errorToVerifyResult(v.name, v.verifierType, fmt.Errorf("no valid signatures found")) @@ -486,16 +486,15 @@ func staticLayerOpts(desc imgspec.Descriptor) ([]static.Option, error) { // ErrorToVerifyResult returns a verifier result with the error message and isSuccess set to false func errorToVerifyResult(name string, verifierType string, err error) verifier.VerifierResult { verifierErr := re.ErrorCodeVerifyReferenceFailure.WithDetail("Verification failed").WithError(err) - return verifier.VerifierResult{ - IsSuccess: false, - Name: name, // Deprecating Name in v2, switch to VerifierName instead. - Type: verifierType, // Deprecating Type in v2, switch to VerifierType instead. - VerifierName: name, - VerifierType: verifierType, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - } + return verifier.NewVerifierResult( + "", + name, + verifierType, + "", + false, + &verifierErr, + nil, + ) } // decodeASN1Signature decodes the ASN.1 signature to raw signature bytes diff --git a/pkg/verifier/notation/notation.go b/pkg/verifier/notation/notation.go index e9e42cbd6..ef412b9f0 100644 --- a/pkg/verifier/notation/notation.go +++ b/pkg/verifier/notation/notation.go @@ -174,15 +174,7 @@ func (v *notationPluginVerifier) Verify(ctx context.Context, extensions["SN"] = cert.Subject.String() } - return verifier.VerifierResult{ - Name: v.name, // Deprecating Name in v2, switch to VerifierName instead. - Type: v.verifierType, // Deprecating Type in v2, switch to VerifierType instead. - VerifierName: v.name, - VerifierType: v.verifierType, - IsSuccess: true, - Message: "Signature verification success", - Extensions: extensions, - }, nil + return verifier.NewVerifierResult("", v.name, v.verifierType, "Signature verification success", true, nil, extensions), nil } func getVerifierService(conf *NotationPluginVerifierConfig, pluginDirectory string) (notation.Verifier, error) { diff --git a/pkg/verifier/result.go b/pkg/verifier/result.go new file mode 100644 index 000000000..dff28e9e8 --- /dev/null +++ b/pkg/verifier/result.go @@ -0,0 +1,62 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package verifier + +import "github.com/ratify-project/ratify/errors" + +// VerifierResult describes the result of verifying a reference manifest for a subject. +// Note: This struct is used to represent the result of verification in v0. +type VerifierResult struct { //nolint:revive // ignore linter to have unique type name + Subject string `json:"subject,omitempty"` + IsSuccess bool `json:"isSuccess"` + // Name will be deprecated in v2, tracking issue: https://github.com/ratify-project/ratify/issues/1707 + Name string `json:"name,omitempty"` + VerifierName string `json:"verifierName,omitempty"` + // Type will be deprecated in v2, tracking issue: https://github.com/ratify-project/ratify/issues/1707 + Type string `json:"type,omitempty"` + VerifierType string `json:"verifierType,omitempty"` + ReferenceDigest string `json:"referenceDigest,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` + Message string `json:"message,omitempty"` + ErrorReason string `json:"errorReason,omitempty"` + Remediation string `json:"remediation,omitempty"` + Extensions interface{} `json:"extensions,omitempty"` + NestedResults []VerifierResult `json:"nestedResults,omitempty"` +} + +// NewVerifierResult creates a new VerifierResult object with the given parameters. +func NewVerifierResult(subject, verifierName, verifierType, message string, isSuccess bool, err *errors.Error, extensions interface{}) VerifierResult { + var errorReason, remediation string + if err != nil { + if err.GetDetail() != "" { + message = err.GetDetail() + } + errorReason = err.GetErrorReason() + remediation = err.GetRemediation() + } + return VerifierResult{ + Subject: subject, + IsSuccess: isSuccess, + Name: verifierName, + Type: verifierType, + VerifierName: verifierName, + VerifierType: verifierType, + Message: message, + ErrorReason: errorReason, + Remediation: remediation, + Extensions: extensions, + } +} diff --git a/pkg/verifier/result_test.go b/pkg/verifier/result_test.go new file mode 100644 index 000000000..64efd2c52 --- /dev/null +++ b/pkg/verifier/result_test.go @@ -0,0 +1,83 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package verifier + +import ( + "fmt" + "testing" + + "github.com/ratify-project/ratify/errors" +) + +const ( + testMsg1 = "test message 1" + testMsg2 = "test message 2" + testErrReason = "test error reason" + testRemediation = "test remediation" +) + +func TestNewVerifierResult(t *testing.T) { + tests := []struct { + name string + message string + err errors.Error + expectedMsg string + expectedErrReason string + expectedRemediation string + }{ + { + name: "nil error", + message: testMsg1, + err: errors.Error{}, + expectedMsg: testMsg1, + }, + { + name: "error without detail", + message: testMsg1, + err: errors.ErrorCodeUnknown.WithError(fmt.Errorf(testErrReason)).WithRemediation(testRemediation), + expectedMsg: testMsg1, + expectedErrReason: testErrReason, + expectedRemediation: testRemediation, + }, + { + name: "error with detail", + message: testMsg1, + err: errors.ErrorCodeUnknown.WithError(fmt.Errorf(testErrReason)).WithRemediation(testRemediation).WithDetail(testMsg2), + expectedMsg: testMsg2, + expectedErrReason: testErrReason, + expectedRemediation: testRemediation, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := &tt.err + if tt.err == (errors.Error{}) { + err = nil + } + + result := NewVerifierResult("", "", "", tt.message, false, err, nil) + if result.Message != tt.expectedMsg { + t.Errorf("expected message %s, got %s", tt.expectedMsg, result.Message) + } + if result.ErrorReason != tt.expectedErrReason { + t.Errorf("expected error reason %s, got %s", tt.expectedErrReason, result.ErrorReason) + } + if result.Remediation != tt.expectedRemediation { + t.Errorf("expected remediation %s, got %s", tt.expectedRemediation, result.Remediation) + } + }) + } +} diff --git a/pkg/verifier/types/types.go b/pkg/verifier/types/types.go index eceab1f4a..446813056 100644 --- a/pkg/verifier/types/types.go +++ b/pkg/verifier/types/types.go @@ -19,6 +19,7 @@ import ( "encoding/json" "io" + "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/verifier" ) @@ -80,6 +81,30 @@ func WriteVerifyResultResult(result *verifier.VerifierResult, w io.Writer) error return json.NewEncoder(w).Encode(result) } +// CreateVerifierResult creates a new verifier result object from given input. +func CreateVerifierResult(verifierName, verifierType, message string, isSuccess bool, err *errors.Error) VerifierResult { + var errorReason string + var remediation string + if err != nil { + if err.GetDetail() != "" { + message = err.GetDetail() + } + errorReason = err.GetErrorReason() + remediation = err.GetRemediation() + } + + return VerifierResult{ + IsSuccess: isSuccess, + Name: verifierName, // Name will be deprecated in v2, tracking issue: https://github.com/ratify-project/ratify/issues/1707 + Type: verifierType, // Type will be deprecated in v2, tracking issue: https://github.com/ratify-project/ratify/issues/1707 + VerifierName: verifierName, + VerifierType: verifierType, + Message: message, + ErrorReason: errorReason, + Remediation: remediation, + } +} + // NewVerifierResult creates a new verifier result object from the given // verifier.VerifierResult. func NewVerifierResult(result verifier.VerifierResult) VerifierResult { diff --git a/pkg/verifier/types/types_test.go b/pkg/verifier/types/types_test.go new file mode 100644 index 000000000..ce1cd39f6 --- /dev/null +++ b/pkg/verifier/types/types_test.go @@ -0,0 +1,83 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "fmt" + "testing" + + "github.com/ratify-project/ratify/errors" +) + +const ( + testMsg1 = "test message 1" + testMsg2 = "test message 2" + testErrReason = "test error reason" + testRemediation = "test remediation" +) + +func TestCreateVerifierResult(t *testing.T) { + tests := []struct { + name string + message string + err errors.Error + expectedMsg string + expectedErrReason string + expectedRemediation string + }{ + { + name: "nil error", + message: testMsg1, + err: errors.Error{}, + expectedMsg: testMsg1, + }, + { + name: "error without detail", + message: testMsg1, + err: errors.ErrorCodeUnknown.WithError(fmt.Errorf(testErrReason)).WithRemediation(testRemediation), + expectedMsg: testMsg1, + expectedErrReason: testErrReason, + expectedRemediation: testRemediation, + }, + { + name: "error with detail", + message: testMsg1, + err: errors.ErrorCodeUnknown.WithError(fmt.Errorf(testErrReason)).WithRemediation(testRemediation).WithDetail(testMsg2), + expectedMsg: testMsg2, + expectedErrReason: testErrReason, + expectedRemediation: testRemediation, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := &tt.err + if tt.err == (errors.Error{}) { + err = nil + } + + result := CreateVerifierResult("", "", tt.message, false, err) + if result.Message != tt.expectedMsg { + t.Errorf("expected message %s, got %s", tt.expectedMsg, result.Message) + } + if result.ErrorReason != tt.expectedErrReason { + t.Errorf("expected error reason %s, got %s", tt.expectedErrReason, result.ErrorReason) + } + if result.Remediation != tt.expectedRemediation { + t.Errorf("expected remediation %s, got %s", tt.expectedRemediation, result.Remediation) + } + }) + } +} diff --git a/plugins/verifier/sbom/sbom.go b/plugins/verifier/sbom/sbom.go index d94bbc656..70a18ad2a 100644 --- a/plugins/verifier/sbom/sbom.go +++ b/plugins/verifier/sbom/sbom.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" + "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/common" "github.com/ratify-project/ratify/pkg/ocispecs" "github.com/ratify-project/ratify/pkg/referrerstore" @@ -84,28 +85,14 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe referenceManifest, err := referrerStore.GetReferenceManifest(ctx, subjectReference, referenceDescriptor) if err != nil { storeErr := re.ErrorCodeGetReferenceManifestFailure.WithDetail(fmt.Sprintf("Failed to fetch reference manifest for subject: %s reference descriptor: %v", subjectReference, referenceDescriptor.Descriptor)).WithError(err) - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: storeErr.GetDetail(), - ErrorReason: storeErr.GetErrorReason(), - Remediation: storeErr.GetRemediation(), - }, nil + result := verifier.NewVerifierResult("", input.Name, verifierType, "", false, &storeErr, nil) + return &result, nil } if len(referenceManifest.Blobs) == 0 { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: "SBOM validation failed", - ErrorReason: fmt.Sprintf("No layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), - }, nil + noBlobErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("No layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String())) + result := verifier.NewVerifierResult("", input.Name, verifierType, "SBOM validation failed", false, &noBlobErr, nil) + return &result, nil } artifactType := referenceDescriptor.ArtifactType @@ -114,42 +101,21 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe if err != nil { storeErr := re.ErrorCodeGetBlobContentFailure.WithDetail(fmt.Sprintf("Failed to fetch blob for subject: %s digest: %s", subjectReference, blobDesc.Digest)).WithError(err) - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: storeErr.GetDetail(), - ErrorReason: storeErr.GetErrorReason(), - Remediation: storeErr.GetRemediation(), - }, nil + result := verifier.NewVerifierResult("", input.Name, verifierType, "", false, &storeErr, nil) + return &result, nil } switch artifactType { case SpdxJSONMediaType: return processSpdxJSONMediaType(input.Name, verifierType, refBlob, input.DisallowedLicenses, input.DisallowedPackages), nil default: - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: "Failed to process SBOM blobs.", - ErrorReason: fmt.Sprintf("Unsupported artifactType: %s", artifactType), - }, nil + storeErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Unsupported artifactType: %s", artifactType)) + result := verifier.NewVerifierResult("", input.Name, verifierType, "Failed to process SBOM blobs.", false, &storeErr, nil) + return &result, nil } } - - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: true, - Message: "SBOM verification success. No license or package violation found.", - }, nil + result := verifier.NewVerifierResult("", input.Name, verifierType, "SBOM verification success. No license or package violation found.", true, nil, nil) + return &result, nil } // getViolations returns the package and license violations based on the deny list @@ -197,37 +163,26 @@ func processSpdxJSONMediaType(name string, verifierType string, refBlob []byte, } if len(licenseViolation) != 0 || len(packageViolation) != 0 { - return &verifier.VerifierResult{ - Name: name, - IsSuccess: false, - Extensions: extensionData, - Message: "SBOM validation failed.", - ErrorReason: "License or package violation found.", - Remediation: "Please review extensions data for license and package violation found.", - } + sbomErr := errors.ErrorCodeVerifyPluginFailure.WithDetail("License or package violation found.").WithRemediation("Please review extensions data for license and package violation found.") + result := verifier.NewVerifierResult("", name, verifierType, "SBOM validation failed", false, &sbomErr, extensionData) + return &result } } - return &verifier.VerifierResult{ - Name: name, - Type: verifierType, - IsSuccess: true, - Extensions: map[string]interface{}{ - CreationInfo: spdxDoc.CreationInfo, - }, - Message: "SBOM verification success. No license or package violation found.", - } + result := verifier.NewVerifierResult( + "", + name, + verifierType, + "SBOM verification success. No license or package violation found.", + true, + nil, + map[string]interface{}{CreationInfo: spdxDoc.CreationInfo}, + ) + return &result } verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("failed to verify artifact: %s", name)).WithError(err) - return &verifier.VerifierResult{ - Name: name, - Type: verifierType, - VerifierName: name, - VerifierType: verifierType, - IsSuccess: false, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - } + result := verifier.NewVerifierResult("", name, verifierType, "", false, &verifierErr, nil) + return &result } // iterate through all package info and check against the deny list diff --git a/plugins/verifier/schemavalidator/schema_validator.go b/plugins/verifier/schemavalidator/schema_validator.go index 0c8cf7806..d31886222 100644 --- a/plugins/verifier/schemavalidator/schema_validator.go +++ b/plugins/verifier/schemavalidator/schema_validator.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" + "github.com/ratify-project/ratify/errors" re "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/common" "github.com/ratify-project/ratify/pkg/ocispecs" @@ -72,14 +73,9 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe } if len(referenceManifest.Blobs) == 0 { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("No blobs found for referrer %s@%s.", subjectReference.Path, referenceDescriptor.Digest.String()), - }, nil + noBlobErr := errors.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("No blobs found for referrer %s@%s.", subjectReference.Path, referenceDescriptor.Digest.String())) + result := verifier.NewVerifierResult("", input.Name, verifierType, "", false, &noBlobErr, nil) + return &result, nil } for _, blobDesc := range referenceManifest.Blobs { @@ -91,27 +87,13 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe err = processMediaType(schemaMap, blobDesc.MediaType, refBlob) if err != nil { verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("schema validation failed for digest:[%s], media type:[%s].", blobDesc.Digest, blobDesc.MediaType)).WithError(err) - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - }, nil + result := verifier.NewVerifierResult("", input.Name, verifierType, "", false, &verifierErr, nil) + return &result, nil } } - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: true, - Message: "schema validation passed for configured media types", - }, nil + result := verifier.NewVerifierResult("", input.Name, verifierType, "schema validation passed for configured media types", true, nil, nil) + return &result, nil } func processMediaType(schemaMap map[string]string, mediaType string, refBlob []byte) error { diff --git a/plugins/verifier/schemavalidator/schema_validator_test.go b/plugins/verifier/schemavalidator/schema_validator_test.go index d0f28f60c..3448ec897 100644 --- a/plugins/verifier/schemavalidator/schema_validator_test.go +++ b/plugins/verifier/schemavalidator/schema_validator_test.go @@ -87,7 +87,7 @@ func TestVerifyReference(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("No blobs found for referrer %s@%s.", "test_subject_path", manifestDigest.String()), + errorReason: fmt.Sprintf("No blobs found for referrer %s@%s.", "test_subject_path", manifestDigest.String()), }, }, { diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report.go b/plugins/verifier/vulnerabilityreport/vulnerability_report.go index a8422a0fc..ec4004f8f 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report.go @@ -26,6 +26,7 @@ import ( imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/owenrumney/go-sarif/v2/sarif" + "github.com/ratify-project/ratify/errors" re "github.com/ratify-project/ratify/errors" "github.com/ratify-project/ratify/pkg/common" "github.com/ratify-project/ratify/pkg/ocispecs" @@ -95,16 +96,8 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe createdTime, err := extractCreationTimestamp(input.CreatedAnnotationName, referenceDescriptor) if err != nil { verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to create timestamp annotation.").WithError(err) - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - }, nil + result := verifier.NewVerifierResult("", input.Name, verifierType, "", false, &verifierErr, nil) + return &result, nil } // check report is newer than allowed maximum age @@ -112,32 +105,29 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe ok, err := validateMaximumAge(input.MaximumAge, createdTime) if err != nil { verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to validate maximum age.").WithError(err) - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } if !ok { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Report is older than maximum age:[%s].", input.MaximumAge), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Report is older than maximum age:[%s].", input.MaximumAge)) + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } } @@ -146,104 +136,95 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe referenceManifest, err := referrerStore.GetReferenceManifest(ctx, subjectReference, referenceDescriptor) if err != nil { verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Failed to fetch reference manifest for subject: %s, reference descriptor: %v.", subjectReference, referenceDescriptor)).WithError(err) - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } if len(referenceManifest.Blobs) == 0 { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("No layers found in manifest for referrer %s@%s.", subjectReference.Path, referenceDescriptor.Digest.String()), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + noBlobErr := errors.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("No layers found in manifest for referrer %s@%s.", subjectReference.Path, referenceDescriptor.Digest.String())) + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &noBlobErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } blobDesc := referenceManifest.Blobs[0] refBlob, err := referrerStore.GetBlobContent(ctx, subjectReference, blobDesc.Digest) if err != nil { verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Failed to fetch blob for subject:[%s] digest:[%s].", subjectReference, blobDesc.Digest)).WithError(err) - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } // skip all validation if passthrough is enabled if input.Passthrough { - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: true, - Message: "Validation skipped. passthrough enabled", - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "Validation skipped. passthrough enabled", + true, + nil, + map[string]interface{}{ CreatedAnnotation: createdTime, "passthrough": true, "report": string(refBlob), }, - }, nil + ) + return &result, nil } // validate json schema if err := verifyJSONSchema(referenceDescriptor.ArtifactType, refBlob, input.SchemaURL); err != nil { verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail(fmt.Sprintf("Schema validation failed for digest:[%s],artifact type:[%s].", blobDesc.Digest, referenceDescriptor.ArtifactType)).WithError(err) - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: false, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } if referenceDescriptor.ArtifactType == SarifArtifactType { return processSarifReport(input, input.Name, verifierType, refBlob, createdTime) } - return &verifier.VerifierResult{ - Name: input.Name, - Type: verifierType, - VerifierName: input.Name, - VerifierType: verifierType, - IsSuccess: true, - Message: "Validation succeeded", - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + result := verifier.NewVerifierResult( + "", + input.Name, + verifierType, + "Validation succeeded", + true, + nil, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } // verifyJSONSchema validates the json schema of the report @@ -269,33 +250,29 @@ func processSarifReport(input *PluginConfig, verifierName string, verifierType s sarifReport, err := sarif.FromBytes(blob) if err != nil { verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to parse sarif report.").WithError(err) - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: false, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } // verify that there is at least one run in the report if len(sarifReport.Runs) < 1 { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: false, - Message: "No runs found in sarif report.", - Extensions: map[string]interface{}{ - CreatedAnnotation: createdTime, - }, - }, nil + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "No runs found in sarif report.", + false, + nil, + map[string]interface{}{CreatedAnnotation: createdTime}, + ) + return &result, nil } scannerName := strings.ToLower(sarifReport.Runs[0].Tool.Driver.Name) if len(input.DenylistCVEs) > 0 { @@ -317,18 +294,19 @@ func processSarifReport(input *PluginConfig, verifierName string, verifierType s } } - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: true, - Message: "Validation succeeded", - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "Validation succeeded", + true, + nil, + map[string]interface{}{ CreatedAnnotation: createdTime, "scanner": scannerName, }, - }, nil + ) + return &result, nil } // verifyDenyListCVEs verifies that the report does not contain any deny-listed CVEs @@ -344,18 +322,19 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st // iterate over the results and check which cves are deny-listed for _, result := range sarifReport.Runs[0].Results { if result.RuleID == nil || *result.RuleID == "" { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Rule id not found for result:[%v].", result), - Extensions: map[string]interface{}{ + verifierResult := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + fmt.Sprintf("Rule id not found for result:[%v].", result), + false, + nil, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &verifierResult, nil } ruleIDLower := strings.ToLower(*result.RuleID) if _, ok := denylistCVESet[ruleIDLower]; ok { @@ -371,34 +350,36 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st } if len(denylistViolations) > 0 { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: false, - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "Found denied CVEs. See extensions field for details.", + false, + nil, + map[string]interface{}{ "scanner": scannerName, "denylistCVEs": denylistCVEs, "cveViolations": denylistViolations, CreatedAnnotation: createdTime, }, - Message: "Found denied CVEs. See extensions field for details.", - }, nil + ) + return &result, nil } - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: true, - Message: "Validation succeeded", - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "Validation succeeded", + true, + nil, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &result, nil } // verifyDisallowedSeverities verifies that the report does not contain any disallowed severity levels @@ -412,51 +393,52 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne // iterate over the results and check if the severity is disallowed for _, result := range sarifReport.Runs[0].Results { if result.RuleID == nil || *result.RuleID == "" { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Rule id not found for result:[%v].", result), - Extensions: map[string]interface{}{ + verifierResult := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + fmt.Sprintf("Rule id not found for result:[%v].", result), + false, + nil, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &verifierResult, nil } rule, ok := ruleMap[*result.RuleID] if !ok { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: false, - Message: fmt.Sprintf("Rule not found for result:[%v].", result), - Extensions: map[string]interface{}{ + verifierResult := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + fmt.Sprintf("Rule not found for result:[%v].", result), + false, + nil, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &verifierResult, nil } severity, err := extractSeverity(scannerName, *rule) if err != nil { verifierErr := re.ErrorCodeVerifyPluginFailure.WithDetail("Failed to extract severity.").WithError(err) - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: false, - Message: verifierErr.GetDetail(), - ErrorReason: verifierErr.GetErrorReason(), - Remediation: verifierErr.GetRemediation(), - Extensions: map[string]interface{}{ + verifierResult := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "", + false, + &verifierErr, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &verifierResult, nil } // check if the severity is disallowed and add it to the map of violating CVE IDs for _, disallowed := range disallowedSeverities { @@ -467,33 +449,35 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne } // if there are violating rules, return them as custom extension field if len(violatingRules) > 0 { - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: false, - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "Found disallowed severities. See extensions field for details.", + false, + nil, + map[string]interface{}{ "scanner": scannerName, "disallowedSeverities": disallowedSeverities, "severityViolations": violatingRules, CreatedAnnotation: createdTime, }, - Message: "Found disallowed severities. See extensions field for details.", - }, nil + ) + return &result, nil } - return &verifier.VerifierResult{ - Name: verifierName, - Type: verifierType, - VerifierName: verifierName, - VerifierType: verifierType, - IsSuccess: true, - Message: "Validation succeeded", - Extensions: map[string]interface{}{ + result := verifier.NewVerifierResult( + "", + verifierName, + verifierType, + "Validation succeeded", + true, + nil, + map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, }, - }, nil + ) + return &result, nil } // extractSeverity extracts the severity from the rule help text using regex diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go index 84728f529..0823d2ab9 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go @@ -136,7 +136,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "Report is older than maximum age:[24h].", + errorReason: "Report is older than maximum age:[24h].", }, }, { @@ -151,7 +151,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("No layers found in manifest for referrer %s@%s.", "test_subject_path", manifestDigest.String()), + errorReason: fmt.Sprintf("No layers found in manifest for referrer %s@%s.", "test_subject_path", manifestDigest.String()), }, }, {