From d4aa9b13c40343586bc70be08b66e2bb09ed84f6 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Wed, 19 Jul 2023 10:17:14 +0800 Subject: [PATCH] Add vulnerability search API (#18924) use q.Query to pass all query conditions Signed-off-by: stonezdj --- api/v2.0/swagger.yaml | 106 ++++++++- .../postgresql/0120_2.9.0_schema.up.sql | 2 + src/common/rbac/system/policies.go | 1 + src/controller/securityhub/controller.go | 61 +++++- src/controller/securityhub/controller_test.go | 77 +++++-- src/pkg/scan/scanner/manager.go | 15 ++ src/pkg/scan/scanner/manager_test.go | 9 + src/pkg/securityhub/dao/security.go | 207 +++++++++++++++++- src/pkg/securityhub/dao/security_test.go | 88 +++++++- src/pkg/securityhub/manager.go | 12 + src/pkg/securityhub/model/model.go | 10 + src/server/v2.0/handler/security.go | 51 +++++ .../controller/securityhub/controller.go | 52 +++++ src/testing/pkg/scan/scanner/manager.go | 24 ++ src/testing/pkg/securityhub/manager.go | 50 +++++ 15 files changed, 736 insertions(+), 29 deletions(-) diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 7f6f36f484b..1f06b719266 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -6063,13 +6063,13 @@ paths: - $ref: '#/parameters/requestId' - name: with_dangerous_cve in: query - description: Specify whether the dangerous CVE is include in the security summary + description: Specify whether the dangerous CVEs are included inside summary information type: boolean required: false default: false - name: with_dangerous_artifact in: query - description: Specify whether the dangerous artifacts is include in the security summary + description: Specify whether the dangerous Artifact are included inside summary information type: boolean required: false default: false @@ -6086,6 +6086,61 @@ paths: $ref: '#/responses/404' '500': $ref: '#/responses/500' + + /security/vul: + get: + summary: Get the vulnerability list. + description: | + Get the vulnerability list. use q to pass the query condition, + supported conditions: + cve_id(exact match) + cvss_score_v3(range condition) + severity(exact match) + repository_name(exact match) + project_id(exact match) + package(exact match) + and tag(exact match) + tags: + - securityhub + operationId: ListVulnerabilities + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/query' + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + - name: tune_count + in: query + description: Enable to ignore X-Total-Count when the total count > 1000, if the total count is less than 1000, the real total count is returned, else -1. + type: boolean + required: false + default: false + - name: with_tag + in: query + description: Specify whether the tag information is included inside vulnerability information + type: boolean + required: false + default: false + responses: + '200': + description: The vulnerability list. + schema: + type: array + items: + $ref: '#/definitions/VulnerabilityItem' + headers: + X-Total-Count: + description: The total count of vulnerabilities + type: integer + Link: + description: Link refers to the previous page and next page + type: string + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '500': + $ref: '#/responses/500' + parameters: query: name: q @@ -9760,3 +9815,50 @@ definitions: type: integer x-omitempty: false description: the count of medium vulnerabilities + + VulnerabilityItem: + type: object + description: the vulnerability item info + properties: + project_id: + type: integer + format: int64 + description: the project ID of the artifact + repository_name: + type: string + description: the repository name of the artifact + digest: + type: string + description: the digest of the artifact + tags: + type: array + items: + type: string + description: the tags of the artifact + cve_id: + type: string + description: the CVE id of the vulnerability. + severity: + type: string + description: the severity of the vulnerability + cvss_v3_score: + type: number + format: float + description: the nvd cvss v3 score of the vulnerability + package: + type: string + description: the package of the vulnerability + version: + type: string + description: the version of the package + fixed_version: + type: string + description: the fixed version of the package + desc: + type: string + description: The description of the vulnerability + links: + type: array + items: + type: string + description: Links of the vulnerability diff --git a/make/migrations/postgresql/0120_2.9.0_schema.up.sql b/make/migrations/postgresql/0120_2.9.0_schema.up.sql index 8204b0c75c1..899231317bf 100644 --- a/make/migrations/postgresql/0120_2.9.0_schema.up.sql +++ b/make/migrations/postgresql/0120_2.9.0_schema.up.sql @@ -11,6 +11,8 @@ UPDATE vulnerability_record SET cvss_score_v3 = (vendor_attributes->'CVSS'->'nvd'->>'V3Score')::double precision WHERE jsonb_path_exists(vendor_attributes::jsonb, '$.CVSS.nvd.V3Score'); +CREATE INDEX IF NOT EXISTS idx_vulnerability_record_cvss_score_v3 ON vulnerability_record (cvss_score_v3); + /* add summary information in scan_report */ ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS critical_cnt BIGINT; ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS high_cnt BIGINT; diff --git a/src/common/rbac/system/policies.go b/src/common/rbac/system/policies.go index 8fd769380f9..204d3e9775d 100644 --- a/src/common/rbac/system/policies.go +++ b/src/common/rbac/system/policies.go @@ -86,5 +86,6 @@ var ( {Resource: rbac.ResourceJobServiceMonitor, Action: rbac.ActionStop}, {Resource: rbac.ResourceSecurityHub, Action: rbac.ActionRead}, + {Resource: rbac.ResourceSecurityHub, Action: rbac.ActionList}, } ) diff --git a/src/controller/securityhub/controller.go b/src/controller/securityhub/controller.go index f83af4956c0..2501d2d7d92 100644 --- a/src/controller/securityhub/controller.go +++ b/src/controller/securityhub/controller.go @@ -23,6 +23,7 @@ import ( "github.com/goharbor/harbor/src/pkg/scan/scanner" "github.com/goharbor/harbor/src/pkg/securityhub" secHubModel "github.com/goharbor/harbor/src/pkg/securityhub/model" + "github.com/goharbor/harbor/src/pkg/tag" ) // Ctl is the global controller for security hub @@ -63,26 +64,32 @@ func WithArtifact(enable bool) Option { type Controller interface { // SecuritySummary returns the security summary of the specified project. SecuritySummary(ctx context.Context, projectID int64, options ...Option) (*secHubModel.Summary, error) + // ListVuls list vulnerabilities by query + ListVuls(ctx context.Context, scannerUUID string, projectID int64, withTag bool, query *q.Query) ([]*secHubModel.VulnerabilityItem, error) + // CountVuls get all vulnerability count by query + CountVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) } type controller struct { artifactMgr artifact.Manager scannerMgr scanner.Manager secHubMgr securityhub.Manager + tagMgr tag.Manager } // NewController ... func NewController() Controller { return &controller{ artifactMgr: pkg.ArtifactMgr, - scannerMgr: scanner.New(), + scannerMgr: scanner.Mgr, secHubMgr: securityhub.Mgr, + tagMgr: tag.Mgr, } } func (c *controller) SecuritySummary(ctx context.Context, projectID int64, options ...Option) (*secHubModel.Summary, error) { opts := newOptions(options...) - scannerUUID, err := c.defaultScannerUUID(ctx) + scannerUUID, err := c.scannerMgr.DefaultScannerUUID(ctx) if err != nil { return nil, err } @@ -114,7 +121,7 @@ func (c *controller) SecuritySummary(ctx context.Context, projectID int64, optio } func (c *controller) scannedArtifactCount(ctx context.Context, projectID int64) (int64, error) { - scannerUUID, err := c.defaultScannerUUID(ctx) + scannerUUID, err := c.scannerMgr.DefaultScannerUUID(ctx) if err != nil { return 0, err } @@ -128,11 +135,49 @@ func (c *controller) totalArtifactCount(ctx context.Context, projectID int64) (i return c.artifactMgr.Count(ctx, q.New(q.KeyWords{"project_id": projectID})) } -// defaultScannerUUID returns the default scanner uuid. -func (c *controller) defaultScannerUUID(ctx context.Context) (string, error) { - reg, err := c.scannerMgr.GetDefault(ctx) +func (c *controller) ListVuls(ctx context.Context, scannerUUID string, projectID int64, withTag bool, query *q.Query) ([]*secHubModel.VulnerabilityItem, error) { + vuls, err := c.secHubMgr.ListVuls(ctx, scannerUUID, projectID, query) if err != nil { - return "", err + return nil, err + } + if withTag { + return c.attachTags(ctx, vuls) + } + return vuls, nil +} + +func (c *controller) attachTags(ctx context.Context, vuls []*secHubModel.VulnerabilityItem) ([]*secHubModel.VulnerabilityItem, error) { + // get all artifact_ids + artifactTagMap := make(map[int64][]string, 0) + for _, v := range vuls { + artifactTagMap[v.ArtifactID] = make([]string, 0) + } + + // get tags in the artifact list + var artifactIds []interface{} + for k := range artifactTagMap { + artifactIds = append(artifactIds, k) } - return reg.UUID, nil + query := q.New(q.KeyWords{"artifact_id": q.NewOrList(artifactIds)}) + tags, err := c.tagMgr.List(ctx, query) + if err != nil { + return vuls, err + } + for _, tag := range tags { + artifactTagMap[tag.ArtifactID] = append(artifactTagMap[tag.ArtifactID], tag.Name) + } + + // attach tags, only show 10 tags + for _, v := range vuls { + if len(artifactTagMap[v.ArtifactID]) > 10 { + v.Tags = artifactTagMap[v.ArtifactID][:10] + continue + } + v.Tags = artifactTagMap[v.ArtifactID] + } + return vuls, nil +} + +func (c *controller) CountVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) { + return c.secHubMgr.TotalVuls(ctx, scannerUUID, projectID, tuneCount, query) } diff --git a/src/controller/securityhub/controller_test.go b/src/controller/securityhub/controller_test.go index 987a7619da8..773d9b1c5ba 100644 --- a/src/controller/securityhub/controller_test.go +++ b/src/controller/securityhub/controller_test.go @@ -21,13 +21,14 @@ import ( "github.com/stretchr/testify/suite" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" - "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/goharbor/harbor/src/pkg/securityhub/model" + "github.com/goharbor/harbor/src/pkg/tag/model/tag" htesting "github.com/goharbor/harbor/src/testing" "github.com/goharbor/harbor/src/testing/mock" artifactMock "github.com/goharbor/harbor/src/testing/pkg/artifact" scannerMock "github.com/goharbor/harbor/src/testing/pkg/scan/scanner" securityMock "github.com/goharbor/harbor/src/testing/pkg/securityhub" + tagMock "github.com/goharbor/harbor/src/testing/pkg/tag" ) var sum = &model.Summary{ @@ -45,6 +46,7 @@ type ControllerTestSuite struct { artifactMgr *artifactMock.Manager scannerMgr *scannerMock.Manager secHubMgr *securityMock.Manager + tagMgr *tagMock.FakeManager } // TestController is the entry of controller test suite @@ -57,10 +59,13 @@ func (suite *ControllerTestSuite) SetupTest() { suite.artifactMgr = &artifactMock.Manager{} suite.secHubMgr = &securityMock.Manager{} suite.scannerMgr = &scannerMock.Manager{} + suite.tagMgr = &tagMock.FakeManager{} + suite.c = &controller{ artifactMgr: suite.artifactMgr, secHubMgr: suite.secHubMgr, scannerMgr: suite.scannerMgr, + tagMgr: suite.tagMgr, } } @@ -74,7 +79,7 @@ func (suite *ControllerTestSuite) TestSecuritySummary() { mock.OnAnything(suite.artifactMgr, "Count").Return(int64(1234), nil) mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil) mock.OnAnything(suite.secHubMgr, "Summary").Return(sum, nil).Twice() - mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: "ruuid"}, nil) + mock.OnAnything(suite.scannerMgr, "DefaultScannerUUID").Return("ruuid", nil) summary, err := suite.c.SecuritySummary(ctx, 0, WithArtifact(false), WithCVE(false)) suite.NoError(err) suite.NotNil(summary) @@ -119,7 +124,7 @@ func (suite *ControllerTestSuite) TestSecuritySummary() { // TestSecuritySummaryError tests the security summary with error func (suite *ControllerTestSuite) TestSecuritySummaryError() { ctx := suite.Context() - mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: "ruuid"}, nil) + mock.OnAnything(suite.scannerMgr, "DefaultScannerUUID").Return("ruuid", nil) mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil) mock.OnAnything(suite.secHubMgr, "Summary").Return(nil, errors.New("invalid project")).Once() summary, err := suite.c.SecuritySummary(ctx, 0, WithCVE(false), WithArtifact(false)) @@ -133,25 +138,63 @@ func (suite *ControllerTestSuite) TestSecuritySummaryError() { } -// TestGetDefaultScanner tests the get default scanner -func (suite *ControllerTestSuite) TestGetDefaultScanner() { +func (suite *ControllerTestSuite) TestScannedArtifact() { ctx := suite.Context() - mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: ""}, nil).Once() - scanner, err := suite.c.defaultScannerUUID(ctx) + mock.OnAnything(suite.scannerMgr, "DefaultScannerUUID").Return("ruuid", nil) + mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil) + scanned, err := suite.c.scannedArtifactCount(ctx, 0) suite.NoError(err) - suite.Equal("", scanner) + suite.Equal(int64(1000), scanned) +} - mock.OnAnything(suite.scannerMgr, "GetDefault").Return(nil, errors.New("failed to get scanner")).Once() - scanner, err = suite.c.defaultScannerUUID(ctx) - suite.Error(err) - suite.Equal("", scanner) +// TestAttachTags test the attachTags +func (suite *ControllerTestSuite) TestAttachTags() { + ctx := suite.Context() + tagList := []*tag.Tag{ + {ArtifactID: int64(1), Name: "latest"}, + {ArtifactID: int64(1), Name: "tag1"}, + {ArtifactID: int64(1), Name: "tag2"}, + {ArtifactID: int64(1), Name: "tag3"}, + {ArtifactID: int64(1), Name: "tag4"}, + {ArtifactID: int64(1), Name: "tag5"}, + {ArtifactID: int64(1), Name: "tag6"}, + {ArtifactID: int64(1), Name: "tag7"}, + {ArtifactID: int64(1), Name: "tag8"}, + {ArtifactID: int64(1), Name: "tag9"}, + {ArtifactID: int64(1), Name: "tag10"}, + } + vulItems := []*model.VulnerabilityItem{ + {ArtifactID: int64(1)}, + } + mock.OnAnything(suite.c.tagMgr, "List").Return(tagList, nil).Once() + resultItems, err := suite.c.attachTags(ctx, vulItems) + suite.NoError(err) + suite.Equal(len(vulItems), len(resultItems)) + suite.Equal([]string{"latest"}, resultItems[0].Tags[:1]) + suite.Equal(10, len(resultItems[0].Tags)) } -func (suite *ControllerTestSuite) TestScannedArtifact() { +// TestListVuls tests the list vulnerabilities +func (suite *ControllerTestSuite) TestListVuls() { ctx := suite.Context() - mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: "ruuid"}, nil) - mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil) - scanned, err := suite.c.scannedArtifactCount(ctx, 0) + vulItems := []*model.VulnerabilityItem{ + {ArtifactID: int64(1)}, + } + tagList := []*tag.Tag{ + {ArtifactID: int64(1), Name: "latest"}, + } + mock.OnAnything(suite.c.secHubMgr, "ListVuls").Return(vulItems, nil) + mock.OnAnything(suite.c.tagMgr, "List").Return(tagList, nil).Once() + vulResult, err := suite.c.ListVuls(ctx, "", 0, true, nil) suite.NoError(err) - suite.Equal(int64(1000), scanned) + suite.Equal(1, len(vulResult)) + suite.Equal(int64(1), vulResult[0].ArtifactID) +} + +func (suite *ControllerTestSuite) TestCountVuls() { + ctx := suite.Context() + mock.OnAnything(suite.c.secHubMgr, "TotalVuls").Return(int64(10), nil) + count, err := suite.c.CountVuls(ctx, "", 0, true, nil) + suite.NoError(err) + suite.Equal(int64(10), count) } diff --git a/src/pkg/scan/scanner/manager.go b/src/pkg/scan/scanner/manager.go index bf2983480a4..96598f20b1c 100644 --- a/src/pkg/scan/scanner/manager.go +++ b/src/pkg/scan/scanner/manager.go @@ -24,6 +24,9 @@ import ( "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" ) +// Mgr is the global manager for scanner +var Mgr = New() + // Manager defines the related scanner API endpoints type Manager interface { // Count returns the total count of scanner registrations according to the query. @@ -52,6 +55,9 @@ type Manager interface { // GetDefault returns the default scanner registration or `nil` if there are no registrations configured. GetDefault(ctx context.Context) (*scanner.Registration, error) + + // DefaultScannerUUID get default scanner UUID + DefaultScannerUUID(ctx context.Context) (string, error) } // basicManager is the default implementation of Manager @@ -139,3 +145,12 @@ func (bm *basicManager) SetAsDefault(ctx context.Context, registrationUUID strin func (bm *basicManager) GetDefault(ctx context.Context) (*scanner.Registration, error) { return scanner.GetDefaultRegistration(ctx) } + +// DefaultScannerUUID returns the default scanner uuid. +func (bm *basicManager) DefaultScannerUUID(ctx context.Context) (string, error) { + reg, err := bm.GetDefault(ctx) + if err != nil { + return "", err + } + return reg.UUID, nil +} diff --git a/src/pkg/scan/scanner/manager_test.go b/src/pkg/scan/scanner/manager_test.go index b3c7a76fb5e..742aae302bb 100644 --- a/src/pkg/scan/scanner/manager_test.go +++ b/src/pkg/scan/scanner/manager_test.go @@ -111,3 +111,12 @@ func (suite *BasicManagerTestSuite) TestDefault() { require.NotNil(suite.T(), dr) assert.Equal(suite.T(), true, dr.IsDefault) } + +// TestGetDefaultScanner tests the get default scanner +func (suite *BasicManagerTestSuite) TestGetDefaultScanner() { + ctx := suite.Context() + suite.mgr.SetAsDefault(ctx, suite.sampleUUID) + scanner, err := suite.mgr.DefaultScannerUUID(ctx) + suite.NoError(err) + suite.Equal(suite.sampleUUID, scanner) +} diff --git a/src/pkg/securityhub/dao/security.go b/src/pkg/securityhub/dao/security.go index 8adc67f0ac2..227cd3a076f 100644 --- a/src/pkg/securityhub/dao/security.go +++ b/src/pkg/securityhub/dao/security.go @@ -16,7 +16,10 @@ package dao import ( "context" + "fmt" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" @@ -24,6 +27,7 @@ import ( ) const ( + // sql to query the security summary summarySQL = `select sum(s.critical_cnt) critical_cnt, sum(s.high_cnt) high_cnt, sum(s.medium_cnt) medium_cnt, @@ -34,7 +38,7 @@ const ( from artifact a left join scan_report s on a.digest = s.digest where s.registration_uuid = ?` - + // sql to query the dangerous artifact dangerousArtifactSQL = `select a.project_id project, a.repository_name repository, a.digest, s.critical_cnt, s.high_cnt, s.medium_cnt, s.low_cnt from artifact a, scan_report s @@ -43,19 +47,95 @@ where a.digest = s.digest order by s.critical_cnt desc, s.high_cnt desc, s.medium_cnt desc, s.low_cnt desc limit 5` + // sql to query the scanned artifact count scannedArtifactCountSQL = `select count(1) from artifact a left join scan_report s on a.digest = s.digest where s.registration_uuid= ? and s.uuid is not null` + // sql to query the dangerous CVEs dangerousCVESQL = `select vr.* from vulnerability_record vr where vr.cvss_score_v3 is not null and vr.registration_uuid = ? order by vr.cvss_score_v3 desc limit 5` + + // sql to query vulnerabilities + vulnerabilitySQL = `select vr.cve_id, vr.cvss_score_v3, vr.package, a.repository_name, a.id artifact_id, a.digest, vr.package, vr.package_version, vr.severity, vr.fixed_version, vr.description, vr.urls, a.project_id +from artifact a, + scan_report s, + report_vulnerability_record rvr, + vulnerability_record vr +where a.digest = s.digest + and s.uuid = rvr.report_uuid + and rvr.vuln_record_id = vr.id + and rvr.report_uuid is not null + and vr.registration_uuid = ? ` + + stringType = "string" + intType = "int" + rangeType = "range" ) +type filterMetaData struct { + DataType string + FilterFunc func(ctx context.Context, key string, query *q.Query) (sqlStr string, params []interface{}) +} + +var filterMap = map[string]*filterMetaData{ + "cve_id": &filterMetaData{DataType: stringType, FilterFunc: exactMatchFilter}, + "severity": &filterMetaData{DataType: stringType, FilterFunc: exactMatchFilter}, + "cvss_score_v3": &filterMetaData{DataType: rangeType, FilterFunc: rangeFilter}, + "project_id": &filterMetaData{DataType: stringType, FilterFunc: exactMatchFilter}, + "repository_name": &filterMetaData{DataType: stringType, FilterFunc: exactMatchFilter}, + "package": &filterMetaData{DataType: stringType, FilterFunc: exactMatchFilter}, + "tag": &filterMetaData{DataType: stringType, FilterFunc: tagFilter}, +} + +var applyFilterFunc func(ctx context.Context, key string, query *q.Query) (sqlStr string, params []interface{}) + +func exactMatchFilter(ctx context.Context, key string, query *q.Query) (sqlStr string, params []interface{}) { + if query == nil { + return + } + if val, ok := query.Keywords[key]; ok { + sqlStr = fmt.Sprintf(" and %v = ?", key) + params = append(params, val) + return + } + return +} + +func rangeFilter(ctx context.Context, key string, query *q.Query) (sqlStr string, params []interface{}) { + if query == nil { + return + } + if val, ok := query.Keywords[key]; ok { + if r, ok := val.(*q.Range); ok { + sqlStr = fmt.Sprintf(" and %v between ? and ?", key) + params = append(params, r.Min, r.Max) + } + } + return +} + +func tagFilter(ctx context.Context, key string, query *q.Query) (sqlStr string, params []interface{}) { + if query == nil { + return + } + if val, ok := query.Keywords["tag"]; ok { + inClause, err := orm.CreateInClause(ctx, `SELECT artifact_id FROM tag + WHERE tag.name = ?`, val) + if err != nil { + log.Errorf("failed to create in clause: %v, skip this condition", err) + } else { + sqlStr = " and a.id " + inClause + } + } + return +} + // SecurityHubDao defines the interface to access security hub data. type SecurityHubDao interface { // Summary returns the summary of the scan cve reports. @@ -66,6 +146,10 @@ type SecurityHubDao interface { DangerousArtifacts(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*model.DangerousArtifact, error) // ScannedArtifactsCount return the count of scanned artifacts. ScannedArtifactsCount(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) (int64, error) + // ListVulnerabilities search vulnerability record by cveID + ListVulnerabilities(ctx context.Context, registrationUUID string, projectID int64, query *q.Query) ([]*model.VulnerabilityItem, error) + // CountVulnerabilities count the total vulnerabilities + CountVulnerabilities(ctx context.Context, registrationUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) } // New creates a new SecurityHubDao instance. @@ -131,3 +215,124 @@ func (d *dao) DangerousCVEs(ctx context.Context, scannerUUID string, projectID i _, err = o.Raw(dangerousCVESQL, scannerUUID).QueryRows(&cves) return cves, err } + +func countSQL(strSQL string) string { + return fmt.Sprintf(`select count(1) cnt from (%v) as t`, strSQL) +} + +func (d *dao) CountVulnerabilities(ctx context.Context, registrationUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) { + o, err := orm.FromContext(ctx) + if err != nil { + return 0, err + } + sqlStr := vulnerabilitySQL + params := []interface{}{registrationUUID} + if err := checkQFilter(query, filterMap); err != nil { + return 0, err + } + sqlStr, params = applyVulFilter(ctx, sqlStr, query, params) + if tuneCount { + exceedLimit, err := d.countExceedLimit(ctx, sqlStr, params) + if err != nil { + return 0, err + } + if exceedLimit { + log.Warning("the count is exceed to limit 1000 due to the tuneCount is enabled, return count with -1 instead") + return -1, nil + } + } + var cnt int64 + err = o.Raw(countSQL(sqlStr), params).QueryRow(&cnt) + return cnt, err +} + +// countExceedLimit check if the count is exceed to limit 1000, avoid count all record for large table +func (d *dao) countExceedLimit(ctx context.Context, sqlStr string, params []interface{}) (bool, error) { + o, err := orm.FromContext(ctx) + if err != nil { + return false, err + } + queryExceed := fmt.Sprintf(`SELECT EXISTS (%s LIMIT 1 OFFSET 1000)`, sqlStr) + var exceed bool + err = o.Raw(queryExceed, params).QueryRow(&exceed) + if err != nil { + return false, err + } + return exceed, nil +} + +func (d *dao) ListVulnerabilities(ctx context.Context, registrationUUID string, projectID int64, query *q.Query) ([]*model.VulnerabilityItem, error) { + o, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + sqlStr := vulnerabilitySQL + params := []interface{}{registrationUUID} + if err := checkQFilter(query, filterMap); err != nil { + return nil, err + } + sqlStr, params = applyVulFilter(ctx, sqlStr, query, params) + sqlStr, params = applyVulPagination(sqlStr, query, params) + vulnRecs := make([]*model.VulnerabilityItem, 0) + _, err = o.Raw(sqlStr, params).QueryRows(&vulnRecs) + return vulnRecs, err +} + +func applyVulFilter(ctx context.Context, sqlStr string, query *q.Query, params []interface{}) (queryStr string, newParam []interface{}) { + if query == nil { + return sqlStr, params + } + queryStr = sqlStr + newParam = params + for k, m := range filterMap { + s, p := m.FilterFunc(ctx, k, query) + queryStr = queryStr + s + newParam = append(newParam, p...) + } + return queryStr, newParam +} + +// applyVulPagination apply pagination to the query and sort by cvss_score_v3 desc +func applyVulPagination(sqlStr string, query *q.Query, params []interface{}) (string, []interface{}) { + offSet := int64(0) + pageSize := int64(15) + if query != nil && query.PageNumber > 1 { + offSet = (query.PageNumber - 1) * query.PageSize + } + if query != nil && query.PageSize > 0 { + pageSize = query.PageSize + } + params = append(params, pageSize, offSet) + return fmt.Sprintf("%v order by cvss_score_v3 desc nulls last limit ? offset ? ", sqlStr), params +} + +func checkQFilter(query *q.Query, filterMap map[string]*filterMetaData) error { + if query == nil { + return nil + } + if len(query.Keywords) == 0 { + return nil + } + for k := range query.Keywords { + if metadata, exist := filterMap[k]; exist { + typeName := metadata.DataType + switch typeName { + case rangeType: + if _, ok := query.Keywords[k].(*q.Range); !ok { + return errors.BadRequestError(fmt.Errorf("keyword: %v, the query type is not allowed", k)) + } + case stringType: + if _, ok := query.Keywords[k].(string); !ok { + return errors.BadRequestError(fmt.Errorf("keyword: %v, the query type is not allowed", k)) + } + case intType: + if _, ok := query.Keywords[k].(int); !ok { + return errors.BadRequestError(fmt.Errorf("keyword: %v, the query type is not allowed", k)) + } + } + } else { + return errors.BadRequestError(fmt.Errorf("keyword: %v is not allowed", k)) + } + } + return nil +} diff --git a/src/pkg/securityhub/dao/security_test.go b/src/pkg/securityhub/dao/security_test.go index 7cab1ab4a78..2ffdd2a57e5 100644 --- a/src/pkg/securityhub/dao/security_test.go +++ b/src/pkg/securityhub/dao/security_test.go @@ -15,12 +15,14 @@ package dao import ( + "context" "testing" "github.com/stretchr/testify/suite" testDao "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" htesting "github.com/goharbor/harbor/src/testing" ) @@ -61,7 +63,6 @@ values (1, 'library/hello-world', 'digest1001', 'IMAGE', '2023-06-02 09:16:47.8 }) } -// TearDownTest clears enf for test case. func (suite *SecurityDaoTestSuite) TearDownTest() { testDao.ExecuteBatchSQL([]string{ `delete from scan_report where uuid = 'uuid'`, @@ -102,3 +103,88 @@ func (suite *SecurityDaoTestSuite) TestGetDangerousCVEs() { suite.NoError(err, "Error when fetching most dangerous artifact") suite.Equal(5, len(records)) } + +func Test_checkQFilter(t *testing.T) { + type args struct { + query *q.Query + filterMap map[string]*filterMetaData + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"happy_path", args{q.New(q.KeyWords{"sample": 1}), map[string]*filterMetaData{"sample": &filterMetaData{intType, exactMatchFilter}}}, false}, + {"happy_path_cve_id", args{q.New(q.KeyWords{"cve_id": "CVE-2023-2345"}), map[string]*filterMetaData{"cve_id": &filterMetaData{stringType, exactMatchFilter}}}, false}, + {"happy_path_severity", args{q.New(q.KeyWords{"severity": "Critical"}), map[string]*filterMetaData{"severity": &filterMetaData{stringType, exactMatchFilter}}}, false}, + {"happy_path_cvss_score_v3", args{q.New(q.KeyWords{"cvss_score_v3": &q.Range{Min: 2.0, Max: 3.0}}), map[string]*filterMetaData{"cvss_score_v3": &filterMetaData{rangeType, rangeFilter}}}, false}, + {"unhappy_path", args{q.New(q.KeyWords{"sample": 1}), map[string]*filterMetaData{"a": &filterMetaData{DataType: intType}}}, true}, + {"unhappy_path2", args{q.New(q.KeyWords{"cve_id": 1}), map[string]*filterMetaData{"cve_id": &filterMetaData{stringType, exactMatchFilter}}}, true}, + {"unhappy_path3", args{q.New(q.KeyWords{"severity": &q.Range{Min: 2.0, Max: 10.0}}), map[string]*filterMetaData{"severity": &filterMetaData{stringType, exactMatchFilter}}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := checkQFilter(tt.args.query, tt.args.filterMap); (err != nil) != tt.wantErr { + t.Errorf("checkQFilter() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func (suite *SecurityDaoTestSuite) TestExacthMatchFilter() { + type args struct { + ctx context.Context + key string + query *q.Query + } + tests := []struct { + name string + args args + wantSQLStr string + wantParams []interface{} + }{ + {"normal", args{suite.Context(), "cve_id", q.New(q.KeyWords{"cve_id": "CVE-2023-2345"})}, " and cve_id = ?", []interface{}{"CVE-2023-2345"}}, + } + for _, tt := range tests { + suite.Run(tt.name, func() { + gotSQLStr, gotParams := exactMatchFilter(tt.args.ctx, tt.args.key, tt.args.query) + suite.Equal(gotSQLStr, tt.wantSQLStr, "exactMatchFilter() gotSqlStr = %v, want %v", gotSQLStr, tt.wantSQLStr) + suite.Equal(gotParams, tt.wantParams, "exactMatchFilter() gotParams = %v, want %v", gotParams, tt.wantParams) + }) + } +} + +func (suite *SecurityDaoTestSuite) TestRangeFilter() { + type args struct { + ctx context.Context + key string + query *q.Query + } + tests := []struct { + name string + args args + wantSQLStr string + wantParams []interface{} + }{ + {"normal", args{suite.Context(), "cvss_score_v3", q.New(q.KeyWords{"cvss_score_v3": &q.Range{1.0, 2.0}})}, " and cvss_score_v3 between ? and ?", []interface{}{1.0, 2.0}}, + } + for _, tt := range tests { + suite.Run(tt.name, func() { + gotSQLStr, gotParams := rangeFilter(tt.args.ctx, tt.args.key, tt.args.query) + suite.Equal(tt.wantSQLStr, gotSQLStr, "exactMatchFilter() gotSqlStr = %v, want %v", gotSQLStr, tt.wantSQLStr) + suite.Equal(tt.wantParams, gotParams, "exactMatchFilter() gotParams = %v, want %v", gotParams, tt.wantParams) + }) + } +} + +func (suite *SecurityDaoTestSuite) TestCountVul() { + count, err := suite.dao.CountVulnerabilities(suite.Context(), "ruuid", 0, true, nil) + suite.NoError(err) + suite.Equal(int64(1), count) +} + +func (suite *SecurityDaoTestSuite) TestListVul() { + vuls, err := suite.dao.ListVulnerabilities(suite.Context(), "ruuid", 0, nil) + suite.NoError(err) + suite.Equal(1, len(vuls)) +} diff --git a/src/pkg/securityhub/manager.go b/src/pkg/securityhub/manager.go index a71ebdf3976..679367d6254 100644 --- a/src/pkg/securityhub/manager.go +++ b/src/pkg/securityhub/manager.go @@ -38,6 +38,10 @@ type Manager interface { ScannedArtifactsCount(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) (int64, error) // DangerousCVEs returns the most dangerous CVEs for the given scanner. DangerousCVEs(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*scan.VulnerabilityRecord, error) + // TotalVuls return the count of vulnerabilities + TotalVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) + // ListVuls returns vulnerabilities list + ListVuls(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*model.VulnerabilityItem, error) } // NewManager news security manager. @@ -67,3 +71,11 @@ func (s *securityManager) ScannedArtifactsCount(ctx context.Context, scannerUUID func (s *securityManager) DangerousCVEs(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*scan.VulnerabilityRecord, error) { return s.dao.DangerousCVEs(ctx, scannerUUID, projectID, query) } + +func (s *securityManager) TotalVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) { + return s.dao.CountVulnerabilities(ctx, scannerUUID, projectID, tuneCount, query) +} + +func (s *securityManager) ListVuls(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*model.VulnerabilityItem, error) { + return s.dao.ListVulnerabilities(ctx, scannerUUID, projectID, query) +} diff --git a/src/pkg/securityhub/model/model.go b/src/pkg/securityhub/model/model.go index 2bb85cb5b27..403be5f63e1 100644 --- a/src/pkg/securityhub/model/model.go +++ b/src/pkg/securityhub/model/model.go @@ -42,3 +42,13 @@ type DangerousArtifact struct { MediumCnt int64 `json:"medium_cnt" orm:"column(medium_cnt)"` LowCnt int64 `json:"low_cnt" orm:"column(low_cnt)"` } + +// VulnerabilityItem is the item of vulnerability +type VulnerabilityItem struct { + scan.VulnerabilityRecord + ArtifactID int64 `orm:"column(artifact_id)"` + RepositoryName string `orm:"column(repository_name)"` + Digest string `orm:"column(digest)"` + Tags []string `orm:"-"` + ProjectID int64 `orm:"column(project_id)"` +} diff --git a/src/server/v2.0/handler/security.go b/src/server/v2.0/handler/security.go index 5865874518c..27aba998fcf 100644 --- a/src/server/v2.0/handler/security.go +++ b/src/server/v2.0/handler/security.go @@ -16,10 +16,12 @@ package handler import ( "context" + "strings" "github.com/go-openapi/runtime/middleware" "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/pkg/scan/scanner" "github.com/goharbor/harbor/src/server/v2.0/models" securityModel "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/securityhub" @@ -96,3 +98,52 @@ func toDangerousCves(cves []*scan.VulnerabilityRecord) []*models.DangerousCVE { } return result } + +func (s *securityAPI) ListVulnerabilities(ctx context.Context, params securityModel.ListVulnerabilitiesParams) middleware.Responder { + if err := s.RequireSystemAccess(ctx, rbac.ActionList, rbac.ResourceSecurityHub); err != nil { + return s.SendError(ctx, err) + } + query, err := s.BuildQuery(ctx, params.Q, nil, params.Page, params.PageSize) + if err != nil { + return s.SendError(ctx, err) + } + scannerUUID, err := scanner.Mgr.DefaultScannerUUID(ctx) + if err != nil { + return s.SendError(ctx, err) + } + cnt, err := s.controller.CountVuls(ctx, scannerUUID, 0, *params.TuneCount, query) + if err != nil { + return s.SendError(ctx, err) + } + vuls, err := s.controller.ListVuls(ctx, scannerUUID, 0, *params.WithTag, query) + if err != nil { + return s.SendError(ctx, err) + } + link := s.Links(ctx, params.HTTPRequest.URL, cnt, query.PageNumber, query.PageSize).String() + return securityModel.NewListVulnerabilitiesOK().WithPayload(toVulnerabilities(vuls)).WithLink(link).WithXTotalCount(cnt) +} + +func toVulnerabilities(vuls []*secHubModel.VulnerabilityItem) []*models.VulnerabilityItem { + result := make([]*models.VulnerabilityItem, 0) + for _, item := range vuls { + score := float32(0) + if item.CVE3Score != nil { + score = float32(*item.CVE3Score) + } + result = append(result, &models.VulnerabilityItem{ + ProjectID: item.ProjectID, + RepositoryName: item.RepositoryName, + Digest: item.Digest, + CVEID: item.CVEID, + Severity: item.Severity, + Package: item.Package, + Tags: item.Tags, + Version: item.PackageVersion, + FixedVersion: item.Fix, + Desc: item.Description, + CvssV3Score: score, + Links: strings.Split(item.URLs, "|"), + }) + } + return result +} diff --git a/src/testing/controller/securityhub/controller.go b/src/testing/controller/securityhub/controller.go index 6c2f5cd8390..f324e69e20e 100644 --- a/src/testing/controller/securityhub/controller.go +++ b/src/testing/controller/securityhub/controller.go @@ -8,6 +8,8 @@ import ( model "github.com/goharbor/harbor/src/pkg/securityhub/model" mock "github.com/stretchr/testify/mock" + q "github.com/goharbor/harbor/src/lib/q" + securityhub "github.com/goharbor/harbor/src/controller/securityhub" ) @@ -16,6 +18,56 @@ type Controller struct { mock.Mock } +// CountVuls provides a mock function with given fields: ctx, scannerUUID, projectID, tuneCount, query +func (_m *Controller) CountVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) { + ret := _m.Called(ctx, scannerUUID, projectID, tuneCount, query) + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) (int64, error)); ok { + return rf(ctx, scannerUUID, projectID, tuneCount, query) + } + if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) int64); ok { + r0 = rf(ctx, scannerUUID, projectID, tuneCount, query) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, int64, bool, *q.Query) error); ok { + r1 = rf(ctx, scannerUUID, projectID, tuneCount, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListVuls provides a mock function with given fields: ctx, scannerUUID, projectID, withTag, query +func (_m *Controller) ListVuls(ctx context.Context, scannerUUID string, projectID int64, withTag bool, query *q.Query) ([]*model.VulnerabilityItem, error) { + ret := _m.Called(ctx, scannerUUID, projectID, withTag, query) + + var r0 []*model.VulnerabilityItem + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) ([]*model.VulnerabilityItem, error)); ok { + return rf(ctx, scannerUUID, projectID, withTag, query) + } + if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) []*model.VulnerabilityItem); ok { + r0 = rf(ctx, scannerUUID, projectID, withTag, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.VulnerabilityItem) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, int64, bool, *q.Query) error); ok { + r1 = rf(ctx, scannerUUID, projectID, withTag, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // SecuritySummary provides a mock function with given fields: ctx, projectID, options func (_m *Controller) SecuritySummary(ctx context.Context, projectID int64, options ...securityhub.Option) (*model.Summary, error) { _va := make([]interface{}, len(options)) diff --git a/src/testing/pkg/scan/scanner/manager.go b/src/testing/pkg/scan/scanner/manager.go index 56f7584101f..8920ecee516 100644 --- a/src/testing/pkg/scan/scanner/manager.go +++ b/src/testing/pkg/scan/scanner/manager.go @@ -64,6 +64,30 @@ func (_m *Manager) Create(ctx context.Context, registration *daoscanner.Registra return r0, r1 } +// DefaultScannerUUID provides a mock function with given fields: ctx +func (_m *Manager) DefaultScannerUUID(ctx context.Context) (string, error) { + ret := _m.Called(ctx) + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (string, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) string); ok { + r0 = rf(ctx) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Delete provides a mock function with given fields: ctx, registrationUUID func (_m *Manager) Delete(ctx context.Context, registrationUUID string) error { ret := _m.Called(ctx, registrationUUID) diff --git a/src/testing/pkg/securityhub/manager.go b/src/testing/pkg/securityhub/manager.go index cb06f31dfdb..027f95e0d1d 100644 --- a/src/testing/pkg/securityhub/manager.go +++ b/src/testing/pkg/securityhub/manager.go @@ -70,6 +70,32 @@ func (_m *Manager) DangerousCVEs(ctx context.Context, scannerUUID string, projec return r0, r1 } +// ListVuls provides a mock function with given fields: ctx, scannerUUID, projectID, query +func (_m *Manager) ListVuls(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*model.VulnerabilityItem, error) { + ret := _m.Called(ctx, scannerUUID, projectID, query) + + var r0 []*model.VulnerabilityItem + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, int64, *q.Query) ([]*model.VulnerabilityItem, error)); ok { + return rf(ctx, scannerUUID, projectID, query) + } + if rf, ok := ret.Get(0).(func(context.Context, string, int64, *q.Query) []*model.VulnerabilityItem); ok { + r0 = rf(ctx, scannerUUID, projectID, query) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.VulnerabilityItem) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, int64, *q.Query) error); ok { + r1 = rf(ctx, scannerUUID, projectID, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ScannedArtifactsCount provides a mock function with given fields: ctx, scannerUUID, projectID, query func (_m *Manager) ScannedArtifactsCount(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) (int64, error) { ret := _m.Called(ctx, scannerUUID, projectID, query) @@ -120,6 +146,30 @@ func (_m *Manager) Summary(ctx context.Context, scannerUUID string, projectID in return r0, r1 } +// TotalVuls provides a mock function with given fields: ctx, scannerUUID, projectID, tuneCount, query +func (_m *Manager) TotalVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) { + ret := _m.Called(ctx, scannerUUID, projectID, tuneCount, query) + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) (int64, error)); ok { + return rf(ctx, scannerUUID, projectID, tuneCount, query) + } + if rf, ok := ret.Get(0).(func(context.Context, string, int64, bool, *q.Query) int64); ok { + r0 = rf(ctx, scannerUUID, projectID, tuneCount, query) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, int64, bool, *q.Query) error); ok { + r1 = rf(ctx, scannerUUID, projectID, tuneCount, query) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + type mockConstructorTestingTNewManager interface { mock.TestingT Cleanup(func())