Add vulnerability search API

  use q.Query to pass all query conditions

Signed-off-by: stonezdj <[email protected]>

api/v2.0/swagger.yaml

@@ -6063,13 +6063,13 @@ paths:
         - $ref: '#/parameters/requestId'
         - name: with_dangerous_cve
           in: query
-          description: Specify whether the dangerous CVE is include in the security summary
+          description: Specify whether the dangerous CVEs are included inside summary information
           type: boolean
           required: false
           default: false
         - name: with_dangerous_artifact
           in: query
-          description: Specify whether the dangerous artifacts is include in the security summary
+          description: Specify whether the dangerous Artifact are included inside summary information
           type: boolean
           required: false
           default: false                  
@@ -6086,6 +6086,55 @@ paths:
           $ref: '#/responses/404'
         '500':
           $ref: '#/responses/500'
+
+  /security/vul:
+    get:
+      summary: Get the vulnerability list.
+      description: |
+        Get the vulnerability list. use q to pass the query condition, 
+        supported conditions:
+        cve_id(exact match)
+        cvss_score_v3(range condition)
+        severity(exact match)
+        repository(exact match) 
+        project_id(exact match) 
+        package(exact match)
+        and tag(exact match) 
+      tags:
+        - securityhub
+      operationId: ListVulnerabilities
+      parameters:
+        - $ref: '#/parameters/requestId'
+        - $ref: '#/parameters/query'
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/pageSize'
+        - name: tune_count
+          in: query
+          description: Enable to ignore X-Total-Count when the total count > 1000, if the total count is less than 1000, the real total count is returned, else -1.
+          type: boolean
+          required: false
+          default: false
+      responses:
+        '200':
+          description: The vulnerability list.
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/VulnerabilityItem'
+          headers:
+            X-Total-Count:
+              description: The total count of vulnerabilities
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
+        '400':
+          $ref: '#/responses/400'
+        '401':
+          $ref: '#/responses/401'
+        '500':
+          $ref: '#/responses/500'          
+
 parameters:
   query:
     name: q
@@ -9760,3 +9809,50 @@ definitions:
         type: integer
         x-omitempty: false
         description: the count of medium vulnerabilities
+
+  VulnerabilityItem:
+    type: object
+    description: the vulnerability item info
+    properties:
+      project_id:
+        type: integer
+        format: int64
+        description: the project ID of the artifact
+      repository_name:
+        type: string
+        description: the repository name of the artifact
+      digest:
+        type: string
+        description: the digest of the artifact
+      tags:
+        type: array
+        items:
+          type: string
+        description: the tags of the artifact
+      cve_id:
+        type: string
+        description: the CVE id of the vulnerability.
+      severity:
+        type: string
+        description: the severity of the vulnerability
+      cvss_v3_score:
+        type: number
+        format: float
+        description: the nvd cvss v3 score of the vulnerability
+      package:
+        type: string
+        description: the package of the vulnerability
+      version:
+        type: string
+        description: the version of the package
+      fixed_version:
+        type: string
+        description: the fixed version of the package
+      desc:
+        type: string
+        description: The description of the vulnerability
+      links:
+        type: array
+        items: 
+          type: string
+        description: Links of the vulnerability

make/migrations/postgresql/0120_2.9.0_schema.up.sql

@@ -11,6 +11,8 @@ UPDATE vulnerability_record
 SET cvss_score_v3 = (vendor_attributes->'CVSS'->'nvd'->>'V3Score')::double precision
 WHERE jsonb_path_exists(vendor_attributes::jsonb, '$.CVSS.nvd.V3Score');
 
+CREATE INDEX IF NOT EXISTS idx_vulnerability_record_cvss_score_v3 ON vulnerability_record (cvss_score_v3);
+
 /* add summary information in scan_report */
 ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS critical_cnt BIGINT;
 ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS high_cnt BIGINT;

src/controller/securityhub/controller.go

@@ -23,6 +23,7 @@ import (
 	"github.com/goharbor/harbor/src/pkg/scan/scanner"
 	"github.com/goharbor/harbor/src/pkg/securityhub"
 	secHubModel "github.com/goharbor/harbor/src/pkg/securityhub/model"
+	"github.com/goharbor/harbor/src/pkg/tag"
 )
 
 // Ctl is the global controller for security hub
@@ -63,12 +64,17 @@ func WithArtifact(enable bool) Option {
 type Controller interface {
 	// SecuritySummary returns the security summary of the specified project.
 	SecuritySummary(ctx context.Context, projectID int64, options ...Option) (*secHubModel.Summary, error)
+	// ListVuls list vulnerabilities by query
+	ListVuls(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*secHubModel.VulnerabilityItem, error)
+	// CountVuls get all vulnerability count by query
+	CountVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error)
 }
 
 type controller struct {
 	artifactMgr artifact.Manager
 	scannerMgr  scanner.Manager
 	secHubMgr   securityhub.Manager
+	tagMgr      tag.Manager
 }
 
 // NewController ...
@@ -77,12 +83,13 @@ func NewController() Controller {
 		artifactMgr: pkg.ArtifactMgr,
 		scannerMgr:  scanner.New(),
 		secHubMgr:   securityhub.Mgr,
+		tagMgr:      tag.Mgr,
 	}
 }
 
 func (c *controller) SecuritySummary(ctx context.Context, projectID int64, options ...Option) (*secHubModel.Summary, error) {
 	opts := newOptions(options...)
-	scannerUUID, err := c.defaultScannerUUID(ctx)
+	scannerUUID, err := c.scannerMgr.DefaultScannerUUID(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -114,7 +121,7 @@ func (c *controller) SecuritySummary(ctx context.Context, projectID int64, optio
 }
 
 func (c *controller) scannedArtifactCount(ctx context.Context, projectID int64) (int64, error) {
-	scannerUUID, err := c.defaultScannerUUID(ctx)
+	scannerUUID, err := c.scannerMgr.DefaultScannerUUID(ctx)
 	if err != nil {
 		return 0, err
 	}
@@ -128,11 +135,50 @@ func (c *controller) totalArtifactCount(ctx context.Context, projectID int64) (i
 	return c.artifactMgr.Count(ctx, q.New(q.KeyWords{"project_id": projectID}))
 }
 
-// defaultScannerUUID returns the default scanner uuid.
-func (c *controller) defaultScannerUUID(ctx context.Context) (string, error) {
-	reg, err := c.scannerMgr.GetDefault(ctx)
+func (c *controller) ListVuls(ctx context.Context, scannerUUID string, projectID int64, query *q.Query) ([]*secHubModel.VulnerabilityItem, error) {
+	vuls, err := c.secHubMgr.ListVuls(ctx, scannerUUID, projectID, query)
 	if err != nil {
-		return "", err
+		return nil, err
+	}
+	resultList, err := c.attachTags(ctx, vuls)
+	if err != nil {
+		return nil, err
+	}
+	return resultList, nil
+}
+
+func (c *controller) attachTags(ctx context.Context, vuls []*secHubModel.VulnerabilityItem) ([]*secHubModel.VulnerabilityItem, error) {
+	// get all artifact_ids
+	artifactTagMap := make(map[int64][]string, 0)
+	for _, v := range vuls {
+		artifactTagMap[v.ArtifactID] = make([]string, 0)
+	}
+
+	// get tags in the artifact list
+	var artifactIds []interface{}
+	for k := range artifactTagMap {
+		artifactIds = append(artifactIds, k)
 	}
-	return reg.UUID, nil
+	query := q.New(q.KeyWords{"artifact_id": q.NewOrList(artifactIds)})
+	tags, err := c.tagMgr.List(ctx, query)
+	if err != nil {
+		return vuls, err
+	}
+	for _, tag := range tags {
+		artifactTagMap[tag.ArtifactID] = append(artifactTagMap[tag.ArtifactID], tag.Name)
+	}
+
+	// attach tags, only show 10 tags
+	for _, v := range vuls {
+		if len(artifactTagMap[v.ArtifactID]) > 10 {
+			v.Tags = artifactTagMap[v.ArtifactID][:10]
+			continue
+		}
+		v.Tags = artifactTagMap[v.ArtifactID]
+	}
+	return vuls, nil
+}
+
+func (c *controller) CountVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) {
+	return c.secHubMgr.TotalVuls(ctx, scannerUUID, projectID, tuneCount, query)
 }

src/controller/securityhub/controller_test.go

@@ -23,11 +23,13 @@ import (
 	"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
 	"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
 	"github.com/goharbor/harbor/src/pkg/securityhub/model"
+	"github.com/goharbor/harbor/src/pkg/tag/model/tag"
 	htesting "github.com/goharbor/harbor/src/testing"
 	"github.com/goharbor/harbor/src/testing/mock"
 	artifactMock "github.com/goharbor/harbor/src/testing/pkg/artifact"
 	scannerMock "github.com/goharbor/harbor/src/testing/pkg/scan/scanner"
 	securityMock "github.com/goharbor/harbor/src/testing/pkg/securityhub"
+	tagMock "github.com/goharbor/harbor/src/testing/pkg/tag"
 )
 
 var sum = &model.Summary{
@@ -45,6 +47,7 @@ type ControllerTestSuite struct {
 	artifactMgr *artifactMock.Manager
 	scannerMgr  *scannerMock.Manager
 	secHubMgr   *securityMock.Manager
+	tagMgr      *tagMock.FakeManager
 }
 
 // TestController is the entry of controller test suite
@@ -57,10 +60,13 @@ func (suite *ControllerTestSuite) SetupTest() {
 	suite.artifactMgr = &artifactMock.Manager{}
 	suite.secHubMgr = &securityMock.Manager{}
 	suite.scannerMgr = &scannerMock.Manager{}
+	suite.tagMgr = &tagMock.FakeManager{}
+
 	suite.c = &controller{
 		artifactMgr: suite.artifactMgr,
 		secHubMgr:   suite.secHubMgr,
 		scannerMgr:  suite.scannerMgr,
+		tagMgr:      suite.tagMgr,
 	}
 }
 
@@ -133,20 +139,6 @@ func (suite *ControllerTestSuite) TestSecuritySummaryError() {
 
 }
 
-// TestGetDefaultScanner tests the get default scanner
-func (suite *ControllerTestSuite) TestGetDefaultScanner() {
-	ctx := suite.Context()
-	mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: ""}, nil).Once()
-	scanner, err := suite.c.defaultScannerUUID(ctx)
-	suite.NoError(err)
-	suite.Equal("", scanner)
-
-	mock.OnAnything(suite.scannerMgr, "GetDefault").Return(nil, errors.New("failed to get scanner")).Once()
-	scanner, err = suite.c.defaultScannerUUID(ctx)
-	suite.Error(err)
-	suite.Equal("", scanner)
-}
-
 func (suite *ControllerTestSuite) TestScannedArtifact() {
 	ctx := suite.Context()
 	mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: "ruuid"}, nil)
@@ -155,3 +147,55 @@ func (suite *ControllerTestSuite) TestScannedArtifact() {
 	suite.NoError(err)
 	suite.Equal(int64(1000), scanned)
 }
+
+// TestAttachTags test the attachTags
+func (suite *ControllerTestSuite) TestAttachTags() {
+	ctx := suite.Context()
+	tagList := []*tag.Tag{
+		{ArtifactID: int64(1), Name: "latest"},
+		{ArtifactID: int64(1), Name: "tag1"},
+		{ArtifactID: int64(1), Name: "tag2"},
+		{ArtifactID: int64(1), Name: "tag3"},
+		{ArtifactID: int64(1), Name: "tag4"},
+		{ArtifactID: int64(1), Name: "tag5"},
+		{ArtifactID: int64(1), Name: "tag6"},
+		{ArtifactID: int64(1), Name: "tag7"},
+		{ArtifactID: int64(1), Name: "tag8"},
+		{ArtifactID: int64(1), Name: "tag9"},
+		{ArtifactID: int64(1), Name: "tag10"},
+	}
+	vulItems := []*model.VulnerabilityItem{
+		{ArtifactID: int64(1)},
+	}
+	mock.OnAnything(suite.c.tagMgr, "List").Return(tagList, nil).Once()
+	resultItems, err := suite.c.attachTags(ctx, vulItems)
+	suite.NoError(err)
+	suite.Equal(len(vulItems), len(resultItems))
+	suite.Equal([]string{"latest"}, resultItems[0].Tags[:1])
+	suite.Equal(10, len(resultItems[0].Tags))
+}
+
+// TestListVuls tests the list vulnerabilities
+func (suite *ControllerTestSuite) TestListVuls() {
+	ctx := suite.Context()
+	vulItems := []*model.VulnerabilityItem{
+		{ArtifactID: int64(1)},
+	}
+	tagList := []*tag.Tag{
+		{ArtifactID: int64(1), Name: "latest"},
+	}
+	mock.OnAnything(suite.c.secHubMgr, "ListVuls").Return(vulItems, nil)
+	mock.OnAnything(suite.c.tagMgr, "List").Return(tagList, nil).Once()
+	vulResult, err := suite.c.ListVuls(ctx, "", 0, nil)
+	suite.NoError(err)
+	suite.Equal(1, len(vulResult))
+	suite.Equal(int64(1), vulResult[0].ArtifactID)
+}
+
+func (suite *ControllerTestSuite) TestCountVuls() {
+	ctx := suite.Context()
+	mock.OnAnything(suite.c.secHubMgr, "TotalVuls").Return(int64(10), nil)
+	count, err := suite.c.CountVuls(ctx, "", 0, true, nil)
+	suite.NoError(err)
+	suite.Equal(int64(10), count)
+}

src/pkg/scan/scanner/manager.go

@@ -24,6 +24,9 @@ import (
 	"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
 )
 
+// Mgr is the global manager for scanner
+var Mgr = New()
+
 // Manager defines the related scanner API endpoints
 type Manager interface {
 	// Count returns the total count of scanner registrations according to the query.
@@ -52,6 +55,9 @@ type Manager interface {
 
 	// GetDefault returns the default scanner registration or `nil` if there are no registrations configured.
 	GetDefault(ctx context.Context) (*scanner.Registration, error)
+
+	// DefaultScannerUUID get default scanner UUID
+	DefaultScannerUUID(ctx context.Context) (string, error)
 }
 
 // basicManager is the default implementation of Manager
@@ -139,3 +145,12 @@ func (bm *basicManager) SetAsDefault(ctx context.Context, registrationUUID strin
 func (bm *basicManager) GetDefault(ctx context.Context) (*scanner.Registration, error) {
 	return scanner.GetDefaultRegistration(ctx)
 }
+
+// DefaultScannerUUID returns the default scanner uuid.
+func (bm *basicManager) DefaultScannerUUID(ctx context.Context) (string, error) {
+	reg, err := bm.GetDefault(ctx)
+	if err != nil {
+		return "", err
+	}
+	return reg.UUID, nil
+}

src/pkg/scan/scanner/manager_test.go

@@ -111,3 +111,12 @@ func (suite *BasicManagerTestSuite) TestDefault() {
 	require.NotNil(suite.T(), dr)
 	assert.Equal(suite.T(), true, dr.IsDefault)
 }
+
+// TestGetDefaultScanner tests the get default scanner
+func (suite *BasicManagerTestSuite) TestGetDefaultScanner() {
+	ctx := suite.Context()
+	suite.mgr.SetAsDefault(ctx, suite.sampleUUID)
+	scanner, err := suite.mgr.DefaultScannerUUID(ctx)
+	suite.NoError(err)
+	suite.Equal(suite.sampleUUID, scanner)
+}