Add vulnerability search API (#18924)

use q.Query to pass all query conditions

Signed-off-by: stonezdj <[email protected]>

api/v2.0/swagger.yaml

@@ -6063,13 +6063,13 @@ paths:
         - $ref: '#/parameters/requestId'
         - name: with_dangerous_cve
           in: query
-          description: Specify whether the dangerous CVE is include in the security summary
+          description: Specify whether the dangerous CVEs are included inside summary information
           type: boolean
           required: false
           default: false
         - name: with_dangerous_artifact
           in: query
-          description: Specify whether the dangerous artifacts is include in the security summary
+          description: Specify whether the dangerous Artifact are included inside summary information
           type: boolean
           required: false
           default: false                  
@@ -6086,6 +6086,61 @@ paths:
           $ref: '#/responses/404'
         '500':
           $ref: '#/responses/500'
+
+  /security/vul:
+    get:
+      summary: Get the vulnerability list.
+      description: |
+        Get the vulnerability list. use q to pass the query condition, 
+        supported conditions:
+        cve_id(exact match)
+        cvss_score_v3(range condition)
+        severity(exact match)
+        repository_name(exact match) 
+        project_id(exact match) 
+        package(exact match)
+        and tag(exact match) 
+      tags:
+        - securityhub
+      operationId: ListVulnerabilities
+      parameters:
+        - $ref: '#/parameters/requestId'
+        - $ref: '#/parameters/query'
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/pageSize'
+        - name: tune_count
+          in: query
+          description: Enable to ignore X-Total-Count when the total count > 1000, if the total count is less than 1000, the real total count is returned, else -1.
+          type: boolean
+          required: false
+          default: false
+        - name: with_tag
+          in: query
+          description: Specify whether the tag information is included inside vulnerability information
+          type: boolean
+          required: false
+          default: false
+      responses:
+        '200':
+          description: The vulnerability list.
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/VulnerabilityItem'
+          headers:
+            X-Total-Count:
+              description: The total count of vulnerabilities
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
+        '400':
+          $ref: '#/responses/400'
+        '401':
+          $ref: '#/responses/401'
+        '500':
+          $ref: '#/responses/500'          
+
 parameters:
   query:
     name: q
@@ -9760,3 +9815,50 @@ definitions:
         type: integer
         x-omitempty: false
         description: the count of medium vulnerabilities
+
+  VulnerabilityItem:
+    type: object
+    description: the vulnerability item info
+    properties:
+      project_id:
+        type: integer
+        format: int64
+        description: the project ID of the artifact
+      repository_name:
+        type: string
+        description: the repository name of the artifact
+      digest:
+        type: string
+        description: the digest of the artifact
+      tags:
+        type: array
+        items:
+          type: string
+        description: the tags of the artifact
+      cve_id:
+        type: string
+        description: the CVE id of the vulnerability.
+      severity:
+        type: string
+        description: the severity of the vulnerability
+      cvss_v3_score:
+        type: number
+        format: float
+        description: the nvd cvss v3 score of the vulnerability
+      package:
+        type: string
+        description: the package of the vulnerability
+      version:
+        type: string
+        description: the version of the package
+      fixed_version:
+        type: string
+        description: the fixed version of the package
+      desc:
+        type: string
+        description: The description of the vulnerability
+      links:
+        type: array
+        items: 
+          type: string
+        description: Links of the vulnerability

make/migrations/postgresql/0120_2.9.0_schema.up.sql

@@ -11,6 +11,8 @@ UPDATE vulnerability_record
 SET cvss_score_v3 = (vendor_attributes->'CVSS'->'nvd'->>'V3Score')::double precision
 WHERE jsonb_path_exists(vendor_attributes::jsonb, '$.CVSS.nvd.V3Score');
 
+CREATE INDEX IF NOT EXISTS idx_vulnerability_record_cvss_score_v3 ON vulnerability_record (cvss_score_v3);
+
 /* add summary information in scan_report */
 ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS critical_cnt BIGINT;
 ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS high_cnt BIGINT;

src/common/rbac/system/policies.go

@@ -86,5 +86,6 @@ var (
 		{Resource: rbac.ResourceJobServiceMonitor, Action: rbac.ActionStop},
 
 		{Resource: rbac.ResourceSecurityHub, Action: rbac.ActionRead},
+		{Resource: rbac.ResourceSecurityHub, Action: rbac.ActionList},
 	}
 )

src/controller/securityhub/controller.go

@@ -23,6 +23,7 @@ import (
 	"github.com/goharbor/harbor/src/pkg/scan/scanner"
 	"github.com/goharbor/harbor/src/pkg/securityhub"
 	secHubModel "github.com/goharbor/harbor/src/pkg/securityhub/model"
+	"github.com/goharbor/harbor/src/pkg/tag"
 )
 
 // Ctl is the global controller for security hub
@@ -63,26 +64,32 @@ func WithArtifact(enable bool) Option {
 type Controller interface {
 	// SecuritySummary returns the security summary of the specified project.
 	SecuritySummary(ctx context.Context, projectID int64, options ...Option) (*secHubModel.Summary, error)
+	// ListVuls list vulnerabilities by query
+	ListVuls(ctx context.Context, scannerUUID string, projectID int64, withTag bool, query *q.Query) ([]*secHubModel.VulnerabilityItem, error)
+	// CountVuls get all vulnerability count by query
+	CountVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error)
 }
 
 type controller struct {
 	artifactMgr artifact.Manager
 	scannerMgr  scanner.Manager
 	secHubMgr   securityhub.Manager
+	tagMgr      tag.Manager
 }
 
 // NewController ...
 func NewController() Controller {
 	return &controller{
 		artifactMgr: pkg.ArtifactMgr,
-		scannerMgr:  scanner.New(),
+		scannerMgr:  scanner.Mgr,
 		secHubMgr:   securityhub.Mgr,
+		tagMgr:      tag.Mgr,
 	}
 }
 
 func (c *controller) SecuritySummary(ctx context.Context, projectID int64, options ...Option) (*secHubModel.Summary, error) {
 	opts := newOptions(options...)
-	scannerUUID, err := c.defaultScannerUUID(ctx)
+	scannerUUID, err := c.scannerMgr.DefaultScannerUUID(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -114,7 +121,7 @@ func (c *controller) SecuritySummary(ctx context.Context, projectID int64, optio
 }
 
 func (c *controller) scannedArtifactCount(ctx context.Context, projectID int64) (int64, error) {
-	scannerUUID, err := c.defaultScannerUUID(ctx)
+	scannerUUID, err := c.scannerMgr.DefaultScannerUUID(ctx)
 	if err != nil {
 		return 0, err
 	}
@@ -128,11 +135,49 @@ func (c *controller) totalArtifactCount(ctx context.Context, projectID int64) (i
 	return c.artifactMgr.Count(ctx, q.New(q.KeyWords{"project_id": projectID}))
 }
 
-// defaultScannerUUID returns the default scanner uuid.
-func (c *controller) defaultScannerUUID(ctx context.Context) (string, error) {
-	reg, err := c.scannerMgr.GetDefault(ctx)
+func (c *controller) ListVuls(ctx context.Context, scannerUUID string, projectID int64, withTag bool, query *q.Query) ([]*secHubModel.VulnerabilityItem, error) {
+	vuls, err := c.secHubMgr.ListVuls(ctx, scannerUUID, projectID, query)
 	if err != nil {
-		return "", err
+		return nil, err
+	}
+	if withTag {
+		return c.attachTags(ctx, vuls)
+	}
+	return vuls, nil
+}
+
+func (c *controller) attachTags(ctx context.Context, vuls []*secHubModel.VulnerabilityItem) ([]*secHubModel.VulnerabilityItem, error) {
+	// get all artifact_ids
+	artifactTagMap := make(map[int64][]string, 0)
+	for _, v := range vuls {
+		artifactTagMap[v.ArtifactID] = make([]string, 0)
+	}
+
+	// get tags in the artifact list
+	var artifactIds []interface{}
+	for k := range artifactTagMap {
+		artifactIds = append(artifactIds, k)
 	}
-	return reg.UUID, nil
+	query := q.New(q.KeyWords{"artifact_id": q.NewOrList(artifactIds)})
+	tags, err := c.tagMgr.List(ctx, query)
+	if err != nil {
+		return vuls, err
+	}
+	for _, tag := range tags {
+		artifactTagMap[tag.ArtifactID] = append(artifactTagMap[tag.ArtifactID], tag.Name)
+	}
+
+	// attach tags, only show 10 tags
+	for _, v := range vuls {
+		if len(artifactTagMap[v.ArtifactID]) > 10 {
+			v.Tags = artifactTagMap[v.ArtifactID][:10]
+			continue
+		}
+		v.Tags = artifactTagMap[v.ArtifactID]
+	}
+	return vuls, nil
+}
+
+func (c *controller) CountVuls(ctx context.Context, scannerUUID string, projectID int64, tuneCount bool, query *q.Query) (int64, error) {
+	return c.secHubMgr.TotalVuls(ctx, scannerUUID, projectID, tuneCount, query)
 }

src/controller/securityhub/controller_test.go

@@ -21,13 +21,14 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
-	"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
 	"github.com/goharbor/harbor/src/pkg/securityhub/model"
+	"github.com/goharbor/harbor/src/pkg/tag/model/tag"
 	htesting "github.com/goharbor/harbor/src/testing"
 	"github.com/goharbor/harbor/src/testing/mock"
 	artifactMock "github.com/goharbor/harbor/src/testing/pkg/artifact"
 	scannerMock "github.com/goharbor/harbor/src/testing/pkg/scan/scanner"
 	securityMock "github.com/goharbor/harbor/src/testing/pkg/securityhub"
+	tagMock "github.com/goharbor/harbor/src/testing/pkg/tag"
 )
 
 var sum = &model.Summary{
@@ -45,6 +46,7 @@ type ControllerTestSuite struct {
 	artifactMgr *artifactMock.Manager
 	scannerMgr  *scannerMock.Manager
 	secHubMgr   *securityMock.Manager
+	tagMgr      *tagMock.FakeManager
 }
 
 // TestController is the entry of controller test suite
@@ -57,10 +59,13 @@ func (suite *ControllerTestSuite) SetupTest() {
 	suite.artifactMgr = &artifactMock.Manager{}
 	suite.secHubMgr = &securityMock.Manager{}
 	suite.scannerMgr = &scannerMock.Manager{}
+	suite.tagMgr = &tagMock.FakeManager{}
+
 	suite.c = &controller{
 		artifactMgr: suite.artifactMgr,
 		secHubMgr:   suite.secHubMgr,
 		scannerMgr:  suite.scannerMgr,
+		tagMgr:      suite.tagMgr,
 	}
 }
 
@@ -74,7 +79,7 @@ func (suite *ControllerTestSuite) TestSecuritySummary() {
 	mock.OnAnything(suite.artifactMgr, "Count").Return(int64(1234), nil)
 	mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil)
 	mock.OnAnything(suite.secHubMgr, "Summary").Return(sum, nil).Twice()
-	mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: "ruuid"}, nil)
+	mock.OnAnything(suite.scannerMgr, "DefaultScannerUUID").Return("ruuid", nil)
 	summary, err := suite.c.SecuritySummary(ctx, 0, WithArtifact(false), WithCVE(false))
 	suite.NoError(err)
 	suite.NotNil(summary)
@@ -119,7 +124,7 @@ func (suite *ControllerTestSuite) TestSecuritySummary() {
 // TestSecuritySummaryError tests the security summary with error
 func (suite *ControllerTestSuite) TestSecuritySummaryError() {
 	ctx := suite.Context()
-	mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: "ruuid"}, nil)
+	mock.OnAnything(suite.scannerMgr, "DefaultScannerUUID").Return("ruuid", nil)
 	mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil)
 	mock.OnAnything(suite.secHubMgr, "Summary").Return(nil, errors.New("invalid project")).Once()
 	summary, err := suite.c.SecuritySummary(ctx, 0, WithCVE(false), WithArtifact(false))
@@ -133,25 +138,63 @@ func (suite *ControllerTestSuite) TestSecuritySummaryError() {
 
 }
 
-// TestGetDefaultScanner tests the get default scanner
-func (suite *ControllerTestSuite) TestGetDefaultScanner() {
+func (suite *ControllerTestSuite) TestScannedArtifact() {
 	ctx := suite.Context()
-	mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: ""}, nil).Once()
-	scanner, err := suite.c.defaultScannerUUID(ctx)
+	mock.OnAnything(suite.scannerMgr, "DefaultScannerUUID").Return("ruuid", nil)
+	mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil)
+	scanned, err := suite.c.scannedArtifactCount(ctx, 0)
 	suite.NoError(err)
-	suite.Equal("", scanner)
+	suite.Equal(int64(1000), scanned)
+}
 
-	mock.OnAnything(suite.scannerMgr, "GetDefault").Return(nil, errors.New("failed to get scanner")).Once()
-	scanner, err = suite.c.defaultScannerUUID(ctx)
-	suite.Error(err)
-	suite.Equal("", scanner)
+// TestAttachTags test the attachTags
+func (suite *ControllerTestSuite) TestAttachTags() {
+	ctx := suite.Context()
+	tagList := []*tag.Tag{
+		{ArtifactID: int64(1), Name: "latest"},
+		{ArtifactID: int64(1), Name: "tag1"},
+		{ArtifactID: int64(1), Name: "tag2"},
+		{ArtifactID: int64(1), Name: "tag3"},
+		{ArtifactID: int64(1), Name: "tag4"},
+		{ArtifactID: int64(1), Name: "tag5"},
+		{ArtifactID: int64(1), Name: "tag6"},
+		{ArtifactID: int64(1), Name: "tag7"},
+		{ArtifactID: int64(1), Name: "tag8"},
+		{ArtifactID: int64(1), Name: "tag9"},
+		{ArtifactID: int64(1), Name: "tag10"},
+	}
+	vulItems := []*model.VulnerabilityItem{
+		{ArtifactID: int64(1)},
+	}
+	mock.OnAnything(suite.c.tagMgr, "List").Return(tagList, nil).Once()
+	resultItems, err := suite.c.attachTags(ctx, vulItems)
+	suite.NoError(err)
+	suite.Equal(len(vulItems), len(resultItems))
+	suite.Equal([]string{"latest"}, resultItems[0].Tags[:1])
+	suite.Equal(10, len(resultItems[0].Tags))
 }
 
-func (suite *ControllerTestSuite) TestScannedArtifact() {
+// TestListVuls tests the list vulnerabilities
+func (suite *ControllerTestSuite) TestListVuls() {
 	ctx := suite.Context()
-	mock.OnAnything(suite.scannerMgr, "GetDefault").Return(&scanner.Registration{UUID: "ruuid"}, nil)
-	mock.OnAnything(suite.secHubMgr, "ScannedArtifactsCount").Return(int64(1000), nil)
-	scanned, err := suite.c.scannedArtifactCount(ctx, 0)
+	vulItems := []*model.VulnerabilityItem{
+		{ArtifactID: int64(1)},
+	}
+	tagList := []*tag.Tag{
+		{ArtifactID: int64(1), Name: "latest"},
+	}
+	mock.OnAnything(suite.c.secHubMgr, "ListVuls").Return(vulItems, nil)
+	mock.OnAnything(suite.c.tagMgr, "List").Return(tagList, nil).Once()
+	vulResult, err := suite.c.ListVuls(ctx, "", 0, true, nil)
 	suite.NoError(err)
-	suite.Equal(int64(1000), scanned)
+	suite.Equal(1, len(vulResult))
+	suite.Equal(int64(1), vulResult[0].ArtifactID)
+}
+
+func (suite *ControllerTestSuite) TestCountVuls() {
+	ctx := suite.Context()
+	mock.OnAnything(suite.c.secHubMgr, "TotalVuls").Return(int64(10), nil)
+	count, err := suite.c.CountVuls(ctx, "", 0, true, nil)
+	suite.NoError(err)
+	suite.Equal(int64(10), count)
 }