Support Debricked Config (#218)

* Add config parsing and send as json

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/vifraa/gopom v0.2.1
 	lukechampine.com/blake3 v1.2.1
+	sigs.k8s.io/yaml v1.4.0
 )
 
 require (

go.sum

@@ -131,6 +131,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -618,3 +619,5 @@ lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

internal/cmd/scan/scan.go

@@ -79,7 +79,6 @@ If the given path contains a git repository all flags but "integration" will be
 		`name of integration used to trigger scan. For example "GitHub Actions"`,
 	)
 	cmd.Flags().StringVarP(&jsonFilePath, JsonFilePathFlag, "j", "", "write upload result as json to provided path")
-
 	fileExclusionExample := filepath.Join("*", "**.lock")
 	dirExclusionExample := filepath.Join("**", "node_modules", "**")
 	exampleFlags := fmt.Sprintf("-e \"%s\" -e \"%s\"", fileExclusionExample, dirExclusionExample)

internal/file/finder.go

@@ -25,6 +25,7 @@ const SupportedFormatsUri = "/api/1.0/open/files/supported-formats"
 type IFinder interface {
 	GetGroups(rootPath string, exclusions []string, lockfileOnly bool, strictness int) (Groups, error)
 	GetSupportedFormats() ([]*CompiledFormat, error)
+	GetConfigPath(rootPath string, exclusions []string) string
 }
 
 type Finder struct {
@@ -40,6 +41,30 @@ func NewFinder(c client.IDebClient, fs ioFs.IFileSystem) (*Finder, error) {
 	return &Finder{c, fs}, nil
 }
 
+func (finder *Finder) GetConfigPath(rootPath string, exclusions []string) string {
+	var configPath string
+	err := filepath.Walk(
+		rootPath,
+		func(path string, fileInfo os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if !fileInfo.IsDir() && !Excluded(exclusions, path) {
+				if filepath.Base(path) == "debricked-config.yaml" {
+					configPath = path
+				}
+			}
+
+			return nil
+		},
+	)
+	if err != nil {
+		return ""
+	}
+
+	return configPath
+}
+
 // GetGroups return all file groups in specified path recursively.
 func (finder *Finder) GetGroups(rootPath string, exclusions []string, lockfileOnly bool, strictness int) (Groups, error) {
 	var groups Groups

internal/file/finder_test.go

@@ -214,6 +214,12 @@ func TestGetGroupsWithTwoFileMatchesInSameDir(t *testing.T) {
 	assert.Contains(t, files, "requirements-dev.txt")
 }
 
+func TestGetDebrickedConfig(t *testing.T) {
+	path := "testdata"
+	configPath := finder.GetConfigPath(path, nil)
+	assert.Equal(t, filepath.Join("testdata", "misc", "debricked-config.yaml"), configPath)
+}
+
 func TestGetGroupsWithStrictFlag(t *testing.T) {
 	setUp(true)
 	cases := []struct {

internal/file/testdata/finder_mock.go

@@ -23,6 +23,10 @@ func (f *FinderMock) GetGroups(_ string, _ []string, _ bool, _ int) (file.Groups
 	return f.groups, f.error
 }
 
+func (f *FinderMock) GetConfigPath(_ string, _ []string) string {
+	return ""
+}
+
 func (f *FinderMock) GetSupportedFormats() ([]*file.CompiledFormat, error) {
 	return f.compiledFormats, f.error
 }

internal/scan/scanner.go

@@ -208,6 +208,7 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g
 		IntegrationsName:       options.IntegrationName,
 		CallGraphUploadTimeout: options.CallGraphUploadTimeout,
 		VersionHint:            options.VersionHint,
+		DebrickedConfig:        dScanner.getDebrickedConfig(options.Path, options.Exclusions),
 	}
 	result, err := (*dScanner.uploader).Upload(uploaderOptions)
 	if err != nil {
@@ -217,6 +218,15 @@ func (dScanner *DebrickedScanner) scan(options DebrickedOptions, gitMetaObject g
 	return result, nil
 }
 
+func (dScanner *DebrickedScanner) getDebrickedConfig(path string, exclusions []string) upload.DebrickedConfig {
+	configPath := dScanner.finder.GetConfigPath(path, exclusions)
+	if configPath == "" {
+		return upload.DebrickedConfig{}
+	}
+
+	return upload.GetDebrickedConfig(configPath)
+}
+
 func (dScanner *DebrickedScanner) handleScanError(err error, passOnTimeOut bool) error {
 	if err == client.NoResErr && passOnTimeOut {
 		fmt.Println(err)

internal/upload/batch.go

@@ -20,6 +20,7 @@ import (
 	"github.com/debricked/cli/internal/git"
 	"github.com/debricked/cli/internal/tui"
 	"github.com/fatih/color"
+	"gopkg.in/yaml.v3"
 )
 
 var (
@@ -39,10 +40,23 @@ type uploadBatch struct {
 	ciUploadId       int
 	callGraphTimeout int
 	versionHint      bool
+	debrickedConfig  DebrickedConfig // JSON Config
 }
 
-func newUploadBatch(client *client.IDebClient, fileGroups file.Groups, gitMetaObject *git.MetaObject, integrationName string, callGraphTimeout int, versionHint bool) *uploadBatch {
-	return &uploadBatch{client: client, fileGroups: fileGroups, gitMetaObject: gitMetaObject, integrationName: integrationName, ciUploadId: 0, callGraphTimeout: callGraphTimeout, versionHint: versionHint}
+func newUploadBatch(
+	client *client.IDebClient, fileGroups file.Groups, gitMetaObject *git.MetaObject,
+	integrationName string, callGraphTimeout int, versionHint bool, debrickedConfig DebrickedConfig,
+) *uploadBatch {
+	return &uploadBatch{
+		client:           client,
+		fileGroups:       fileGroups,
+		gitMetaObject:    gitMetaObject,
+		integrationName:  integrationName,
+		ciUploadId:       0,
+		callGraphTimeout: callGraphTimeout,
+		versionHint:      versionHint,
+		debrickedConfig:  debrickedConfig,
+	}
 }
 
 // upload concurrently posts all file groups to Debricked
@@ -162,12 +176,14 @@ func (uploadBatch *uploadBatch) initAnalysis() error {
 		CommitName:           uploadBatch.gitMetaObject.CommitName,
 		Author:               uploadBatch.gitMetaObject.Author,
 		VersionHint:          uploadBatch.versionHint,
+		DebrickedConfig:      uploadBatch.debrickedConfig,
 		DebrickedIntegration: "cli",
 	})
 
 	if err != nil {
 		return err
 	}
+
 	response, err := (*uploadBatch.client).Post(
 		"/api/1.0/open/finishes/dependencies/files/uploads",
 		"application/json",
@@ -269,14 +285,38 @@ type uploadedFile struct {
 	EstimateDaysLeft     int    `json:"estimateDaysLeft"`
 }
 
+type boolOrString struct {
+	Version    string `json:"version"`
+	HasVersion bool   `json:"hasVersion"`
+}
+
+func (boolOrString *boolOrString) MarshalJSON() ([]byte, error) {
+	if !boolOrString.HasVersion {
+		return json.Marshal(&boolOrString.HasVersion)
+	}
+
+	return json.Marshal(&boolOrString.Version)
+}
+
+type purlConfig struct {
+	PackageURL string       `json:"pURL" yaml:"pURL"`
+	Version    boolOrString `json:"version" yaml:"version"` // Either false or version string
+	FileRegex  string       `json:"fileRegex" yaml:"fileRegex"`
+}
+
+type DebrickedConfig struct {
+	Overrides []purlConfig `json:"overrides" yaml:"overrides"`
+}
+
 type uploadFinish struct {
-	CiUploadId           string `json:"ciUploadId"`
-	RepositoryName       string `json:"repositoryName"`
-	IntegrationName      string `json:"integrationName"`
-	CommitName           string `json:"commitName"`
-	Author               string `json:"author"`
-	DebrickedIntegration string `json:"debrickedIntegration"`
-	VersionHint          bool   `json:"versionHint"`
+	CiUploadId           string          `json:"ciUploadId"`
+	RepositoryName       string          `json:"repositoryName"`
+	IntegrationName      string          `json:"integrationName"`
+	CommitName           string          `json:"commitName"`
+	Author               string          `json:"author"`
+	DebrickedIntegration string          `json:"debrickedIntegration"`
+	VersionHint          bool            `json:"versionHint"`
+	DebrickedConfig      DebrickedConfig `json:"debrickedConfig"`
 }
 
 func getRelativeFilePath(filePath string) string {
@@ -291,3 +331,42 @@ func getRelativeFilePath(filePath string) string {
 func printSuccessfulUpload(f string) {
 	fmt.Printf("Successfully uploaded: %s\n", color.YellowString(f))
 }
+
+type DebrickedConfigYAML struct {
+	Overrides []map[string]string `yaml:"overrides"`
+}
+
+func GetDebrickedConfig(path string) DebrickedConfig {
+	var overrides []purlConfig
+	var yamlConfig DebrickedConfigYAML
+	yamlFile, err := os.ReadFile(path)
+	if err != nil {
+		fmt.Printf("Failed to read debricked config file on path \"%s\"", path)
+
+		return DebrickedConfig{Overrides: nil}
+	}
+	err = yaml.Unmarshal(yamlFile, &yamlConfig)
+	if err != nil {
+		fmt.Printf("Failed to unmarshal debricked config, is the format correct? \"%s\"", yamlFile)
+
+		return DebrickedConfig{Overrides: nil}
+	}
+	for _, entry := range yamlConfig.Overrides {
+		var version string
+		var exist bool
+		pURL := entry["pURL"]
+		fileRegex := entry["fileRegex"]
+		if _, exist = entry["version"]; exist {
+			version = entry["version"]
+			exist = true
+		} else {
+			version = ""
+			exist = false
+		}
+		overrides = append(overrides, purlConfig{PackageURL: pURL, Version: boolOrString{Version: version, HasVersion: exist}, FileRegex: fileRegex})
+	}
+
+	return DebrickedConfig{
+		Overrides: overrides,
+	}
+}

internal/upload/batch_test.go

@@ -2,11 +2,13 @@ package upload
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"io"
 	"log"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -36,7 +38,7 @@ func TestUploadWithBadFiles(t *testing.T) {
 	clientMock.AddMockResponse(mockRes)
 	clientMock.AddMockResponse(mockRes)
 	c = clientMock
-	batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true)
+	batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true, DebrickedConfig{})
 	var buf bytes.Buffer
 	log.SetOutput(&buf)
 	err = batch.upload()
@@ -48,7 +50,7 @@ func TestUploadWithBadFiles(t *testing.T) {
 }
 
 func TestInitAnalysisWithoutAnyFiles(t *testing.T) {
-	batch := newUploadBatch(nil, file.Groups{}, nil, "CLI", 10*60, true)
+	batch := newUploadBatch(nil, file.Groups{}, nil, "CLI", 10*60, true, DebrickedConfig{})
 	err := batch.initAnalysis()
 
 	assert.ErrorContains(t, err, "failed to find dependency files")
@@ -71,7 +73,7 @@ func TestWaitWithPollingTerminatedError(t *testing.T) {
 	}
 	clientMock.AddMockResponse(mockRes)
 	c = clientMock
-	batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true)
+	batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true, DebrickedConfig{})
 
 	uploadResult, err := batch.wait()
 
@@ -96,7 +98,7 @@ func TestInitUploadBadFile(t *testing.T) {
 	clientMock.AddMockResponse(mockRes)
 
 	var c client.IDebClient = clientMock
-	batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true)
+	batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true, DebrickedConfig{})
 
 	files, err := batch.initUpload()
 
@@ -123,11 +125,73 @@ func TestInitUpload(t *testing.T) {
 	clientMock.AddMockResponse(mockRes)
 
 	var c client.IDebClient = clientMock
-	batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true)
+	batch := newUploadBatch(&c, groups, metaObj, "CLI", 10*60, true, DebrickedConfig{})
 
 	files, err := batch.initUpload()
 
 	assert.Len(t, files, 1, "failed to assert that the init deleted one file from the files to be uploaded")
 	assert.NoError(t, err)
 	assert.Equal(t, 1, batch.ciUploadId)
 }
+
+func TestGetDebrickedConfig(t *testing.T) {
+	config := GetDebrickedConfig(filepath.Join("testdata", "debricked-config.yaml"))
+	configJSON, err := json.Marshal(config)
+	assert.Nil(t, err)
+	expectedJSON, err := json.Marshal(DebrickedConfig{
+		Overrides: []purlConfig{
+			{
+				PackageURL: "pkg:npm/lodash",
+				Version:    boolOrString{Version: "1.0.0", HasVersion: true},
+				FileRegex:  ".*/lodash/.*",
+			},
+			{
+				PackageURL: "pkg:maven/org.openjfx/javafx-base",
+				Version:    boolOrString{Version: "", HasVersion: false},
+				FileRegex:  "subpath/org.openjfx/.*",
+			},
+		},
+	})
+	assert.Nil(t, err)
+	assert.JSONEq(t, string(configJSON), string(expectedJSON))
+}
+
+func TestGetDebrickedConfigUnmarshalError(t *testing.T) {
+	config := GetDebrickedConfig(filepath.Join("testdata", "debricked-config-error.yaml"))
+	configJSON, err := json.Marshal(config)
+	assert.Nil(t, err)
+	expectedJSON, err := json.Marshal(DebrickedConfig{
+		Overrides: nil})
+	assert.Nil(t, err)
+	assert.JSONEq(t, string(configJSON), string(expectedJSON))
+}
+
+func TestGetDebrickedConfigFailure(t *testing.T) {
+	config := GetDebrickedConfig("")
+	configJSON, err := json.Marshal(config)
+	assert.Nil(t, err)
+	expectedJSON, err := json.Marshal(DebrickedConfig{
+		Overrides: nil})
+	assert.Nil(t, err)
+	assert.JSONEq(t, string(configJSON), string(expectedJSON))
+}
+
+func TestMarshalJSONDebrickedConfig(t *testing.T) {
+	config, err := json.Marshal(DebrickedConfig{
+		Overrides: []purlConfig{
+			{
+				PackageURL: "pkg:npm/lodash",
+				Version:    boolOrString{Version: "1.0.0", HasVersion: true},
+				FileRegex:  ".*/lodash/.*",
+			},
+			{
+				PackageURL: "pkg:maven/org.openjfx/javafx-base",
+				Version:    boolOrString{Version: "", HasVersion: false},
+				FileRegex:  "subpath/org.openjfx/.*",
+			},
+		},
+	})
+	expectedJSON := "{\"overrides\":[{\"pURL\":\"pkg:npm/lodash\",\"version\":\"1.0.0\",\"fileRegex\":\".*/lodash/.*\"},{\"pURL\":\"pkg:maven/org.openjfx/javafx-base\",\"version\":false,\"fileRegex\":\"subpath/org.openjfx/.*\"}]}"
+	assert.Nil(t, err)
+	assert.Equal(t, []byte(expectedJSON), config)
+}

internal/upload/testdata/debricked-config-error.yaml

@@ -0,0 +1,6 @@
+overrides:
+  - pURL: "pkg:npm/lodash"
+    version: "1.0.0" # optional: if left out, we will decide version
+    fileRegex: ".*/lodash/.*" # PCRE2 
+  - pURL: "pkg:maven/org.openjfx/javafx-base" % ERROR
+¤¤

internal/upload/testdata/debricked-config.yaml

@@ -0,0 +1,6 @@
+overrides:
+  - pURL: "pkg:npm/lodash"
+    version: "1.0.0" # optional: if left out, we will decide version
+    fileRegex: ".*/lodash/.*" # PCRE2 
+  - pURL: "pkg:maven/org.openjfx/javafx-base"
+    fileRegex: "subpath/org.openjfx/.*"