Skip to content

Commit

Permalink
Support JFrog Apps Config file
Browse files Browse the repository at this point in the history
  • Loading branch information
yahavi committed Sep 11, 2023
1 parent 257da28 commit 75378ec
Show file tree
Hide file tree
Showing 14 changed files with 374 additions and 53 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/jedib0t/go-pretty/v6 v6.4.7
github.com/jfrog/build-info-go v1.9.10
github.com/jfrog/gofrog v1.3.0
github.com/jfrog/jfrog-apps-config v1.0.1
github.com/jfrog/jfrog-client-go v1.31.6
github.com/magiconair/properties v1.8.7
github.com/manifoldco/promptui v0.9.0
Expand Down
4 changes: 3 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,8 @@ github.com/jfrog/build-info-go v1.9.10 h1:uXnDLVxpqxoAMpXcki00QaBB+M2BoGMMpHODPk
github.com/jfrog/build-info-go v1.9.10/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg=
github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk=
github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0=
github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=
github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w=
github.com/jfrog/jfrog-client-go v1.28.1-0.20230910192358-6994626b2069 h1:vk+P6jK4Zv8+F44ZnRxXUPT14BQxjJtNKdpGdemci7A=
github.com/jfrog/jfrog-client-go v1.28.1-0.20230910192358-6994626b2069/go.mod h1:362+oa7uTTYurzBs1L0dmUTlLo7uhpAU/pwM5Zb9clg=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
Expand Down Expand Up @@ -320,7 +322,7 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+
github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
Expand Down
16 changes: 10 additions & 6 deletions xray/commands/audit/jas/applicability/applicabilitymanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package applicability
import (
"path/filepath"

jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go"
"github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas"

"github.com/jfrog/gofrog/datastructures"
Expand Down Expand Up @@ -98,19 +99,22 @@ func isDirectComponents(components []string, directDependencies []string) bool {
return false
}

func (asm *ApplicabilityScanManager) Run(wd string) (err error) {
if len(asm.scanner.WorkingDirs) > 1 {
log.Info("Running applicability scanning in the", wd, "directory...")
func (asm *ApplicabilityScanManager) Run(module jfrogappsconfig.Module) (err error) {
if jas.ShouldSkipScanner(module, utils.Applicability) {
return
}
if len(asm.scanner.JFrogAppsConfig.Modules) > 1 {
log.Info("Running applicability scanning in the", module.SourceRoot, "directory...")
} else {
log.Info("Running applicability scanning...")
}
if err = asm.createConfigFile(wd); err != nil {
if err = asm.createConfigFile(module.SourceRoot); err != nil {
return
}
if err = asm.runAnalyzerManager(); err != nil {
return
}
workingDirResults, err := jas.ReadJasScanRunsFromFile(asm.scanner.ResultsFileName, wd)
workingDirResults, err := jas.ReadJasScanRunsFromFile(asm.scanner.ResultsFileName, module.SourceRoot)
if err != nil {
return
}
Expand Down Expand Up @@ -148,7 +152,7 @@ func (asm *ApplicabilityScanManager) createConfigFile(workingDir string) error {
Type: applicabilityScanType,
GrepDisable: false,
CveWhitelist: asm.directDependenciesCves,
SkippedDirs: jas.SkippedDirs,
SkippedDirs: jas.DefaultExcludePatterns,
},
},
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
package applicability

import (
"os"
"path/filepath"
"testing"

"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas"
"github.com/jfrog/jfrog-client-go/xray/services"
"github.com/stretchr/testify/assert"
"os"
"path/filepath"
"testing"
)

var mockDirectDependencies = []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}
Expand Down Expand Up @@ -36,7 +37,7 @@ func TestNewApplicabilityScanManager_DependencyTreeDoesntExist(t *testing.T) {
// Assert
if assert.NotNil(t, applicabilityManager) {
assert.NotNil(t, applicabilityManager.scanner.ScannerDirCleanupFunc)
assert.Len(t, applicabilityManager.scanner.WorkingDirs, 1)
assert.Len(t, applicabilityManager.scanner.JFrogAppsConfig.Modules, 1)
assert.NotEmpty(t, applicabilityManager.scanner.ConfigFileName)
assert.NotEmpty(t, applicabilityManager.scanner.ResultsFileName)
assert.Empty(t, applicabilityManager.directDependenciesCves)
Expand Down Expand Up @@ -276,7 +277,7 @@ func TestParseResults_EmptyResults_AllCvesShouldGetUnknown(t *testing.T) {

// Act
var err error
applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.WorkingDirs[0])
applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot)

if assert.NoError(t, err) {
assert.Len(t, applicabilityManager.applicabilityScanResults, 1)
Expand All @@ -293,7 +294,7 @@ func TestParseResults_ApplicableCveExist(t *testing.T) {

// Act
var err error
applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.WorkingDirs[0])
applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot)

if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) {
assert.Len(t, applicabilityManager.applicabilityScanResults, 1)
Expand All @@ -310,7 +311,7 @@ func TestParseResults_AllCvesNotApplicable(t *testing.T) {

// Act
var err error
applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.WorkingDirs[0])
applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot)

if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) {
assert.Len(t, applicabilityManager.applicabilityScanResults, 1)
Expand Down
72 changes: 66 additions & 6 deletions xray/commands/audit/jas/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,29 @@ package jas

import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"

jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go"
rtutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-cli-core/v2/xray/utils"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
"github.com/jfrog/jfrog-client-go/utils/log"
"github.com/jfrog/jfrog-client-go/xray/services"
"github.com/owenrumney/go-sarif/v2/sarif"
"github.com/stretchr/testify/assert"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v3"
)

var (
SkippedDirs = []string{"**/*test*/**", "**/*venv*/**", "**/*node_modules*/**", "**/*target*/**"}
DefaultExcludePatterns = []string{"**/*test*/**", "**/*venv*/**", "**/*node_modules*/**", "**/*target*/**"}

mapSeverityToScore = map[string]string{
"": "0.0",
Expand All @@ -37,7 +41,7 @@ type JasScanner struct {
ResultsFileName string
AnalyzerManager utils.AnalyzerManager
ServerDetails *config.ServerDetails
WorkingDirs []string
JFrogAppsConfig *jfrogappsconfig.JFrogAppsConfig
ScannerDirCleanupFunc func() error
}

Expand All @@ -56,21 +60,41 @@ func NewJasScanner(workingDirs []string, serverDetails *config.ServerDetails) (s
scanner.ServerDetails = serverDetails
scanner.ConfigFileName = filepath.Join(tempDir, "config.yaml")
scanner.ResultsFileName = filepath.Join(tempDir, "results.sarif")
scanner.WorkingDirs, err = coreutils.GetFullPathsWorkingDirs(workingDirs)
scanner.JFrogAppsConfig, err = createJFrogAppsConfig(workingDirs)
return
}

func createJFrogAppsConfig(workingDirs []string) (*jfrogappsconfig.JFrogAppsConfig, error) {
if jfrogAppsConfig, err := jfrogappsconfig.LoadConfigIfExist(); err != nil {
return nil, errorutils.CheckError(err)
} else if jfrogAppsConfig != nil {
// jfrog-apps-config.yml exist in the workspace
return jfrogAppsConfig, nil
}

// jfrog-apps-config.yml does not exist in the workspace
fullPathsWorkingDirs, err := coreutils.GetFullPathsWorkingDirs(workingDirs)
if err != nil {
return nil, err
}
jfrogAppsConfig := new(jfrogappsconfig.JFrogAppsConfig)
for _, workingDir := range fullPathsWorkingDirs {
jfrogAppsConfig.Modules = append(jfrogAppsConfig.Modules, jfrogappsconfig.Module{SourceRoot: workingDir})
}
return jfrogAppsConfig, nil
}

type ScannerCmd interface {
Run(wd string) (err error)
Run(module jfrogappsconfig.Module) (err error)
}

func (a *JasScanner) Run(scannerCmd ScannerCmd) (err error) {
for _, workingDir := range a.WorkingDirs {
for _, module := range a.JFrogAppsConfig.Modules {
func() {
defer func() {
err = errors.Join(err, deleteJasProcessFiles(a.ConfigFileName, a.ResultsFileName))
}()
if err = scannerCmd.Run(workingDir); err != nil {
if err = scannerCmd.Run(module); err != nil {
return
}
}()
Expand Down Expand Up @@ -153,6 +177,7 @@ func CreateScannersConfigFile(fileName string, fileContent interface{}) error {
if errorutils.CheckError(err) != nil {
return err
}
log.Debug("Input YAML:\n" + string(yamlData))
err = os.WriteFile(fileName, yamlData, 0644)
return errorutils.CheckError(err)
}
Expand Down Expand Up @@ -191,3 +216,38 @@ func InitJasTest(t *testing.T, workingDirs ...string) (*JasScanner, func()) {
func GetTestDataPath() string {
return filepath.Join("..", "..", "..", "testdata")
}

func ShouldSkipScanner(module jfrogappsconfig.Module, scanType utils.JasScanType) bool {
lowerScanType := strings.ToLower(string(scanType))
if slices.Contains(module.ExcludeScanners, lowerScanType) {
log.Info(fmt.Sprintf("Skipping %s scanning", scanType))
return true
}
return false
}

func GetSourceRoots(module jfrogappsconfig.Module, scanner *jfrogappsconfig.Scanner) ([]string, error) {
root, err := filepath.Abs(module.SourceRoot)
if err != nil {
return []string{}, errorutils.CheckError(err)
}
if scanner == nil || len(scanner.WorkingDirs) == 0 {
return []string{root}, errorutils.CheckError(err)
}
var roots []string
for _, workingDir := range scanner.WorkingDirs {
roots = append(roots, filepath.Join(root, workingDir))
}
return roots, nil
}

func GetExcludePatterns(module jfrogappsconfig.Module, scanner *jfrogappsconfig.Scanner) []string {
excludePatterns := module.ExcludePatterns
if scanner != nil {
excludePatterns = append(excludePatterns, scanner.ExcludePatterns...)
}
if len(excludePatterns) == 0 {
return DefaultExcludePatterns
}
return excludePatterns
}
129 changes: 129 additions & 0 deletions xray/commands/audit/jas/commons_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
package jas

import (
"fmt"
"os"
"path/filepath"
"testing"

jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go"
clientTestUtils "github.com/jfrog/jfrog-client-go/utils/tests"

"github.com/jfrog/jfrog-cli-core/v2/xray/utils"
"github.com/stretchr/testify/assert"
)

var createJFrogAppsConfigCases = []struct {
workingDirs []string
}{
{workingDirs: []string{}},
{workingDirs: []string{"working-dir"}},
{workingDirs: []string{"working-dir-1", "working-dir-2"}},
}

func TestCreateJFrogAppsConfig(t *testing.T) {
wd, err := os.Getwd()
assert.NoError(t, err)

for _, testCase := range createJFrogAppsConfigCases {
t.Run(fmt.Sprintf("%v", testCase.workingDirs), func(t *testing.T) {
jfrogAppsConfig, err := createJFrogAppsConfig(testCase.workingDirs)
assert.NoError(t, err)
assert.NotNil(t, jfrogAppsConfig)
if len(testCase.workingDirs) == 0 {
assert.Len(t, jfrogAppsConfig.Modules, 1)
assert.Equal(t, wd, jfrogAppsConfig.Modules[0].SourceRoot)
return
}
assert.Len(t, jfrogAppsConfig.Modules, len(testCase.workingDirs))
for i, workingDir := range testCase.workingDirs {
assert.Equal(t, filepath.Join(wd, workingDir), jfrogAppsConfig.Modules[i].SourceRoot)
}
})
}
}

func TestCreateJFrogAppsConfigWithConfig(t *testing.T) {
wd, err := os.Getwd()
assert.NoError(t, err)
chdirCallback := clientTestUtils.ChangeDirWithCallback(t, wd, "testdata")
defer chdirCallback()

jfrogAppsConfig, err := createJFrogAppsConfig([]string{})
assert.NoError(t, err)
assert.NotNil(t, jfrogAppsConfig)
assert.Equal(t, "1.0", jfrogAppsConfig.Version)
assert.Len(t, jfrogAppsConfig.Modules, 1)
}

func TestShouldSkipScanner(t *testing.T) {
module := jfrogappsconfig.Module{}
assert.False(t, ShouldSkipScanner(module, utils.IaC))

module = jfrogappsconfig.Module{ExcludeScanners: []string{"sast"}}
assert.False(t, ShouldSkipScanner(module, utils.IaC))
assert.True(t, ShouldSkipScanner(module, utils.Sast))
}

var getSourceRootsCases = []struct {
scanner *jfrogappsconfig.Scanner
}{
{scanner: nil},
{&jfrogappsconfig.Scanner{WorkingDirs: []string{"working-dir"}}},
{&jfrogappsconfig.Scanner{WorkingDirs: []string{"working-dir-1", "working-dir-2"}}},
}

func TestGetSourceRoots(t *testing.T) {
testGetSourceRoots(t, "source-root")
}

func TestGetSourceRootsEmptySourceRoot(t *testing.T) {
testGetSourceRoots(t, "")
}

func testGetSourceRoots(t *testing.T, sourceRoot string) {
sourceRoot, err := filepath.Abs(sourceRoot)
assert.NoError(t, err)
module := jfrogappsconfig.Module{SourceRoot: sourceRoot}
for _, testCase := range getSourceRootsCases {
t.Run("", func(t *testing.T) {
scanner := testCase.scanner
actualSourceRoots, err := GetSourceRoots(module, scanner)
assert.NoError(t, err)
if scanner == nil {
assert.ElementsMatch(t, []string{module.SourceRoot}, actualSourceRoots)
return
}
expectedWorkingDirs := []string{}
for _, workingDir := range scanner.WorkingDirs {
expectedWorkingDirs = append(expectedWorkingDirs, filepath.Join(module.SourceRoot, workingDir))
}
assert.ElementsMatch(t, actualSourceRoots, expectedWorkingDirs)
})
}
}

var getExcludePatternsCases = []struct {
scanner *jfrogappsconfig.Scanner
}{
{scanner: nil},
{&jfrogappsconfig.Scanner{WorkingDirs: []string{"exclude-dir"}}},
{&jfrogappsconfig.Scanner{WorkingDirs: []string{"exclude-dir-1", "exclude-dir-2"}}},
}

func TestGetExcludePatterns(t *testing.T) {
module := jfrogappsconfig.Module{ExcludePatterns: []string{"exclude-root"}}
for _, testCase := range getExcludePatternsCases {
t.Run("", func(t *testing.T) {
scanner := testCase.scanner
actualExcludePatterns := GetExcludePatterns(module, scanner)
if scanner == nil {
assert.ElementsMatch(t, module.ExcludePatterns, actualExcludePatterns)
return
}
expectedExcludePatterns := module.ExcludePatterns
expectedExcludePatterns = append(expectedExcludePatterns, scanner.ExcludePatterns...)
assert.ElementsMatch(t, actualExcludePatterns, expectedExcludePatterns)
})
}
}
Loading

0 comments on commit 75378ec

Please sign in to comment.