diff --git a/pkg/analyzer/analyzer.go b/pkg/analyzer/analyzer.go index 73bdf21905c..605c46d690f 100644 --- a/pkg/analyzer/analyzer.go +++ b/pkg/analyzer/analyzer.go @@ -246,15 +246,18 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) { // start metrics for file analyzer metrics.Metric.Start("file_type_analyzer") returnAnalyzedPaths := model.AnalyzedPaths{ - Types: make([]string, 0), - Exc: make([]string, 0), + Types: make([]string, 0), + Exc: make([]string, 0), + ExpectedLOC: 0, } var files []string var wg sync.WaitGroup // results is the channel shared by the workers that contains the types found results := make(chan string) + locCount := make(chan int) ignoreFiles := make([]string, 0) + done := make(chan bool) hasGitIgnoreFile, gitIgnore := shouldConsiderGitIgnoreFile(a.Paths[0], a.GitIgnoreFileName, a.ExcludeGitIgnore) // get all the files inside the given paths @@ -291,6 +294,7 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) { a.Types[i] = strings.ToLower(a.Types[i]) } + // Start the workers for _, file := range files { wg.Add(1) // analyze the files concurrently @@ -298,7 +302,7 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) { typesFlag: a.Types, filePath: file, } - go a.worker(results, unwanted, &wg) + go a.worker(results, unwanted, locCount, &wg) } go func() { @@ -306,16 +310,18 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) { defer func() { close(unwanted) close(results) + close(locCount) }() wg.Wait() + done <- true }() - availableTypes := createSlice(results) + availableTypes, unwantedPaths, loc := computeValues(results, unwanted, locCount, done) multiPlatformTypeCheck(&availableTypes) - unwantedPaths := createSlice(unwanted) unwantedPaths = append(unwantedPaths, ignoreFiles...) returnAnalyzedPaths.Types = availableTypes returnAnalyzedPaths.Exc = unwantedPaths + returnAnalyzedPaths.ExpectedLOC = loc // stop metrics for file analyzer metrics.Metric.Stop() return returnAnalyzedPaths, nil @@ -324,11 +330,11 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) { // worker determines the type of the file by ext (dockerfile and terraform)/content and // writes the answer to the results channel // if no types were found, the worker will write the path of the file in the unwanted channel -func (a *analyzerInfo) worker(results, unwanted chan<- string, wg *sync.WaitGroup) { +func (a *analyzerInfo) worker(results, unwanted chan<- string, locCount chan<- int, wg *sync.WaitGroup) { defer wg.Done() ext := utils.GetExtension(a.filePath) - + linesCount, _ := utils.LineCounter(a.filePath) typesFlag := a.typesFlag switch ext { @@ -336,11 +342,13 @@ func (a *analyzerInfo) worker(results, unwanted chan<- string, wg *sync.WaitGrou case ".dockerfile", "Dockerfile": if typesFlag[0] == "" || utils.Contains(dockerfile, typesFlag) { results <- dockerfile + locCount <- linesCount } // Dockerfile (indirect identification) case "possibleDockerfile", ".ubi8", ".debian": if (typesFlag[0] == "" || utils.Contains(dockerfile, typesFlag)) && isDockerfile(a.filePath) { results <- dockerfile + locCount <- linesCount } else { unwanted <- a.filePath } @@ -348,15 +356,17 @@ func (a *analyzerInfo) worker(results, unwanted chan<- string, wg *sync.WaitGrou case ".tf", "tfvars": if typesFlag[0] == "" || utils.Contains(terraform, typesFlag) { results <- terraform + locCount <- linesCount } // GRPC case ".proto": if typesFlag[0] == "" || utils.Contains(grpc, typesFlag) { results <- grpc + locCount <- linesCount } // Cloud Formation, Ansible, OpenAPI, Buildah case yaml, yml, json, sh: - a.checkContent(results, unwanted, ext) + a.checkContent(results, unwanted, locCount, linesCount, ext) } } @@ -396,7 +406,7 @@ func needsOverride(check bool, returnType, key, ext string) bool { // checkContent will determine the file type by content when worker was unable to // determine by ext, if no type was determined checkContent adds it to unwanted channel -func (a *analyzerInfo) checkContent(results, unwanted chan<- string, ext string) { +func (a *analyzerInfo) checkContent(results, unwanted chan<- string, locCount chan<- int, linesCount int, ext string) { typesFlag := a.typesFlag // get file content content, err := os.ReadFile(a.filePath) @@ -438,6 +448,7 @@ func (a *analyzerInfo) checkContent(results, unwanted chan<- string, ext string) if returnType != "" { if typesFlag[0] == "" || utils.Contains(returnType, typesFlag) { results <- returnType + locCount <- linesCount return } } @@ -495,15 +506,28 @@ func checkYamlPlatform(content []byte, path string) string { return ansible } -// createSlice creates a slice from the channel given removing any duplicates -func createSlice(chanel chan string) []string { - slice := make([]string, 0) - for i := range chanel { - if !utils.Contains(i, slice) { - slice = append(slice, i) +// computeValues computes expected Lines of Code to be scanned from locCount channel +// and creates the types and unwanted slices from the channels removing any duplicates +func computeValues(types, unwanted chan string, locCount chan int, done chan bool) (typesS, unwantedS []string, locTotal int) { + var val int + unwantedSlice := make([]string, 0) + typeSlice := make([]string, 0) + for { + select { + case i := <-locCount: + val += i + case i := <-unwanted: + if !utils.Contains(i, unwantedSlice) { + unwantedSlice = append(unwantedSlice, i) + } + case i := <-types: + if !utils.Contains(i, typeSlice) { + typeSlice = append(typeSlice, i) + } + case <-done: + return typeSlice, unwantedSlice, val } } - return slice } // getKeysFromTypesFlag gets all the regexes keys related to the types flag diff --git a/pkg/analyzer/analyzer_test.go b/pkg/analyzer/analyzer_test.go index 5d6b5f41b7f..9d5be355281 100644 --- a/pkg/analyzer/analyzer_test.go +++ b/pkg/analyzer/analyzer_test.go @@ -14,6 +14,7 @@ func TestAnalyzer_Analyze(t *testing.T) { paths []string wantTypes []string wantExclude []string + wantLOC int wantErr bool gitIgnoreFileName string excludeGitIgnore bool @@ -23,6 +24,7 @@ func TestAnalyzer_Analyze(t *testing.T) { paths: []string{filepath.FromSlash("../../test/fixtures/analyzer_test")}, wantTypes: []string{"dockerfile", "googledeploymentmanager", "cloudformation", "crossplane", "knative", "kubernetes", "openapi", "terraform", "ansible", "azureresourcemanager", "dockercompose", "pulumi", "serverlessfw"}, wantExclude: []string{filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json")}, + wantLOC: 563, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -32,6 +34,7 @@ func TestAnalyzer_Analyze(t *testing.T) { paths: []string{filepath.FromSlash("../../test/fixtures/analyzer_test/helm")}, wantTypes: []string{"kubernetes"}, wantExclude: []string{}, + wantLOC: 118, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -43,6 +46,7 @@ func TestAnalyzer_Analyze(t *testing.T) { filepath.FromSlash("../../test/fixtures/analyzer_test/terraform.tf")}, wantTypes: []string{"dockerfile", "terraform"}, wantExclude: []string{}, + wantLOC: 13, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -53,6 +57,7 @@ func TestAnalyzer_Analyze(t *testing.T) { filepath.FromSlash("../../test/fixtures/analyzer_test/openAPI_test")}, wantTypes: []string{"openapi"}, wantExclude: []string{}, + wantLOC: 107, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -63,6 +68,7 @@ func TestAnalyzer_Analyze(t *testing.T) { filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json")}, wantTypes: []string{}, wantExclude: []string{filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json")}, + wantLOC: 0, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -74,6 +80,7 @@ func TestAnalyzer_Analyze(t *testing.T) { filepath.FromSlash("../../test/fixtures/analyzer_test/terraform.tf")}, wantTypes: []string{}, wantExclude: []string{}, + wantLOC: 0, wantErr: true, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -85,6 +92,7 @@ func TestAnalyzer_Analyze(t *testing.T) { }, wantTypes: []string{}, wantExclude: []string{filepath.FromSlash("../../test/fixtures/type-test01/template01/metadata.json")}, + wantLOC: 0, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -96,6 +104,7 @@ func TestAnalyzer_Analyze(t *testing.T) { }, wantTypes: []string{"terraform"}, wantExclude: []string{}, + wantLOC: 26, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -109,6 +118,7 @@ func TestAnalyzer_Analyze(t *testing.T) { wantExclude: []string{filepath.FromSlash("../../test/fixtures/gitignore/positive.dockerfile"), filepath.FromSlash("../../test/fixtures/gitignore/secrets.tf"), filepath.FromSlash("../../test/fixtures/gitignore/gitignore")}, + wantLOC: 13, wantErr: false, gitIgnoreFileName: "gitignore", excludeGitIgnore: false, @@ -120,6 +130,7 @@ func TestAnalyzer_Analyze(t *testing.T) { }, wantTypes: []string{"dockerfile", "kubernetes", "terraform"}, wantExclude: []string{filepath.FromSlash("../../test/fixtures/gitignore/gitignore")}, + wantLOC: 42, wantErr: false, gitIgnoreFileName: "gitignore", excludeGitIgnore: true, @@ -131,6 +142,7 @@ func TestAnalyzer_Analyze(t *testing.T) { }, wantTypes: []string{"knative", "kubernetes"}, wantExclude: []string{}, + wantLOC: 15, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -142,6 +154,7 @@ func TestAnalyzer_Analyze(t *testing.T) { }, wantTypes: []string{"serverlessfw", "cloudformation"}, wantExclude: []string{}, + wantLOC: 88, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -153,6 +166,7 @@ func TestAnalyzer_Analyze(t *testing.T) { }, wantTypes: []string{"ansible"}, wantExclude: []string{}, + wantLOC: 1, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -182,6 +196,8 @@ func TestAnalyzer_Analyze(t *testing.T) { sort.Strings(got.Exc) require.Equal(t, tt.wantTypes, got.Types, "wrong types from analyzer") require.Equal(t, tt.wantExclude, got.Exc, "wrong excludes from analyzer") + require.Equal(t, tt.wantLOC, got.ExpectedLOC, "wrong loc from analyzer") + }) } } diff --git a/pkg/model/model.go b/pkg/model/model.go index a96355ad247..7dbad360115 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -276,8 +276,9 @@ func (m FileMetadatas) Combine(lineInfo bool) Documents { // AnalyzedPaths is a slice of types and excluded files obtained from the Analyzer type AnalyzedPaths struct { - Types []string - Exc []string + Types []string + Exc []string + ExpectedLOC int } // ResolvedFileSplit is a struct that contains the information of a resolved file, the path and the lines of the file diff --git a/pkg/scan/utils_test.go b/pkg/scan/utils_test.go index f7f3e6c7fe7..e5efb15ff29 100644 --- a/pkg/scan/utils_test.go +++ b/pkg/scan/utils_test.go @@ -169,7 +169,7 @@ func Test_GetTotalFiles(t *testing.T) { { name: "count utils folder files", paths: []string{filepath.Join("..", "..", "pkg", "utils")}, - expectedOutput: 12, + expectedOutput: 14, }, { name: "count progress folder files", @@ -179,7 +179,7 @@ func Test_GetTotalFiles(t *testing.T) { { name: "count progress and utils folder files", paths: []string{filepath.Join("..", "..", "pkg", "progress"), filepath.Join("..", "..", "pkg", "utils")}, - expectedOutput: 18, + expectedOutput: 20, }, { name: "count invalid folder", diff --git a/pkg/utils/line_counter.go b/pkg/utils/line_counter.go new file mode 100644 index 00000000000..c2a982c772d --- /dev/null +++ b/pkg/utils/line_counter.go @@ -0,0 +1,35 @@ +// Package utils contains various utility functions to use in other packages +package utils + +import ( + "bufio" + "os" + "path/filepath" + + "github.com/rs/zerolog/log" +) + +// LineCounter get the number of lines of a given file +func LineCounter(path string) (int, error) { + file, err := os.Open(filepath.Clean(path)) + if err != nil { + return 0, err + } + defer func() { + if err := file.Close(); err != nil { + log.Err(err).Msgf("failed to close '%s'", filepath.Clean(path)) + } + }() + + scanner := bufio.NewScanner(file) + lineCount := 0 + for scanner.Scan() { + lineCount++ + } + + if err := scanner.Err(); err != nil { + return 0, err + } + + return lineCount, nil +} diff --git a/pkg/utils/line_counter_test.go b/pkg/utils/line_counter_test.go new file mode 100644 index 00000000000..9470590c734 --- /dev/null +++ b/pkg/utils/line_counter_test.go @@ -0,0 +1,49 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLineCounter(t *testing.T) { + tests := []struct { + name string + want int + filePath string + wantError bool + }{ + { + name: "Get lines from non existent file", + want: 0, + filePath: "../../Dockerfile2", + wantError: true, + }, + { + name: "Get lines from a dockerfile file", + want: 7, + filePath: "../../test/fixtures/dockerfile/Dockerfile-example", + wantError: false, + }, + { + name: "Get lines from a yaml file", + want: 25, + filePath: "../../test/assets/sample_K8S_CONFIG_FILE.yaml", + wantError: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := LineCounter(test.filePath) + if test.wantError { + require.NotEqual(t, err, nil) + require.Equal(t, test.want, got) + } else { + require.Equal(t, test.want, got) + require.Equal(t, err, nil) + } + + }) + } +}