feat(analyzer): add expected lines of code in analyzer

pkg/analyzer/analyzer.go

@@ -246,15 +246,18 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) {
 	// start metrics for file analyzer
 	metrics.Metric.Start("file_type_analyzer")
 	returnAnalyzedPaths := model.AnalyzedPaths{
-		Types: make([]string, 0),
-		Exc:   make([]string, 0),
+		Types:       make([]string, 0),
+		Exc:         make([]string, 0),
+		ExpectedLOC: 0,
 	}
 
 	var files []string
 	var wg sync.WaitGroup
 	// results is the channel shared by the workers that contains the types found
 	results := make(chan string)
+	locCount := make(chan int)
 	ignoreFiles := make([]string, 0)
+	done := make(chan bool)
 	hasGitIgnoreFile, gitIgnore := shouldConsiderGitIgnoreFile(a.Paths[0], a.GitIgnoreFileName, a.ExcludeGitIgnore)
 
 	// get all the files inside the given paths
@@ -291,31 +294,34 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) {
 		a.Types[i] = strings.ToLower(a.Types[i])
 	}
 
+	// Start the workers
 	for _, file := range files {
 		wg.Add(1)
 		// analyze the files concurrently
 		a := &analyzerInfo{
 			typesFlag: a.Types,
 			filePath:  file,
 		}
-		go a.worker(results, unwanted, &wg)
+		go a.worker(results, unwanted, locCount, &wg)
 	}
 
 	go func() {
 		// close channel results when the worker has finished writing into it
 		defer func() {
 			close(unwanted)
 			close(results)
+			close(locCount)
 		}()
 		wg.Wait()
+		done <- true
 	}()
 
-	availableTypes := createSlice(results)
+	availableTypes, unwantedPaths, loc := computeValues(results, unwanted, locCount, done)
 	multiPlatformTypeCheck(&availableTypes)
-	unwantedPaths := createSlice(unwanted)
 	unwantedPaths = append(unwantedPaths, ignoreFiles...)
 	returnAnalyzedPaths.Types = availableTypes
 	returnAnalyzedPaths.Exc = unwantedPaths
+	returnAnalyzedPaths.ExpectedLOC = loc
 	// stop metrics for file analyzer
 	metrics.Metric.Stop()
 	return returnAnalyzedPaths, nil
@@ -324,39 +330,43 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) {
 // worker determines the type of the file by ext (dockerfile and terraform)/content and
 // writes the answer to the results channel
 // if no types were found, the worker will write the path of the file in the unwanted channel
-func (a *analyzerInfo) worker(results, unwanted chan<- string, wg *sync.WaitGroup) {
+func (a *analyzerInfo) worker(results, unwanted chan<- string, locCount chan<- int, wg *sync.WaitGroup) {
 	defer wg.Done()
 
 	ext := utils.GetExtension(a.filePath)
-
+	linesCount, _ := utils.LineCounter(a.filePath)
 	typesFlag := a.typesFlag
 
 	switch ext {
 	// Dockerfile (direct identification)
 	case ".dockerfile", "Dockerfile":
 		if typesFlag[0] == "" || utils.Contains(dockerfile, typesFlag) {
 			results <- dockerfile
+			locCount <- linesCount
 		}
 	// Dockerfile (indirect identification)
 	case "possibleDockerfile", ".ubi8", ".debian":
 		if (typesFlag[0] == "" || utils.Contains(dockerfile, typesFlag)) && isDockerfile(a.filePath) {
 			results <- dockerfile
+			locCount <- linesCount
 		} else {
 			unwanted <- a.filePath
 		}
 	// Terraform
 	case ".tf", "tfvars":
 		if typesFlag[0] == "" || utils.Contains(terraform, typesFlag) {
 			results <- terraform
+			locCount <- linesCount
 		}
 	// GRPC
 	case ".proto":
 		if typesFlag[0] == "" || utils.Contains(grpc, typesFlag) {
 			results <- grpc
+			locCount <- linesCount
 		}
 	// Cloud Formation, Ansible, OpenAPI, Buildah
 	case yaml, yml, json, sh:
-		a.checkContent(results, unwanted, ext)
+		a.checkContent(results, unwanted, locCount, linesCount, ext)
 	}
 }
 
@@ -396,7 +406,7 @@ func needsOverride(check bool, returnType, key, ext string) bool {
 
 // checkContent will determine the file type by content when worker was unable to
 // determine by ext, if no type was determined checkContent adds it to unwanted channel
-func (a *analyzerInfo) checkContent(results, unwanted chan<- string, ext string) {
+func (a *analyzerInfo) checkContent(results, unwanted chan<- string, locCount chan<- int, linesCount int, ext string) {
 	typesFlag := a.typesFlag
 	// get file content
 	content, err := os.ReadFile(a.filePath)
@@ -438,6 +448,7 @@ func (a *analyzerInfo) checkContent(results, unwanted chan<- string, ext string)
 	if returnType != "" {
 		if typesFlag[0] == "" || utils.Contains(returnType, typesFlag) {
 			results <- returnType
+			locCount <- linesCount
 			return
 		}
 	}
@@ -495,15 +506,28 @@ func checkYamlPlatform(content []byte, path string) string {
 	return ansible
 }
 
-// createSlice creates a slice from the channel given removing any duplicates
-func createSlice(chanel chan string) []string {
-	slice := make([]string, 0)
-	for i := range chanel {
-		if !utils.Contains(i, slice) {
-			slice = append(slice, i)
+// computeValues computes expected Lines of Code to be scanned from locCount channel
+// and creates the types and unwanted slices from the channels removing any duplicates
+func computeValues(types, unwanted chan string, locCount chan int, done chan bool) (typesS, unwantedS []string, locTotal int) {
+	var val int
+	unwantedSlice := make([]string, 0)
+	typeSlice := make([]string, 0)
+	for {
+		select {
+		case i := <-locCount:
+			val += i
+		case i := <-unwanted:
+			if !utils.Contains(i, unwantedSlice) {
+				unwantedSlice = append(unwantedSlice, i)
+			}
+		case i := <-types:
+			if !utils.Contains(i, typeSlice) {
+				typeSlice = append(typeSlice, i)
+			}
+		case <-done:
+			return typeSlice, unwantedSlice, val
 		}
 	}
-	return slice
 }
 
 // getKeysFromTypesFlag gets all the regexes keys related to the types flag

pkg/analyzer/analyzer_test.go

@@ -14,6 +14,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 		paths             []string
 		wantTypes         []string
 		wantExclude       []string
+		wantLOC           int
 		wantErr           bool
 		gitIgnoreFileName string
 		excludeGitIgnore  bool
@@ -23,6 +24,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			paths:             []string{filepath.FromSlash("../../test/fixtures/analyzer_test")},
 			wantTypes:         []string{"dockerfile", "googledeploymentmanager", "cloudformation", "crossplane", "knative", "kubernetes", "openapi", "terraform", "ansible", "azureresourcemanager", "dockercompose", "pulumi", "serverlessfw"},
 			wantExclude:       []string{filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json")},
+			wantLOC:           563,
 			wantErr:           false,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -32,6 +34,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			paths:             []string{filepath.FromSlash("../../test/fixtures/analyzer_test/helm")},
 			wantTypes:         []string{"kubernetes"},
 			wantExclude:       []string{},
+			wantLOC:           118,
 			wantErr:           false,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -43,6 +46,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 				filepath.FromSlash("../../test/fixtures/analyzer_test/terraform.tf")},
 			wantTypes:         []string{"dockerfile", "terraform"},
 			wantExclude:       []string{},
+			wantLOC:           13,
 			wantErr:           false,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -53,6 +57,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 				filepath.FromSlash("../../test/fixtures/analyzer_test/openAPI_test")},
 			wantTypes:         []string{"openapi"},
 			wantExclude:       []string{},
+			wantLOC:           107,
 			wantErr:           false,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -63,6 +68,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 				filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json")},
 			wantTypes:         []string{},
 			wantExclude:       []string{filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json")},
+			wantLOC:           0,
 			wantErr:           false,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -74,6 +80,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 				filepath.FromSlash("../../test/fixtures/analyzer_test/terraform.tf")},
 			wantTypes:         []string{},
 			wantExclude:       []string{},
+			wantLOC:           0,
 			wantErr:           true,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -85,6 +92,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			},
 			wantTypes:         []string{},
 			wantExclude:       []string{filepath.FromSlash("../../test/fixtures/type-test01/template01/metadata.json")},
+			wantLOC:           0,
 			wantErr:           false,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -96,6 +104,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			},
 			wantTypes:         []string{"terraform"},
 			wantExclude:       []string{},
+			wantLOC:           26,
 			wantErr:           false,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -109,6 +118,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			wantExclude: []string{filepath.FromSlash("../../test/fixtures/gitignore/positive.dockerfile"),
 				filepath.FromSlash("../../test/fixtures/gitignore/secrets.tf"),
 				filepath.FromSlash("../../test/fixtures/gitignore/gitignore")},
+			wantLOC:           13,
 			wantErr:           false,
 			gitIgnoreFileName: "gitignore",
 			excludeGitIgnore:  false,
@@ -120,6 +130,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			},
 			wantTypes:         []string{"dockerfile", "kubernetes", "terraform"},
 			wantExclude:       []string{filepath.FromSlash("../../test/fixtures/gitignore/gitignore")},
+			wantLOC:           42,
 			wantErr:           false,
 			gitIgnoreFileName: "gitignore",
 			excludeGitIgnore:  true,
@@ -131,6 +142,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			},
 			wantTypes:         []string{"knative", "kubernetes"},
 			wantExclude:       []string{},
+			wantLOC:           15,
 			wantErr:           false,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -142,6 +154,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			},
 			wantTypes:         []string{"serverlessfw", "cloudformation"},
 			wantExclude:       []string{},
+			wantLOC:           88,
 			wantErr:           false,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -153,6 +166,7 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			},
 			wantTypes:         []string{"ansible"},
 			wantExclude:       []string{},
+			wantLOC:           1,
 			wantErr:           false,
 			gitIgnoreFileName: "",
 			excludeGitIgnore:  false,
@@ -182,6 +196,8 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			sort.Strings(got.Exc)
 			require.Equal(t, tt.wantTypes, got.Types, "wrong types from analyzer")
 			require.Equal(t, tt.wantExclude, got.Exc, "wrong excludes from analyzer")
+			require.Equal(t, tt.wantLOC, got.ExpectedLOC, "wrong loc from analyzer")
+
 		})
 	}
 }

pkg/model/model.go

@@ -276,8 +276,9 @@ func (m FileMetadatas) Combine(lineInfo bool) Documents {
 
 // AnalyzedPaths is a slice of types and excluded files obtained from the Analyzer
 type AnalyzedPaths struct {
-	Types []string
-	Exc   []string
+	Types       []string
+	Exc         []string
+	ExpectedLOC int
 }
 
 // ResolvedFileSplit is a struct that contains the information of a resolved file, the path and the lines of the file

pkg/scan/utils_test.go

@@ -169,7 +169,7 @@ func Test_GetTotalFiles(t *testing.T) {
 		{
 			name:           "count utils folder files",
 			paths:          []string{filepath.Join("..", "..", "pkg", "utils")},
-			expectedOutput: 12,
+			expectedOutput: 14,
 		},
 		{
 			name:           "count progress folder files",
@@ -179,7 +179,7 @@ func Test_GetTotalFiles(t *testing.T) {
 		{
 			name:           "count progress and utils folder files",
 			paths:          []string{filepath.Join("..", "..", "pkg", "progress"), filepath.Join("..", "..", "pkg", "utils")},
-			expectedOutput: 18,
+			expectedOutput: 20,
 		},
 		{
 			name:           "count invalid folder",

pkg/utils/line_counter.go

@@ -0,0 +1,28 @@
+// Package that contains
+package utils
+
+import (
+	"bufio"
+	"os"
+)
+
+// Get file number of lines
+func LineCounter(path string) (int, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return 0, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	lineCount := 0
+	for scanner.Scan() {
+		lineCount++
+	}
+
+	if err := scanner.Err(); err != nil {
+		return 0, err
+	}
+
+	return lineCount, nil
+}

pkg/utils/line_counter_test.go

@@ -0,0 +1,49 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLineCounter(t *testing.T) {
+	tests := []struct {
+		name      string
+		want      int
+		filePath  string
+		wantError bool
+	}{
+		{
+			name:      "Get lines from non existent file",
+			want:      0,
+			filePath:  "../../Dockerfile2",
+			wantError: true,
+		},
+		{
+			name:      "Get lines from a dockerfile file",
+			want:      7,
+			filePath:  "../../test/fixtures/dockerfile/Dockerfile-example",
+			wantError: false,
+		},
+		{
+			name:      "Get lines from a yaml file",
+			want:      25,
+			filePath:  "../../test/assets/sample_K8S_CONFIG_FILE.yaml",
+			wantError: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := LineCounter(test.filePath)
+			if test.wantError {
+				require.NotEqual(t, err, nil)
+				require.Equal(t, test.want, got)
+			} else {
+				require.Equal(t, test.want, got)
+				require.Equal(t, err, nil)
+			}
+
+		})
+	}
+}