Include offenderEntropy in the JSON output (#549)

* Pass the entropy data back to the Leak struct

Do this to make it easier to tune entropy checks and make decisions in
systems consuming the output.

~ B'ezrat Hashem ~

* Return negative number when entropy not checked

That way you can tell the difference between not checking or an
actual entropy level of 0

~ B'ezrat Hashem ~

* Make sure to handle range checks properly

Make sure to show when something had an entropy returned but was outside
range, or didn't have a hit at all, etc...

~ B'zrat Hashem ~

* Add a few doc strings

Follow the project's conventiona add a comment above the methods

~ B'ezrat Hashem ~

* Update tests and get them to pass

~ B'ezrat Hashem ~

* Remove checked in `.got` files

~ B'ezrat Hashem ~

* Add `*.got` to the `.gitignore`

Make sure the test output files aren't checked-in

~ B'ezrat Hashem ~

.gitignore

@@ -6,6 +6,7 @@
 *.dylib
 *.DS_STORE
 *.idea
+*.got
 gitleaks
 build
 

Makefile

@@ -10,15 +10,16 @@ test-cover:
 	go test ./... --race $(COVER) $(PKG) -v
 	go tool cover -html=cover.out
 
-test:
-	go get golang.org/x/lint/golint
+format:
 	go fmt ./...
+
+test: format
+	go get golang.org/x/lint/golint
 	go vet ./...
 	golint ./...
 	go test ./... --race $(PKG) -v
 
-build:
-	go fmt ./...
+build: format
 	golint ./...
 	go vet ./...
 	go mod tidy
@@ -41,9 +42,9 @@ release-builds:
 
 deploy:
 	@echo "$(DOCKER_PASSWORD)" | docker login -u "$(DOCKER_USERNAME)" --password-stdin
-	docker build --build-arg ldflags=$(_LDFLAGS) -f Dockerfile -t zricethezav/gitleaks:latest -t zricethezav/gitleaks:$(VERSION) . 
+	docker build --build-arg ldflags=$(_LDFLAGS) -f Dockerfile -t zricethezav/gitleaks:latest -t zricethezav/gitleaks:$(VERSION) .
 	echo "Pushing zricethezav/gitleaks:$(VERSION) and zricethezav/gitleaks:latest"
 	docker push zricethezav/gitleaks
 
-dockerbuild: 
-	docker build --build-arg ldflags=$(_LDFLAGS) -f Dockerfile -t zricethezav/gitleaks:latest -t zricethezav/gitleaks:$(VERSION) . 
+dockerbuild:
+	docker build --build-arg ldflags=$(_LDFLAGS) -f Dockerfile -t zricethezav/gitleaks:latest -t zricethezav/gitleaks:$(VERSION) .

config/rule.go

@@ -6,6 +6,23 @@ import (
 	"regexp"
 )
 
+// Offender is a struct that contains the information matched when searching
+// content and information on why it matched (i.e. the EntropyLevel)
+type Offender struct {
+	Match        string
+	EntropyLevel float64
+}
+
+// IsEmpty checks to see if nothing was found in the match
+func (o *Offender) IsEmpty() bool {
+	return o.Match == ""
+}
+
+// ToString the contents of the match
+func (o *Offender) ToString() string {
+	return o.Match
+}
+
 // Rule is a struct that contains information that is loaded from a gitleaks config.
 // This struct is used in the Config struct as an array of Rules and is iterated
 // over during an scan. Each rule will be checked. If a regex match is found AND
@@ -23,28 +40,46 @@ type Rule struct {
 }
 
 // Inspect checks the content of a line for a leak
-func (r *Rule) Inspect(line string) string {
-	offender := r.Regex.FindString(line)
-	if offender == "" {
-		return ""
+func (r *Rule) Inspect(line string) *Offender {
+	match := r.Regex.FindString(line)
+
+	// EntropyLevel -1 means not checked
+	if match == "" {
+		return &Offender{
+			Match:        "",
+			EntropyLevel: -1,
+		}
 	}
 
 	// check if offender is allowed
+	// EntropyLevel -1 means not checked
 	if r.RegexAllowed(line) {
-		return ""
+		return &Offender{
+			Match:        "",
+			EntropyLevel: -1,
+		}
 	}
 
 	// check entropy
-	groups := r.Regex.FindStringSubmatch(offender)
-	if len(r.Entropies) != 0 && !r.ContainsEntropyLeak(groups) {
-		return ""
+	groups := r.Regex.FindStringSubmatch(match)
+	entropyWithinRange, entropyLevel := r.CheckEntropy(groups)
+
+	if len(r.Entropies) != 0 && !entropyWithinRange {
+		return &Offender{
+			Match:        "",
+			EntropyLevel: entropyLevel,
+		}
 	}
 
 	// 0 is a match for the full regex pattern
 	if 0 < r.ReportGroup && r.ReportGroup < len(groups) {
-		offender = groups[r.ReportGroup]
+		match = groups[r.ReportGroup]
+	}
+
+	return &Offender{
+		Match:        match,
+		EntropyLevel: entropyLevel,
 	}
-	return offender
 }
 
 // RegexAllowed checks if the content is allowlisted
@@ -57,17 +92,28 @@ func (r *Rule) CommitAllowed(commit string) bool {
 	return r.AllowList.CommitAllowed(commit)
 }
 
-// ContainsEntropyLeak checks if there is an entropy leak
-func (r *Rule) ContainsEntropyLeak(groups []string) bool {
+// CheckEntropy checks if there is an entropy leak
+func (r *Rule) CheckEntropy(groups []string) (bool, float64) {
+	var highestFound float64 = 0
+
 	for _, e := range r.Entropies {
 		if len(groups) > e.Group {
 			entropy := shannonEntropy(groups[e.Group])
 			if entropy >= e.Min && entropy <= e.Max {
-				return true
+				return true, entropy
+			} else if entropy > highestFound {
+				highestFound = entropy
 			}
 		}
 	}
-	return false
+
+	if len(r.Entropies) == 0 {
+		// entropies not checked
+		return false, -1
+	}
+
+	// entropies checked but not within the range
+	return false, highestFound
 }
 
 // HasFileOrPathLeakOnly first checks if there are no entropy/regex rules, then checks if

scan/commit.go

@@ -121,7 +121,7 @@ func (cs *CommitScanner) Scan() (Report, error) {
 							continue
 						}
 						offender := rule.Inspect(line)
-						if offender == "" {
+						if offender.IsEmpty() {
 							continue
 						}
 
@@ -136,7 +136,7 @@ func (cs *CommitScanner) Scan() (Report, error) {
 							continue
 						}
 
-						leak := NewLeak(line, offender, defaultLineNumber).WithCommit(cs.commit)
+						leak := NewLeak(line, offender.ToString(), defaultLineNumber).WithCommit(cs.commit).WithEntropy(offender.EntropyLevel)
 						leak.File = to.Path()
 						leak.LineNumber = extractLine(patchContent, leak, lineLookup)
 						leak.RepoURL = cs.opts.RepoURL

scan/filesatcommit.go

@@ -97,8 +97,7 @@ func (fs *FilesAtCommitScanner) Scan() (Report, error) {
 				}
 
 				offender := rule.Inspect(line)
-
-				if offender == "" {
+				if offender.IsEmpty() {
 					continue
 				}
 
@@ -113,7 +112,7 @@ func (fs *FilesAtCommitScanner) Scan() (Report, error) {
 					continue
 				}
 
-				leak := NewLeak(line, offender, defaultLineNumber).WithCommit(fs.commit)
+				leak := NewLeak(line, offender.ToString(), defaultLineNumber).WithCommit(fs.commit).WithEntropy(offender.EntropyLevel)
 				leak.File = f.Name
 				leak.LineNumber = i + 1
 				leak.RepoURL = fs.opts.RepoURL

scan/leak.go

@@ -3,6 +3,7 @@ package scan
 import (
 	"encoding/json"
 	"fmt"
+	"math"
 	"strings"
 	"time"
 
@@ -14,20 +15,21 @@ import (
 // Leak is a struct that contains information about some line of code that contains
 // sensitive information as determined by the rules set in a gitleaks config
 type Leak struct {
-	Line       string    `json:"line"`
-	LineNumber int       `json:"lineNumber"`
-	Offender   string    `json:"offender"`
-	Commit     string    `json:"commit"`
-	Repo       string    `json:"repo"`
-	RepoURL    string    `json:"repoURL"`
-	LeakURL    string    `json:"leakURL"`
-	Rule       string    `json:"rule"`
-	Message    string    `json:"commitMessage"`
-	Author     string    `json:"author"`
-	Email      string    `json:"email"`
-	File       string    `json:"file"`
-	Date       time.Time `json:"date"`
-	Tags       string    `json:"tags"`
+	Line            string    `json:"line"`
+	LineNumber      int       `json:"lineNumber"`
+	Offender        string    `json:"offender"`
+	OffenderEntropy float64   `json:"offenderEntropy"`
+	Commit          string    `json:"commit"`
+	Repo            string    `json:"repo"`
+	RepoURL         string    `json:"repoURL"`
+	LeakURL         string    `json:"leakURL"`
+	Rule            string    `json:"rule"`
+	Message         string    `json:"commitMessage"`
+	Author          string    `json:"author"`
+	Email           string    `json:"email"`
+	File            string    `json:"file"`
+	Date            time.Time `json:"date"`
+	Tags            string    `json:"tags"`
 }
 
 // RedactLeak will replace the offending string with "REDACTED" in both
@@ -41,9 +43,10 @@ func RedactLeak(leak Leak) Leak {
 // NewLeak creates a new leak from common data all leaks must have, line, offender, linenumber
 func NewLeak(line string, offender string, lineNumber int) Leak {
 	return Leak{
-		Line:       line,
-		Offender:   offender,
-		LineNumber: lineNumber,
+		Line:            line,
+		Offender:        offender,
+		LineNumber:      lineNumber,
+		OffenderEntropy: -1, // -1 means not checked
 	}
 }
 
@@ -57,6 +60,12 @@ func (leak Leak) WithCommit(commit *object.Commit) Leak {
 	return leak
 }
 
+// WithEntropy adds OffenderEntropy data to the leak
+func (leak Leak) WithEntropy(entropyLevel float64) Leak {
+	leak.OffenderEntropy = math.Round(entropyLevel*1000) / 1000
+	return leak
+}
+
 // Log logs a leak and redacts if necessary
 func (leak Leak) Log(opts options.Options) {
 	if !opts.Quiet && !opts.Verbose {

scan/nogit.go

@@ -105,7 +105,7 @@ func (ngs *NoGitScanner) Scan() (Report, error) {
 					}
 
 					offender := rule.Inspect(line)
-					if offender == "" {
+					if offender.IsEmpty() {
 						continue
 					}
 					if ngs.cfg.Allowlist.RegexAllowed(line) {
@@ -119,7 +119,7 @@ func (ngs *NoGitScanner) Scan() (Report, error) {
 						continue
 					}
 
-					leak := NewLeak(line, offender, defaultLineNumber)
+					leak := NewLeak(line, offender.ToString(), defaultLineNumber).WithEntropy(offender.EntropyLevel)
 					relPath, err := filepath.Rel(ngs.opts.Path, p)
 					if err != nil {
 						leak.File = p

scan/scan_test.go

@@ -760,7 +760,7 @@ func fileCheck(wantPath, gotPath string) error {
 			}
 		}
 		if !found {
-			return fmt.Errorf("unable to find %+v in got leaks", wantLeak)
+			return fmt.Errorf("unable to find %+v in %s", wantLeak, gotPath)
 		}
 	}
 
@@ -779,6 +779,10 @@ func same(l1, l2 Leak) bool {
 		return false
 	}
 
+	if l1.OffenderEntropy != l2.OffenderEntropy {
+		return false
+	}
+
 	if l1.Line != l2.Line {
 		return false
 	}

scan/unstaged.go

@@ -87,7 +87,7 @@ func (us *UnstagedScanner) Scan() (Report, error) {
 				lineNumber++
 				for _, rule := range us.cfg.Rules {
 					offender := rule.Inspect(line)
-					if offender == "" {
+					if offender.IsEmpty() {
 						continue
 					}
 					if us.cfg.Allowlist.RegexAllowed(line) ||
@@ -101,7 +101,7 @@ func (us *UnstagedScanner) Scan() (Report, error) {
 					if rule.Path.String() != "" && !rule.HasFilePathLeak(filepath.Base(workTreeFile.Name())) {
 						continue
 					}
-					leak := NewLeak(line, offender, defaultLineNumber).WithCommit(emptyCommit())
+					leak := NewLeak(line, offender.ToString(), defaultLineNumber).WithCommit(emptyCommit()).WithEntropy(offender.EntropyLevel)
 					leak.File = workTreeFile.Name()
 					leak.LineNumber = lineNumber
 					leak.Repo = us.repoName
@@ -206,7 +206,7 @@ func (us *UnstagedScanner) Scan() (Report, error) {
 			for _, line := range strings.Split(diffContents, "\n") {
 				for _, rule := range us.cfg.Rules {
 					offender := rule.Inspect(line)
-					if offender == "" {
+					if offender.IsEmpty() {
 						continue
 					}
 					if us.cfg.Allowlist.RegexAllowed(line) ||
@@ -220,7 +220,7 @@ func (us *UnstagedScanner) Scan() (Report, error) {
 					if rule.Path.String() != "" && !rule.HasFilePathLeak(filepath.Base(filename)) {
 						continue
 					}
-					leak := NewLeak(line, offender, defaultLineNumber).WithCommit(emptyCommit())
+					leak := NewLeak(line, offender.ToString(), defaultLineNumber).WithCommit(emptyCommit()).WithEntropy(offender.EntropyLevel)
 					leak.File = filename
 					leak.LineNumber = extractLine(prettyDiff, leak, lineLookup) + 1
 					leak.Repo = us.repoName

test_data/test_additional_config.json

@@ -3,6 +3,7 @@
   "line": "",
   "lineNumber": 1,
   "offender": "Filename or path offender: tmp/bad.docx",
+  "offenderEntropy": -1,
   "commit": "b0f9b62dfe12e4e10de180359c6b9276472494f8",
   "repo": "test_repo_10",
   "repoURL": "",

test_data/test_allow_list_docx_no_git.json

@@ -3,6 +3,7 @@
   "line": "",
   "lineNumber": 1,
   "offender": "Filename or path offender: ../test_data/test_repos/test_repo_10/.gitignore",
+  "offenderEntropy": -1,
   "commit": "",
   "repo": "",
   "repoURL": "",
@@ -19,6 +20,7 @@
   "line": "",
   "lineNumber": 1,
   "offender": "Filename or path offender: ../test_data/test_repos/test_repo_10/bad.zip",
+  "offenderEntropy": -1,
   "commit": "",
   "repo": "",
   "repoURL": "",
@@ -35,6 +37,7 @@
   "line": "",
   "lineNumber": 1,
   "offender": "Filename or path offender: ../test_data/test_repos/test_repo_10/gitfile.txt",
+  "offenderEntropy": -1,
   "commit": "",
   "repo": "",
   "repoURL": "",
@@ -51,6 +54,7 @@
   "line": "",
   "lineNumber": 1,
   "offender": "Filename or path offender: ../test_data/test_repos/test_repo_10/creds.git",
+  "offenderEntropy": -1,
   "commit": "",
   "repo": "",
   "repoURL": "",
@@ -67,6 +71,7 @@
   "line": "",
   "lineNumber": 1,
   "offender": "Filename or path offender: ../test_data/test_repos/test_repo_10/somedir.git/secret.key",
+  "offenderEntropy": -1,
   "commit": "",
   "repo": "",
   "repoURL": "",

test_data/test_allow_list_file.json

@@ -3,6 +3,7 @@
   "line": "",
   "lineNumber": 1,
   "offender": "Filename or path offender: tmp/bad.docx",
+  "offenderEntropy": -1,
   "commit": "b0f9b62dfe12e4e10de180359c6b9276472494f8",
   "repo": "test_repo_10",
   "repoURL": "",