Merge branch 'main' into map-aws-id-to-account-id

pkg/detectors/falsepositives.go

@@ -5,6 +5,7 @@ import (
 	"math"
 	"strings"
 	"unicode"
+	"unicode/utf8"
 )
 
 var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample", "www"}
@@ -21,9 +22,9 @@ var wordList []byte
 var programmingBookWords []byte
 
 type Wordlists struct {
-	wordList             []string
-	badList              []string
-	programmingBookWords []string
+	wordList             map[string]struct{}
+	badList              map[string]struct{}
+	programmingBookWords map[string]struct{}
 }
 
 var FalsePositiveWordlists = Wordlists{
@@ -36,36 +37,29 @@ var FalsePositiveWordlists = Wordlists{
 // Currently that includes: No number, english word in key, or matches common example pattens.
 // Only the secret key material should be passed into this function
 func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) bool {
-
+	if !utf8.ValidString(match) {
+		return true
+	}
+	lower := strings.ToLower(match)
 	for _, fp := range falsePositives {
-		if strings.Contains(strings.ToLower(match), string(fp)) {
+		if strings.Contains(lower, string(fp)) {
 			return true
 		}
 	}
 
 	if wordCheck {
 		// check against common substring badlist
-		if hasDictWord(FalsePositiveWordlists.badList, match) {
+		if _, ok := FalsePositiveWordlists.badList[lower]; ok {
 			return true
 		}
 
 		// check for dictionary word substrings
-		if hasDictWord(FalsePositiveWordlists.wordList, match) {
+		if _, ok := FalsePositiveWordlists.wordList[lower]; ok {
 			return true
 		}
 
 		// check for programming book token substrings
-		if hasDictWord(FalsePositiveWordlists.programmingBookWords, match) {
-			return true
-		}
-	}
-	return false
-}
-
-func hasDictWord(wordList []string, token string) bool {
-	lower := strings.ToLower(token)
-	for _, word := range wordList {
-		if strings.Contains(lower, word) {
+		if _, ok := FalsePositiveWordlists.programmingBookWords[lower]; ok {
 			return true
 		}
 	}
@@ -82,11 +76,11 @@ func HasDigit(key string) bool {
 	return false
 }
 
-func bytesToCleanWordList(data []byte) []string {
-	words := []string{}
+func bytesToCleanWordList(data []byte) map[string]struct{} {
+	words := make(map[string]struct{})
 	for _, word := range strings.Split(string(data), "\n") {
 		if strings.TrimSpace(word) != "" {
-			words = append(words, strings.TrimSpace(strings.ToLower(word)))
+			words[strings.TrimSpace(strings.ToLower(word))] = struct{}{}
 		}
 	}
 	return words

pkg/detectors/falsepositives_test.go

@@ -90,3 +90,10 @@ func TestStringShannonEntropy(t *testing.T) {
 		})
 	}
 }
+
+func BenchmarkDefaultIsKnownFalsePositive(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		// Use a string that won't be found in any dictionary for the worst case check.
+		IsKnownFalsePositive("aoeuaoeuaoeuaoeuaoeuaoeu", DefaultFalsePositives, true)
+	}
+}

pkg/engine/defaults.go

@@ -569,7 +569,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperbox"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapersite"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapestack"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapfly"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapingant"
@@ -1315,7 +1314,6 @@ func DefaultDetectors() []detectors.Detector {
 		zenkitapi.Scanner{},
 		sherpadesk.Scanner{},
 		shotstack.Scanner{},
-		scrapersite.Scanner{},
 		luno.Scanner{},
 		apacta.Scanner{},
 		fmfw.Scanner{},

proto/detectors.proto

@@ -667,7 +667,7 @@ enum DetectorType {
   Meistertask = 656;
   Mindmeister = 657;
   PeopleDataLabs = 658;
-  ScraperSite = 659;
+  ScraperSite = 659  [deprecated = true];
   Scrapfly = 660;
   SimplyNoted = 661;
   TravelPayouts = 662;