From e98f48a4aa8c95821a88b21448bee97f29d2f65e Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Thu, 8 Feb 2024 13:55:25 -0600 Subject: [PATCH 01/10] add prefixregex to guardian api --- pkg/detectors/guardianapi/guardianapi.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/detectors/guardianapi/guardianapi.go b/pkg/detectors/guardianapi/guardianapi.go index 549492e9a846..1c55c8f4902a 100644 --- a/pkg/detectors/guardianapi/guardianapi.go +++ b/pkg/detectors/guardianapi/guardianapi.go @@ -2,10 +2,11 @@ package guardianapi import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -20,7 +21,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b([0-9Aa-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"guardian"}) + `\b([0-9Aa-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`) ) // Keywords are used for efficiently pre-filtering chunks. From f6880b634282fee24e4db26b14c564189ecdcea6 Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Fri, 9 Feb 2024 15:08:18 -0600 Subject: [PATCH 02/10] better boundaries and other tweaks --- pkg/detectors/accuweather/accuweather.go | 2 +- pkg/detectors/aeroworkflow/aeroworkflow.go | 2 +- pkg/detectors/aiven/aiven.go | 5 +++-- pkg/detectors/alibaba/alibaba.go | 2 +- pkg/detectors/appointedd/appointedd.go | 5 +++-- pkg/detectors/artifactory/artifactory.go | 2 +- pkg/detectors/azurebatch/azurebatch.go | 6 ++++-- pkg/detectors/beamer/beamer.go | 5 +++-- pkg/detectors/bitmex/bitmex.go | 7 ++++--- pkg/detectors/circleci/circleci.go | 5 +++-- .../cloudflareglobalapikey/cloudflareglobalapikey.go | 5 +++-- pkg/detectors/d7network/d7network.go | 3 ++- pkg/detectors/datadogtoken/datadogtoken.go | 7 ++++--- pkg/detectors/dovico/dovico.go | 7 ++++--- pkg/detectors/formio/formio.go | 5 +++-- pkg/detectors/gemini/gemini.go | 9 +++++---- pkg/detectors/gengo/gengo.go | 7 ++++--- pkg/detectors/getgist/getgist.go | 5 +++-- pkg/detectors/github/github.go | 3 ++- 19 files changed, 54 insertions(+), 38 deletions(-) diff --git a/pkg/detectors/accuweather/accuweather.go b/pkg/detectors/accuweather/accuweather.go index e7ebadae6fba..bd1ad5ee90d4 100644 --- a/pkg/detectors/accuweather/accuweather.go +++ b/pkg/detectors/accuweather/accuweather.go @@ -26,7 +26,7 @@ var ( defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"accuweather"}) + `([a-z0-9A-Z\%]{35})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"accuweather"}) + `\b([a-z0-9A-Z\%]{35})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/aeroworkflow/aeroworkflow.go b/pkg/detectors/aeroworkflow/aeroworkflow.go index 780a7826528f..aea1e6b7c7ab 100644 --- a/pkg/detectors/aeroworkflow/aeroworkflow.go +++ b/pkg/detectors/aeroworkflow/aeroworkflow.go @@ -26,7 +26,7 @@ var ( defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `([a-zA-Z0-9^!?#:*;]{20})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([a-zA-Z0-9^!?#:*;]{20})\b`) idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([0-9]{1,})\b`) ) diff --git a/pkg/detectors/aiven/aiven.go b/pkg/detectors/aiven/aiven.go index 451ed8ad0f05..a0b0585bfad8 100644 --- a/pkg/detectors/aiven/aiven.go +++ b/pkg/detectors/aiven/aiven.go @@ -3,10 +3,11 @@ package aiven import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -21,7 +22,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aiven"}) + `([a-zA-Z0-9/+=]{372})`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aiven"}) + `\b([a-zA-Z0-9/+=]{372})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/alibaba/alibaba.go b/pkg/detectors/alibaba/alibaba.go index 26570c6471f2..d1dd74a81b74 100644 --- a/pkg/detectors/alibaba/alibaba.go +++ b/pkg/detectors/alibaba/alibaba.go @@ -43,7 +43,7 @@ var ( // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{30})\b`) - idPat = regexp.MustCompile(`\b(LTAI[a-zA-Z0-9]{17,21})[\"';\s]*`) + idPat = regexp.MustCompile(`\b(LTAI[a-zA-Z0-9]{17,21})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/appointedd/appointedd.go b/pkg/detectors/appointedd/appointedd.go index 6c6e3fe383a6..8dd1a84f4e89 100644 --- a/pkg/detectors/appointedd/appointedd.go +++ b/pkg/detectors/appointedd/appointedd.go @@ -2,11 +2,12 @@ package appointedd import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -21,7 +22,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appointedd"}) + `\b([a-zA-Z0-9=+]{88})`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appointedd"}) + `\b([a-zA-Z0-9=+]{88})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/artifactory/artifactory.go b/pkg/detectors/artifactory/artifactory.go index accdc385ea0a..54473a02cecf 100644 --- a/pkg/detectors/artifactory/artifactory.go +++ b/pkg/detectors/artifactory/artifactory.go @@ -24,7 +24,7 @@ var ( defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73}|\b[a-zA-Z0-9]{64})`) + keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73}|[a-zA-Z0-9]{64})\b`) URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.jfrog\.io)`) ) diff --git a/pkg/detectors/azurebatch/azurebatch.go b/pkg/detectors/azurebatch/azurebatch.go index 927052ae8abb..b5336765cc41 100644 --- a/pkg/detectors/azurebatch/azurebatch.go +++ b/pkg/detectors/azurebatch/azurebatch.go @@ -27,8 +27,10 @@ var _ detectors.Detector = (*Scanner)(nil) var ( defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - urlPat = regexp.MustCompile(`https://(.{1,50})\.(.{1,50})\.batch\.azure\.com`) - secretPat = regexp.MustCompile(`[A-Za-z0-9+/=]{88}`) + urlPat = regexp.MustCompile(`https://(.{1,50})\.(.{1,50})\.batch\.azure\.com`) + + // examples we have end with == so /b won't work for the end boundary, gotta specifically match double "=" + secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/\=]{86}={0,2})`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/beamer/beamer.go b/pkg/detectors/beamer/beamer.go index f6b8d5fa5217..6ad7f582ece4 100644 --- a/pkg/detectors/beamer/beamer.go +++ b/pkg/detectors/beamer/beamer.go @@ -2,10 +2,11 @@ package beamer import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -20,7 +21,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"beamer"}) + `\b([a-zA-Z0-9_+/]{45}=)`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"beamer"}) + `\b([a-zA-Z0-9_+/]{45}=)\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/bitmex/bitmex.go b/pkg/detectors/bitmex/bitmex.go index a2fa65a6badf..e1f872cc60e5 100644 --- a/pkg/detectors/bitmex/bitmex.go +++ b/pkg/detectors/bitmex/bitmex.go @@ -5,13 +5,14 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" - regexp "github.com/wasilibs/go-re2" "net/http" "net/url" "strconv" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -26,8 +27,8 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + `([ \r\n]{1}[0-9a-zA-Z\-\_]{24}[ \r\n]{1})`) - secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + `([ \r\n]{1}[0-9a-zA-Z\-\_]{48}[ \r\n]{1})`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + `\b([0-9a-zA-Z\-\_]{24})\b`) + secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bitmex"}) + `\b([0-9a-zA-Z\-\_]{48})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/circleci/circleci.go b/pkg/detectors/circleci/circleci.go index 22ab9cdced66..2f52aef9a12d 100644 --- a/pkg/detectors/circleci/circleci.go +++ b/pkg/detectors/circleci/circleci.go @@ -2,9 +2,10 @@ package circleci import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -17,7 +18,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"circle"}) + `([a-fA-F0-9]{40})`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"circle_", "circle-", "circleci"}) + `\b([a-fA-F0-9]{40})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go b/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go index b0d6b46bd7dd..f5086ee61b1f 100644 --- a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go +++ b/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey.go @@ -2,10 +2,11 @@ package cloudflareglobalapikey import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -19,7 +20,7 @@ var _ detectors.Detector = (*Scanner)(nil) var ( client = common.SaneHttpClient() - apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloudflare"}) + `([A-Za-z0-9_-]{37})`) + apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"cloudflare"}) + `\b([A-Za-z0-9_-]{37})\b`) // email pattern thanks https://golangcode.com/validate-an-email-address/ // emailPat = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$") diff --git a/pkg/detectors/d7network/d7network.go b/pkg/detectors/d7network/d7network.go index 722ab4981e0b..8f817b5a12fa 100644 --- a/pkg/detectors/d7network/d7network.go +++ b/pkg/detectors/d7network/d7network.go @@ -2,10 +2,11 @@ package d7network import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) diff --git a/pkg/detectors/datadogtoken/datadogtoken.go b/pkg/detectors/datadogtoken/datadogtoken.go index c588864a4a63..fcf1ae11dc76 100644 --- a/pkg/detectors/datadogtoken/datadogtoken.go +++ b/pkg/detectors/datadogtoken/datadogtoken.go @@ -2,10 +2,11 @@ package datadogtoken import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -20,8 +21,8 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`) - apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`) + appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog"}) + `\b([a-zA-Z-0-9]{40})\b`) + apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog"}) + `\b([a-zA-Z-0-9]{32})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/dovico/dovico.go b/pkg/detectors/dovico/dovico.go index 766c76db3d2a..87dbeb401773 100644 --- a/pkg/detectors/dovico/dovico.go +++ b/pkg/detectors/dovico/dovico.go @@ -3,10 +3,11 @@ package dovico import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -21,8 +22,8 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dovico"}) + `\b([0-9a-z]{32}\.[0-9a-z]{1,}\b)`) - userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dovico"}) + `\b([0-9a-z]{32}\.[0-9a-z]{1,}\b)`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dovico"}) + `\b([0-9a-z]{32}\.[0-9a-z]{1,})\b`) + userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dovico"}) + `\b([0-9a-z]{32}\.[0-9a-z]{1,})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/formio/formio.go b/pkg/detectors/formio/formio.go index 3c9aba537481..26a976667af5 100644 --- a/pkg/detectors/formio/formio.go +++ b/pkg/detectors/formio/formio.go @@ -2,10 +2,11 @@ package formio import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -20,7 +21,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"formio"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[0-9A-Za-z]{220,310}\.[0-9A-Z-a-z\-_]{43}[ \r\n]{1})`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"formio"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[0-9A-Za-z]{220,310}\.[0-9A-Z-a-z\-_]{43})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/gemini/gemini.go b/pkg/detectors/gemini/gemini.go index 668763e6a43e..0e0b3f92df62 100644 --- a/pkg/detectors/gemini/gemini.go +++ b/pkg/detectors/gemini/gemini.go @@ -8,11 +8,12 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -33,14 +34,14 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b((?:master-|account-)[0-9A-Za-z]{20})\b`) - secretPat = regexp.MustCompile(`[A-Za-z0-9]{27,28}`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gemini"}) + `\b((?:master-|account-)[0-9A-Za-z]{20})\b`) + secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gemini"}) + `\b([A-Za-z0-9]{27,28})\b`) ) // Keywords are used for efficiently pre-filtering chunks. // Use identifiers in the secret preferably, or the provider name. func (s Scanner) Keywords() []string { - return []string{"master-", "account-"} + return []string{"master-", "account-", "gemini"} } // FromData will find and optionally verify Gemini secrets in a given set of bytes. diff --git a/pkg/detectors/gengo/gengo.go b/pkg/detectors/gengo/gengo.go index e3c908b643ac..40ddefdc010a 100644 --- a/pkg/detectors/gengo/gengo.go +++ b/pkg/detectors/gengo/gengo.go @@ -7,13 +7,14 @@ import ( "encoding/hex" "encoding/json" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strconv" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -29,8 +30,8 @@ var ( // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. // Removed bounds since there are some cases where the start and end of the token is a special character - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([ ]{0,1}[0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64}[ \r\n]{1})`) - secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([ ]{0,1}[0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64}[ \r\n]{1})`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64})\b`) + secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/getgist/getgist.go b/pkg/detectors/getgist/getgist.go index b3a985217781..80c6f043e37a 100644 --- a/pkg/detectors/getgist/getgist.go +++ b/pkg/detectors/getgist/getgist.go @@ -3,10 +3,11 @@ package getgist import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -21,7 +22,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getgist"}) + `\b([a-z0-9A-Z+=]{68})`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getgist"}) + `\b([a-z0-9A-Z+=]{67}=)`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/github/github.go b/pkg/detectors/github/github.go index 038cd08c2124..237aae9c0806 100644 --- a/pkg/detectors/github/github.go +++ b/pkg/detectors/github/github.go @@ -4,9 +4,10 @@ import ( "context" "encoding/json" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" From e83f233f6ba1d28ccddda6130a15223c6170c8a1 Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Fri, 9 Feb 2024 15:12:48 -0600 Subject: [PATCH 03/10] dd_, dd- for datadog, double equals for azure --- pkg/detectors/azurebatch/azurebatch.go | 2 +- pkg/detectors/datadogtoken/datadogtoken.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/detectors/azurebatch/azurebatch.go b/pkg/detectors/azurebatch/azurebatch.go index b5336765cc41..b7fbfc95a2be 100644 --- a/pkg/detectors/azurebatch/azurebatch.go +++ b/pkg/detectors/azurebatch/azurebatch.go @@ -30,7 +30,7 @@ var ( urlPat = regexp.MustCompile(`https://(.{1,50})\.(.{1,50})\.batch\.azure\.com`) // examples we have end with == so /b won't work for the end boundary, gotta specifically match double "=" - secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/\=]{86}={0,2})`) + secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/\=]{86}==)`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/datadogtoken/datadogtoken.go b/pkg/detectors/datadogtoken/datadogtoken.go index fcf1ae11dc76..b1bc26349553 100644 --- a/pkg/detectors/datadogtoken/datadogtoken.go +++ b/pkg/detectors/datadogtoken/datadogtoken.go @@ -21,8 +21,8 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog"}) + `\b([a-zA-Z-0-9]{40})\b`) - apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog"}) + `\b([a-zA-Z-0-9]{32})\b`) + appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd_", "dd-"}) + `\b([a-zA-Z-0-9]{40})\b`) + apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd_", "dd-"}) + `\b([a-zA-Z-0-9]{32})\b`) ) // Keywords are used for efficiently pre-filtering chunks. From 62363f28c0f97b4e3041cdaeb94394c4a014b20d Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Fri, 9 Feb 2024 15:15:53 -0600 Subject: [PATCH 04/10] unnecessary boundary --- pkg/detectors/beamer/beamer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/detectors/beamer/beamer.go b/pkg/detectors/beamer/beamer.go index 6ad7f582ece4..c9600a0ba84f 100644 --- a/pkg/detectors/beamer/beamer.go +++ b/pkg/detectors/beamer/beamer.go @@ -21,7 +21,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"beamer"}) + `\b([a-zA-Z0-9_+/]{45}=)\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"beamer"}) + `\b([a-zA-Z0-9_+/]{45}=)`) ) // Keywords are used for efficiently pre-filtering chunks. From 7697ae2330c06d12785a2d01a6db526c66fa992d Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Fri, 9 Feb 2024 15:18:56 -0600 Subject: [PATCH 05/10] leave circleci alone! (read in voice) --- pkg/detectors/circleci/circleci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/detectors/circleci/circleci.go b/pkg/detectors/circleci/circleci.go index 2f52aef9a12d..5f38db581105 100644 --- a/pkg/detectors/circleci/circleci.go +++ b/pkg/detectors/circleci/circleci.go @@ -18,7 +18,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"circle_", "circle-", "circleci"}) + `\b([a-fA-F0-9]{40})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"circle"}) + `\b([a-fA-F0-9]{40})\b`) ) // Keywords are used for efficiently pre-filtering chunks. From d97260cbc71414c3ced6aa39b5643751febb26b9 Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Fri, 9 Feb 2024 15:24:20 -0600 Subject: [PATCH 06/10] dovico, you're out of control --- pkg/detectors/dovico/dovico.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/detectors/dovico/dovico.go b/pkg/detectors/dovico/dovico.go index 87dbeb401773..17d3dacef11b 100644 --- a/pkg/detectors/dovico/dovico.go +++ b/pkg/detectors/dovico/dovico.go @@ -22,8 +22,8 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dovico"}) + `\b([0-9a-z]{32}\.[0-9a-z]{1,})\b`) - userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dovico"}) + `\b([0-9a-z]{32}\.[0-9a-z]{1,})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dovico"}) + `\b([0-9a-z]{32}\.[0-9a-z]{1,100})\b`) + userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"dovico"}) + `\b([0-9a-z]{32}\.[0-9a-z]{1,100})\b`) ) // Keywords are used for efficiently pre-filtering chunks. From e31f0a4322ca440405cd339b79a76849031553dc Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Fri, 9 Feb 2024 15:35:21 -0600 Subject: [PATCH 07/10] appointed ending --- pkg/detectors/appointedd/appointedd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/detectors/appointedd/appointedd.go b/pkg/detectors/appointedd/appointedd.go index 8dd1a84f4e89..1997d4797034 100644 --- a/pkg/detectors/appointedd/appointedd.go +++ b/pkg/detectors/appointedd/appointedd.go @@ -22,7 +22,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appointedd"}) + `\b([a-zA-Z0-9=+]{88})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appointedd"}) + `\b([a-zA-Z0-9=+]{87}=)`) ) // Keywords are used for efficiently pre-filtering chunks. From cd6b6d99c7fe46d8fe928fdc129a6ddd8cbd01c3 Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Wed, 14 Feb 2024 13:38:32 -0600 Subject: [PATCH 08/10] lazy matchin ftw --- pkg/detectors/aiven/aiven.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/detectors/aiven/aiven.go b/pkg/detectors/aiven/aiven.go index a0b0585bfad8..054d59c43116 100644 --- a/pkg/detectors/aiven/aiven.go +++ b/pkg/detectors/aiven/aiven.go @@ -22,7 +22,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aiven"}) + `\b([a-zA-Z0-9/+=]{372})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aiven"}) + `?\b([a-zA-Z0-9/+=]{350,380})`) ) // Keywords are used for efficiently pre-filtering chunks. From 9a925db375cdf5b05bd4d6580fa04b918a31c2bc Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Wed, 14 Feb 2024 15:33:53 -0600 Subject: [PATCH 09/10] no boundary for gengo --- pkg/detectors/gengo/gengo.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/detectors/gengo/gengo.go b/pkg/detectors/gengo/gengo.go index 40ddefdc010a..841122fbca65 100644 --- a/pkg/detectors/gengo/gengo.go +++ b/pkg/detectors/gengo/gengo.go @@ -30,8 +30,8 @@ var ( // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. // Removed bounds since there are some cases where the start and end of the token is a special character - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64})\b`) - secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64})`) + secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64})`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -43,6 +43,7 @@ func (s Scanner) Keywords() []string { // FromData will find and optionally verify Gengo secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) + fmt.Println(keyPat.String()) matches := keyPat.FindAllStringSubmatch(dataStr, -1) secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1) From 0b1c277683f5975236132ab21af24882913e7216 Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Wed, 14 Feb 2024 15:42:38 -0600 Subject: [PATCH 10/10] appointedd boundary --- pkg/detectors/appointedd/appointedd.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/detectors/appointedd/appointedd.go b/pkg/detectors/appointedd/appointedd.go index 1997d4797034..cad05aa559e8 100644 --- a/pkg/detectors/appointedd/appointedd.go +++ b/pkg/detectors/appointedd/appointedd.go @@ -2,6 +2,7 @@ package appointedd import ( "context" + "fmt" "io" "net/http" "strings" @@ -22,7 +23,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appointedd"}) + `\b([a-zA-Z0-9=+]{87}=)`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"appointedd"}) + `\b([a-zA-Z0-9=+]{88)`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -34,6 +35,8 @@ func (s Scanner) Keywords() []string { // FromData will find and optionally verify appointedd secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) + fmt.Println(keyPat.String()) + fmt.Println("data", dataStr) matches := keyPat.FindAllStringSubmatch(dataStr, -1) @@ -42,6 +45,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } resMatch := strings.TrimSpace(match[1]) + fmt.Println(resMatch) s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Appointedd,