From d6edbe8ae35d4fb4a1a3b2f198e3a890597fe2db Mon Sep 17 00:00:00 2001 From: Richard Gomez Date: Sun, 2 Jun 2024 11:01:48 -0400 Subject: [PATCH] feat: add flag to disable ssl verify --- main.go | 11 +++- pkg/common/http.go | 37 ++++++++----- pkg/detectors/aha/aha.go | 15 ++---- pkg/detectors/apiflash/apiflash.go | 8 +-- pkg/detectors/artifactory/artifactory.go | 17 ++---- .../auth0managementapitoken.go | 8 +-- pkg/detectors/azure_batch/azurebatch.go | 8 ++- pkg/detectors/azure_storage/storage.go | 9 ++-- .../azuresearchquerykey.go | 8 ++- pkg/detectors/caspio/caspio.go | 11 ++-- .../databrickstoken/databrickstoken.go | 9 ++-- pkg/detectors/deputy/deputy.go | 9 ++-- .../discordwebhook/discordwebhook.go | 12 +++-- pkg/detectors/fibery/fibery.go | 12 ++--- pkg/detectors/freshbooks/freshbooks.go | 8 ++- .../grafanaserviceaccount.go | 8 ++- pkg/detectors/http.go | 54 +++++++++++++------ pkg/detectors/invoiceocean/invoiceocean.go | 11 ++-- pkg/detectors/jdbc/mysql.go | 12 ++++- pkg/detectors/jdbc/postgres.go | 5 +- pkg/detectors/jdbc/sqlserver.go | 3 +- pkg/detectors/jiratoken/v1/jiratoken.go | 15 ++---- pkg/detectors/jiratoken/v2/jiratoken_v2.go | 15 ++---- pkg/detectors/kanban/kanban.go | 11 ++-- pkg/detectors/kanbantool/kanbantool.go | 9 ++-- pkg/detectors/liveagent/liveagent.go | 11 ++-- pkg/detectors/loggly/loggly.go | 8 ++- pkg/detectors/metabase/metabase.go | 11 ++-- .../microsoftteamswebhook.go | 9 ++-- pkg/detectors/mite/mite.go | 9 ++-- pkg/detectors/mongodb/mongodb.go | 6 +++ pkg/detectors/okta/okta.go | 8 ++- pkg/detectors/openvpn/openvpn.go | 9 ++-- .../planviewleankit/planviewleankit.go | 11 ++-- pkg/detectors/portainer/portainer.go | 8 ++- .../portainertoken/portainertoken.go | 8 ++- pkg/detectors/repairshopr/repairshopr.go | 9 ++-- pkg/detectors/salesforce/salesforce.go | 8 ++- pkg/detectors/salesmate/salesmate.go | 11 ++-- pkg/detectors/shopify/shopify.go | 9 ++-- pkg/detectors/signalwire/signalwire.go | 9 ++-- pkg/detectors/slackwebhook/slackwebhook.go | 9 ++-- pkg/detectors/sugester/sugester.go | 9 ++-- pkg/detectors/thinkific/thinkific.go | 9 ++-- pkg/detectors/tineswebhook/tineswebhook.go | 12 +++-- pkg/detectors/uri/uri.go | 6 +-- pkg/detectors/zendeskapi/zendeskapi.go | 11 ++-- pkg/detectors/zulipchat/zulipchat.go | 9 ++-- pkg/feature/feature.go | 2 + 49 files changed, 301 insertions(+), 235 deletions(-) diff --git a/main.go b/main.go index 9f7a66597382d..3bc29057cdfe6 100644 --- a/main.go +++ b/main.go @@ -20,10 +20,11 @@ import ( "github.com/go-logr/logr" "github.com/jpillora/overseer" "github.com/mattn/go-isatty" + "go.uber.org/automaxprocs/maxprocs" + "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache" - "go.uber.org/automaxprocs/maxprocs" "github.com/trufflesecurity/trufflehog/v3/pkg/analyzer" "github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp" @@ -78,6 +79,7 @@ var ( includeDetectors = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String() excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String() jobReportFile = cli.Flag("output-report", "Write a scan report to the provided path.").Hidden().OpenFile(os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + SslVerify = cli.Flag("ssl-verify", "Whether to verify the SSL certificates when making requests.").Default("true").Bool() noVerificationCache = cli.Flag("no-verification-cache", "Disable verification caching").Bool() @@ -283,7 +285,7 @@ func init() { cmd = kingpin.MustParse(cli.Parse(os.Args[1:])) - // Configure logging. + // Configure log level. switch { case *trace: log.SetLevel(5) @@ -304,6 +306,11 @@ func init() { log.SetLevel(l) } } + + // Disable certificate validation, if specified. + if !*SslVerify { + feature.NoVerifySsl.Store(true) + } } func main() { diff --git a/pkg/common/http.go b/pkg/common/http.go index 782ac1747c7b6..4630e8afa7ef1 100644 --- a/pkg/common/http.go +++ b/pkg/common/http.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hashicorp/go-retryablehttp" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" ) @@ -204,23 +205,31 @@ func RetryableHTTPClientTimeout(timeOutSeconds int64, opts ...ClientOption) *htt const DefaultResponseTimeout = 5 * time.Second -var saneTransport = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 2 * time.Second, - KeepAlive: 5 * time.Second, - }).DialContext, - MaxIdleConns: 5, - IdleConnTimeout: 5 * time.Second, - TLSHandshakeTimeout: 3 * time.Second, - ExpectContinueTimeout: 1 * time.Second, +func saneTransport() *http.Transport { + t := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 2 * time.Second, + KeepAlive: 5 * time.Second, + }).DialContext, + MaxIdleConns: 5, + IdleConnTimeout: 5 * time.Second, + TLSHandshakeTimeout: 3 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + + // Disable TLS certificate validation. + if feature.NoVerifySsl.Load() { + t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + return t } func SaneHttpClient() *http.Client { - httpClient := &http.Client{} - httpClient.Timeout = DefaultResponseTimeout - httpClient.Transport = NewCustomTransport(saneTransport) - return httpClient + client := &http.Client{} + client.Timeout = DefaultResponseTimeout + client.Transport = NewCustomTransport(saneTransport()) + return client } // SaneHttpClientTimeOut adds a custom timeout for some scanners diff --git a/pkg/detectors/aha/aha.go b/pkg/detectors/aha/aha.go index 15c4734d1b4f2..0fef26acc75eb 100644 --- a/pkg/detectors/aha/aha.go +++ b/pkg/detectors/aha/aha.go @@ -21,8 +21,6 @@ var ( // Ensure the Scanner satisfies the interface at compile time. _ detectors.Detector = (*Scanner)(nil) - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aha"}) + `\b([0-9a-f]{64})\b`) URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.aha\.io)`) @@ -34,13 +32,6 @@ func (s Scanner) Keywords() []string { return []string{"aha.io"} } -func (s Scanner) getClient() *http.Client { - if s.client != nil { - return s.client - } - return defaultClient -} - // FromData will find and optionally verify Aha secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -67,8 +58,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.getClient() - isVerified, verificationErr := verifyAha(ctx, client, resMatch, resURLMatch) + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + isVerified, verificationErr := verifyAha(ctx, s.client, resMatch, resURLMatch) s1.Verified = isVerified s1.SetVerificationError(verificationErr, resMatch) } diff --git a/pkg/detectors/apiflash/apiflash.go b/pkg/detectors/apiflash/apiflash.go index d4ef185d213a9..4f88e22e1e72b 100644 --- a/pkg/detectors/apiflash/apiflash.go +++ b/pkg/detectors/apiflash/apiflash.go @@ -14,6 +14,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -21,8 +22,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apiflash"}) + `\b([a-z0-9]{32})\b`) @@ -51,7 +50,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - isVerified, verificationErr := verifyAPIFlash(ctx, client, key) + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + isVerified, verificationErr := verifyAPIFlash(ctx, s.client, key) s1.Verified = isVerified s1.SetVerificationError(verificationErr, key) } diff --git a/pkg/detectors/artifactory/artifactory.go b/pkg/detectors/artifactory/artifactory.go index 92fe66e98f5b8..15d886a3c2af9 100644 --- a/pkg/detectors/artifactory/artifactory.go +++ b/pkg/detectors/artifactory/artifactory.go @@ -23,8 +23,6 @@ var ( _ detectors.Detector = (*Scanner)(nil) _ detectors.EndpointCustomizer = (*Scanner)(nil) - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73}|\b[a-zA-Z0-9]{64})`) URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.jfrog\.io)`) @@ -36,13 +34,6 @@ func (s Scanner) Keywords() []string { return []string{"artifactory"} } -func (s Scanner) getClient() *http.Client { - if s.client != nil { - return s.client - } - return defaultClient -} - // FromData will find and optionally verify Artifactory secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -64,8 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } resMatch := strings.TrimSpace(match[1]) - client := s.getClient() - for _, URL := range s.Endpoints(resURLMatch) { s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, @@ -74,7 +63,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - isVerified, verificationErr := verifyArtifactory(ctx, client, URL, resMatch) + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + + isVerified, verificationErr := verifyArtifactory(ctx, s.client, URL, resMatch) s1.Verified = isVerified s1.SetVerificationError(verificationErr, resMatch) } diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go index 95ccaf0e6f5c9..073f45ca2d3fa 100644 --- a/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go +++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go @@ -13,6 +13,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -21,8 +22,6 @@ var _ detectors.Detector = (*Scanner)(nil) var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithLocalAddresses - // long jwt token but note this is default 8640000 seconds = 24 hours but could be set to maximum 2592000 seconds = 720 hours = 30 days // at https://manage.auth0.com/dashboard/us/dev-63memjo3/apis/management/explorer managementAPITokenPat = regexp.MustCompile(`\b(ey[a-zA-Z0-9._-]+)\b`) @@ -67,6 +66,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithLocalAddresses() + } /* curl -H "Authorization: Bearer $token" https://domain/api/v2/users */ @@ -76,7 +78,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", managementAPITokenRes)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/azure_batch/azurebatch.go b/pkg/detectors/azure_batch/azurebatch.go index 8165dfb44892e..e4eb9427da189 100644 --- a/pkg/detectors/azure_batch/azurebatch.go +++ b/pkg/detectors/azure_batch/azurebatch.go @@ -26,7 +26,6 @@ var _ detectors.Detector = (*Scanner)(nil) var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. urlPat = regexp.MustCompile(`https://(.{1,50})\.(.{1,50})\.batch\.azure\.com`) secretPat = regexp.MustCompile(`[A-Za-z0-9+/=]{88}`) @@ -61,9 +60,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } url := fmt.Sprintf("%s/applications?api-version=2020-09-01.12.0", endpoint) date := time.Now().UTC().Format(http.TimeFormat) @@ -84,7 +82,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("SharedKey %s:%s", accountName, signature)) req.Header.Set("Date", date) - resp, err := client.Do(req) + resp, err := s.client.Do(req) if err != nil { continue } diff --git a/pkg/detectors/azure_storage/storage.go b/pkg/detectors/azure_storage/storage.go index c48496a816106..befc853a33280 100644 --- a/pkg/detectors/azure_storage/storage.go +++ b/pkg/detectors/azure_storage/storage.go @@ -26,8 +26,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses - namePat = regexp.MustCompile(`(?i:Account[_.-]?Name|Storage[_.-]?(?:Account|Name))(?:.|\s){0,20}?\b([a-z0-9]{3,24})\b|([a-z0-9]{3,24})(?i:\.blob\.core\.windows\.net)`) // Names can only be lowercase alphanumeric. keyPat = regexp.MustCompile(`(?i:(?:Access|Account|Storage)[_.-]?Key)(?:.|\s){0,25}?([a-zA-Z0-9+\/-]{86,88}={0,2})`) @@ -105,12 +103,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } - isVerified, verificationErr := s.verifyMatch(ctx, client, name, key, s1.ExtraData) + isVerified, verificationErr := s.verifyMatch(ctx, s.client, name, key, s1.ExtraData) s1.Verified = isVerified s1.SetVerificationError(verificationErr, key) } diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go index b685a56b177aa..5c7bb08f09ec0 100644 --- a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go +++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go @@ -21,7 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z]{52})\b`) urlPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `https:\/\/([0-9a-z]{5,40})\.search\.windows\.net\/indexes\/([0-9a-z]{5,40})\b`) @@ -56,9 +55,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result RawV2: []byte(resMatch + resUrlMatch), } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } req, err := http.NewRequestWithContext(ctx, "GET", resUrlMatch+"/docs/$count?api-version=2023-10-01-Preview", nil) if err != nil { @@ -66,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } req.Header.Add("api-key", resMatch) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/caspio/caspio.go b/pkg/detectors/caspio/caspio.go index 567ce64a394cb..c7dda6d8cd9ca 100644 --- a/pkg/detectors/caspio/caspio.go +++ b/pkg/detectors/caspio/caspio.go @@ -12,7 +12,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -20,8 +21,6 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"caspio"}) + `\b([a-z0-9]{50})\b`) idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"caspio"}) + `\b([a-z0-9]{50})\b`) @@ -69,13 +68,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + payload := strings.NewReader(fmt.Sprintf(`grant_type=client_credentials&client_id=%s&client_secret=%s`, resIdMatch, resMatch)) req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://%s.caspio.com/oauth/token", resDomainMatch), payload) if err != nil { continue } req.Header.Add("Content-Type", "text/plain") - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/databrickstoken/databrickstoken.go b/pkg/detectors/databrickstoken/databrickstoken.go index 246921b105d97..393a12241afad 100644 --- a/pkg/detectors/databrickstoken/databrickstoken.go +++ b/pkg/detectors/databrickstoken/databrickstoken.go @@ -21,8 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. domain = regexp.MustCompile(`\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(cloud\.databricks\.com|gcp\.databricks\.com|azuredatabricks\.net))\b`) keyPat = regexp.MustCompile(`\b(dapi[0-9a-f]{32}(-\d)?)\b`) @@ -54,16 +52,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomainMatch+"/api/2.0/clusters/list", nil) if err != nil { continue } req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/deputy/deputy.go b/pkg/detectors/deputy/deputy.go index d162d705c03d1..1830732b85840 100644 --- a/pkg/detectors/deputy/deputy.go +++ b/pkg/detectors/deputy/deputy.go @@ -13,6 +13,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -20,8 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"deputy"}) + `\b([0-9a-z]{32})\b`) urlPat = regexp.MustCompile(`\b([0-9a-z]{1,}\.as\.deputy\.com)\b`) @@ -58,13 +57,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/api/v1/me", resURL), nil) if err != nil { continue } req.Header.Add("Content-Type", "application/json") req.Header.Add("Authorization", fmt.Sprintf("OAuth %s", resMatch)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/discordwebhook/discordwebhook.go b/pkg/detectors/discordwebhook/discordwebhook.go index 8772199ad43ff..959afc09fd79d 100644 --- a/pkg/detectors/discordwebhook/discordwebhook.go +++ b/pkg/detectors/discordwebhook/discordwebhook.go @@ -11,14 +11,14 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`(https:\/\/discord\.com\/api\/webhooks\/[0-9]{18}\/[0-9a-zA-Z-]{68})`) ) @@ -44,11 +44,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, resMatch, nil) if err != nil { continue } - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/fibery/fibery.go b/pkg/detectors/fibery/fibery.go index 8fdc1093b8b60..8b0f8981eb1dc 100644 --- a/pkg/detectors/fibery/fibery.go +++ b/pkg/detectors/fibery/fibery.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "strings" - "time" regexp "github.com/wasilibs/go-re2" @@ -14,6 +13,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -21,8 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"fibery"}) + `\b([0-9a-f]{8}.[0-9a-f]{35})\b`) domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"fibery", "domain"}) + `\b([0-9A-Za-z]{2,40})\b`) @@ -65,15 +63,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - timeout := 10 * time.Second - client.Timeout = timeout + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://%s.fibery.io/api/commands", resDomainMatch), nil) if err != nil { continue } req.Header.Add("Content-Type", "application/json") req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks.go index a49e87f112291..fc77bf89e87ab 100644 --- a/pkg/detectors/freshbooks/freshbooks.go +++ b/pkg/detectors/freshbooks/freshbooks.go @@ -14,6 +14,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -21,7 +22,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b([0-9a-z]{64})\b`) // TODO: this domain pattern is too restrictive @@ -58,11 +58,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf(`https://auth.freshbooks.com/oauth/authorize?client_id=%s&redirect_uri=%s&response_type=code`, resMatch, resURI), nil) if err != nil { continue } - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() bodyBytes, err := io.ReadAll(res.Body) diff --git a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go index 225582b2c960e..0ad848b3c283b 100644 --- a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go +++ b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go @@ -21,7 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b(glsa_[0-9a-zA-Z_]{41})\b`) domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.grafana\.net)\b`) @@ -59,16 +58,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/api/access-control/user/permissions", nil) if err != nil { continue } req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", key)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/http.go b/pkg/detectors/http.go index 9f556ec2c281c..4d9addf587b15 100644 --- a/pkg/detectors/http.go +++ b/pkg/detectors/http.go @@ -2,16 +2,22 @@ package detectors import ( "context" + "crypto/tls" "errors" "net" "net/http" + "sync" "time" "github.com/trufflesecurity/trufflehog/v3/pkg/feature" ) -var DetectorHttpClientWithNoLocalAddresses *http.Client -var DetectorHttpClientWithLocalAddresses *http.Client +var ( + clientWithNoLocalAddresses *http.Client + noLocalOnce sync.Once + clientWithLocalAddresses *http.Client + localOnce sync.Once +) const DefaultResponseTimeout = 5 * time.Second @@ -22,18 +28,31 @@ func userAgent() string { return "TruffleHog" } -func init() { - DetectorHttpClientWithLocalAddresses = NewDetectorHttpClient( - WithTransport(NewDetectorTransport(nil)), - WithTimeout(DefaultResponseTimeout), - WithNoFollowRedirects(), - ) - DetectorHttpClientWithNoLocalAddresses = NewDetectorHttpClient( - WithTransport(NewDetectorTransport(nil)), - WithTimeout(DefaultResponseTimeout), - WithNoFollowRedirects(), - WithNoLocalIP(), - ) +func GetHttpClientWithNoLocalAddresses() *http.Client { + if clientWithNoLocalAddresses == nil { + noLocalOnce.Do(func() { + clientWithNoLocalAddresses = NewDetectorHttpClient( + WithTransport(NewDetectorTransport(nil)), + WithTimeout(DefaultResponseTimeout), + WithNoFollowRedirects(), + WithNoLocalIP(), + ) + }) + } + return clientWithNoLocalAddresses +} + +func GetHttpClientWithLocalAddresses() *http.Client { + if clientWithLocalAddresses == nil { + localOnce.Do(func() { + clientWithLocalAddresses = NewDetectorHttpClient( + WithTransport(NewDetectorTransport(nil)), + WithTimeout(DefaultResponseTimeout), + WithNoFollowRedirects(), + ) + }) + } + return clientWithLocalAddresses } // ClientOption defines a function type that modifies an http.Client. @@ -64,7 +83,7 @@ var defaultDialer = &net.Dialer{ func NewDetectorTransport(T http.RoundTripper) http.RoundTripper { if T == nil { - T = &http.Transport{ + t := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: defaultDialer.DialContext, MaxIdleConns: 100, @@ -73,6 +92,11 @@ func NewDetectorTransport(T http.RoundTripper) http.RoundTripper { TLSHandshakeTimeout: 3 * time.Second, ExpectContinueTimeout: 1 * time.Second, } + // Disable TLS certificate validation. + if feature.NoVerifySsl.Load() { + t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + T = t } return &detectorTransport{T: T} } diff --git a/pkg/detectors/invoiceocean/invoiceocean.go b/pkg/detectors/invoiceocean/invoiceocean.go index 55791c273973e..2920d9169303d 100644 --- a/pkg/detectors/invoiceocean/invoiceocean.go +++ b/pkg/detectors/invoiceocean/invoiceocean.go @@ -12,7 +12,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -20,8 +21,6 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"invoiceocean"}) + `\b([0-9A-Za-z]{20})\b`) urlPat = regexp.MustCompile(`\b([0-9a-z]{1,}\.invoiceocean\.com)\b`) @@ -58,12 +57,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/invoices.json?period=this_month&api_token=%s", resURL, resMatch), nil) if err != nil { continue } req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/jdbc/mysql.go b/pkg/detectors/jdbc/mysql.go index 60572016078a5..f0f76e7a6fdb6 100644 --- a/pkg/detectors/jdbc/mysql.go +++ b/pkg/detectors/jdbc/mysql.go @@ -8,6 +8,7 @@ import ( "strings" logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/go-sql-driver/mysql" ) @@ -51,12 +52,19 @@ func isMySQLErrorDeterminate(err error) bool { return false } +const defaultParams = "timeout=5s" + func parseMySQL(_ logContext.Context, subname string) (jdbc, error) { // expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]] if !strings.HasPrefix(subname, "//") { return nil, errors.New("expected host to start with //") } + params := defaultParams + if feature.NoVerifySsl.Load() { + params = defaultParams + "&tls=skip-verify" + } + // need for hostnames that have tcp(host:port) format required by this database driver cfg, err := mysql.ParseDSN(strings.TrimPrefix(subname, "//")) if err == nil { @@ -64,7 +72,7 @@ func parseMySQL(_ logContext.Context, subname string) (jdbc, error) { conn: subname[2:], userPass: cfg.User + ":" + cfg.Passwd, host: fmt.Sprintf("tcp(%s)", cfg.Addr), - params: "timeout=5s", + params: params, }, nil } @@ -97,7 +105,7 @@ func parseMySQL(_ logContext.Context, subname string) (jdbc, error) { conn: subname[2:], userPass: userAndPass, host: fmt.Sprintf("tcp(%s)", u.Host), - params: "timeout=5s", + params: params, }, nil } diff --git a/pkg/detectors/jdbc/postgres.go b/pkg/detectors/jdbc/postgres.go index 9e262e85e316b..8d26859dd9783 100644 --- a/pkg/detectors/jdbc/postgres.go +++ b/pkg/detectors/jdbc/postgres.go @@ -8,6 +8,7 @@ import ( "strings" logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/lib/pq" ) @@ -90,7 +91,9 @@ func parsePostgres(_ logContext.Context, subname string) (jdbc, error) { } } - if v := u.Query()["sslmode"]; len(v) > 0 { + if feature.NoVerifySsl.Load() { + params["sslmode"] = "disable" + } else if v := u.Query()["sslmode"]; len(v) > 0 { switch v[0] { // https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION case "disable", "allow", "prefer", diff --git a/pkg/detectors/jdbc/sqlserver.go b/pkg/detectors/jdbc/sqlserver.go index 4178b99f1b2d5..c907e856b2d85 100644 --- a/pkg/detectors/jdbc/sqlserver.go +++ b/pkg/detectors/jdbc/sqlserver.go @@ -8,6 +8,7 @@ import ( "strings" logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" mssql "github.com/microsoft/go-mssqldb" ) @@ -69,7 +70,7 @@ func parseSqlServer(ctx logContext.Context, subname string) (jdbc, error) { } } - urlStr := fmt.Sprintf("sqlserver://sa:%s@%s:%s?database=master&connection+timeout=5", password, host, port) + urlStr := fmt.Sprintf("sqlserver://sa:%s@%s:%s?database=master&connection+timeout=5&TrustServerCertificate=%t", password, host, port, !feature.NoVerifySsl.Load()) jdbcUrl, err := url.Parse(urlStr) if err != nil { ctx.Logger().WithName("jdbc"). diff --git a/pkg/detectors/jiratoken/v1/jiratoken.go b/pkg/detectors/jiratoken/v1/jiratoken.go index 189f12d6e5f4d..7fba83e4c3acb 100644 --- a/pkg/detectors/jiratoken/v1/jiratoken.go +++ b/pkg/detectors/jiratoken/v1/jiratoken.go @@ -25,8 +25,6 @@ var _ detectors.Versioner = (*Scanner)(nil) func (Scanner) Version() int { return 1 } var ( - defaultClient = detectors.DetectorHttpClientWithLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`) domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{5,24}\.[a-zA-Z-0-9]{3,16}\.[a-zA-Z-0-9]{3,16})\b`) @@ -80,8 +78,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.getClient() - isVerified, verificationErr := verifyJiratoken(ctx, client, resEmail, resDomain, resToken) + if s.client == nil { + s.client = detectors.GetHttpClientWithLocalAddresses() + } + isVerified, verificationErr := verifyJiratoken(ctx, s.client, resEmail, resDomain, resToken) s1.Verified = isVerified s1.SetVerificationError(verificationErr, resToken) } @@ -94,13 +94,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } -func (s Scanner) getClient() *http.Client { - if s.client != nil { - return s.client - } - return defaultClient -} - func verifyJiratoken(ctx context.Context, client *http.Client, email, domain, token string) (bool, error) { data := fmt.Sprintf("%s:%s", email, token) sEnc := b64.StdEncoding.EncodeToString([]byte(data)) diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2.go b/pkg/detectors/jiratoken/v2/jiratoken_v2.go index e4b082fd8eb1b..615d2bc0d5f90 100644 --- a/pkg/detectors/jiratoken/v2/jiratoken_v2.go +++ b/pkg/detectors/jiratoken/v2/jiratoken_v2.go @@ -25,8 +25,6 @@ var _ detectors.Versioner = (*Scanner)(nil) func (Scanner) Version() int { return 2 } var ( - defaultClient = detectors.DetectorHttpClientWithLocalAddresses - // https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/ // Tokens created after Jan 18 2023 use a variable length tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})\b`) @@ -81,8 +79,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.getClient() - isVerified, verificationErr := verifyJiratoken(ctx, client, resEmail, resDomain, resToken) + if s.client == nil { + s.client = detectors.GetHttpClientWithLocalAddresses() + } + isVerified, verificationErr := verifyJiratoken(ctx, s.client, resEmail, resDomain, resToken) s1.Verified = isVerified s1.SetVerificationError(verificationErr, resToken) } @@ -97,13 +97,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } -func (s Scanner) getClient() *http.Client { - if s.client != nil { - return s.client - } - return defaultClient -} - func verifyJiratoken(ctx context.Context, client *http.Client, email, domain, token string) (bool, error) { data := fmt.Sprintf("%s:%s", email, token) sEnc := b64.StdEncoding.EncodeToString([]byte(data)) diff --git a/pkg/detectors/kanban/kanban.go b/pkg/detectors/kanban/kanban.go index ba4f1cf2c49cf..62635f4b79393 100644 --- a/pkg/detectors/kanban/kanban.go +++ b/pkg/detectors/kanban/kanban.go @@ -12,7 +12,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -20,8 +21,6 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanban"}) + `\b([0-9A-Z]{12})\b`) urlPat = regexp.MustCompile(`\b([0-9a-z]{1,}\.kanbantool\.com)\b`) @@ -57,13 +56,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/api/v3/users/current.json", resURL), nil) if err != nil { continue } req.Header.Add("Accept", "application/vnd.kanban+json; version=3") req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/kanbantool/kanbantool.go b/pkg/detectors/kanbantool/kanbantool.go index 3715d82362ec0..215741c751d71 100644 --- a/pkg/detectors/kanbantool/kanbantool.go +++ b/pkg/detectors/kanbantool/kanbantool.go @@ -13,6 +13,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -20,8 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanbantool"}) + `\b([0-9A-Z]{12})\b`) domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanbantool"}) + `\b([a-z0-9A-Z]{2,22})\b`) @@ -56,13 +55,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result Raw: []byte(resMatch), } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + url := fmt.Sprintf("https://%s.kanbantool.com/api/v3/users/current.json", resIdMatch) req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { continue } req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/liveagent/liveagent.go b/pkg/detectors/liveagent/liveagent.go index 391dcdcfd0115..6968d20a3e33e 100644 --- a/pkg/detectors/liveagent/liveagent.go +++ b/pkg/detectors/liveagent/liveagent.go @@ -12,7 +12,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -20,8 +21,6 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. domainPat = regexp.MustCompile(`\b(https?://[A-Za-z0-9-]+\.ladesk\.com)\b`) keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"liveagent", "apikey"}) + `\b([a-zA-Z0-9]{32})\b`) @@ -65,12 +64,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "GET", domainRes+"/api/v3/agents", nil) if err != nil { continue } req.Header.Add("apikey", resMatch) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/loggly/loggly.go b/pkg/detectors/loggly/loggly.go index eb7583a7bb2cc..018b9bb9d8032 100644 --- a/pkg/detectors/loggly/loggly.go +++ b/pkg/detectors/loggly/loggly.go @@ -21,7 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.loggly\.com)\b`) keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loggly"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`) @@ -60,16 +59,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/apiv2/customer", nil) if err != nil { continue } req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", key)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/metabase/metabase.go b/pkg/detectors/metabase/metabase.go index ebd7647d782eb..9c3a4e484dfc5 100644 --- a/pkg/detectors/metabase/metabase.go +++ b/pkg/detectors/metabase/metabase.go @@ -13,7 +13,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -21,8 +22,6 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"metabase"}) + `\b([a-zA-Z0-9-]{36})\b`) @@ -67,13 +66,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithLocalAddresses() + } + u.Path = "/api/user/current" req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { continue } req.Header.Add("X-Metabase-Session", resMatch) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() body, err := io.ReadAll(res.Body) diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go index c58a59205dc30..1103f71dd1841 100644 --- a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go +++ b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go @@ -22,8 +22,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`) ) @@ -55,12 +53,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } - isVerified, verificationErr := verifyWebhook(ctx, client, resMatch) + isVerified, verificationErr := verifyWebhook(ctx, s.client, resMatch) s1.Verified = isVerified s1.SetVerificationError(verificationErr, resMatch) } diff --git a/pkg/detectors/mite/mite.go b/pkg/detectors/mite/mite.go index efb144660b2f0..67fa949febdfc 100644 --- a/pkg/detectors/mite/mite.go +++ b/pkg/detectors/mite/mite.go @@ -13,6 +13,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -20,8 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mite"}) + `\b([0-9a-z]{16})\b`) urlPat = regexp.MustCompile(`\b([0-9a-z-]{1,}.mite.yo.lk)\b`) @@ -57,13 +56,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/account.json", resURL), nil) if err != nil { continue } req.Header.Add("Content-Type", "application/json") req.Header.Add("X-MiteApiKey", resMatch) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.go index 290e8c373b541..70744bf2c57f6 100644 --- a/pkg/detectors/mongodb/mongodb.go +++ b/pkg/detectors/mongodb/mongodb.go @@ -9,6 +9,7 @@ import ( logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" regexp "github.com/wasilibs/go-re2" @@ -131,6 +132,11 @@ func verifyUri(ctx context.Context, connStr string, timeout time.Duration) (bool return false, err } + // Disable TLS certificate validation. + if feature.NoVerifySsl.Load() { + clientOptions.TLSConfig.InsecureSkipVerify = true + } + client, err := mongo.Connect(ctx, clientOptions) if err != nil { return false, err diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.go index f72792b6b8480..4f823654876d2 100644 --- a/pkg/detectors/okta/okta.go +++ b/pkg/detectors/okta/okta.go @@ -13,7 +13,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -47,6 +48,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } // curl -v -X GET \ // -H "Accept: application/json" \ // -H "Content-Type: application/json" \ @@ -63,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", fmt.Sprintf("SSWS %s", token)) - resp, err := detectors.DetectorHttpClientWithNoLocalAddresses.Do(req) + resp, err := s.client.Do(req) if err != nil { continue } diff --git a/pkg/detectors/openvpn/openvpn.go b/pkg/detectors/openvpn/openvpn.go index 21900342097ca..5bceb0ebd3a57 100644 --- a/pkg/detectors/openvpn/openvpn.go +++ b/pkg/detectors/openvpn/openvpn.go @@ -22,8 +22,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses - clientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openvpn"}) + `\b([A-Za-z0-9-]{3,40}\.[A-Za-z0-9-]{3,40})\b`) clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`) domainPat = regexp.MustCompile(`\b(https?://[A-Za-z0-9-]+\.api\.openvpn\.com)\b`) @@ -57,9 +55,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } payload := strings.NewReader("grant_type=client_credentials") @@ -75,7 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) req.Header.Add("content-type", "application/x-www-form-urlencoded") - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/planviewleankit/planviewleankit.go b/pkg/detectors/planviewleankit/planviewleankit.go index 2d9466c388515..b9e80075e349f 100644 --- a/pkg/detectors/planviewleankit/planviewleankit.go +++ b/pkg/detectors/planviewleankit/planviewleankit.go @@ -12,7 +12,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -20,8 +21,6 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"planviewleankit", "planview"}) + `\b([0-9a-f]{128})\b`) subDomainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"planviewleankit", "planview"}) + `(?:subdomain).\b([a-zA-Z][a-zA-Z0-9.-]{1,23}[a-zA-Z0-9])\b`) @@ -58,12 +57,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.leankit.com/io/account", resSubdomainMatch), nil) if err != nil { continue } req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/portainer/portainer.go b/pkg/detectors/portainer/portainer.go index 8cb20a5f7166a..5cd523306eec3 100644 --- a/pkg/detectors/portainer/portainer.go +++ b/pkg/detectors/portainer/portainer.go @@ -21,7 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`) tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[0-9A-Za-z]{50,310}\.[0-9A-Z-a-z\-_]{43})\b`) @@ -63,16 +62,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithLocalAddresses() } req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) if err != nil { continue } req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/portainertoken/portainertoken.go b/pkg/detectors/portainertoken/portainertoken.go index 0f0c6fec3fad2..ebc9ce715cb88 100644 --- a/pkg/detectors/portainertoken/portainertoken.go +++ b/pkg/detectors/portainertoken/portainertoken.go @@ -21,7 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainertoken"}) + `\b(ptr_[A-Za-z0-9\/_\-+=]{20,60})`) endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`) @@ -64,9 +63,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithLocalAddresses() } req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) if err != nil { @@ -75,7 +73,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-API-Key", resMatch) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() diff --git a/pkg/detectors/repairshopr/repairshopr.go b/pkg/detectors/repairshopr/repairshopr.go index 7d1f8987de77a..fa04e94462601 100644 --- a/pkg/detectors/repairshopr/repairshopr.go +++ b/pkg/detectors/repairshopr/repairshopr.go @@ -13,6 +13,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -20,8 +21,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"repairshopr"}) + `\b([a-zA-Z0-9-]{51})\b`) domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"repairshopr"}) + `\b([a-zA-Z0-9_.!+$#^*]{3,32})\b`) @@ -57,6 +56,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomainMatch+".repairshopr.com/api/v1/appointment_types", nil) if err != nil { continue @@ -64,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Accept", "application/vnd.sugester+json; version=3") req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/salesforce/salesforce.go b/pkg/detectors/salesforce/salesforce.go index 09b7ade923a68..52577886a9e77 100644 --- a/pkg/detectors/salesforce/salesforce.go +++ b/pkg/detectors/salesforce/salesforce.go @@ -26,7 +26,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. accessTokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9]{13}![a-zA-Z0-9_.]{96}\b`) instancePat = regexp.MustCompile(`\bhttps://[0-9a-zA-Z-\.]{1,100}\.my\.salesforce\.com\b`) @@ -65,9 +64,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } req, err := http.NewRequestWithContext(ctx, "GET", instanceMatch+"/services/data/v"+currentVersion+"/query?q=SELECT+name+from+Account", nil) @@ -76,7 +74,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } req.Header.Set("Authorization", "Bearer "+tokenMatch) - res, err := client.Do(req) + res, err := s.client.Do(req) if err != nil { // End execution, append Detector Result if request fails to prevent panic on response body checks diff --git a/pkg/detectors/salesmate/salesmate.go b/pkg/detectors/salesmate/salesmate.go index b7d2ac758373b..406f6ae7b1612 100644 --- a/pkg/detectors/salesmate/salesmate.go +++ b/pkg/detectors/salesmate/salesmate.go @@ -12,7 +12,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -20,8 +21,6 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"salesmate"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`) domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"salesmate"}) + `\b([a-z0-9A-Z]{3,22})\b`) @@ -56,6 +55,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + url := fmt.Sprintf("https://%s.salesmate.io/apis/v3/companies/1?trackingRecentSearch=true", resIdMatch) req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { @@ -64,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("Content-Type", "application/json") req.Header.Add("sessionToken", resMatch) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go index 08a29d9bbc928..d546ec65144e5 100644 --- a/pkg/detectors/shopify/shopify.go +++ b/pkg/detectors/shopify/shopify.go @@ -13,6 +13,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -21,8 +22,6 @@ var _ detectors.Detector = (*Scanner)(nil) var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b(shppa_|shpat_)([0-9A-Fa-f]{32})\b`) domainPat = regexp.MustCompile(`[a-zA-Z0-9-]+\.myshopify\.com`) @@ -54,12 +53,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/admin/oauth/access_scopes.json", nil) if err != nil { continue } req.Header.Add("X-Shopify-Access-Token", key) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { if res.StatusCode >= 200 && res.StatusCode < 300 { shopifyTokenAccessScopes := shopifyTokenAccessScopes{} diff --git a/pkg/detectors/signalwire/signalwire.go b/pkg/detectors/signalwire/signalwire.go index 6f13c554c324c..34e1727030e1b 100644 --- a/pkg/detectors/signalwire/signalwire.go +++ b/pkg/detectors/signalwire/signalwire.go @@ -14,6 +14,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -21,8 +22,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"signalwire"}) + `\b([0-9A-Za-z]{50})\b`) idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"signalwire"}) + `\b([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`) @@ -67,6 +66,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + data := fmt.Sprintf("%s:%s", resID, resMatch) sEnc := b64.StdEncoding.EncodeToString([]byte(data)) req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/api/laml/2010-04-01/Accounts", resURL), nil) @@ -75,7 +78,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } req.Header.Add("Content-Type", "application/json") req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/slackwebhook/slackwebhook.go b/pkg/detectors/slackwebhook/slackwebhook.go index f77551cf305b5..9b7863272e391 100644 --- a/pkg/detectors/slackwebhook/slackwebhook.go +++ b/pkg/detectors/slackwebhook/slackwebhook.go @@ -23,7 +23,6 @@ var _ detectors.Detector = (*Scanner)(nil) var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPats = map[string]*regexp.Regexp{ "Slack Service Web Hook": regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`), @@ -59,10 +58,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } // We don't want to actually send anything to webhooks we find. To verify them without spamming them, we @@ -73,7 +70,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() bodyBytes, err := io.ReadAll(res.Body) diff --git a/pkg/detectors/sugester/sugester.go b/pkg/detectors/sugester/sugester.go index e3288db8027d0..66a6ccfedd6b8 100644 --- a/pkg/detectors/sugester/sugester.go +++ b/pkg/detectors/sugester/sugester.go @@ -12,6 +12,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -19,8 +20,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sugester"}) + `\b([a-zA-Z0-9]{32})\b`) domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sugester"}) + `\b([a-zA-Z0-9_.!+$#^*%]{3,32})\b`) @@ -56,12 +55,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomainMatch+".sugester.com/app/clients.json?api_token="+resMatch, nil) if err != nil { continue } req.Header.Add("Accept", "application/vnd.sugester+json; version=3") - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/thinkific/thinkific.go b/pkg/detectors/thinkific/thinkific.go index 5cb43c985a481..87d956005c327 100644 --- a/pkg/detectors/thinkific/thinkific.go +++ b/pkg/detectors/thinkific/thinkific.go @@ -14,6 +14,7 @@ import ( ) type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -21,8 +22,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"thinkific"}) + `\b([0-9a-f]{32})\b`) domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"thinkific"}) + `\b([0-9A-Za-z]{4,40})\b`) @@ -58,6 +57,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + domainRes := fmt.Sprintf("%s-s-school", resDomainMatch) req, err := http.NewRequestWithContext(ctx, "GET", "https://api.thinkific.com/api/public/v1/collections", nil) if err != nil { @@ -66,7 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result req.Header.Add("X-Auth-API-Key", resMatch) req.Header.Add("X-Auth-Subdomain", domainRes) req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() bodyBytes, err := io.ReadAll(res.Body) diff --git a/pkg/detectors/tineswebhook/tineswebhook.go b/pkg/detectors/tineswebhook/tineswebhook.go index 5ce4bd3fa5011..ee3f51c827692 100644 --- a/pkg/detectors/tineswebhook/tineswebhook.go +++ b/pkg/detectors/tineswebhook/tineswebhook.go @@ -11,14 +11,14 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`(https://[\w-]+\.tines\.com/webhook/[a-z0-9]{32}/[a-z0-9]{32})`) ) @@ -47,12 +47,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + payload := strings.NewReader(``) req, err := http.NewRequestWithContext(ctx, "GET", resMatch, payload) if err != nil { continue } - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go index f97f3198ad1c7..3eece99f60684 100644 --- a/pkg/detectors/uri/uri.go +++ b/pkg/detectors/uri/uri.go @@ -24,9 +24,6 @@ var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil) var ( keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`) - - // TODO: make local addr opt-out - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses ) // Keywords are used for efficiently pre-filtering chunks. @@ -82,8 +79,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + // TODO: make local addr opt-out if s.client == nil { - s.client = defaultClient + s.client = detectors.GetHttpClientWithNoLocalAddresses() } isVerified, verificationError := verifyURL(ctx, s.client, parsedURL) s1.Verified = isVerified diff --git a/pkg/detectors/zendeskapi/zendeskapi.go b/pkg/detectors/zendeskapi/zendeskapi.go index 2962171099c64..e923c0cdf6e0f 100644 --- a/pkg/detectors/zendeskapi/zendeskapi.go +++ b/pkg/detectors/zendeskapi/zendeskapi.go @@ -13,7 +13,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { + client *http.Client detectors.DefaultMultiPartCredentialProvider } @@ -21,8 +22,6 @@ type Scanner struct{ var _ detectors.Detector = (*Scanner)(nil) var ( - client = detectors.DetectorHttpClientWithNoLocalAddresses - token = regexp.MustCompile(detectors.PrefixRegex([]string{"zendesk"}) + `([A-Za-z0-9_-]{40})`) email = regexp.MustCompile(`\b([a-zA-Z-0-9-]{5,16}\@[a-zA-Z-0-9]{4,16}\.[a-zA-Z-0-9]{3,6})\b`) domain = regexp.MustCompile(`\b([a-zA-Z-0-9]{3,16}\.zendesk\.com)\b`) @@ -67,6 +66,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() + } + data := fmt.Sprintf("%s/token:%s", resEmail, resMatch) sEnc := b64.StdEncoding.EncodeToString([]byte(data)) req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/api/v2/users.json", nil) @@ -74,7 +77,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) - res, err := client.Do(req) + res, err := s.client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/zulipchat/zulipchat.go b/pkg/detectors/zulipchat/zulipchat.go index 7388e96a951b3..58245a35bcacf 100644 --- a/pkg/detectors/zulipchat/zulipchat.go +++ b/pkg/detectors/zulipchat/zulipchat.go @@ -23,8 +23,6 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(common.BuildRegex(common.AlphaNumPattern, "", 32)) idPat = regexp.MustCompile(`\b([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})\b`) @@ -68,11 +66,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - client := s.client - if client == nil { - client = defaultClient + if s.client == nil { + s.client = detectors.GetHttpClientWithNoLocalAddresses() } - verified, verificationErr := verifyResult(ctx, client, domain, id, key) + verified, verificationErr := verifyResult(ctx, s.client, domain, id, key) s1.Verified = verified s1.SetVerificationError(verificationErr) } diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index db8036f5d3869..bc67bb1d08824 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -8,6 +8,8 @@ var ( SkipAdditionalRefs atomic.Bool EnableAPKHandler atomic.Bool UserAgentSuffix AtomicString + // NoVerifySsl instructs supported detectors to skip SSL/TLS certificate verification. + NoVerifySsl atomic.Bool ) type AtomicString struct {