diff --git a/pkg/detectors/abbysale/abbysale.go b/pkg/detectors/abbysale/abbysale.go index 7f816d40f203..0bea49261a01 100644 --- a/pkg/detectors/abbysale/abbysale.go +++ b/pkg/detectors/abbysale/abbysale.go @@ -2,22 +2,28 @@ package abbysale import ( "context" - regexp "github.com/wasilibs/go-re2" + "fmt" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +const abbysaleURL = "https://api.abyssale.com" var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"abbysale"}) + `\b([a-z0-9A-Z]{40})\b`) @@ -29,6 +35,13 @@ func (s Scanner) Keywords() []string { return []string{"abbysale"} } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify Abbysale secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -47,23 +60,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.abyssale.com/ready", nil) - if err != nil { - continue - } - req.Header.Add("x-api-key", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyAbbysale(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } results = append(results, s1) @@ -72,6 +77,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyAbbysale(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + // https://developers.abyssale.com/rest-api/authentication + req, err := http.NewRequestWithContext(ctx, http.MethodGet, abbysaleURL+"/ready", nil) + if err != nil { + return false, err + } + req.Header.Add("x-api-key", resMatch) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Abbysale } diff --git a/pkg/detectors/abbysale/abbysale_test.go b/pkg/detectors/abbysale/abbysale_test.go index d45feadd4db4..8f4744985aae 100644 --- a/pkg/detectors/abbysale/abbysale_test.go +++ b/pkg/detectors/abbysale/abbysale_test.go @@ -9,7 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -43,7 +44,7 @@ func TestAbbysale_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a abbysale secret %s within", secret)), + data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)), verify: true, }, want: []detectors.Result{ @@ -54,12 +55,48 @@ func TestAbbysale_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Abbysale, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Abbysale, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", inactiveSecret)), // the secret would satisfy the regex but not pass validation verify: true, }, want: []detectors.Result{ @@ -84,8 +121,7 @@ func TestAbbysale_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Abbysale.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -94,9 +130,21 @@ func TestAbbysale_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Abbysale.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/abstract/abstract.go b/pkg/detectors/abstract/abstract.go index 0005e6326ecd..b88883a7bff7 100644 --- a/pkg/detectors/abstract/abstract.go +++ b/pkg/detectors/abstract/abstract.go @@ -12,13 +12,17 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +const abstractURL = "https://exchange-rates.abstractapi.com" var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"abstract"}) + `\b([0-9a-z]{32})\b`) @@ -30,6 +34,13 @@ func (s Scanner) Keywords() []string { return []string{"abstract"} } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify Abstract secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -48,23 +59,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://exchange-rates.abstractapi.com/v1/live/?api_key=%s&base=USD", resMatch), nil) - if err != nil { - continue - } - req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyAbstract(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } results = append(results, s1) @@ -73,6 +76,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyAbstract(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + // https://docs.abstractapi.com/exchange-rates#response-and-error-codes + req, err := http.NewRequestWithContext(ctx, http.MethodGet, abstractURL+fmt.Sprintf("/v1/live/?api_key=%s&base=USD", resMatch), nil) + if err != nil { + return false, err + } + req.Header.Add("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + // https://docs.abstractapi.com/exchange-rates#response-and-error-codes + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Abstract } diff --git a/pkg/detectors/abstract/abstract_test.go b/pkg/detectors/abstract/abstract_test.go index 69f205061f14..2ed3939c9445 100644 --- a/pkg/detectors/abstract/abstract_test.go +++ b/pkg/detectors/abstract/abstract_test.go @@ -9,7 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -19,7 +20,7 @@ import ( func TestAbstract_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } @@ -43,7 +44,7 @@ func TestAbstract_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a abstract secret %s within", secret)), + data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)), verify: true, }, want: []detectors.Result{ @@ -54,12 +55,48 @@ func TestAbstract_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Abstract, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a abstract secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Abstract, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a abstract secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("You can find a abstract secret %s within but not valid", inactiveSecret)), verify: true, }, want: []detectors.Result{ @@ -84,8 +121,8 @@ func TestAbstract_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + time.Sleep(900 * time.Millisecond) // avoid rate limiting + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Abstract.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -94,9 +131,20 @@ func TestAbstract_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Abstract.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/abuseipdb/abuseipdb.go b/pkg/detectors/abuseipdb/abuseipdb.go index dfb40ec6ec88..83febcd1375d 100644 --- a/pkg/detectors/abuseipdb/abuseipdb.go +++ b/pkg/detectors/abuseipdb/abuseipdb.go @@ -1,25 +1,31 @@ package abuseipdb import ( + "bytes" "context" + "fmt" "io" "net/http" - // "log" - regexp "github.com/wasilibs/go-re2" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +const abuseipdbURL = "https://api.abuseipdb.com" var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"abuseipdb"}) + `\b([a-z0-9]{80})\b`) @@ -31,6 +37,13 @@ func (s Scanner) Keywords() []string { return []string{"abuseipdb"} } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify AbuseIPDB secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -49,41 +62,55 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.abuseipdb.com/api/v2/check?ipAddress=118.25.6.39", nil) - if err != nil { - continue - } - req.Header.Add("Key", resMatch) - res, err := client.Do(req) - if err == nil { - bodyBytes, err := io.ReadAll(res.Body) - if err == nil { - bodyString := string(bodyBytes) - validResponse := strings.Contains(bodyString, `ipAddress`) - // errCode := strings.Contains(bodyString, `AbuseIPDB APIv2 Server.`) - - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if validResponse { - s1.Verified = true - } else { - s1.Verified = false - } - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyAbuseIPDB(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } + results = append(results, s1) } return results, nil } +func verifyAbuseIPDB(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + // https://docs.abuseipdb.com/#check-endpoint + req, err := http.NewRequestWithContext(ctx, http.MethodGet, abuseipdbURL+"/api/v2/check?ipAddress=8.8.8.8", nil) + if err != nil { + return false, err + } + req.Header.Add("Key", resMatch) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusOK: + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, err + } + validResponse := bytes.Contains(bodyBytes, []byte("ipAddress")) + if validResponse { + return true, nil + } else { + return false, nil + } + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_AbuseIPDB } diff --git a/pkg/detectors/abuseipdb/abuseipdb_test.go b/pkg/detectors/abuseipdb/abuseipdb_test.go index 1f83936a6739..bd481f08059d 100644 --- a/pkg/detectors/abuseipdb/abuseipdb_test.go +++ b/pkg/detectors/abuseipdb/abuseipdb_test.go @@ -9,10 +9,11 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -43,7 +44,7 @@ func TestAbuseIPDB_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)), + data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within but verified", secret)), verify: true, }, want: []detectors.Result{ @@ -54,12 +55,48 @@ func TestAbuseIPDB_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_AbuseIPDB, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_AbuseIPDB, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", inactiveSecret)), // the secret would satisfy the regex but not pass validation verify: true, }, want: []detectors.Result{ @@ -84,8 +121,7 @@ func TestAbuseIPDB_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("AbuseIPDB.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -94,9 +130,20 @@ func TestAbuseIPDB_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("AbuseIPDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/accuweather/accuweather.go b/pkg/detectors/accuweather/accuweather.go index 77ad2543025f..e7ebadae6fba 100644 --- a/pkg/detectors/accuweather/accuweather.go +++ b/pkg/detectors/accuweather/accuweather.go @@ -2,22 +2,28 @@ package accuweather import ( "context" - regexp "github.com/wasilibs/go-re2" + "fmt" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +const accuweatherURL = "https://dataservice.accuweather.com" var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"accuweather"}) + `([a-z0-9A-Z\%]{35})\b`) @@ -29,6 +35,13 @@ func (s Scanner) Keywords() []string { return []string{"accuweather"} } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify Accuweather secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -47,22 +60,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://dataservice.accuweather.com/locations/v1/cities/autocomplete?apikey="+resMatch+"&q=----&language=en-us", nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyAccuweather(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } results = append(results, s1) @@ -71,6 +77,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyAccuweather(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, accuweatherURL+"/locations/v1/cities/autocomplete?apikey="+resMatch+"&q=----&language=en-us", nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + // https://developer.accuweather.com/accuweather-locations-api/apis/get/locations/v1/cities/autocomplete + switch res.StatusCode { + case http.StatusOK, http.StatusForbidden: + // 403 indicates lack of permission, but valid token + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Accuweather } diff --git a/pkg/detectors/accuweather/accuweather_test.go b/pkg/detectors/accuweather/accuweather_test.go index 8b8d87330ebb..8dc6f8421c2f 100644 --- a/pkg/detectors/accuweather/accuweather_test.go +++ b/pkg/detectors/accuweather/accuweather_test.go @@ -9,10 +9,11 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -43,7 +44,7 @@ func TestAccuweather_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)), + data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", secret)), verify: true, }, want: []detectors.Result{ @@ -54,12 +55,48 @@ func TestAccuweather_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Accuweather, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Accuweather, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", inactiveSecret)), verify: true, }, want: []detectors.Result{ @@ -84,8 +121,7 @@ func TestAccuweather_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Accuweather.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -94,9 +130,21 @@ func TestAccuweather_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } got[i].Raw = nil } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Accuweather.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/adafruitio/adafruitio.go b/pkg/detectors/adafruitio/adafruitio.go index b699af6bdff8..d38a579bcb8b 100644 --- a/pkg/detectors/adafruitio/adafruitio.go +++ b/pkg/detectors/adafruitio/adafruitio.go @@ -2,22 +2,28 @@ package adafruitio import ( "context" - regexp "github.com/wasilibs/go-re2" + "fmt" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +const adafruitioURL = "https://io.adafruit.com" var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b(aio\_[a-zA-Z0-9]{28})\b`) @@ -29,6 +35,13 @@ func (s Scanner) Keywords() []string { return []string{"aio_"} } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify AdafruitIO secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -47,22 +60,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://io.adafruit.com/api/v2/ladybugtest/feeds/?x-aio-key="+resMatch, nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyAdafruitIO(ctx, client, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } results = append(results, s1) @@ -71,6 +77,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyAdafruitIO(ctx context.Context, client *http.Client, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, adafruitioURL+"/api/v2/ladybugtest/feeds/?x-aio-key="+resMatch, nil) + if err != nil { + return false, err + } + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + // https://learn.adafruit.com/adafruit-io/http-status-codes + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_AdafruitIO } diff --git a/pkg/detectors/adafruitio/adafruitio_test.go b/pkg/detectors/adafruitio/adafruitio_test.go index 92d461b7a377..82702d0e9858 100644 --- a/pkg/detectors/adafruitio/adafruitio_test.go +++ b/pkg/detectors/adafruitio/adafruitio_test.go @@ -9,7 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -54,6 +55,42 @@ func TestAdafruitIO_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(10 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a adafruitio secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_AdafruitIO, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a adafruitio secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_AdafruitIO, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, @@ -84,8 +121,7 @@ func TestAdafruitIO_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("AdafruitIO.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -94,9 +130,20 @@ func TestAdafruitIO_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("AdafruitIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/adzuna/adzuna.go b/pkg/detectors/adzuna/adzuna.go index adf07c597780..eb13aad414a6 100644 --- a/pkg/detectors/adzuna/adzuna.go +++ b/pkg/detectors/adzuna/adzuna.go @@ -12,13 +12,17 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +const adzunaURL = "https://api.adzuna.com" var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"adzuna"}) + `\b([a-z0-9]{32})\b`) @@ -31,6 +35,13 @@ func (s Scanner) Keywords() []string { return []string{"adzuna"} } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify Adzuna secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -56,22 +67,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.adzuna.com/v1/api/jobs/gb/search/1?app_id=%s&app_key=%s", resIdMatch, resMatch), nil) - if err != nil { - continue - } - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyAdzuna(ctx, client, resMatch, resIdMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } results = append(results, s1) @@ -81,6 +85,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyAdzuna(ctx context.Context, client *http.Client, resMatch, resIdMatch string) (bool, error) { + // https://developer.adzuna.com/activedocs#!/adzuna/search + req, err := http.NewRequestWithContext(ctx, http.MethodGet, adzunaURL+fmt.Sprintf("/v1/api/jobs/us/search/1?app_id=%s&app_key=%s", resIdMatch, resMatch), nil) + if err != nil { + return false, err + } + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + // https://developer.adzuna.com/overview + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Adzuna } diff --git a/pkg/detectors/adzuna/adzuna_test.go b/pkg/detectors/adzuna/adzuna_test.go index 6a804fc7b22b..f2a1bf630dd0 100644 --- a/pkg/detectors/adzuna/adzuna_test.go +++ b/pkg/detectors/adzuna/adzuna_test.go @@ -9,10 +9,11 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -55,12 +56,48 @@ func TestAdzuna_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Adzuna, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Adzuna, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s but not valid", inactiveSecret, id)), verify: true, }, want: []detectors.Result{ @@ -85,8 +122,7 @@ func TestAdzuna_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Adzuna.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -95,9 +131,20 @@ func TestAdzuna_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Adzuna.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/aeroworkflow/aeroworkflow.go b/pkg/detectors/aeroworkflow/aeroworkflow.go index ef534924a65d..780a7826528f 100644 --- a/pkg/detectors/aeroworkflow/aeroworkflow.go +++ b/pkg/detectors/aeroworkflow/aeroworkflow.go @@ -2,25 +2,31 @@ package aeroworkflow import ( "context" - regexp "github.com/wasilibs/go-re2" + "fmt" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +const aeroworkflowURL = "https://api.aeroworkflow.com" var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([a-zA-Z0-9^!]{20})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `([a-zA-Z0-9^!?#:*;]{20})\b`) idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aeroworkflow"}) + `\b([0-9]{1,})\b`) ) @@ -30,6 +36,13 @@ func (s Scanner) Keywords() []string { return []string{"aeroworkflow"} } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify Aeroworkflow secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -56,26 +69,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequest("GET", "https://api.aeroworkflow.com/api/"+resIdMatch+"/v1/AeroAppointments", nil) - if err != nil { - continue - } - req.Header.Add("Accept", "application/json") - req.Header.Add("apikey", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyAeroworkflow(ctx, client, resMatch, resIdMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue + } results = append(results, s1) } @@ -84,6 +87,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyAeroworkflow(ctx context.Context, client *http.Client, resMatch, resIdMatch string) (bool, error) { + req, err := http.NewRequest(http.MethodGet, aeroworkflowURL+"/api/"+resIdMatch+"/v1/AeroAppointments", nil) + if err != nil { + return false, err + } + req.Header.Add("Accept", "application/json") + req.Header.Add("apikey", resMatch) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + // https://api.aeroworkflow.com/swagger/index.html + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + // 401 for invalid API key + // 403 for invalid Account ID + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Aeroworkflow } diff --git a/pkg/detectors/aeroworkflow/aeroworkflow_test.go b/pkg/detectors/aeroworkflow/aeroworkflow_test.go index 404f0ddaead9..ba46cd93e328 100644 --- a/pkg/detectors/aeroworkflow/aeroworkflow_test.go +++ b/pkg/detectors/aeroworkflow/aeroworkflow_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" - + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -19,13 +19,12 @@ import ( func TestAeroworkflow_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } - secret := testSecrets.MustGetField("AEROWORKFLOW") + secret := testSecrets.MustGetField("AEROWORKFLOW_SECRET") id := testSecrets.MustGetField("AEROWORKFLOW_ID") - inactiveSecret := testSecrets.MustGetField("AEROWORKFLOW_INACTIVE") type args struct { @@ -56,6 +55,42 @@ func TestAeroworkflow_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Aeroworkflow, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Aeroworkflow, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, @@ -86,8 +121,7 @@ func TestAeroworkflow_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Aeroworkflow.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -96,9 +130,20 @@ func TestAeroworkflow_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Aeroworkflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/agora/agora.go b/pkg/detectors/agora/agora.go index 132274a17227..ce1e101b87aa 100644 --- a/pkg/detectors/agora/agora.go +++ b/pkg/detectors/agora/agora.go @@ -2,22 +2,28 @@ package agora import ( "context" - regexp "github.com/wasilibs/go-re2" + "fmt" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +const agoraURL = "https://api.agora.io" var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"agora"}) + `\b([a-z0-9]{32})\b`) @@ -30,6 +36,13 @@ func (s Scanner) Keywords() []string { return []string{"agora"} } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify Agora secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -57,24 +70,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.agora.io/dev/v1/projects", nil) - if err != nil { - continue - } - req.SetBasicAuth(resSecret, resMatch) - res, err := client.Do(req) - - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyAgora(ctx, client, resMatch, resSecret) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } results = append(results, s1) @@ -84,6 +88,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyAgora(ctx context.Context, client *http.Client, resMatch, resSecret string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, agoraURL+"/dev/v1/projects", nil) + if err != nil { + return false, err + } + req.SetBasicAuth(resSecret, resMatch) + res, err := client.Do(req) + + if err != nil { + return false, err + } + defer res.Body.Close() + + // https://docs.agora.io/en/voice-calling/reference/agora-console-rest-api#get-all-projects + switch res.StatusCode { + case http.StatusOK, http.StatusCreated: + return true, nil + case http.StatusUnauthorized, http.StatusForbidden: + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Agora } diff --git a/pkg/detectors/agora/agora_test.go b/pkg/detectors/agora/agora_test.go index 2e8329aed4f0..db0f64ac9575 100644 --- a/pkg/detectors/agora/agora_test.go +++ b/pkg/detectors/agora/agora_test.go @@ -9,7 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -48,13 +49,61 @@ func TestAgora_FromChunk(t *testing.T) { verify: true, }, want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Agora, + Verified: false, + }, { DetectorType: detectorspb.DetectorType_Agora, Verified: true, }, + { + DetectorType: detectorspb.DetectorType_Agora, + Verified: false, + }, + { + DetectorType: detectorspb.DetectorType_Agora, + Verified: false, + }, }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Agora, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r, r, r, r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Agora, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r, r, r, r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, @@ -68,6 +117,18 @@ func TestAgora_FromChunk(t *testing.T) { DetectorType: detectorspb.DetectorType_Agora, Verified: false, }, + { + DetectorType: detectorspb.DetectorType_Agora, + Verified: false, + }, + { + DetectorType: detectorspb.DetectorType_Agora, + Verified: false, + }, + { + DetectorType: detectorspb.DetectorType_Agora, + Verified: false, + }, }, wantErr: false, }, @@ -85,8 +146,7 @@ func TestAgora_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Agora.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -95,9 +155,20 @@ func TestAgora_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatal("no raw secret present") } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Agora.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/aha/aha.go b/pkg/detectors/aha/aha.go index 27659d21a2e6..41aa29c0e65b 100644 --- a/pkg/detectors/aha/aha.go +++ b/pkg/detectors/aha/aha.go @@ -3,25 +3,29 @@ package aha import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +type Scanner struct { + client *http.Client +} var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aha"}) + `\b([0-9a-f]{64})\b`) + URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.aha\.io)`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -30,11 +34,26 @@ func (s Scanner) Keywords() []string { return []string{"aha"} } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify Aha secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) + URLmatches := URLPat.FindAllStringSubmatch(dataStr, -1) + + resURLMatch := "aha.io" + for _, URLmatch := range URLmatches { + if len(URLmatch) != 2 { + continue + } + resURLMatch = strings.TrimSpace(URLmatch[1]) + } for _, match := range matches { if len(match) != 2 { @@ -48,24 +67,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://company.aha.io/api/v1/me", nil) - if err != nil { - continue - } - req.Header.Add("Accept", "application/vnd.aha+json; version=3") - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyAha(ctx, client, resMatch, resURLMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) + } + + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue } results = append(results, s1) @@ -74,6 +84,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyAha(ctx context.Context, client *http.Client, resMatch, resURLMatch string) (bool, error) { + url := fmt.Sprintf("https://%s/api/v1/me", resURLMatch) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return false, err + } + req.Header.Add("Accept", "application/vnd.aha+json; version=3") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch)) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + // https://www.aha.io/api + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusUnauthorized, http.StatusNotFound, http.StatusForbidden: + // 403 is a known case where an account is inactive bc of a trial ending or payment issue + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Aha } diff --git a/pkg/detectors/aha/aha_test.go b/pkg/detectors/aha/aha_test.go index 0ecdf5ead7fe..5e3a9378438f 100644 --- a/pkg/detectors/aha/aha_test.go +++ b/pkg/detectors/aha/aha_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" - + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -20,11 +20,12 @@ import ( func TestAha_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } - secret := testSecrets.MustGetField("AHA") + domain := testSecrets.MustGetField("AHA_DOMAIN") + secret := testSecrets.MustGetField("AHA_SECRET") inactiveSecret := testSecrets.MustGetField("AHA_INACTIVE") type args struct { @@ -41,10 +42,10 @@ func TestAha_FromChunk(t *testing.T) { }{ { name: "found, verified", - s: Scanner{}, + s: Scanner{client: common.ConstantResponseHttpClient(200, "{}")}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aha secret %s within", secret)), + data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)), verify: true, }, want: []detectors.Result{ @@ -55,12 +56,48 @@ func TestAha_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Aha, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Aha, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aha secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("You can find a aha secret %s within but not valid domain %s", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation verify: true, }, want: []detectors.Result{ @@ -85,8 +122,7 @@ func TestAha_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Aha.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -95,9 +131,20 @@ func TestAha_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Aha.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/alibaba/alibaba.go b/pkg/detectors/alibaba/alibaba.go index 2e03009aa83d..26570c6471f2 100644 --- a/pkg/detectors/alibaba/alibaba.go +++ b/pkg/detectors/alibaba/alibaba.go @@ -6,25 +6,40 @@ import ( "crypto/rand" "crypto/sha1" "encoding/base64" - regexp "github.com/wasilibs/go-re2" + "encoding/json" + "fmt" "net/http" "net/url" "strconv" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} + +type alibabaResp struct { + RequestId string `json:"RequestId"` + Message string `json:"Message"` + Recommend string `json:"Recommend"` + HostId string `json:"HostId"` + Code string `json:"Code"` +} -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +const alibabaURL = "https://ecs.aliyuncs.com" var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{30})\b`) @@ -53,6 +68,7 @@ func GetSignature(input, key string) string { h.Write([]byte(input)) return base64.StdEncoding.EncodeToString(h.Sum(nil)) } + func buildStringToSign(method, input string) string { filter := strings.Replace(input, "+", "%20", -1) filter = strings.Replace(filter, "%7E", "~", -1) @@ -61,6 +77,13 @@ func buildStringToSign(method, input string) string { return filter } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify Alibaba secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -87,42 +110,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://ecs.aliyuncs.com/?", nil) - if err != nil { - continue - } - dateISO := time.Now().UTC().Format("2006-01-02T15:04:05Z07:00") - params := req.URL.Query() - params.Add("AccessKeyId", resIdMatch) - params.Add("Action", "DescribeRegions") - params.Add("Format", "JSON") - params.Add("SignatureMethod", "HMAC-SHA1") - params.Add("SignatureNonce", randString(16)) - params.Add("SignatureVersion", "1.0") - params.Add("Timestamp", dateISO) - params.Add("Version", "2014-05-26") - - stringToSign := buildStringToSign(req.Method, params.Encode()) - signature := GetSignature(stringToSign, resMatch+"&") // Get Signature HMAC SHA1 - params.Add("Signature", signature) - req.URL.RawQuery = params.Encode() - - req.Header.Add("Content-Type", "text/xml;charset=utf-8") - req.Header.Add("Content-Length", strconv.Itoa(len(params.Encode()))) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyAlibaba(ctx, client, resIdMatch, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue + } results = append(results, s1) } } @@ -130,6 +127,58 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } +func verifyAlibaba(ctx context.Context, client *http.Client, resIdMatch, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, alibabaURL, nil) + if err != nil { + return false, err + } + + dateISO := time.Now().UTC().Format("2006-01-02T15:04:05Z07:00") + params := req.URL.Query() + params.Add("AccessKeyId", resIdMatch) + params.Add("Action", "DescribeRegions") + params.Add("Format", "JSON") + params.Add("SignatureMethod", "HMAC-SHA1") + params.Add("SignatureNonce", randString(16)) + params.Add("SignatureVersion", "1.0") + params.Add("Timestamp", dateISO) + params.Add("Version", "2014-05-26") + + stringToSign := buildStringToSign(req.Method, params.Encode()) + signature := GetSignature(stringToSign, resMatch+"&") // Get Signature HMAC SHA1 + params.Add("Signature", signature) + req.URL.RawQuery = params.Encode() + + req.Header.Add("Content-Type", "text/xml;charset=utf-8") + req.Header.Add("Content-Length", strconv.Itoa(len(params.Encode()))) + + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + var alibabaResp alibabaResp + if err = json.NewDecoder(res.Body).Decode(&alibabaResp); err != nil { + return false, err + } + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound, http.StatusBadRequest: + // 400 used for most of error cases + // 404 used if the AccessKeyId is not valid + return false, nil + default: + err := fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + if alibabaResp.Message != "" { + err = fmt.Errorf("%s: %s, %s", err, alibabaResp.Message, alibabaResp.Code) + } + return false, err + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_Alibaba } diff --git a/pkg/detectors/alibaba/alibaba_test.go b/pkg/detectors/alibaba/alibaba_test.go index f6cf9bb4d190..31b81881e0e7 100644 --- a/pkg/detectors/alibaba/alibaba_test.go +++ b/pkg/detectors/alibaba/alibaba_test.go @@ -9,7 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -55,6 +56,60 @@ func TestAlibaba_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Alibaba, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Alibaba, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to broken json", + s: Scanner{client: common.ConstantResponseHttpClient(418, "{")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Alibaba, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected EOF")) + return []detectors.Result{r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, @@ -85,8 +140,7 @@ func TestAlibaba_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Alibaba.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -95,10 +149,20 @@ func TestAlibaba_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil - got[i].RawV2 = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Alibaba.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/artifactory/artifactory.go b/pkg/detectors/artifactory/artifactory.go index a5bf889943cd..accdc385ea0a 100644 --- a/pkg/detectors/artifactory/artifactory.go +++ b/pkg/detectors/artifactory/artifactory.go @@ -2,25 +2,29 @@ package artifactory import ( "context" - regexp "github.com/wasilibs/go-re2" + "fmt" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) +type Scanner struct { + client *http.Client +} var ( - client = common.SaneHttpClient() + // Ensure the Scanner satisfies the interface at compile time. + _ detectors.Detector = (*Scanner)(nil) + + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73})`) + keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73}|\b[a-zA-Z0-9]{64})`) URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.jfrog\.io)`) ) @@ -30,6 +34,13 @@ func (s Scanner) Keywords() []string { return []string{"artifactory"} } +func (s Scanner) getClient() *http.Client { + if s.client != nil { + return s.client + } + return defaultClient +} + // FromData will find and optionally verify Artifactory secrets in a given set of bytes. func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) @@ -42,7 +53,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result continue } resURLMatch = strings.TrimSpace(URLmatch[1]) - } for _, match := range matches { @@ -58,32 +68,46 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resURLMatch+"/artifactory/api/storageinfo", nil) - if err != nil { - continue - } - - req.Header.Add("X-JFrog-Art-Api", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } - } - } + client := s.getClient() + isVerified, verificationErr := verifyArtifactory(ctx, client, resURLMatch, resMatch) + s1.Verified = isVerified + s1.SetVerificationError(verificationErr, resMatch) } + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue + } results = append(results, s1) } return results, nil } +func verifyArtifactory(ctx context.Context, client *http.Client, resURLMatch, resMatch string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+resURLMatch+"/artifactory/api/storageinfo", nil) + if err != nil { + return false, err + } + + req.Header.Add("X-JFrog-Art-Api", resMatch) + res, err := client.Do(req) + if err != nil { + return false, err + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusForbidden: + // https://jfrog.com/help/r/jfrog-rest-apis/error-responses + return false, nil + default: + return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } +} + func (s Scanner) Type() detectorspb.DetectorType { return detectorspb.DetectorType_ArtifactoryAccessToken } diff --git a/pkg/detectors/artifactory/artifactory_test.go b/pkg/detectors/artifactory/artifactory_test.go index c33dcdc30042..deab1d778658 100644 --- a/pkg/detectors/artifactory/artifactory_test.go +++ b/pkg/detectors/artifactory/artifactory_test.go @@ -9,7 +9,8 @@ import ( "testing" "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -19,7 +20,7 @@ import ( func TestArtifactory_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } @@ -55,6 +56,42 @@ func TestArtifactory_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, { name: "found, unverified", s: Scanner{}, @@ -85,8 +122,7 @@ func TestArtifactory_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Artifactory.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -95,9 +131,20 @@ func TestArtifactory_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Artifactory.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } })