From 9095845958d9cabdc354ddf372123b5399a5d334 Mon Sep 17 00:00:00 2001 From: Kashif Khan <70996046+kashifkhan0771@users.noreply.github.com> Date: Thu, 7 Nov 2024 02:45:47 +0500 Subject: [PATCH] added pattern test cases for detectors starting with b (#3559) * added pattern test cases for detectors starting with b * last brick --- pkg/detectors/bannerbear/bannerbear.go | 3 +- .../bannerbear/bannerbear_integration_test.go | 120 +++++++++ pkg/detectors/bannerbear/bannerbear_test.go | 171 ++++++------ pkg/detectors/baremetrics/baremetrics.go | 3 +- .../baremetrics_integration_test.go | 120 +++++++++ pkg/detectors/baremetrics/baremetrics_test.go | 171 ++++++------ pkg/detectors/beamer/beamer.go | 3 +- .../beamer/beamer_integration_test.go | 120 +++++++++ pkg/detectors/beamer/beamer_test.go | 170 ++++++------ pkg/detectors/beebole/beebole.go | 3 +- .../beebole/beebole_integration_test.go | 120 +++++++++ pkg/detectors/beebole/beebole_test.go | 171 ++++++------ pkg/detectors/besnappy/besnappy.go | 3 +- .../besnappy/besnappy_integration_test.go | 120 +++++++++ pkg/detectors/besnappy/besnappy_test.go | 171 ++++++------ pkg/detectors/besttime/besttime.go | 3 +- .../besttime/besttime_integration_test.go | 120 +++++++++ pkg/detectors/besttime/besttime_test.go | 168 ++++++------ pkg/detectors/betterstack/betterstack.go | 3 +- .../betterstack_integration_test.go | 128 +++++++++ pkg/detectors/betterstack/betterstack_test.go | 173 ++++++------- pkg/detectors/billomat/billomat.go | 5 +- .../billomat/billomat_integration_test.go | 121 +++++++++ pkg/detectors/billomat/billomat_test.go | 174 ++++++------- .../bitbar/bitbar_integration_test.go | 120 +++++++++ pkg/detectors/bitbar/bitbar_test.go | 172 ++++++------ .../bitcoinaverage/bitcoinaverage.go | 3 +- .../bitcoinaverage_integration_test.go | 120 +++++++++ .../bitcoinaverage/bitcoinaverage_test.go | 174 ++++++------- pkg/detectors/bitfinex/bitfinex.go | 6 +- .../bitfinex/bitfinex_integration_test.go | 122 +++++++++ pkg/detectors/bitfinex/bitfinex_test.go | 154 +++++------ .../bitlyaccesstoken/bitlyaccesstoken.go | 3 +- .../bitlyaccesstoken_integration_test.go | 120 +++++++++ .../bitlyaccesstoken/bitlyaccesstoken_test.go | 171 ++++++------ pkg/detectors/bitmex/bitmex.go | 5 +- .../bitmex/bitmex_integration_test.go | 122 +++++++++ pkg/detectors/bitmex/bitmex_test.go | 183 +++++++------ .../blazemeter/blazemeter_integration_test.go | 120 +++++++++ pkg/detectors/blazemeter/blazemeter_test.go | 186 ++++++------- pkg/detectors/blitapp/blitapp.go | 3 +- .../blitapp/blitapp_integration_test.go | 120 +++++++++ pkg/detectors/blitapp/blitapp_test.go | 171 ++++++------ pkg/detectors/blocknative/blocknative.go | 3 +- .../blocknative_integration_test.go | 120 +++++++++ pkg/detectors/blocknative/blocknative_test.go | 171 ++++++------ pkg/detectors/blogger/blogger.go | 3 +- .../blogger/blogger_integration_test.go | 120 +++++++++ pkg/detectors/blogger/blogger_test.go | 168 ++++++------ pkg/detectors/bombbomb/bombbomb.go | 3 +- .../bombbomb/bombbomb_integration_test.go | 120 +++++++++ pkg/detectors/bombbomb/bombbomb_test.go | 169 ++++++------ pkg/detectors/boostnote/boostnote.go | 3 +- .../boostnote/boostnote_integration_test.go | 117 +++++++++ pkg/detectors/boostnote/boostnote_test.go | 166 ++++++------ pkg/detectors/borgbase/borgbase.go | 3 +- .../borgbase/borgbase_integration_test.go | 117 +++++++++ pkg/detectors/borgbase/borgbase_test.go | 167 ++++++------ pkg/detectors/box/box_test.go | 3 - pkg/detectors/boxoauth/boxoauth_test.go | 3 - .../braintreepayments_integration_test.go | 164 ++++++++++++ .../braintreepayments_test.go | 210 ++++++--------- pkg/detectors/brandfetch/brandfetch.go | 3 +- .../brandfetch/brandfetch_integration_test.go | 120 +++++++++ pkg/detectors/brandfetch/brandfetch_test.go | 171 ++++++------ pkg/detectors/browserstack/browserstack.go | 5 +- .../browserstack_integration_test.go | 194 ++++++++++++++ .../browserstack/browserstack_test.go | 244 ++++++------------ pkg/detectors/browshot/browshot.go | 3 +- .../browshot/browshot_integration_test.go | 120 +++++++++ pkg/detectors/browshot/browshot_test.go | 168 ++++++------ pkg/detectors/bscscan/bscscan.go | 3 +- .../bscscan/bscscan_integration_test.go | 120 +++++++++ pkg/detectors/bscscan/bscscan_test.go | 168 ++++++------ pkg/detectors/buddyns/buddyns.go | 3 +- .../buddyns/buddyns_integration_test.go | 120 +++++++++ pkg/detectors/buddyns/buddyns_test.go | 171 ++++++------ pkg/detectors/budibase/budibase.go | 3 +- .../budibase/budibase_integration_test.go | 130 ++++++++++ pkg/detectors/budibase/budibase_test.go | 174 ++++++------- .../bugherd/bugherd_integration_test.go | 120 +++++++++ pkg/detectors/bugherd/bugherd_test.go | 171 ++++++------ pkg/detectors/bugsnag/bugsnag.go | 3 +- .../bugsnag/bugsnag_integration_test.go | 120 +++++++++ pkg/detectors/bugsnag/bugsnag_test.go | 171 ++++++------ pkg/detectors/buildkite/buildkite.go | 3 +- .../buildkite/buildkite_integration_test.go | 120 +++++++++ pkg/detectors/buildkite/buildkite_test.go | 169 ++++++------ pkg/detectors/buildkitev2/buildkite_test.go | 169 ++++++------ .../buildkitev2_integration_test.go | 120 +++++++++ .../bulbul/bulbul_integration_test.go | 120 +++++++++ pkg/detectors/bulbul/bulbul_test.go | 168 ++++++------ .../bulksms/bulksms_integration_test.go | 122 +++++++++ pkg/detectors/bulksms/bulksms_test.go | 176 ++++++------- .../buttercms/buttercms_integration_test.go | 120 +++++++++ pkg/detectors/buttercms/buttercms_test.go | 168 ++++++------ 96 files changed, 7046 insertions(+), 3183 deletions(-) create mode 100644 pkg/detectors/bannerbear/bannerbear_integration_test.go create mode 100644 pkg/detectors/baremetrics/baremetrics_integration_test.go create mode 100644 pkg/detectors/beamer/beamer_integration_test.go create mode 100644 pkg/detectors/beebole/beebole_integration_test.go create mode 100644 pkg/detectors/besnappy/besnappy_integration_test.go create mode 100644 pkg/detectors/besttime/besttime_integration_test.go create mode 100644 pkg/detectors/betterstack/betterstack_integration_test.go create mode 100644 pkg/detectors/billomat/billomat_integration_test.go create mode 100644 pkg/detectors/bitbar/bitbar_integration_test.go create mode 100644 pkg/detectors/bitcoinaverage/bitcoinaverage_integration_test.go create mode 100644 pkg/detectors/bitfinex/bitfinex_integration_test.go create mode 100644 pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_integration_test.go create mode 100644 pkg/detectors/bitmex/bitmex_integration_test.go create mode 100644 pkg/detectors/blazemeter/blazemeter_integration_test.go create mode 100644 pkg/detectors/blitapp/blitapp_integration_test.go create mode 100644 pkg/detectors/blocknative/blocknative_integration_test.go create mode 100644 pkg/detectors/blogger/blogger_integration_test.go create mode 100644 pkg/detectors/bombbomb/bombbomb_integration_test.go create mode 100644 pkg/detectors/boostnote/boostnote_integration_test.go create mode 100644 pkg/detectors/borgbase/borgbase_integration_test.go create mode 100644 pkg/detectors/braintreepayments/braintreepayments_integration_test.go create mode 100644 pkg/detectors/brandfetch/brandfetch_integration_test.go create mode 100644 pkg/detectors/browserstack/browserstack_integration_test.go create mode 100644 pkg/detectors/browshot/browshot_integration_test.go create mode 100644 pkg/detectors/bscscan/bscscan_integration_test.go create mode 100644 pkg/detectors/buddyns/buddyns_integration_test.go create mode 100644 pkg/detectors/budibase/budibase_integration_test.go create mode 100644 pkg/detectors/bugherd/bugherd_integration_test.go create mode 100644 pkg/detectors/bugsnag/bugsnag_integration_test.go create mode 100644 pkg/detectors/buildkite/buildkite_integration_test.go create mode 100644 pkg/detectors/buildkitev2/buildkitev2_integration_test.go create mode 100644 pkg/detectors/bulbul/bulbul_integration_test.go create mode 100644 pkg/detectors/bulksms/bulksms_integration_test.go create mode 100644 pkg/detectors/buttercms/buttercms_integration_test.go diff --git a/pkg/detectors/bannerbear/bannerbear.go b/pkg/detectors/bannerbear/bannerbear.go index 0d5c8f24f72b..6b4a4c3209f1 100644 --- a/pkg/detectors/bannerbear/bannerbear.go +++ b/pkg/detectors/bannerbear/bannerbear.go @@ -3,10 +3,11 @@ package bannerbear import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/bannerbear/bannerbear_integration_test.go b/pkg/detectors/bannerbear/bannerbear_integration_test.go new file mode 100644 index 000000000000..263daa202cfc --- /dev/null +++ b/pkg/detectors/bannerbear/bannerbear_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package bannerbear + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBannerbear_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BANNERBEAR") + inactiveSecret := testSecrets.MustGetField("BANNERBEAR_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bannerbear secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bannerbear, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bannerbear secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bannerbear, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Bannerbear.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Bannerbear.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bannerbear/bannerbear_test.go b/pkg/detectors/bannerbear/bannerbear_test.go index 263daa202cfc..4cac36af2043 100644 --- a/pkg/detectors/bannerbear/bannerbear_test.go +++ b/pkg/detectors/bannerbear/bannerbear_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package bannerbear import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBannerbear_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BANNERBEAR") - inactiveSecret := testSecrets.MustGetField("BANNERBEAR_INACTIVE") +var ( + validPattern = "yvxpthLIcYpZweFpPOVeCOtt" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte("{}"))) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + bannerBearToken := "Bearer yvxpthLIcYpZweFpPOVeCOtt" + req.Header.Set("Authorization", bannerBearToken) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "yvxpthLIcYpZweFpPOVeCOtot" +) + +func TestBannerBear_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bannerbear secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bannerbear, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("bannerbear credentials: %s", validPattern), + want: []string{"yvxpthLIcYpZweFpPOVeCOtt"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bannerbear secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bannerbear, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"yvxpthLIcYpZweFpPOVeCOtt"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("bannerbear credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Bannerbear.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Bannerbear.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/baremetrics/baremetrics.go b/pkg/detectors/baremetrics/baremetrics.go index 39798ab5e0ee..0d1f014df1e9 100644 --- a/pkg/detectors/baremetrics/baremetrics.go +++ b/pkg/detectors/baremetrics/baremetrics.go @@ -3,10 +3,11 @@ package baremetrics import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/baremetrics/baremetrics_integration_test.go b/pkg/detectors/baremetrics/baremetrics_integration_test.go new file mode 100644 index 000000000000..87fde4b24191 --- /dev/null +++ b/pkg/detectors/baremetrics/baremetrics_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package baremetrics + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBaremetrics_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BAREMETRICS") + inactiveSecret := testSecrets.MustGetField("BAREMETRICS_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a baremetrics secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Baremetrics, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a baremetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Baremetrics, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Baremetrics.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Baremetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/baremetrics/baremetrics_test.go b/pkg/detectors/baremetrics/baremetrics_test.go index 87fde4b24191..eb0d1ad8cbcb 100644 --- a/pkg/detectors/baremetrics/baremetrics_test.go +++ b/pkg/detectors/baremetrics/baremetrics_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package baremetrics import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBaremetrics_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BAREMETRICS") - inactiveSecret := testSecrets.MustGetField("BAREMETRICS_INACTIVE") +var ( + validPattern = "FNdRZCc2qoPLrwrLJIBE6vovy" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + baremetricsToken := "Bearer FN_RZCc2qoPLrwrLJIBE6vovy" + req.Header.Set("Authorization", baremetricsToken) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "FNdRZCc-qoPLrwrLJIBE6vovyu" +) + +func TestBareMetrics_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a baremetrics secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Baremetrics, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("baremetrics credentials: %s", validPattern), + want: []string{"FNdRZCc2qoPLrwrLJIBE6vovy"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a baremetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Baremetrics, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"FN_RZCc2qoPLrwrLJIBE6vovy"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("baremetrics credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Baremetrics.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Baremetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/beamer/beamer.go b/pkg/detectors/beamer/beamer.go index 2a00920b0713..490bcb081c54 100644 --- a/pkg/detectors/beamer/beamer.go +++ b/pkg/detectors/beamer/beamer.go @@ -2,10 +2,11 @@ package beamer import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/beamer/beamer_integration_test.go b/pkg/detectors/beamer/beamer_integration_test.go new file mode 100644 index 000000000000..2fd40a34aed9 --- /dev/null +++ b/pkg/detectors/beamer/beamer_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package beamer + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBeamer_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BEAMER_TOKEN") + inactiveSecret := testSecrets.MustGetField("BEAMER_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a beamer secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Beamer, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a beamer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Beamer, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Beamer.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Beamer.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/beamer/beamer_test.go b/pkg/detectors/beamer/beamer_test.go index 2fd40a34aed9..aa40186d80d1 100644 --- a/pkg/detectors/beamer/beamer_test.go +++ b/pkg/detectors/beamer/beamer_test.go @@ -1,120 +1,112 @@ -//go:build detectors -// +build detectors - package beamer import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBeamer_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BEAMER_TOKEN") - inactiveSecret := testSecrets.MustGetField("BEAMER_INACTIVE") +var ( + validPattern = "DyVdf7+cAXw4MH9gT1CPotU31RMl__aLKbrABRWvT7TyO=" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + req.Header.Set("Beamer-Api-Key", "DyVdf7+cAXw4MH9gT1CPotU31RMl__aLKbrABRWvT7TyO=") + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "DyVdf7%c^AXw4MH9gT1CPotU31RMl__aLKbrABRWvT7TyO" +) + +func TestBeamer_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a beamer secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Beamer, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("beamer credentials: %s", validPattern), + want: []string{"DyVdf7+cAXw4MH9gT1CPotU31RMl__aLKbrABRWvT7TyO="}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a beamer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Beamer, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"DyVdf7+cAXw4MH9gT1CPotU31RMl__aLKbrABRWvT7TyO="}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("beamer credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Beamer.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Beamer.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/beebole/beebole.go b/pkg/detectors/beebole/beebole.go index 765c92d48dc7..091f4f19c5ad 100644 --- a/pkg/detectors/beebole/beebole.go +++ b/pkg/detectors/beebole/beebole.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/beebole/beebole_integration_test.go b/pkg/detectors/beebole/beebole_integration_test.go new file mode 100644 index 000000000000..8dc52fc458b5 --- /dev/null +++ b/pkg/detectors/beebole/beebole_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package beebole + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBeebole_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BEEBOLE") + inactiveSecret := testSecrets.MustGetField("BEEBOLE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a beebole secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Beebole, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a beebole secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Beebole, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Beebole.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Beebole.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/beebole/beebole_test.go b/pkg/detectors/beebole/beebole_test.go index 8dc52fc458b5..a68222075051 100644 --- a/pkg/detectors/beebole/beebole_test.go +++ b/pkg/detectors/beebole/beebole_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package beebole import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBeebole_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BEEBOLE") - inactiveSecret := testSecrets.MustGetField("BEEBOLE_INACTIVE") +var ( + validPattern = "bn6htprmfpukfalts4muwalxh9j15ucvnrfdme8t" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + beeboleAuth := b51rul64exy5yz0gupfm0bbh0b5efg2sc6kex9rk + req.Header.Set("Authorization", "Basic " + beeboleAuth) // beebole authorization header + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "DyVdf7%c^AXw4MH9gT1CPotU31RMl__aLKbrABRWvT7TyO" +) + +func TestBeeBole_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a beebole secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Beebole, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("beebole credentials: %s", validPattern), + want: []string{"bn6htprmfpukfalts4muwalxh9j15ucvnrfdme8t"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a beebole secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Beebole, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"b51rul64exy5yz0gupfm0bbh0b5efg2sc6kex9rk"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("beebole credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Beebole.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Beebole.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/besnappy/besnappy.go b/pkg/detectors/besnappy/besnappy.go index 0a7154692513..e5afc0bcb469 100644 --- a/pkg/detectors/besnappy/besnappy.go +++ b/pkg/detectors/besnappy/besnappy.go @@ -4,10 +4,11 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/besnappy/besnappy_integration_test.go b/pkg/detectors/besnappy/besnappy_integration_test.go new file mode 100644 index 000000000000..bab5c5563160 --- /dev/null +++ b/pkg/detectors/besnappy/besnappy_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package besnappy + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBesnappy_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BESNAPPY") + inactiveSecret := testSecrets.MustGetField("BESNAPPY_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a besnappy secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Besnappy, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a besnappy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Besnappy, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Besnappy.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Besnappy.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/besnappy/besnappy_test.go b/pkg/detectors/besnappy/besnappy_test.go index bab5c5563160..1e8d1da3e671 100644 --- a/pkg/detectors/besnappy/besnappy_test.go +++ b/pkg/detectors/besnappy/besnappy_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package besnappy import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBesnappy_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BESNAPPY") - inactiveSecret := testSecrets.MustGetField("BESNAPPY_INACTIVE") +var ( + validPattern = "f58c5d37d7876d32cfdd823f8fe4ded364a8d483b5dbfadcc55ad801b3be8523" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte("{}"))) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + beSnappyToken := f58c5d37d7876d32cfdd823f8fe4ded364a8d483b5dbfadcc55ad801b3be8523 + req.Header.Set("Authorization", "Basic " + beSnappyToken) // authorization header + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "f58c5d37d7876d32cf__f8fe4ded364a8d483b5db+adcc55ad801b3be8523" +) + +func TestBeSnappy_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a besnappy secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Besnappy, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("besnappy credentials: %s", validPattern), + want: []string{"f58c5d37d7876d32cfdd823f8fe4ded364a8d483b5dbfadcc55ad801b3be8523"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a besnappy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Besnappy, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"f58c5d37d7876d32cfdd823f8fe4ded364a8d483b5dbfadcc55ad801b3be8523"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("besnappy credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Besnappy.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Besnappy.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/besttime/besttime.go b/pkg/detectors/besttime/besttime.go index 0d3b05ca184f..38edf917630e 100644 --- a/pkg/detectors/besttime/besttime.go +++ b/pkg/detectors/besttime/besttime.go @@ -2,11 +2,12 @@ package besttime import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/besttime/besttime_integration_test.go b/pkg/detectors/besttime/besttime_integration_test.go new file mode 100644 index 000000000000..902377d769fa --- /dev/null +++ b/pkg/detectors/besttime/besttime_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package besttime + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBesttime_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BESTTIME") + inactiveSecret := testSecrets.MustGetField("BESTTIME_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a besttime secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Besttime, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a besttime secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Besttime, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Besttime.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Besttime.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/besttime/besttime_test.go b/pkg/detectors/besttime/besttime_test.go index 902377d769fa..fef150cf55ea 100644 --- a/pkg/detectors/besttime/besttime_test.go +++ b/pkg/detectors/besttime/besttime_test.go @@ -1,120 +1,110 @@ -//go:build detectors -// +build detectors - package besttime import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBesttime_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BESTTIME") - inactiveSecret := testSecrets.MustGetField("BESTTIME_INACTIVE") +var ( + validPattern = "4K1WTb2ysVeg_jHDwtwhH68K9MuOjiTtXQCS" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/besttime/keys/4K1WTb2ysVeg_jHDwtwhH68K9MuOjiTtXQCS" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "4K1WTb2ysVeg^jHD*wtwhH68K9MuOjiTtXQCS" +) + +func TestBestTime_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a besttime secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Besttime, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("besttime credentials: %s", validPattern), + want: []string{"4K1WTb2ysVeg_jHDwtwhH68K9MuOjiTtXQCS"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a besttime secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Besttime, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"4K1WTb2ysVeg_jHDwtwhH68K9MuOjiTtXQCS"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("besttime credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Besttime.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Besttime.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/betterstack/betterstack.go b/pkg/detectors/betterstack/betterstack.go index 7c3c4d16dcdb..0f4ebda8254f 100644 --- a/pkg/detectors/betterstack/betterstack.go +++ b/pkg/detectors/betterstack/betterstack.go @@ -3,10 +3,11 @@ package betterstack import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/betterstack/betterstack_integration_test.go b/pkg/detectors/betterstack/betterstack_integration_test.go new file mode 100644 index 000000000000..4273df1f291c --- /dev/null +++ b/pkg/detectors/betterstack/betterstack_integration_test.go @@ -0,0 +1,128 @@ +//go:build detectors +// +build detectors + +package betterstack + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBetterstack_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BETTERSTACK") + inactiveSecret := testSecrets.MustGetField("BETTERSTACK_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a betterstack secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BetterStack, + Verified: true, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a betterstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BetterStack, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Betterstack.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Betterstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/betterstack/betterstack_test.go b/pkg/detectors/betterstack/betterstack_test.go index 4273df1f291c..7c14514425ed 100644 --- a/pkg/detectors/betterstack/betterstack_test.go +++ b/pkg/detectors/betterstack/betterstack_test.go @@ -1,127 +1,112 @@ -//go:build detectors -// +build detectors - package betterstack import ( "context" "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBetterstack_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BETTERSTACK") - inactiveSecret := testSecrets.MustGetField("BETTERSTACK_INACTIVE") +var ( + validPattern = "ntJD0ER8QpuT0O1WqsclApO2" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + req.Header.Set("Authorization", "Bearer " + getbetterStackToken()) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + func getBetterStackToken() string{ return "ntJD0ER8QpuT0O1WqsclApO2" } + ` + invalidPattern = "DyntJD0ER8QpuT0O1WqsclApO2" +) + +func TestBetterStack_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a betterstack secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BetterStack, - Verified: true, - }, - }, - wantErr: false, - wantVerificationErr: false, + name: "valid pattern", + input: fmt.Sprintf("betterstack credentials: %s", validPattern), + want: []string{"ntJD0ER8QpuT0O1WqsclApO2"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a betterstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BetterStack, - Verified: false, - }, - }, - wantErr: false, - wantVerificationErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"ntJD0ER8QpuT0O1WqsclApO2"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - wantVerificationErr: false, + name: "invalid pattern", + input: fmt.Sprintf("betterstack credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Betterstack.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("Betterstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/billomat/billomat.go b/pkg/detectors/billomat/billomat.go index b0d316c9a09e..3afe44c972ce 100644 --- a/pkg/detectors/billomat/billomat.go +++ b/pkg/detectors/billomat/billomat.go @@ -3,16 +3,17 @@ package billomat import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { detectors.DefaultMultiPartCredentialProvider } diff --git a/pkg/detectors/billomat/billomat_integration_test.go b/pkg/detectors/billomat/billomat_integration_test.go new file mode 100644 index 000000000000..745e16291021 --- /dev/null +++ b/pkg/detectors/billomat/billomat_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package billomat + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBillomat_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BILLOMAT") + id := testSecrets.MustGetField("BILLOMAT_ID") + inactiveSecret := testSecrets.MustGetField("BILLOMAT_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a billomat secret %s within billomat id %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Billomat, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a billomat secret %s within billomat id %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Billomat, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Billomat.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Billomat.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/billomat/billomat_test.go b/pkg/detectors/billomat/billomat_test.go index 745e16291021..6ecf9c516363 100644 --- a/pkg/detectors/billomat/billomat_test.go +++ b/pkg/detectors/billomat/billomat_test.go @@ -1,121 +1,117 @@ -//go:build detectors -// +build detectors - package billomat import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBillomat_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BILLOMAT") - id := testSecrets.MustGetField("BILLOMAT_ID") - inactiveSecret := testSecrets.MustGetField("BILLOMAT_INACTIVE") +var ( + validPattern = "billomatKey: xv3khh5klgzztdmptrgbqhkr0ucvr67i / billomatID: s2mels7c75tnsbs7ldu0wmjofzmugkg7vb" + complexPattern = ` + func main() { + url := "https://api.billomat.net/v2/s2mels7c75tnsbs7ldu0wmjofzmugkg7vb" - type args struct { - ctx context.Context - data []byte - verify bool + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + req.Header.Set("X-BillomatApiKey", "xv3khh5klgzztdmptrgbqhkr0ucvr67i") + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "billomat_creds: s2mels7c75tnsbs7ldu0wmjofzmugkg7vb" +) + +func TestBilloMat_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a billomat secret %s within billomat id %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Billomat, - Verified: true, - }, + name: "valid pattern", + input: validPattern, + want: []string{ + "xv3khh5klgzztdmptrgbqhkr0ucvr67is2mels7c75tnsbs7ldu0wmjofzmugkg7vb", + "xv3khh5klgzztdmptrgbqhkr0ucvr67ixv3khh5klgzztdmptrgbqhkr0ucvr67i", }, - wantErr: false, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a billomat secret %s within billomat id %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation - verify: true, + name: "valid pattern - complex", + input: complexPattern, + want: []string{ + "xv3khh5klgzztdmptrgbqhkr0ucvr67inet", + "xv3khh5klgzztdmptrgbqhkr0ucvr67ixv3khh5klgzztdmptrgbqhkr0ucvr67i", }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Billomat, - Verified: false, - }, - }, - wantErr: false, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Billomat.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Billomat.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/bitbar/bitbar_integration_test.go b/pkg/detectors/bitbar/bitbar_integration_test.go new file mode 100644 index 000000000000..743a0369e9c8 --- /dev/null +++ b/pkg/detectors/bitbar/bitbar_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package bitbar + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBitbar_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BITBAR") + inactiveSecret := testSecrets.MustGetField("BITBAR_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bitbar secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bitbar, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bitbar secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bitbar, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Bitbar.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Bitbar.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bitbar/bitbar_test.go b/pkg/detectors/bitbar/bitbar_test.go index 743a0369e9c8..17201c0178f7 100644 --- a/pkg/detectors/bitbar/bitbar_test.go +++ b/pkg/detectors/bitbar/bitbar_test.go @@ -1,120 +1,116 @@ -//go:build detectors -// +build detectors - package bitbar import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBitbar_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BITBAR") - inactiveSecret := testSecrets.MustGetField("BITBAR_INACTIVE") +var ( + validPattern = "64pq66z15thg8fh3acd00l35lpyg7c82" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + bitBarSecret := os.GetEnv("BITBAR_SECRET") + if bitBarSecret == ""{ + bitBarSecret = "64pq66z15thg8fh3acd00l35lpyg7c82" + } + req.Header.Set("Authorization", "Basic " + bitBarSecret) - type args struct { - ctx context.Context - data []byte - verify bool + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "DyV64pq66z15thg8fh3&cd00l35lpyg7c82$" +) + +func TestBitBar_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bitbar secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bitbar, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("bitbar credentials: %s", validPattern), + want: []string{"64pq66z15thg8fh3acd00l35lpyg7c82"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bitbar secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bitbar, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"64pq66z15thg8fh3acd00l35lpyg7c82"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("bitbar credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Bitbar.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Bitbar.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/bitcoinaverage/bitcoinaverage.go b/pkg/detectors/bitcoinaverage/bitcoinaverage.go index c326f709b0eb..938f1810a305 100644 --- a/pkg/detectors/bitcoinaverage/bitcoinaverage.go +++ b/pkg/detectors/bitcoinaverage/bitcoinaverage.go @@ -3,10 +3,11 @@ package bitcoinaverage import ( "context" "encoding/json" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/bitcoinaverage/bitcoinaverage_integration_test.go b/pkg/detectors/bitcoinaverage/bitcoinaverage_integration_test.go new file mode 100644 index 000000000000..87a78cd52c20 --- /dev/null +++ b/pkg/detectors/bitcoinaverage/bitcoinaverage_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package bitcoinaverage + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBitcoinAverage_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BITCOINAVERAGE") + inactiveSecret := testSecrets.MustGetField("BITCOINAVERAGE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bitcoinaverage secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BitcoinAverage, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bitcoinaverage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BitcoinAverage, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("BitcoinAverage.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("BitcoinAverage.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go b/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go index 87a78cd52c20..38c906e1d21f 100644 --- a/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go +++ b/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go @@ -1,120 +1,116 @@ -//go:build detectors -// +build detectors - package bitcoinaverage import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBitcoinAverage_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BITCOINAVERAGE") - inactiveSecret := testSecrets.MustGetField("BITCOINAVERAGE_INACTIVE") +var ( + validPattern = "zt7WwXWk7MYMNB3l2QdtB5itl9uvfhJ8s6h0hibid5E" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + secret := os.GetEnv("BITCOINAVERAGE") + if secret == ""{ + secret = "zt7WwXWk7MYMNB3l2QdtB5itl9uvfhJ8s6h0hibid5E" // bitcoin average secret + } + req.Header.Set("x-ba-key", secret) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "DyV64pq66z15thg8fh3&cd00l35lpyg7c82$" +) + +func TestBitCoinAverage_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bitcoinaverage secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BitcoinAverage, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("bitcoinaverage credentials: %s", validPattern), + want: []string{"zt7WwXWk7MYMNB3l2QdtB5itl9uvfhJ8s6h0hibid5E"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bitcoinaverage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BitcoinAverage, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"zt7WwXWk7MYMNB3l2QdtB5itl9uvfhJ8s6h0hibid5E"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("bitcoinaverage credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("BitcoinAverage.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("BitcoinAverage.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/bitfinex/bitfinex.go b/pkg/detectors/bitfinex/bitfinex.go index 50239e0e3ac6..7df7fd4104f8 100644 --- a/pkg/detectors/bitfinex/bitfinex.go +++ b/pkg/detectors/bitfinex/bitfinex.go @@ -3,18 +3,18 @@ package bitfinex import ( "context" "flag" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" "github.com/bitfinexcom/bitfinex-api-go/v2/rest" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + regexp "github.com/wasilibs/go-re2" "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { detectors.DefaultMultiPartCredentialProvider } diff --git a/pkg/detectors/bitfinex/bitfinex_integration_test.go b/pkg/detectors/bitfinex/bitfinex_integration_test.go new file mode 100644 index 000000000000..479f9a964f04 --- /dev/null +++ b/pkg/detectors/bitfinex/bitfinex_integration_test.go @@ -0,0 +1,122 @@ +//go:build detectors +// +build detectors + +package bitfinex + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBitfinex_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + apiKey := testSecrets.MustGetField("BITFINEX_API_KEY") + inactiveApiKey := testSecrets.MustGetField("BITFINEX_API_KEY_INACTIVE") + apiSecret := testSecrets.MustGetField("BITFINEX_API_SECRET") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bitfinex api key %s within bitfinex api secret %s", apiKey, apiSecret)), + verify: true, + }, + want: []detectors.Result{ + // will try to scan (apiKey, apiSecret) which will verify then (apiSecret, apiKey) which will not since both parameters have equal length + { + DetectorType: detectorspb.DetectorType_Bitfinex, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bitfinex api key %s within bitfinex api secret %s", inactiveApiKey, apiSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bitfinex, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Bitfinex.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Bitfinex.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bitfinex/bitfinex_test.go b/pkg/detectors/bitfinex/bitfinex_test.go index 479f9a964f04..1403b96fff83 100644 --- a/pkg/detectors/bitfinex/bitfinex_test.go +++ b/pkg/detectors/bitfinex/bitfinex_test.go @@ -1,122 +1,94 @@ -//go:build detectors -// +build detectors - package bitfinex import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBitfinex_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) +var ( + validPattern = "bitfinexKey: HxfuG198amaeCcYkASkto5VuIO-oXcplDV6JZ7OIEQZ / bitfinexSecret: Pf3-3v989gPbJT54D3oDBiFZmJoLpWoTHGvF8xuSBPP" + complexPattern = ` + func main() { + bitfinexKey := "HxfuG198amaeCcYkASkto5VuIO-oXcplDV6JZ7OIEQZ" + bitfinexSecret := "Pf3-3v989gPbJT54D3oDBiFZmJoLpWoTHGvF8xuSBPP" + http.DefaultClient = client + c := rest.NewClientWithURL(*api).Credentials(key, secret) } - apiKey := testSecrets.MustGetField("BITFINEX_API_KEY") - inactiveApiKey := testSecrets.MustGetField("BITFINEX_API_KEY_INACTIVE") - apiSecret := testSecrets.MustGetField("BITFINEX_API_SECRET") + ` + invalidPattern = "kASkto5VuIO%c^HxfuG198amaeCcYkASkto5VuIO" +) + +func TestBitFinex_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bitfinex api key %s within bitfinex api secret %s", apiKey, apiSecret)), - verify: true, - }, - want: []detectors.Result{ - // will try to scan (apiKey, apiSecret) which will verify then (apiSecret, apiKey) which will not since both parameters have equal length - { - DetectorType: detectorspb.DetectorType_Bitfinex, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("bitfinex credentials: %s", validPattern), + want: []string{"HxfuG198amaeCcYkASkto5VuIO-oXcplDV6JZ7OIEQZ", "Pf3-3v989gPbJT54D3oDBiFZmJoLpWoTHGvF8xuSBPP"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bitfinex api key %s within bitfinex api secret %s", inactiveApiKey, apiSecret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bitfinex, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"Pf3-3v989gPbJT54D3oDBiFZmJoLpWoTHGvF8xuSBPP", "HxfuG198amaeCcYkASkto5VuIO-oXcplDV6JZ7OIEQZ"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("bitfinex credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Bitfinex.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Bitfinex.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken.go b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken.go index f0f553e30365..f270c68519da 100644 --- a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken.go +++ b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken.go @@ -3,10 +3,11 @@ package bitlyaccesstoken import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_integration_test.go b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_integration_test.go new file mode 100644 index 000000000000..e4625d2639be --- /dev/null +++ b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package bitlyaccesstoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBitLyAccessToken_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BITLYACCESSTOKEN_TOKEN") + inactiveSecret := testSecrets.MustGetField("BITLYACCESSTOKEN_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bitlyaccesstoken secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BitLyAccessToken, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bitlyaccesstoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BitLyAccessToken, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("BitLyAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("BitLyAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go index e4625d2639be..cdd0e6bb8202 100644 --- a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go +++ b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package bitlyaccesstoken import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBitLyAccessToken_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BITLYACCESSTOKEN_TOKEN") - inactiveSecret := testSecrets.MustGetField("BITLYACCESSTOKEN_INACTIVE") +var ( + validPattern = "2xN7puShxzNf5fZleQthTg305lKr7KrbW95D3gSD" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + bitlyToken := "2xN7puShxzNf5fZleQthTg305lKr7KrbW95D3gSD" + req.Header.Set("Authorization", "Bearer " + bitlyToken) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "2xN7puShxzNf5fZleQthTg305l95D3gSD%c^" +) + +func TestBitlyAccessToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bitlyaccesstoken secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BitLyAccessToken, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("bitly credentials: %s", validPattern), + want: []string{"2xN7puShxzNf5fZleQthTg305lKr7KrbW95D3gSD"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bitlyaccesstoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BitLyAccessToken, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"2xN7puShxzNf5fZleQthTg305lKr7KrbW95D3gSD"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("bitly credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("BitLyAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("BitLyAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/bitmex/bitmex.go b/pkg/detectors/bitmex/bitmex.go index bf02fa37b720..0d2771c9426d 100644 --- a/pkg/detectors/bitmex/bitmex.go +++ b/pkg/detectors/bitmex/bitmex.go @@ -5,19 +5,20 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" - regexp "github.com/wasilibs/go-re2" "net/http" "net/url" "strconv" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { detectors.DefaultMultiPartCredentialProvider } diff --git a/pkg/detectors/bitmex/bitmex_integration_test.go b/pkg/detectors/bitmex/bitmex_integration_test.go new file mode 100644 index 000000000000..9aefeabd7df9 --- /dev/null +++ b/pkg/detectors/bitmex/bitmex_integration_test.go @@ -0,0 +1,122 @@ +//go:build detectors +// +build detectors + +package bitmex + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBitmex_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + key := testSecrets.MustGetField("BITMEX_KEY") + inactiveKey := testSecrets.MustGetField("BITMEX_KEY_INACTIVE") + secret := testSecrets.MustGetField("BITMEX") + inactiveSecret := testSecrets.MustGetField("BITMEX_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bitmex key %s with bitmex secret %s within", key, secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bitmex, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bitmex key %s with bitmex secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bitmex, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Bitmex.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Bitmex.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bitmex/bitmex_test.go b/pkg/detectors/bitmex/bitmex_test.go index 9aefeabd7df9..123020939613 100644 --- a/pkg/detectors/bitmex/bitmex_test.go +++ b/pkg/detectors/bitmex/bitmex_test.go @@ -1,122 +1,119 @@ -//go:build detectors -// +build detectors - package bitmex import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBitmex_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - key := testSecrets.MustGetField("BITMEX_KEY") - inactiveKey := testSecrets.MustGetField("BITMEX_KEY_INACTIVE") - secret := testSecrets.MustGetField("BITMEX") - inactiveSecret := testSecrets.MustGetField("BITMEX_INACTIVE") - - type args struct { - ctx context.Context - data []byte - verify bool +var ( + validPattern = "bitmexkey: I4_XpH-fJJiLFn--Wo7rnlXE + bitmexsecret: W_HlMtrmELzXm4bSlWv49JLcgvg5hvu467WbbnpmoEA-RjrY " + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + bitmexKey := " I4_XpH-fJJiLFn--Wo7rnlXE " + bitmexSecret := " W_HlMtrmELzXm4bSlWv49JLcgvg5hvu467WbbnpmoEA-RjrY " + + signature, err := generateSecretSignature(bitmexKey, bitmexSecret) + if err != nil{ + return err + } + + req.Header.Set("api-signature", signature) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "bitmexkey: mELzXm4bSlWv49JLc%c^ bitmexsecret: IXpH-fJJiLFn--Wo7rnlXE" +) + +func TestBitmex_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bitmex key %s with bitmex secret %s within", key, secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bitmex, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"I4_XpH-fJJiLFn--Wo7rnlXEW_HlMtrmELzXm4bSlWv49JLcgvg5hvu467WbbnpmoEA-RjrY"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bitmex key %s with bitmex secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bitmex, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"I4_XpH-fJJiLFn--Wo7rnlXEW_HlMtrmELzXm4bSlWv49JLcgvg5hvu467WbbnpmoEA-RjrY"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Bitmex.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Bitmex.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/blazemeter/blazemeter_integration_test.go b/pkg/detectors/blazemeter/blazemeter_integration_test.go new file mode 100644 index 000000000000..12d958cb23a8 --- /dev/null +++ b/pkg/detectors/blazemeter/blazemeter_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package blazemeter + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBlazemeter_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BLAZEMETER") + inactiveSecret := testSecrets.MustGetField("BLAZEMETER_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a blazemeter secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Blazemeter, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a blazemeter secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Blazemeter, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Blazemeter.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Blazemeter.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/blazemeter/blazemeter_test.go b/pkg/detectors/blazemeter/blazemeter_test.go index 12d958cb23a8..a7ccff213796 100644 --- a/pkg/detectors/blazemeter/blazemeter_test.go +++ b/pkg/detectors/blazemeter/blazemeter_test.go @@ -1,120 +1,128 @@ -//go:build detectors -// +build detectors - package blazemeter import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBlazemeter_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BLAZEMETER") - inactiveSecret := testSecrets.MustGetField("BLAZEMETER_INACTIVE") +var ( + validPattern = "sjbuxa3m-vs4n-ykl8-8jpv-i09hdidciolp" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + blazemeterToken := "sjbuxa3m-vs4n-ykl8-8jpv-i09hdidciolp" + req.Header.Set("Authorization", "Bearer " + blazemeterToken) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else if resp.StatusCode == http.UnAuthorized { + // if auth failed try runscope token + runScope := "q2n4fn1k-wly8-11pm-tfw6-hnltkxgemtpk" + req.Header.Set("Authorization", "Bearer " + runScope) + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "sjbuxa3m-vs4n- ykl8-8jpv#i09hdidciolp" +) + +func TestBlazeMeter_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a blazemeter secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Blazemeter, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("blazemeter credentials: %s", validPattern), + want: []string{"sjbuxa3m-vs4n-ykl8-8jpv-i09hdidciolp"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a blazemeter secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Blazemeter, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"q2n4fn1k-wly8-11pm-tfw6-hnltkxgemtpk", "sjbuxa3m-vs4n-ykl8-8jpv-i09hdidciolp"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("blazemeter credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Blazemeter.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Blazemeter.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/blitapp/blitapp.go b/pkg/detectors/blitapp/blitapp.go index 986ebdc8b730..a3824c808beb 100644 --- a/pkg/detectors/blitapp/blitapp.go +++ b/pkg/detectors/blitapp/blitapp.go @@ -2,10 +2,11 @@ package blitapp import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/blitapp/blitapp_integration_test.go b/pkg/detectors/blitapp/blitapp_integration_test.go new file mode 100644 index 000000000000..01e4e9df16a9 --- /dev/null +++ b/pkg/detectors/blitapp/blitapp_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package blitapp + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBlitApp_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BLITAPP") + inactiveSecret := testSecrets.MustGetField("BLITAPP_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a blitapp secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BlitApp, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a blitapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BlitApp, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("BlitApp.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("BlitApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/blitapp/blitapp_test.go b/pkg/detectors/blitapp/blitapp_test.go index 01e4e9df16a9..2728777cdded 100644 --- a/pkg/detectors/blitapp/blitapp_test.go +++ b/pkg/detectors/blitapp/blitapp_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package blitapp import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBlitApp_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BLITAPP") - inactiveSecret := testSecrets.MustGetField("BLITAPP_INACTIVE") +var ( + validPattern = "I_MncTA8nlFcqlBCakI5vwkwFD4_zRUYZKt8hyd" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + blitAppKey := "I_MncTA8nlFcqlBCakI5vwkwFD4_zRUYZKt8hyd" + req.Header.Set("API-Key", blitAppKey) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "%blitAppKey:I_Mn%^&*qlBCakI5vwkwFD4_zRUY" +) + +func TestBlitApp_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a blitapp secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BlitApp, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("blitapp credentials: %s", validPattern), + want: []string{"I_MncTA8nlFcqlBCakI5vwkwFD4_zRUYZKt8hyd"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a blitapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BlitApp, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"I_MncTA8nlFcqlBCakI5vwkwFD4_zRUYZKt8hyd"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("BlitApp.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("BlitApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/blocknative/blocknative.go b/pkg/detectors/blocknative/blocknative.go index 95f63b8cd4ad..7edec375feab 100644 --- a/pkg/detectors/blocknative/blocknative.go +++ b/pkg/detectors/blocknative/blocknative.go @@ -2,11 +2,12 @@ package blocknative import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/blocknative/blocknative_integration_test.go b/pkg/detectors/blocknative/blocknative_integration_test.go new file mode 100644 index 000000000000..d781cc699589 --- /dev/null +++ b/pkg/detectors/blocknative/blocknative_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package blocknative + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBlocknative_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BLOCKNATIVE") + inactiveSecret := testSecrets.MustGetField("BLOCKNATIVE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a blocknative secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BlockNative, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a blocknative secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BlockNative, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Blocknative.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Blocknative.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/blocknative/blocknative_test.go b/pkg/detectors/blocknative/blocknative_test.go index d781cc699589..bdb560f825ab 100644 --- a/pkg/detectors/blocknative/blocknative_test.go +++ b/pkg/detectors/blocknative/blocknative_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package blocknative import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBlocknative_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BLOCKNATIVE") - inactiveSecret := testSecrets.MustGetField("BLOCKNATIVE_INACTIVE") +var ( + validPattern = "76e50995-059f-3d1a-af8e-cc85fc05eb03" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + blocknativeSecret := "76e50995-059f-3d1a-af8e-cc85fc05eb03" + req.Header.Set("Authorization", blocknativeSecret) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "2xN7puShxzNf5fZleQthTg305l95D3gSD%c^" +) + +func TestBlockNative_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a blocknative secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BlockNative, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("blocknative credentials: %s", validPattern), + want: []string{"76e50995-059f-3d1a-af8e-cc85fc05eb03"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a blocknative secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BlockNative, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"76e50995-059f-3d1a-af8e-cc85fc05eb03"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("blocknative credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Blocknative.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Blocknative.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/blogger/blogger.go b/pkg/detectors/blogger/blogger.go index aa6e5914bb10..cb93b93e9397 100644 --- a/pkg/detectors/blogger/blogger.go +++ b/pkg/detectors/blogger/blogger.go @@ -2,10 +2,11 @@ package blogger import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/blogger/blogger_integration_test.go b/pkg/detectors/blogger/blogger_integration_test.go new file mode 100644 index 000000000000..4afd72354d5b --- /dev/null +++ b/pkg/detectors/blogger/blogger_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package blogger + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBlogger_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BLOGGER") + inactiveSecret := testSecrets.MustGetField("BLOGGER_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a blogger secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Blogger, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a blogger secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Blogger, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Blogger.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Blogger.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/blogger/blogger_test.go b/pkg/detectors/blogger/blogger_test.go index 4afd72354d5b..13bc28111e89 100644 --- a/pkg/detectors/blogger/blogger_test.go +++ b/pkg/detectors/blogger/blogger_test.go @@ -1,120 +1,110 @@ -//go:build detectors -// +build detectors - package blogger import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBlogger_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BLOGGER") - inactiveSecret := testSecrets.MustGetField("BLOGGER_INACTIVE") +var ( + validPattern = "fnWLw7pz1tc6uCzq6qocQZIxRF6SqUaOOkLqePY" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/blogger/blogs?key=fnWLw7pz1tc6uCzq6qocQZIxRF6SqUaOOkLqePY" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "fnWL(w7pz1t)6uCz-q6qocQZIxRF6S/UqePY" +) + +func TestBlogger_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a blogger secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Blogger, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("blogger credentials: %s", validPattern), + want: []string{"fnWLw7pz1tc6uCzq6qocQZIxRF6SqUaOOkLqePY"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a blogger secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Blogger, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"fnWLw7pz1tc6uCzq6qocQZIxRF6SqUaOOkLqePY"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("blogger credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Blogger.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Blogger.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/bombbomb/bombbomb.go b/pkg/detectors/bombbomb/bombbomb.go index f73111c5cc97..f8b0600b41d3 100644 --- a/pkg/detectors/bombbomb/bombbomb.go +++ b/pkg/detectors/bombbomb/bombbomb.go @@ -2,10 +2,11 @@ package bombbomb import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/bombbomb/bombbomb_integration_test.go b/pkg/detectors/bombbomb/bombbomb_integration_test.go new file mode 100644 index 000000000000..cae85980792a --- /dev/null +++ b/pkg/detectors/bombbomb/bombbomb_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package bombbomb + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBombBomb_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BOMBBOMB") + inactiveSecret := testSecrets.MustGetField("BOMBBOMB_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bombbomb secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BombBomb, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bombbomb secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BombBomb, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("BombBomb.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("BombBomb.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bombbomb/bombbomb_test.go b/pkg/detectors/bombbomb/bombbomb_test.go index cae85980792a..201b840c1e27 100644 --- a/pkg/detectors/bombbomb/bombbomb_test.go +++ b/pkg/detectors/bombbomb/bombbomb_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package bombbomb import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBombBomb_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BOMBBOMB") - inactiveSecret := testSecrets.MustGetField("BOMBBOMB_INACTIVE") +var ( + validPattern = "HUmGL.17uQMEShYp2RVMR8vypd1iqj6FZcKkQ4SazuMkbEKhzRFKuvOiwYmNWPSvkE4wiLOv-zWTkK1WkVTScRb9_io0_kvhYX31tpwR3lAJUh27RJzf1BehaJTQDXhJB6aT2gQ2LMT7dda-b3vhmEuZHzPV9AMLV6cOrcqOTkK60vMcB0PTLRQ3c_kY.a.9.hRvgogdlI8mQJrzD0myPBY7lMpjpkcskQDpOgz2I37kNDYhf7IxT6sG-a7rI1LdpJ6HhJacktlNJSswST9jbt4A0ropfJJTHGny2aId4WyPpAnQubM98F1BUnyhfkDzenaUuuQ_ZoPn9mAOsdLQUlAyp4I9oLJ_v8yQ0Q4M.Yujscho9G4ZbVTInC2mP8taCPZdRK5qt-UfAF0CX9B4E0F9NItMUbRdbm3xIkl8C6iPUcgY5OTQDBSJRLKBJgIaEyyXe10pPw.qOUhLKNPcg5qPs1xhgBsZKfW2hNTff2dCL5h6E.940ojPuT0Iw90Q8kpQ2UzeUJrhXH9_GUANKA.pjD0-YcGpnlVEDouyXaXowUoh8pLqD-BtBQfteqyFqz7THGDvQKikMy7wiBuJAo0HttMG3jw1zKtA3gM6_VIXo_K4WN6yz8Ow4n5f6Unn5zn4j2haKA4WWI5-1c8-mm7SF5VqYJVz42wBmRqB6MWXegJ7yLt_EoG1tJHftnHZ" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + bombbombToken := "HUmGL.17uQMEShYp2RVMR8vypd1iqj6FZcKkQ4SazuMkbEKhzRFKuvOiwYmNWPSvkE4wiLOv-zWTkK1WkVTScRb9_io0_kvhYX31tpwR3lAJUh27RJzf1BehaJTQDXhJB6aT2gQ2LMT7dda-b3vhmEuZHzPV9AMLV6cOrcqOTkK60vMcB0PTLRQ3c_kY.a.9.hRvgogdlI8mQJrzD0myPBY7lMpjpkcskQDpOgz2I37kNDYhf7IxT6sG-a7rI1LdpJ6HhJacktlNJSswST9jbt4A0ropfJJTHGny2aId4WyPpAnQubM98F1BUnyhfkDzenaUuuQ_ZoPn9mAOsdLQUlAyp4I9oLJ_v8yQ0Q4M.Yujscho9G4ZbVTInC2mP8taCPZdRK5qt-UfAF0CX9B4E0F9NItMUbRdbm3xIkl8C6iPUcgY5OTQDBSJRLKBJgIaEyyXe10pPw.qOUhLKNPcg5qPs1xhgBsZKfW2hNTff2dCL5h6E.940ojPuT0Iw90Q8kpQ2UzeUJrhXH9_GUANKA.pjD0-YcGpnlVEDouyXaXowUoh8pLqD-BtBQfteqyFqz7THGDvQKikMy7wiBuJAo0HttMG3jw1zKtA3gM6_VIXo_K4WN6yz8Ow4n5f6Unn5zn4j2haKA4WWI5-1c8-mm7SF5VqYJVz42wBmRqB6MWXegJ7yLt_EoG1tJHftnHZ" + req.Header.Set("Authorization", bombbombToken) - type args struct { - ctx context.Context - data []byte - verify bool + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "17uQMEShYp2RVMR8vypd1iqj6FZcKkQ4SazuMkbEKhzRFKuvOiwYmNWPSvkE4wiLOv%c^zWTkK1WkVTScRb9_io0_kvhYX31tpwR3lAJUh27RJzf1BehaJTQDXhJB6aT2gQ2LMT7dda" +) + +func TestBombBomb_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bombbomb secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BombBomb, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("bombbomb credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bombbomb secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BombBomb, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("bombbomb credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("BombBomb.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("BombBomb.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/boostnote/boostnote.go b/pkg/detectors/boostnote/boostnote.go index 519b66443421..c292ad570434 100644 --- a/pkg/detectors/boostnote/boostnote.go +++ b/pkg/detectors/boostnote/boostnote.go @@ -3,10 +3,11 @@ package boostnote import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/boostnote/boostnote_integration_test.go b/pkg/detectors/boostnote/boostnote_integration_test.go new file mode 100644 index 000000000000..321b6e4a87a2 --- /dev/null +++ b/pkg/detectors/boostnote/boostnote_integration_test.go @@ -0,0 +1,117 @@ +//go:build detectors +// +build detectors + +package boostnote + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBoostNote_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BOOSTNOTE") + inactiveSecret := testSecrets.MustGetField("BOOSTNOTE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a boostnote secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BoostNote, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a boostnote secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BoostNote, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("BoostNote.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("BoostNote.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + s.FromData(ctx, false, data) + } + }) + } +} diff --git a/pkg/detectors/boostnote/boostnote_test.go b/pkg/detectors/boostnote/boostnote_test.go index 321b6e4a87a2..a64132615735 100644 --- a/pkg/detectors/boostnote/boostnote_test.go +++ b/pkg/detectors/boostnote/boostnote_test.go @@ -1,116 +1,112 @@ -//go:build detectors -// +build detectors - package boostnote import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBoostNote_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BOOSTNOTE") - inactiveSecret := testSecrets.MustGetField("BOOSTNOTE_INACTIVE") +var ( + validPattern = "fb1026ac5994e3ad01799fe040289317ba2594a20e9e45307a143be82b49d213" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + boostnoteKey := "fb1026ac5994e3ad01799fe040289317ba2594a20e9e45307a143be82b49d213" + req.Header.Set("Authorization", "Bearer " + boostnoteKey) - type args struct { - ctx context.Context - data []byte - verify bool + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "#^fb1026ac59=4e3ad01799fe04028931___4a20e9e45307a143be82b49d213$" +) + +func TestBoostNote_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a boostnote secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BoostNote, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("boostnote credentials: %s", validPattern), + want: []string{"fb1026ac5994e3ad01799fe040289317ba2594a20e9e45307a143be82b49d213"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a boostnote secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BoostNote, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"fb1026ac5994e3ad01799fe040289317ba2594a20e9e45307a143be82b49d213"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("boostnote credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("BoostNote.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - got[i].Raw = nil + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("BoostNote.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - s.FromData(ctx, false, data) + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/borgbase/borgbase.go b/pkg/detectors/borgbase/borgbase.go index 2556852a7b2c..c7d33b51b5c8 100644 --- a/pkg/detectors/borgbase/borgbase.go +++ b/pkg/detectors/borgbase/borgbase.go @@ -3,12 +3,13 @@ package borgbase import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/borgbase/borgbase_integration_test.go b/pkg/detectors/borgbase/borgbase_integration_test.go new file mode 100644 index 000000000000..31757adf83cf --- /dev/null +++ b/pkg/detectors/borgbase/borgbase_integration_test.go @@ -0,0 +1,117 @@ +//go:build detectors +// +build detectors + +package borgbase + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBorgbase_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BORGBASE") + inactiveSecret := testSecrets.MustGetField("BORGBASE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a borgbase secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Borgbase, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a borgbase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Borgbase, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Borgbase.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Borgbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + s.FromData(ctx, false, data) + } + }) + } +} diff --git a/pkg/detectors/borgbase/borgbase_test.go b/pkg/detectors/borgbase/borgbase_test.go index 31757adf83cf..fdd7b6597058 100644 --- a/pkg/detectors/borgbase/borgbase_test.go +++ b/pkg/detectors/borgbase/borgbase_test.go @@ -1,116 +1,113 @@ -//go:build detectors -// +build detectors - package borgbase import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBorgbase_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BORGBASE") - inactiveSecret := testSecrets.MustGetField("BORGBASE_INACTIVE") +var ( + validPattern = "FoHclCFSi_aV09jowJQ4RUF_MiqW6ioqq6_OcyB0PFlV-mQ1yoFjk5JLlxbzRUzKTA6vsfR8wq6TNc83rtNKlkD092Sj1c9CbPVBXlHksy.sT2I/so6bMGdPcqxzbjrxYgAUiORgqJDeTet4gKOQlZpt" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + payload := '{"query":"{ sshList {id, name}}"}' + req, err := http.NewRequest("POST", url, payload) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + borgbaseToken := "FoHclCFSi_aV09jowJQ4RUF_MiqW6ioqq6_OcyB0PFlV-mQ1yoFjk5JLlxbzRUzKTA6vsfR8wq6TNc83rtNKlkD092Sj1c9CbPVBXlHksy.sT2I/so6bMGdPcqxzbjrxYgAUiORgqJDeTet4gKOQlZpt" + req.Header.Set("Authorization", "Bearer " + borgbaseToken) - type args struct { - ctx context.Context - data []byte - verify bool + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "mQ1yoFjk5JLlxbzRUzKTA6vsfR8wq,6TNc83rtNKlkD092Sj1c9CbPVBXlHksy%c^so6bMGdPcqxzbjrxYgAUiORgqJDeTet4gKOQlZpt" +) + +func TestBorgBase_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a borgbase secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Borgbase, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("borgbase credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a borgbase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Borgbase, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("borgbase credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Borgbase.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - got[i].Raw = nil + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Borgbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - s.FromData(ctx, false, data) + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/box/box_test.go b/pkg/detectors/box/box_test.go index cea5ee94b503..49b3199b6079 100644 --- a/pkg/detectors/box/box_test.go +++ b/pkg/detectors/box/box_test.go @@ -1,6 +1,3 @@ -//go:build detectors -// +build detectors - package box import ( diff --git a/pkg/detectors/boxoauth/boxoauth_test.go b/pkg/detectors/boxoauth/boxoauth_test.go index ae5d85e6e6d9..583c7304b41b 100644 --- a/pkg/detectors/boxoauth/boxoauth_test.go +++ b/pkg/detectors/boxoauth/boxoauth_test.go @@ -1,6 +1,3 @@ -//go:build detectors -// +build detectors - package boxoauth import ( diff --git a/pkg/detectors/braintreepayments/braintreepayments_integration_test.go b/pkg/detectors/braintreepayments/braintreepayments_integration_test.go new file mode 100644 index 000000000000..06c933c8e94c --- /dev/null +++ b/pkg/detectors/braintreepayments/braintreepayments_integration_test.go @@ -0,0 +1,164 @@ +//go:build detectors +// +build detectors + +package braintreepayments + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBraintreePayments_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BRAINTREEPAYMENTS") + id := testSecrets.MustGetField("BRAINTREEPAYMENTS_USER") + inactiveSecret := testSecrets.MustGetField("BRAINTREEPAYMENTS_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{useTestURL: true}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BraintreePayments, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{ + client: common.ConstantResponseHttpClient(404, ""), + }, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BraintreePayments, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{ + client: common.SaneHttpClientTimeOut(1 * time.Microsecond), + }, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BraintreePayments, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, unverified", + s: Scanner{useTestURL: true}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BraintreePayments, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("BraintreePayments.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("BraintreePayments.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/braintreepayments/braintreepayments_test.go b/pkg/detectors/braintreepayments/braintreepayments_test.go index db4c346dc373..b66611ae4bea 100644 --- a/pkg/detectors/braintreepayments/braintreepayments_test.go +++ b/pkg/detectors/braintreepayments/braintreepayments_test.go @@ -2,160 +2,112 @@ package braintreepayments import ( "context" - "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBraintreePayments_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BRAINTREEPAYMENTS") - id := testSecrets.MustGetField("BRAINTREEPAYMENTS_USER") - inactiveSecret := testSecrets.MustGetField("BRAINTREEPAYMENTS_INACTIVE") +var ( + validPattern = "braintreeKey: f7b3cb83a7fdb915a71ce17ab8a903cc \n braintreeId: kmajpm4h1pqoqxyo" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + braintreeKey := "f7b3cb83a7fdb915a71ce17ab8a903cc" + braintreeId := "kmajpm4h1pqoqxyo" + req.SetBasicAuth(braintreeKey, braintreeId) - type args struct { - ctx context.Context - data []byte - verify bool + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "braintreeCreds: f7b3cb83a7fdb915a71ce17ab8a903cckmajpm4h1pqoqxyo" +) + +func TestBrainTreePayments_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{useTestURL: true}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BraintreePayments, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, verified but unexpected api surface", - s: Scanner{ - client: common.ConstantResponseHttpClient(404, ""), - }, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BraintreePayments, - Verified: false, - }, - }, - wantErr: false, - wantVerificationErr: true, + name: "valid pattern", + input: validPattern, + want: []string{"f7b3cb83a7fdb915a71ce17ab8a903cc"}, }, { - name: "found, would be verified if not for timeout", - s: Scanner{ - client: common.SaneHttpClientTimeOut(1 * time.Microsecond), - }, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BraintreePayments, - Verified: false, - }, - }, - wantErr: false, - wantVerificationErr: true, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"f7b3cb83a7fdb915a71ce17ab8a903cc"}, }, { - name: "found, unverified", - s: Scanner{useTestURL: true}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a braintree secret %s within braintree %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BraintreePayments, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("BraintreePayments.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) - } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("BraintreePayments.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/brandfetch/brandfetch.go b/pkg/detectors/brandfetch/brandfetch.go index 6dc6c3eabb9c..efdf4b9b66ce 100644 --- a/pkg/detectors/brandfetch/brandfetch.go +++ b/pkg/detectors/brandfetch/brandfetch.go @@ -2,10 +2,11 @@ package brandfetch import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/brandfetch/brandfetch_integration_test.go b/pkg/detectors/brandfetch/brandfetch_integration_test.go new file mode 100644 index 000000000000..76fe755e3eae --- /dev/null +++ b/pkg/detectors/brandfetch/brandfetch_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package brandfetch + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBrandfetch_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BRANDFETCH") + inactiveSecret := testSecrets.MustGetField("BRANDFETCH_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a brandfetch secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Brandfetch, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a brandfetch secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Brandfetch, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Brandfetch.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Brandfetch.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/brandfetch/brandfetch_test.go b/pkg/detectors/brandfetch/brandfetch_test.go index 76fe755e3eae..0085ffca2a54 100644 --- a/pkg/detectors/brandfetch/brandfetch_test.go +++ b/pkg/detectors/brandfetch/brandfetch_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package brandfetch import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBrandfetch_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BRANDFETCH") - inactiveSecret := testSecrets.MustGetField("BRANDFETCH_INACTIVE") +var ( + validPattern = "uHOAdwfQ7sD2yOpur72UqyUeIqnFwILOIlEPyBtJ" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + brandfetchAPIKey := "uHOAdwfQ7sD2yOpur72UqyUeIqnFwILOIlEPyBtJ" + req.Header.Set("x-api-key", brandfetchAPIKey) // brandfetch secret + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "yUeIqnFwILOIlEPyBt+=JOAdwfQ7sD2uHOAdwf2U[qy]UeIqnFwILOIlEPyBtJ^" +) + +func TestBrandFetch_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a brandfetch secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Brandfetch, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("brandfetch credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a brandfetch secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Brandfetch, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("brandfetch credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Brandfetch.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Brandfetch.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/browserstack/browserstack.go b/pkg/detectors/browserstack/browserstack.go index 1e93ac0bf924..8d726ee1de22 100644 --- a/pkg/detectors/browserstack/browserstack.go +++ b/pkg/detectors/browserstack/browserstack.go @@ -3,15 +3,16 @@ package browserstack import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "net/http/cookiejar" "strings" + regexp "github.com/wasilibs/go-re2" + "golang.org/x/net/publicsuffix" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" - "golang.org/x/net/publicsuffix" ) type Scanner struct { diff --git a/pkg/detectors/browserstack/browserstack_integration_test.go b/pkg/detectors/browserstack/browserstack_integration_test.go new file mode 100644 index 000000000000..f8185173e137 --- /dev/null +++ b/pkg/detectors/browserstack/browserstack_integration_test.go @@ -0,0 +1,194 @@ +//go:build detectors +// +build detectors + +package browserstack + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBrowserStack_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secretUser := testSecrets.MustGetField("BROWSERSTACK_USER") + secret := testSecrets.MustGetField("BROWSERSTACK") + inactiveSecret := testSecrets.MustGetField("BROWSERSTACK_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BrowserStack, + Verified: true, + RawV2: []byte(fmt.Sprintf("%s%s", secret, secretUser)), + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s but not valid", inactiveSecret, secretUser)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BrowserStack, + Verified: false, + RawV2: []byte(fmt.Sprintf("%s%s", inactiveSecret, secretUser)), + }, + }, + wantErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_BrowserStack, + RawV2: []byte(fmt.Sprintf("%s%s", secret, secretUser)), + Verified: false, + } + r.SetVerificationError(fmt.Errorf("context deadline exceeded"), secret) + results := []detectors.Result{r} + return results + }(), + wantErr: false, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_BrowserStack, + Verified: false, + RawV2: []byte(fmt.Sprintf("%s%s", secret, secretUser)), + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 404"), secret) + results := []detectors.Result{r} + return results + }(), + wantErr: false, + }, + { + name: "found, verified but blocked by browserstack", + s: Scanner{client: common.SaneHttpClient()}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_BrowserStack, + Verified: false, + RawV2: []byte(fmt.Sprintf("%s%s", secret, secretUser)), + } + r.SetVerificationError(fmt.Errorf("blocked by browserstack"), secret) + results := []detectors.Result{r} + return results + }(), + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("BrowserStack.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("BrowserStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/browserstack/browserstack_test.go b/pkg/detectors/browserstack/browserstack_test.go index f8185173e137..4e9f086689b5 100644 --- a/pkg/detectors/browserstack/browserstack_test.go +++ b/pkg/detectors/browserstack/browserstack_test.go @@ -1,193 +1,117 @@ -//go:build detectors -// +build detectors - package browserstack import ( "context" "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBrowserStack_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secretUser := testSecrets.MustGetField("BROWSERSTACK_USER") - secret := testSecrets.MustGetField("BROWSERSTACK") - inactiveSecret := testSecrets.MustGetField("BROWSERSTACK_INACTIVE") +var ( + validPattern = "accessKey: RxLVnOlvj3V4bh4RBwOd / username: Dun88d_-_-.4yggxTlrq" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + if browserstackKey, _ := os.GetEnv("ACCESS_KEY"); browserstackKey != "RxLVnOlvj3V4bh4RBwOd" { + return fmt.Errorf("invalid accessKey: %v expected: %v", "RxLVnOlvj3V4bh4RBwOd", "1YZazUAPFOiaIFljWDhC") + } + + if browserstackUser, _ := os.GetEnv("USER_NAME"); browserstackUser != "Dun88d_-_-.4yggxTlrq" { + return fmt.Errorf("invalid userName: %v", "Dun88d_-_-.4yggxTlrq") + } - type args struct { - ctx context.Context - data []byte - verify bool + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "BS_USERNAME: Dun--d_$_$.4yggxTlrq%c^ \n BS_AUTHKEY: Dun88d_)" +) + +func TestBrowserStack_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BrowserStack, - Verified: true, - RawV2: []byte(fmt.Sprintf("%s%s", secret, secretUser)), - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("browserstack credentials: %s", validPattern), + want: []string{"RxLVnOlvj3V4bh4RBwOdDun88d_-_-.4yggxTlrq"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s but not valid", inactiveSecret, secretUser)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BrowserStack, - Verified: false, - RawV2: []byte(fmt.Sprintf("%s%s", inactiveSecret, secretUser)), - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"RxLVnOlvj3V4bh4RBwOdDun88d_-_-.4yggxTlrq", "RxLVnOlvj3V4bh4RBwOdbrowserstackUser", "RxLVnOlvj3V4bh4RBwOdDun88d_-_-.4yggxTlrq", "RxLVnOlvj3V4bh4RBwOdDun88d_-_-.4yggxTlrq"}, }, { - name: "found, would be verified if not for timeout", - s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)), - verify: true, - }, - want: func() []detectors.Result { - r := detectors.Result{ - DetectorType: detectorspb.DetectorType_BrowserStack, - RawV2: []byte(fmt.Sprintf("%s%s", secret, secretUser)), - Verified: false, - } - r.SetVerificationError(fmt.Errorf("context deadline exceeded"), secret) - results := []detectors.Result{r} - return results - }(), - wantErr: false, - }, - { - name: "found, verified but unexpected api surface", - s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)), - verify: true, - }, - want: func() []detectors.Result { - r := detectors.Result{ - DetectorType: detectorspb.DetectorType_BrowserStack, - Verified: false, - RawV2: []byte(fmt.Sprintf("%s%s", secret, secretUser)), - } - r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 404"), secret) - results := []detectors.Result{r} - return results - }(), - wantErr: false, - }, - { - name: "found, verified but blocked by browserstack", - s: Scanner{client: common.SaneHttpClient()}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a BROWSERSTACK_ACCESS_KEY %s within BROWSERSTACK_USERNAME %s", secret, secretUser)), - verify: true, - }, - want: func() []detectors.Result { - r := detectors.Result{ - DetectorType: detectorspb.DetectorType_BrowserStack, - Verified: false, - RawV2: []byte(fmt.Sprintf("%s%s", secret, secretUser)), - } - r.SetVerificationError(fmt.Errorf("blocked by browserstack"), secret) - results := []detectors.Result{r} - return results - }(), - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("browserstack credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("BrowserStack.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - gotErr := "" - if got[i].VerificationError() != nil { - gotErr = got[i].VerificationError().Error() - } - wantErr := "" - if tt.want[i].VerificationError() != nil { - wantErr = tt.want[i].VerificationError().Error() + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if gotErr != wantErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("BrowserStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/browshot/browshot.go b/pkg/detectors/browshot/browshot.go index 78e4228ea9bb..23d56572ab71 100644 --- a/pkg/detectors/browshot/browshot.go +++ b/pkg/detectors/browshot/browshot.go @@ -2,10 +2,11 @@ package browshot import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/browshot/browshot_integration_test.go b/pkg/detectors/browshot/browshot_integration_test.go new file mode 100644 index 000000000000..401ed17470eb --- /dev/null +++ b/pkg/detectors/browshot/browshot_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package browshot + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBrowshot_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BROWSHOT") + inactiveSecret := testSecrets.MustGetField("BROWSHOT_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a browshot secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Browshot, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a browshot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Browshot, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Browshot.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Browshot.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/browshot/browshot_test.go b/pkg/detectors/browshot/browshot_test.go index 401ed17470eb..19656366b77c 100644 --- a/pkg/detectors/browshot/browshot_test.go +++ b/pkg/detectors/browshot/browshot_test.go @@ -1,120 +1,110 @@ -//go:build detectors -// +build detectors - package browshot import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBrowshot_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BROWSHOT") - inactiveSecret := testSecrets.MustGetField("BROWSHOT_INACTIVE") +var ( + validPattern = "AemQ06R35S1Y8rXnOzYvT8I4-a7u" + complexPattern = ` + func main() { + url := "https://api.browshot.com/v1/instances/list?key=AemQ06R35S1Y8rXnOzYvT8I4-a7u" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "2xN7puShxzNf5fZleQt#hTg305l95D3gSD-c^" +) + +func TestBrowShot_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a browshot secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Browshot, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("browshot credentials: %s", validPattern), + want: []string{"AemQ06R35S1Y8rXnOzYvT8I4-a7u"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a browshot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Browshot, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"AemQ06R35S1Y8rXnOzYvT8I4-a7u"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("browshot credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Browshot.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Browshot.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/bscscan/bscscan.go b/pkg/detectors/bscscan/bscscan.go index 5d7606c2f983..230a00144b36 100644 --- a/pkg/detectors/bscscan/bscscan.go +++ b/pkg/detectors/bscscan/bscscan.go @@ -2,11 +2,12 @@ package bscscan import ( "context" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/bscscan/bscscan_integration_test.go b/pkg/detectors/bscscan/bscscan_integration_test.go new file mode 100644 index 000000000000..b8bacbc2969f --- /dev/null +++ b/pkg/detectors/bscscan/bscscan_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package bscscan + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBscscan_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BSCSCAN") + inactiveSecret := testSecrets.MustGetField("BSCSCAN_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bscscan secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BscScan, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bscscan secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BscScan, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Bscscan.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Bscscan.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bscscan/bscscan_test.go b/pkg/detectors/bscscan/bscscan_test.go index b8bacbc2969f..739f6ce7eab4 100644 --- a/pkg/detectors/bscscan/bscscan_test.go +++ b/pkg/detectors/bscscan/bscscan_test.go @@ -1,120 +1,110 @@ -//go:build detectors -// +build detectors - package bscscan import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBscscan_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BSCSCAN") - inactiveSecret := testSecrets.MustGetField("BSCSCAN_INACTIVE") +var ( + validPattern = "HYZHPP4PBYXCOZAVK4FH55W4MRHYLALPU1" + complexPattern = ` + func main() { + url := "https://api.bscscan.com/v1/resource?apikey=HYZHPP4PBYXCOZAVK4FH55W4MRHYLALPU1" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "2xHYZHPP4PBYXCOZAVK4FH55W4MRHYLALPU1thTg303gSD%c^" +) + +func TestBscScan_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bscscan secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BscScan, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("bscscan credentials: %s", validPattern), + want: []string{"HYZHPP4PBYXCOZAVK4FH55W4MRHYLALPU1"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bscscan secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BscScan, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"HYZHPP4PBYXCOZAVK4FH55W4MRHYLALPU1"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("bscscan credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Bscscan.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Bscscan.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/buddyns/buddyns.go b/pkg/detectors/buddyns/buddyns.go index 9995b56fa891..4cb2349141bf 100644 --- a/pkg/detectors/buddyns/buddyns.go +++ b/pkg/detectors/buddyns/buddyns.go @@ -3,10 +3,11 @@ package buddyns import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/buddyns/buddyns_integration_test.go b/pkg/detectors/buddyns/buddyns_integration_test.go new file mode 100644 index 000000000000..d1d84bf07093 --- /dev/null +++ b/pkg/detectors/buddyns/buddyns_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package buddyns + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBuddyns_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BUDDYNS") + inactiveSecret := testSecrets.MustGetField("BUDDYNS_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a buddyns secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BuddyNS, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a buddyns secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_BuddyNS, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Buddyns.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Buddyns.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/buddyns/buddyns_test.go b/pkg/detectors/buddyns/buddyns_test.go index d1d84bf07093..4cf65107fb89 100644 --- a/pkg/detectors/buddyns/buddyns_test.go +++ b/pkg/detectors/buddyns/buddyns_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package buddyns import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBuddyns_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BUDDYNS") - inactiveSecret := testSecrets.MustGetField("BUDDYNS_INACTIVE") +var ( + validPattern = "kkmvdiolccw4v0tue4lu7l7kmnnb4ao8z25ezink" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + buddynsToken := "kkmvdiolccw4v0tue4lu7l7kmnnb4ao8z25ezink" + req.Header.Set("Authorization", "Token " + buddynsToken) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "diolccw4v0tue4lu7l7kmnnb4ao8z25ezink305l95D3gSD%c^" +) + +func TestBuddyNs_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a buddyns secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BuddyNS, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("buddyns credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a buddyns secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_BuddyNS, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("buddyns credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Buddyns.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Buddyns.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/budibase/budibase.go b/pkg/detectors/budibase/budibase.go index 3960e963807c..7cbf9742b99b 100644 --- a/pkg/detectors/budibase/budibase.go +++ b/pkg/detectors/budibase/budibase.go @@ -3,10 +3,11 @@ package budibase import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/budibase/budibase_integration_test.go b/pkg/detectors/budibase/budibase_integration_test.go new file mode 100644 index 000000000000..ebd9ce6c8149 --- /dev/null +++ b/pkg/detectors/budibase/budibase_integration_test.go @@ -0,0 +1,130 @@ +//go:build detectors +// +build detectors + +package budibase + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBudibase_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BUDIBASE") + inactiveSecret := testSecrets.MustGetField("BUDIBASE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a budibase secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Budibase, + Verified: true, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a budibase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_Budibase, + Verified: true, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 403")) + return []detectors.Result{r} + }(), + wantErr: false, + wantVerificationErr: true, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Budibase.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Budibase.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/budibase/budibase_test.go b/pkg/detectors/budibase/budibase_test.go index ebd9ce6c8149..962e26856228 100644 --- a/pkg/detectors/budibase/budibase_test.go +++ b/pkg/detectors/budibase/budibase_test.go @@ -1,129 +1,111 @@ -//go:build detectors -// +build detectors - package budibase import ( "context" "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBudibase_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BUDIBASE") - inactiveSecret := testSecrets.MustGetField("BUDIBASE_INACTIVE") +var ( + validPattern = "b256def166fcdf4a429a1e83175105d5-fd36f3da1e934bf533cd0e68dbb80ed6a42e1178bd4200428d83e876e7d05e40b21e3a68888f826d" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" - type args struct { - ctx context.Context - data []byte - verify bool + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + req.Header.Set("x-budibase-api-key", "b256def166fcdf4a429a1e83175105d5-fd36f3da1e934bf533cd0e68dbb80ed6a42e1178bd4200428d83e876e7d05e40b21e3a68888f826d") + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "diolccw4v0tue4lu7l7kmnnb4ao8z25ezink305l95D3gSD%c^" +) + +func TestBudiBase_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a budibase secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Budibase, - Verified: true, - }, - }, - wantErr: false, - wantVerificationErr: false, + name: "valid pattern", + input: fmt.Sprintf("budibase credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a budibase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: func() []detectors.Result { - r := detectors.Result{ - DetectorType: detectorspb.DetectorType_Budibase, - Verified: true, - } - r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 403")) - return []detectors.Result{r} - }(), - wantErr: false, - wantVerificationErr: true, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - wantVerificationErr: false, + name: "invalid pattern", + input: fmt.Sprintf("budibase credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Budibase.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("Budibase.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/bugherd/bugherd_integration_test.go b/pkg/detectors/bugherd/bugherd_integration_test.go new file mode 100644 index 000000000000..7d015a448d68 --- /dev/null +++ b/pkg/detectors/bugherd/bugherd_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package bugherd + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBugherd_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BUGHERD") + inactiveSecret := testSecrets.MustGetField("BUGHERD_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bugherd secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bugherd, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bugherd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bugherd, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Bugherd.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Bugherd.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bugherd/bugherd_test.go b/pkg/detectors/bugherd/bugherd_test.go index 7d015a448d68..bdff434cf9ba 100644 --- a/pkg/detectors/bugherd/bugherd_test.go +++ b/pkg/detectors/bugherd/bugherd_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package bugherd import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBugherd_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BUGHERD") - inactiveSecret := testSecrets.MustGetField("BUGHERD_INACTIVE") +var ( + validPattern = "fisy6bbu6il4x96bekx587" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + bugherdToken := "fisy6bbu6il4x96bekx587" + req.Header.Set("Authorization", "Basic " + buddynsToken) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "fisy6bbu+6il4x()96bekx587-7l7kmnnb4ao8z25ezink305l95D3gSD%c^" +) + +func TestBugHerd_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bugherd secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bugherd, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("bugherd credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bugherd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bugherd, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("bugherd credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Bugherd.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Bugherd.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/bugsnag/bugsnag.go b/pkg/detectors/bugsnag/bugsnag.go index 1289bd3ce7dd..28a1e196a3b7 100644 --- a/pkg/detectors/bugsnag/bugsnag.go +++ b/pkg/detectors/bugsnag/bugsnag.go @@ -3,10 +3,11 @@ package bugsnag import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/bugsnag/bugsnag_integration_test.go b/pkg/detectors/bugsnag/bugsnag_integration_test.go new file mode 100644 index 000000000000..2d56674eb5b8 --- /dev/null +++ b/pkg/detectors/bugsnag/bugsnag_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package bugsnag + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBugsnag_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BUGSNAG") + inactiveSecret := testSecrets.MustGetField("BUGSNAG_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bugsnag secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bugsnag, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bugsnag secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bugsnag, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Bugsnag.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Bugsnag.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bugsnag/bugsnag_test.go b/pkg/detectors/bugsnag/bugsnag_test.go index 2d56674eb5b8..a221b3b26d69 100644 --- a/pkg/detectors/bugsnag/bugsnag_test.go +++ b/pkg/detectors/bugsnag/bugsnag_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package bugsnag import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBugsnag_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BUGSNAG") - inactiveSecret := testSecrets.MustGetField("BUGSNAG_INACTIVE") +var ( + validPattern = "wz9450iu-iewm-jonx-eab8-0ibxwadddm8i" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + bugsnagToken := "wz9450iu-iewm-jonx-eab8-0ibxwadddm8i" + req.Header.Set("Authorization", "token " + bugsnagToken) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "bugsnagToken: %c^wz9450iu-iewm-jonx-eab8-" +) + +func TestBugSnag_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bugsnag secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bugsnag, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("bugsnag credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bugsnag secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bugsnag, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("bugsnag credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Bugsnag.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Bugsnag.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/buildkite/buildkite.go b/pkg/detectors/buildkite/buildkite.go index 1b8dd07271eb..e91d3550631f 100644 --- a/pkg/detectors/buildkite/buildkite.go +++ b/pkg/detectors/buildkite/buildkite.go @@ -3,10 +3,11 @@ package buildkite import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/buildkite/buildkite_integration_test.go b/pkg/detectors/buildkite/buildkite_integration_test.go new file mode 100644 index 000000000000..f0d334959cce --- /dev/null +++ b/pkg/detectors/buildkite/buildkite_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package buildkite + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBuildkite_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BUILDKITE_TOKEN") + inactiveSecret := testSecrets.MustGetField("BUILDKITE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a buildkite secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Buildkite, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a buildkite secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Buildkite, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Buildkite.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Buildkite.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/buildkite/buildkite_test.go b/pkg/detectors/buildkite/buildkite_test.go index f0d334959cce..5005b11e003c 100644 --- a/pkg/detectors/buildkite/buildkite_test.go +++ b/pkg/detectors/buildkite/buildkite_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package buildkite import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBuildkite_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BUILDKITE_TOKEN") - inactiveSecret := testSecrets.MustGetField("BUILDKITE_INACTIVE") +var ( + validPattern = "kimu4axq3jxxdj8un0kpo3ua2ucr05zmhh4de0r6" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + buildkite_secret := "kimu4axq3jxxdj8un0kpo3ua2ucr05zmhh4de0r6" + req.Header.Set("Authorization", "Bearer " + buildkite_secret) - type args struct { - ctx context.Context - data []byte - verify bool + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "buildkite: %c^wz9450iu-buildkite_secret-jonx-eab8" +) + +func TestBuildKite_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a buildkite secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Buildkite, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("buildkite credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a buildkite secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Buildkite, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("buildkite credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Buildkite.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Buildkite.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/buildkitev2/buildkite_test.go b/pkg/detectors/buildkitev2/buildkite_test.go index 26c8a2c1d3a5..44b9cc645c3a 100644 --- a/pkg/detectors/buildkitev2/buildkite_test.go +++ b/pkg/detectors/buildkitev2/buildkite_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package buildkitev2 import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBuildkite_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BUILDKITEV2_TOKEN") - inactiveSecret := testSecrets.MustGetField("BUILDKITEV2_INACTIVE") +var ( + validPattern = "bkua_hqlh73m51jtho0jh12wcf2758c8fcdbv05z023ly" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + buildkite_secret := "bkua_hqlh73m51jtho0jh12wcf2758c8fcdbv05z023ly" + req.Header.Set("Authorization", "Bearer " + buildkite_secret) - type args struct { - ctx context.Context - data []byte - verify bool + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "bkua_: hqlh73m51jtho0jh12wcf2758c8fcdbv05z023ly-jonx-eab8" +) + +func TestBuildKiteV2_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a buildkite secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Buildkite, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("buildkite credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a buildkite secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Buildkite, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("buildkite credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Buildkite.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Buildkite.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/buildkitev2/buildkitev2_integration_test.go b/pkg/detectors/buildkitev2/buildkitev2_integration_test.go new file mode 100644 index 000000000000..26c8a2c1d3a5 --- /dev/null +++ b/pkg/detectors/buildkitev2/buildkitev2_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package buildkitev2 + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBuildkite_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BUILDKITEV2_TOKEN") + inactiveSecret := testSecrets.MustGetField("BUILDKITEV2_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a buildkite secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Buildkite, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a buildkite secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Buildkite, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Buildkite.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Buildkite.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bulbul/bulbul_integration_test.go b/pkg/detectors/bulbul/bulbul_integration_test.go new file mode 100644 index 000000000000..e51371cf554e --- /dev/null +++ b/pkg/detectors/bulbul/bulbul_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package bulbul + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBulbul_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BULBUL") + inactiveSecret := testSecrets.MustGetField("BULBUL_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bulbul secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bulbul, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bulbul secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bulbul, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Bulbul.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Bulbul.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bulbul/bulbul_test.go b/pkg/detectors/bulbul/bulbul_test.go index e51371cf554e..c21734f6edb0 100644 --- a/pkg/detectors/bulbul/bulbul_test.go +++ b/pkg/detectors/bulbul/bulbul_test.go @@ -1,120 +1,110 @@ -//go:build detectors -// +build detectors - package bulbul import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBulbul_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BULBUL") - inactiveSecret := testSecrets.MustGetField("BULBUL_INACTIVE") +var ( + validPattern = "3kx19qpx748ldb75lsjicbs6ipit6ssm" + complexPattern = ` + func main() { + url := "https://api.bulbul.com/v1/users?key=3kx19qpx748ldb75lsjicbs6ipit6ssm" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "bulbulKey: %c^wz9450iu-3kx19qcbs6ipit6ssm-jonx-eab8" +) + +func TestBulBul_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bulbul secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bulbul, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("bulbul credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bulbul secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bulbul, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Bulbul.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Bulbul.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/bulksms/bulksms_integration_test.go b/pkg/detectors/bulksms/bulksms_integration_test.go new file mode 100644 index 000000000000..aed029be3775 --- /dev/null +++ b/pkg/detectors/bulksms/bulksms_integration_test.go @@ -0,0 +1,122 @@ +//go:build detectors +// +build detectors + +package bulksms + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestBulksms_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BULKSMS") + inactiveSecret := testSecrets.MustGetField("BULKSMS_INACTIVE") + token := testSecrets.MustGetField("BULKSMS_TOKEN") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bulksms secret %s within bulksms %s", secret, token)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bulksms, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a bulksms secret %s within bulksms but %s not valid", inactiveSecret, token)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Bulksms, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Bulksms.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + got[i].RawV2 = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Bulksms.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/bulksms/bulksms_test.go b/pkg/detectors/bulksms/bulksms_test.go index aed029be3775..9f1734af3259 100644 --- a/pkg/detectors/bulksms/bulksms_test.go +++ b/pkg/detectors/bulksms/bulksms_test.go @@ -1,122 +1,114 @@ -//go:build detectors -// +build detectors - package bulksms import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestBulksms_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BULKSMS") - inactiveSecret := testSecrets.MustGetField("BULKSMS_INACTIVE") - token := testSecrets.MustGetField("BULKSMS_TOKEN") +var ( + validPattern = "bulksmsKey: (QGxPqRyzvt%xEKcV&#ePJGn)k0d9a \n bulksmsID: 381A26C47380B85F2DB572314-ACBDC267B-8" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + bulksmsKey := "(QGxPqRyzvt%xEKcV&#ePJGn)k0d9a" + bulksmsID := "381A26C47380B85F2DB572314-ACBDC267B-8" + + req.SetBasicAuth(bulksmsKey, bulksmsID) - type args struct { - ctx context.Context - data []byte - verify bool + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() + + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "bulksms creds: %c^wz9450iu-iewm-jonx-eab8-/F2DB572314-ACBDC267B-8" +) + +func TestBulkSMS_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bulksms secret %s within bulksms %s", secret, token)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bulksms, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"QGxPqRyzvt%xEKcV&#ePJGn)k0d9a381A26C47380B85F2DB572314-ACBDC267B-8"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a bulksms secret %s within bulksms but %s not valid", inactiveSecret, token)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Bulksms, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"QGxPqRyzvt%xEKcV&#ePJGn)k0d9a381A26C47380B85F2DB572314-ACBDC267B-8"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Bulksms.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - got[i].RawV2 = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Bulksms.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/buttercms/buttercms_integration_test.go b/pkg/detectors/buttercms/buttercms_integration_test.go new file mode 100644 index 000000000000..e4e5d13a7799 --- /dev/null +++ b/pkg/detectors/buttercms/buttercms_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package buttercms + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestButterCMS_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("BUTTERCMS_TOKEN") + inactiveSecret := testSecrets.MustGetField("BUTTERCMS_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a buttercms secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_ButterCMS, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a buttercms secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_ButterCMS, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("ButterCMS.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("ButterCMS.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/buttercms/buttercms_test.go b/pkg/detectors/buttercms/buttercms_test.go index e4e5d13a7799..cf88df9ed972 100644 --- a/pkg/detectors/buttercms/buttercms_test.go +++ b/pkg/detectors/buttercms/buttercms_test.go @@ -1,120 +1,110 @@ -//go:build detectors -// +build detectors - package buttercms import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestButterCMS_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("BUTTERCMS_TOKEN") - inactiveSecret := testSecrets.MustGetField("BUTTERCMS_INACTIVE") +var ( + validPattern = "l7psk7wkedkpiyp4jrx5fjdnno8c89243of6yde8" + complexPattern = ` + func main() { + url := "https://api.buttercms.com/v2/posts?auth_token=l7psk7wkedkpiyp4jrx5fjdnno8c89243of6yde8" + + // Create a new request with the secret as a header + req, err := http.NewRequest("GET", url, http.NoBody) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "butterCMSToken: l7psk7wkedkpiyp4j(rx5fjdnn)" +) + +func TestButterCMS_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a buttercms secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_ButterCMS, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("buttercms credentials: %s", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a buttercms secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_ButterCMS, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("ButterCMS.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("ButterCMS.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } }