diff --git a/pkg/detectors/allsports/allsports_integration_test.go b/pkg/detectors/allsports/allsports_integration_test.go new file mode 100644 index 000000000000..0619c62f7369 --- /dev/null +++ b/pkg/detectors/allsports/allsports_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package allsports + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAllsports_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("ALLSPORTS") + inactiveSecret := testSecrets.MustGetField("ALLSPORTS_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a allsports secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Allsports, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a allsports secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Allsports, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Allsports.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Allsports.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/allsports/allsports_test.go b/pkg/detectors/allsports/allsports_test.go index edf6454fc8b1..81414fd8f655 100644 --- a/pkg/detectors/allsports/allsports_test.go +++ b/pkg/detectors/allsports/allsports_test.go @@ -1,120 +1,86 @@ -//go:build detectors -// +build detectors - package allsports import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "d1f2e3c4b5a6d7e8f9g0h1i2j3k4l5m6n7o8p9q0r1s2t3u4v5w6x7y8z9a0b1ce" + invalidPattern = "Ad1f2e3c4b5a6d7e8f9g0h1i2j3k4l5m6n7o8p9q0r1sRt3u4v5w6x7y8z9a0b1cE" ) -func TestAllsports_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("ALLSPORTS") - inactiveSecret := testSecrets.MustGetField("ALLSPORTS_INACTIVE") +func TestAllSports_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a allsports secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Allsports, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("allsports: '%s'", validPattern), + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a allsports secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Allsports, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - key out of prefix range", + input: fmt.Sprintf("allsports keyword is not close to the real key in the data\n = '%s'", validPattern), + want: nil, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("allsports: '%s'", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Allsports.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Allsports.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/amadeus/amadeus_integration_test.go b/pkg/detectors/amadeus/amadeus_integration_test.go new file mode 100644 index 000000000000..0f87057ca491 --- /dev/null +++ b/pkg/detectors/amadeus/amadeus_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package amadeus + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAmadeus_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + id := testSecrets.MustGetField("AMADEUS") + secret := testSecrets.MustGetField("AMADEUS_SECRET") + inactiveSecret := testSecrets.MustGetField("AMADEUS_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Amadeus, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Amadeus, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Amadeus.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Amadeus.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/amadeus/amadeus_test.go b/pkg/detectors/amadeus/amadeus_test.go index 0f87057ca491..5437f5f067e6 100644 --- a/pkg/detectors/amadeus/amadeus_test.go +++ b/pkg/detectors/amadeus/amadeus_test.go @@ -1,121 +1,86 @@ -//go:build detectors -// +build detectors - package amadeus import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6 / A1b2C3d4E5f6G7ho" + invalidPattern = "A1b2$3d4E5f6G7h8I9j(K1l2M3n4O5)6 / A1b2C3d4E^f6G7#8" ) -func TestAmadeus_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - id := testSecrets.MustGetField("AMADEUS") - secret := testSecrets.MustGetField("AMADEUS_SECRET") - inactiveSecret := testSecrets.MustGetField("AMADEUS_INACTIVE") +func TestAmadeus_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Amadeus, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("amadeus: '%s'", validPattern), + want: []string{"A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6A1b2C3d4E5f6G7ho"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Amadeus, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - key out of prefix range", + input: fmt.Sprintf("amadeus keyword is not close to the real key in the data\n = '%s'", validPattern), + want: nil, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("amadeus: '%s'", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Amadeus.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Amadeus.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/ambee/ambee_integration_test.go b/pkg/detectors/ambee/ambee_integration_test.go new file mode 100644 index 000000000000..6ff21de9dc28 --- /dev/null +++ b/pkg/detectors/ambee/ambee_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package ambee + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAmbee_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AMBEE") + inactiveSecret := testSecrets.MustGetField("AMBEE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a ambee secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Ambee, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a ambee secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Ambee, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Ambee.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Ambee.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/ambee/ambee_test.go b/pkg/detectors/ambee/ambee_test.go index 6ff21de9dc28..12dc72b4552f 100644 --- a/pkg/detectors/ambee/ambee_test.go +++ b/pkg/detectors/ambee/ambee_test.go @@ -1,120 +1,86 @@ -//go:build detectors -// +build detectors - package ambee import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4" + invalidPattern = "X3b4c5d6e7f8a9b0c1d2e3f4a5b6c7Z8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4" ) -func TestAmbee_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AMBEE") - inactiveSecret := testSecrets.MustGetField("AMBEE_INACTIVE") +func TestAmbee_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a ambee secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Ambee, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("ambee: '%s'", validPattern), + want: []string{"a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a ambee secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Ambee, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - key out of prefix range", + input: fmt.Sprintf("ambee keyword is not close to the real key in the data\n = '%s'", validPattern), + want: nil, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("ambee: '%s'", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Ambee.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Ambee.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey.go b/pkg/detectors/amplitudeapikey/amplitudeapikey.go index 7f5a3d6e3e4b..6e041d76052b 100644 --- a/pkg/detectors/amplitudeapikey/amplitudeapikey.go +++ b/pkg/detectors/amplitudeapikey/amplitudeapikey.go @@ -2,16 +2,17 @@ package amplitudeapikey import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { detectors.DefaultMultiPartCredentialProvider } @@ -51,6 +52,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } resSecretMatch := strings.TrimSpace(secretMatch[1]) + // regex for both key and secret are same so the set of strings could possibly be same as well + if resMatch == resSecretMatch { + continue + } + s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_AmplitudeApiKey, Raw: []byte(resMatch), diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey_integration_test.go b/pkg/detectors/amplitudeapikey/amplitudeapikey_integration_test.go new file mode 100644 index 000000000000..9028ed7fb419 --- /dev/null +++ b/pkg/detectors/amplitudeapikey/amplitudeapikey_integration_test.go @@ -0,0 +1,124 @@ +//go:build detectors +// +build detectors + +package amplitudeapikey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAmplitudeApiKey_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + key := testSecrets.MustGetField("AMPLITUDEAPI_KEY") + secret := testSecrets.MustGetField("AMPLITUDEAPI_SECRET") + inactiveKey := testSecrets.MustGetField("AMPLITUDEAPI_KEY_INACTIVE") + inactiveSecret := testSecrets.MustGetField("AMPLITUDEAPI_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within", key, secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AmplitudeApiKey, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + + data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AmplitudeApiKey, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AmplitudeApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("AmplitudeApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go b/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go index 9028ed7fb419..e9fb9c351c89 100644 --- a/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go +++ b/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go @@ -1,124 +1,90 @@ -//go:build detectors -// +build detectors - package amplitudeapikey import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestAmplitudeApiKey_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } +var ( + validPattern = ` + amplitude key = 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d + amplitude secret = 5b3c4d5e5f7a8b9c0d1e2f2f3a4b4c6e + ` + invalidPattern = ` + amplitude key = 1a2b3c4d5e6f7a8g9c0d1e2f3a4b5c6d + amplitude secret = 5b3c4d5e5f7a8b9c0d1r2f2f3a4b4c6e + ` +) - key := testSecrets.MustGetField("AMPLITUDEAPI_KEY") - secret := testSecrets.MustGetField("AMPLITUDEAPI_SECRET") - inactiveKey := testSecrets.MustGetField("AMPLITUDEAPI_KEY_INACTIVE") - inactiveSecret := testSecrets.MustGetField("AMPLITUDEAPI_INACTIVE") +func TestAmplitudeAPIKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within", key, secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AmplitudeApiKey, - Verified: true, - }, + name: "valid pattern", + input: fmt.Sprintf("amplitude: '%s'", validPattern), + want: []string{ + "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d5b3c4d5e5f7a8b9c0d1e2f2f3a4b4c6e", + "5b3c4d5e5f7a8b9c0d1e2f2f3a4b4c6e1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d", }, - wantErr: false, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - - data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AmplitudeApiKey, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("amplitude: '%s'", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AmplitudeApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("AmplitudeApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/anthropic/anthropic_integration_test.go b/pkg/detectors/anthropic/anthropic_integration_test.go new file mode 100644 index 000000000000..64713de61c29 --- /dev/null +++ b/pkg/detectors/anthropic/anthropic_integration_test.go @@ -0,0 +1,162 @@ +//go:build detectors +// +build detectors + +package anthropic + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAnthropic_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("ANTHROPIC") + inactiveSecret := testSecrets.MustGetField("ANTHROPIC_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Anthropic, + Verified: true, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a anthropic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Anthropic, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Anthropic, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Anthropic, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Anthropic.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Anthropic.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/anthropic/anthropic_test.go b/pkg/detectors/anthropic/anthropic_test.go index 64713de61c29..ba1f085ac5af 100644 --- a/pkg/detectors/anthropic/anthropic_test.go +++ b/pkg/detectors/anthropic/anthropic_test.go @@ -1,161 +1,79 @@ -//go:build detectors -// +build detectors - package anthropic import ( "context" - "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "sk-ant-api03-abc123xyz-456def789ghij-klmnopqrstuvwx-3456yza789bcde-1234fghijklmnopby56aaaogaopaaaabc123xyzAA" + invalidPattern = "sk-ant-api03-abc123xyz-456de-klMnopqrstuvwx-3456yza789bcde-1234fghijklmnopAA" ) -func TestAnthropic_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("ANTHROPIC") - inactiveSecret := testSecrets.MustGetField("ANTHROPIC_INACTIVE") +func TestAnthropic_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Anthropic, - Verified: true, - }, - }, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a anthropic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Anthropic, - Verified: false, - }, - }, - wantErr: false, - wantVerificationErr: false, + name: "valid pattern", + input: validPattern, + want: []string{validPattern}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "found, would be verified if not for timeout", - s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Anthropic, - Verified: false, - }, - }, - wantErr: false, - wantVerificationErr: true, - }, - { - name: "found, verified but unexpected api surface", - s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Anthropic, - Verified: false, - }, - }, - wantErr: false, - wantVerificationErr: true, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Anthropic.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("Anthropic.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/anypoint/anypoint.go b/pkg/detectors/anypoint/anypoint.go index c12bcb5bcd0b..d49643ceb8f4 100644 --- a/pkg/detectors/anypoint/anypoint.go +++ b/pkg/detectors/anypoint/anypoint.go @@ -3,10 +3,11 @@ package anypoint import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -51,6 +52,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } orgRes := strings.TrimSpace(orgMatch[1]) + // regex for both key and org are same, so to avoid same string processing + if resMatch == orgRes { + continue + } + s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Anypoint, Raw: []byte(resMatch), diff --git a/pkg/detectors/anypoint/anypoint_integration_test.go b/pkg/detectors/anypoint/anypoint_integration_test.go new file mode 100644 index 000000000000..e073314369c3 --- /dev/null +++ b/pkg/detectors/anypoint/anypoint_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package anypoint + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAnypoint_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("ANYPOINT_TOKEN") + inactiveSecret := testSecrets.MustGetField("ANYPOINT_INACTIVE") + organizationId := testSecrets.MustGetField("ANYPOINT_ORGANIZATIONID") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s", secret, organizationId)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Anypoint, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s but not valid", inactiveSecret, organizationId)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Anypoint, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Anypoint.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Anypoint.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/anypoint/anypoint_test.go b/pkg/detectors/anypoint/anypoint_test.go index e073314369c3..31f5a91dfcb0 100644 --- a/pkg/detectors/anypoint/anypoint_test.go +++ b/pkg/detectors/anypoint/anypoint_test.go @@ -1,121 +1,107 @@ -//go:build detectors -// +build detectors - package anypoint import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = "1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6 / org: abcd1234-ef56-gh78-ij90-klmn1234opqr" + complexPattern = ` + # Secret Configuration File + # Organization details + ORG_NAME=my_organization + ORG_ID=abcd1234-ef56-gh78-ij90-klmn1234opqr + + # Database credentials + DB_USERNAME=iamnotadmin + DB_PASSWORD=8f3b6d3e7c9a2f5e + + # OAuth tokens + ACCESS_TOKEN=abcxyz123 + REFRESH_TOKEN=zyxwvutsrqponmlkji9876543210abcd - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + # API keys + API_KEY=sk-ant-api03-nothing-just-some-random-api-key-1234fghijklmnopAA + SECRET_KEY=1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6 + + # Endpoints + SERVICE_URL=https://api.example.com/v1/resource + ` + invalidPattern = "1a2b3C4d-5E6f-7g8H-9i0J-k1l2M3n4o5p6 / abcd1234-eF56-gH78-ij90-klmn1234opqr" ) -func TestAnypoint_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("ANYPOINT_TOKEN") - inactiveSecret := testSecrets.MustGetField("ANYPOINT_INACTIVE") - organizationId := testSecrets.MustGetField("ANYPOINT_ORGANIZATIONID") +func TestAnypoint_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s", secret, organizationId)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Anypoint, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("anypoint credentials: %s", validPattern), + want: []string{"1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6abcd1234-ef56-gh78-ij90-klmn1234opqr"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s but not valid", inactiveSecret, organizationId)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Anypoint, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: fmt.Sprintf("anypoint credentials: %s", complexPattern), + want: []string{"1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6abcd1234-ef56-gh78-ij90-klmn1234opqr"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("anypoint credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Anypoint.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Anypoint.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apacta/apacta_integration_test.go b/pkg/detectors/apacta/apacta_integration_test.go new file mode 100644 index 000000000000..d80e7f04b6a2 --- /dev/null +++ b/pkg/detectors/apacta/apacta_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package apacta + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestApacta_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APACTA") + inactiveSecret := testSecrets.MustGetField("APACTA_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apacta secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apacta, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apacta secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apacta, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Apacta.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Apacta.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apacta/apacta_test.go b/pkg/detectors/apacta/apacta_test.go index d80e7f04b6a2..8b6fee13bba8 100644 --- a/pkg/detectors/apacta/apacta_test.go +++ b/pkg/detectors/apacta/apacta_test.go @@ -1,120 +1,113 @@ -//go:build detectors -// +build detectors - package apacta import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestApacta_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APACTA") - inactiveSecret := testSecrets.MustGetField("APACTA_INACTIVE") +var ( + validPattern = "abcd1234-ef56-gh78-ij90-klmn1234opqr" + complexPattern = ` + func main() { + url := "https://api.example.com/v1/resource" + + // Create a new request with the secret as a header + req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte("{}"))) + if err != nil { + fmt.Println("Error creating request:", err) + return + } + + apactaSecret := "Bearer abcd1234-ef56-gh78-ij90-klmn1234opqr" + req.Header.Set("Authorization", apactaSecret) + + // Perform the request + client := &http.Client{} + resp, _ := client.Do(req) + defer resp.Body.Close() - type args struct { - ctx context.Context - data []byte - verify bool + // Check response status + if resp.StatusCode == http.StatusOK { + fmt.Println("Request successful!") + } else { + fmt.Println("Request failed with status:", resp.Status) + } } + ` + invalidPattern = "abcD$1234-ef56-gH78-ij90-klmn1234opqr" +) + +func TestApacta_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apacta secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apacta, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("apacta credentials: %s", validPattern), + want: []string{"abcd1234-ef56-gh78-ij90-klmn1234opqr"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apacta secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apacta, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - complex", + input: complexPattern, + want: []string{"abcd1234-ef56-gh78-ij90-klmn1234opqr"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("apacta credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Apacta.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Apacta.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/api2cart/api2cart_integration_test.go b/pkg/detectors/api2cart/api2cart_integration_test.go new file mode 100644 index 000000000000..f0243d52bcc9 --- /dev/null +++ b/pkg/detectors/api2cart/api2cart_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package api2cart + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestApi2Cart_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("API2CART") + inactiveSecret := testSecrets.MustGetField("API2CART_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a api2cart secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Api2Cart, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a api2cart secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Api2Cart, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Api2Cart.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Api2Cart.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/api2cart/api2cart_test.go b/pkg/detectors/api2cart/api2cart_test.go index f0243d52bcc9..c3fa39be9904 100644 --- a/pkg/detectors/api2cart/api2cart_test.go +++ b/pkg/detectors/api2cart/api2cart_test.go @@ -1,120 +1,93 @@ -//go:build detectors -// +build detectors - package api2cart import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "0123456789abcdef0123456789abcdef" + complexPattern = ` + To integrate with API2Cart, ensure you have the following credentials in your configuration file. Your API2CART key is 1234567890abcdef1234567890abcdef, which you will need to access the API securely. + + The following endpoints are available for your use: + - Get Products: https://api.api2cart.com/v1.0/products/get + - Add Product: https://api.api2cart.com/v1.0/products/add + ` + invalidPattern = "0123456789Gbcde^0123456789abcdef" ) -func TestApi2Cart_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("API2CART") - inactiveSecret := testSecrets.MustGetField("API2CART_INACTIVE") +func TestApi2Cart_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a api2cart secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Api2Cart, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("api2cart credentials: %s", validPattern), + want: []string{"0123456789abcdef0123456789abcdef"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a api2cart secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Api2Cart, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern -complex", + input: complexPattern, + want: []string{"1234567890abcdef1234567890abcdef"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("api2cart credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Api2Cart.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Api2Cart.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apideck/apideck_integration_test.go b/pkg/detectors/apideck/apideck_integration_test.go new file mode 100644 index 000000000000..30eb2fa7c989 --- /dev/null +++ b/pkg/detectors/apideck/apideck_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package apideck + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestApiDeck_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APIDECK_TOKEN") + inactiveSecret := testSecrets.MustGetField("APIDECK_INACTIVE") + id := testSecrets.MustGetField("APIDECK_APPID") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_ApiDeck, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_ApiDeck, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("ApiDeck.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("ApiDeck.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apideck/apideck_test.go b/pkg/detectors/apideck/apideck_test.go index 30eb2fa7c989..268ad1b0b82a 100644 --- a/pkg/detectors/apideck/apideck_test.go +++ b/pkg/detectors/apideck/apideck_test.go @@ -1,121 +1,85 @@ -//go:build detectors -// +build detectors - package apideck import ( "context" "fmt" + "strings" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validKeyPattern = "sk_live_abc123def456ghi789jklmno123456pqrstuvwxyz-1234567890123456789012345678901234567890AAui09pU8as" + validIdPattern = "apideck_id: 1a2b3c4d5e6f7g8h9i0j1k2l3m4n5o6p7q8r9s0t" + invalidPattern = strings.ReplaceAll(validKeyPattern, "sk_live", "sk_inactive") + strings.ReplaceAll(validIdPattern, "1", "$") ) -func TestApiDeck_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APIDECK_TOKEN") - inactiveSecret := testSecrets.MustGetField("APIDECK_INACTIVE") - id := testSecrets.MustGetField("APIDECK_APPID") +func TestApiDeck_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_ApiDeck, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation - verify: true, + name: "valid pattern", + input: fmt.Sprintf("credentials: %s \n %s", validKeyPattern, validIdPattern), + want: []string{ + validKeyPattern + "1a2b3c4d5e6f7g8h9i0j1k2l3m4n5o6p7q8r9s0t", }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_ApiDeck, - Verified: false, - }, - }, - wantErr: false, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("ApiDeck.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("ApiDeck.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apiflash/apiflash_integration_test.go b/pkg/detectors/apiflash/apiflash_integration_test.go new file mode 100644 index 000000000000..c457a47225f2 --- /dev/null +++ b/pkg/detectors/apiflash/apiflash_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package apiflash + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestApiflash_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APIFLASH") + url := testSecrets.MustGetField("API_URL") + inactiveSecret := testSecrets.MustGetField("APIFLASH_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s", secret, url)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apiflash, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apiflash, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Apiflash.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Apiflash.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apiflash/apiflash_test.go b/pkg/detectors/apiflash/apiflash_test.go index c457a47225f2..cae1a3b72024 100644 --- a/pkg/detectors/apiflash/apiflash_test.go +++ b/pkg/detectors/apiflash/apiflash_test.go @@ -1,121 +1,84 @@ -//go:build detectors -// +build detectors - package apiflash import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = ` + apiflash_key: a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 + apiflash_url: abc123XYZ456def789ghijklm + ` + invalidPattern = "0123456789Gbcde^0123456789abcdef" ) -func TestApiflash_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APIFLASH") - url := testSecrets.MustGetField("API_URL") - inactiveSecret := testSecrets.MustGetField("APIFLASH_INACTIVE") +func TestApiFlash_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s", secret, url)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apiflash, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("apiflash credentials: %s", validPattern), + want: []string{"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6abc123XYZ456def789ghijklm"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apiflash, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("apiflash credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Apiflash.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Apiflash.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apifonica/apifonica_integration_test.go b/pkg/detectors/apifonica/apifonica_integration_test.go new file mode 100644 index 000000000000..8df2c4cb50a8 --- /dev/null +++ b/pkg/detectors/apifonica/apifonica_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package apifonica + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestApifonica_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APIFONICA_SECRET") + token := testSecrets.MustGetField("APIFONICA_ID") + inactiveSecret := testSecrets.MustGetField("APIFONICA_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s", secret, token)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_ApiFonica, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s but not valid", inactiveSecret, token)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_ApiFonica, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Apifonica.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Apifonica.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apifonica/apifonica_test.go b/pkg/detectors/apifonica/apifonica_test.go index 8df2c4cb50a8..684ce65566c8 100644 --- a/pkg/detectors/apifonica/apifonica_test.go +++ b/pkg/detectors/apifonica/apifonica_test.go @@ -1,121 +1,81 @@ -//go:build detectors -// +build detectors - package apifonica import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "abc123def45-gh67-ij89-kl01-mn234op56789" + invalidPattern = "0123456789Gbcde^0123456789abcdef" ) -func TestApifonica_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APIFONICA_SECRET") - token := testSecrets.MustGetField("APIFONICA_ID") - inactiveSecret := testSecrets.MustGetField("APIFONICA_INACTIVE") +func TestApiFonica_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s", secret, token)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_ApiFonica, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("apifonica credentials: %s", validPattern), + want: []string{"abc123def45-gh67-ij89-kl01-mn234op56789"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s but not valid", inactiveSecret, token)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_ApiFonica, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("apifonica credentials: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Apifonica.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Apifonica.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apify/apify_integration_test.go b/pkg/detectors/apify/apify_integration_test.go new file mode 100644 index 000000000000..79e866953358 --- /dev/null +++ b/pkg/detectors/apify/apify_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package apify + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestApify_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APIFY") + inactiveSecret := testSecrets.MustGetField("APIFY_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apify secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apify, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apify, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Apify.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Apify.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apify/apify_test.go b/pkg/detectors/apify/apify_test.go index 79e866953358..15798000adf5 100644 --- a/pkg/detectors/apify/apify_test.go +++ b/pkg/detectors/apify/apify_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package apify import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "apify_api_ab12CD34ef56GH78ij90KL12mn34OP56qr78" + invalidPattern = "apify_api_ab12C#34ef56GH7$ij90KL12mn34OP56qr7)" ) -func TestApify_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APIFY") - inactiveSecret := testSecrets.MustGetField("APIFY_INACTIVE") +func TestApiFy_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apify secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apify, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{validPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apify, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Apify.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Apify.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apilayer/apilayer_integration_test.go b/pkg/detectors/apilayer/apilayer_integration_test.go new file mode 100644 index 000000000000..ef4b0fe2f0ab --- /dev/null +++ b/pkg/detectors/apilayer/apilayer_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package apilayer + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestApilayer_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APILAYER") + inactiveSecret := testSecrets.MustGetField("APILAYER_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apilayer secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apilayer, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apilayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apilayer, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Apilayer.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Apilayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apilayer/apilayer_test.go b/pkg/detectors/apilayer/apilayer_test.go index e764f3db11b0..7c369b4113b0 100644 --- a/pkg/detectors/apilayer/apilayer_test.go +++ b/pkg/detectors/apilayer/apilayer_test.go @@ -2,116 +2,79 @@ package apilayer import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "apilayerkey: abcdef1234567890abcdef1234567890" + invalidPattern = "apilayerkey: abcdef123$5678*0abcde()1234567890)" ) -func TestApilayer_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APILAYER") - inactiveSecret := testSecrets.MustGetField("APILAYER_INACTIVE") +func TestApiLayer_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apilayer secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apilayer, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apilayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apilayer, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"abcdef1234567890abcdef1234567890"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Apilayer.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Apilayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apimatic/apimatic.go b/pkg/detectors/apimatic/apimatic.go index 2d9eb5a07698..1198e67073b0 100644 --- a/pkg/detectors/apimatic/apimatic.go +++ b/pkg/detectors/apimatic/apimatic.go @@ -2,17 +2,17 @@ package apimatic import ( "context" - regexp "github.com/wasilibs/go-re2" "net/http" - "strings" "time" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { detectors.DefaultMultiPartCredentialProvider } @@ -23,8 +23,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimatic"}) + `\b([a-zA-Z0-9]{3,20}@[a-zA-Z0-9]{2,12}.[a-zA-Z0-9]{2,5})\b`) - passPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimatic"}) + `\b([a-z0-9-\S]{8,32})\b`) + apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimatic", "apikey"}) + `\b([a-zA-Z0-9_-]{64})\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -37,47 +36,43 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - passMatches := passPat.FindAllStringSubmatch(dataStr, -1) + uniqueApiKeys := make(map[string]struct{}) + for _, matches := range apiKeyPat.FindAllStringSubmatch(dataStr, -1) { + uniqueApiKeys[matches[1]] = struct{}{} + } - for _, match := range matches { - if len(match) != 2 { - continue + for apiKey := range uniqueApiKeys { + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_APIMatic, + Raw: []byte(apiKey), } - userPatMatch := strings.TrimSpace(match[1]) - for _, idMatch := range passMatches { - if len(idMatch) != 2 { + if verify { + timeout := 10 * time.Second + client.Timeout = timeout + + // api docs: https://docs.apimatic.io/platform-api/#/http/api-endpoints/code-generation-external-apis/list-all-code-generations + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.apimatic.io/code-generations", http.NoBody) + if err != nil { continue } - passPatMatch := strings.TrimSpace(idMatch[1]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_APIMatic, - Raw: []byte(userPatMatch), - RawV2: []byte(userPatMatch + passPatMatch), - } - if verify { - timeout := 10 * time.Second - client.Timeout = timeout - req, err := http.NewRequestWithContext(ctx, "GET", "https://www.apimatic.io/api/code-generations", nil) - if err != nil { - continue - } - req.SetBasicAuth(userPatMatch, passPatMatch) - res, err := client.Do(req) - - if err == nil { - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } + // authentication documentation: https://docs.apimatic.io/platform-api/#/http/guides/authentication + req.Header.Set("Authorization", "X-Auth-Key "+apiKey) + resp, err := client.Do(req) + if err == nil { + defer resp.Body.Close() + if resp.StatusCode == 200 { + s1.Verified = true } + } else { + // if any error happens during api request capture that as verificaiton error + s1.SetVerificationError(err, apiKey) } - - results = append(results, s1) } + + results = append(results, s1) + } return results, nil } diff --git a/pkg/detectors/apimatic/apimatic_integration_test.go b/pkg/detectors/apimatic/apimatic_integration_test.go new file mode 100644 index 000000000000..7995bfc48a67 --- /dev/null +++ b/pkg/detectors/apimatic/apimatic_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package apimatic + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAPIMatic_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APIMATIC") + pass := testSecrets.MustGetField("APIMATIC_PASS") + inactiveSecret := testSecrets.MustGetField("APIMATIC_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s", secret, pass)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_APIMatic, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s but not valid", inactiveSecret, pass)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_APIMatic, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("APIMatic.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("APIMatic.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apimatic/apimatic_test.go b/pkg/detectors/apimatic/apimatic_test.go index 7995bfc48a67..09a35794d607 100644 --- a/pkg/detectors/apimatic/apimatic_test.go +++ b/pkg/detectors/apimatic/apimatic_test.go @@ -1,121 +1,86 @@ -//go:build detectors -// +build detectors - package apimatic import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestAPIMatic_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APIMATIC") - pass := testSecrets.MustGetField("APIMATIC_PASS") - inactiveSecret := testSecrets.MustGetField("APIMATIC_INACTIVE") +var ( + validPattern = ` + func validateApiMatic() bool { + apiMaticKey := "rc6iLoUEFGGAWNLsuBJnmsh4tZB-oCxcDUmc45HIPcuiQvfUEuqo8wb9YrUd2LyB" - type args struct { - ctx context.Context - data []byte - verify bool + return isActive(apiMaticKey, apiMaticPass) } + ` + invalidPattern = "apiMaticKey: rc6iL$UEFGGAWNLsuBJnmsh4tZB)oCxcDUmc45HIPcuiQvfUEuqo8wb9YrUd2LyB" +) + +func TestApiMatic_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s", secret, pass)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_APIMatic, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"rc6iLoUEFGGAWNLsuBJnmsh4tZB-oCxcDUmc45HIPcuiQvfUEuqo8wb9YrUd2LyB"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s but not valid", inactiveSecret, pass)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_APIMatic, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("APIMatic.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("APIMatic.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apimetrics/apimetrics.go b/pkg/detectors/apimetrics/apimetrics.go index 7cd84d3666c8..98c9ec37fc8f 100644 --- a/pkg/detectors/apimetrics/apimetrics.go +++ b/pkg/detectors/apimetrics/apimetrics.go @@ -22,7 +22,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimetrics"}) + `\b([a-bA-Z0-9\S]{32})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimetrics"}) + `\b([a-zA-Z0-9]{32})\b`) ) // Keywords are used for efficiently pre-filtering chunks. diff --git a/pkg/detectors/apimetrics/apimetrics_integration_test.go b/pkg/detectors/apimetrics/apimetrics_integration_test.go new file mode 100644 index 000000000000..a16dc409337a --- /dev/null +++ b/pkg/detectors/apimetrics/apimetrics_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package apimetrics + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestApiMetrics_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APIMETRICS") + inactiveSecret := testSecrets.MustGetField("APIMETRICS_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_ApiMetrics, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_ApiMetrics, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("ApiMetrics.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("ApiMetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apimetrics/apimetrics_test.go b/pkg/detectors/apimetrics/apimetrics_test.go index a16dc409337a..93e780ba1f73 100644 --- a/pkg/detectors/apimetrics/apimetrics_test.go +++ b/pkg/detectors/apimetrics/apimetrics_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package apimetrics import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "apimetricskey: 5po8TFGawiYNCc1ct4ofWkBqzIfA6IeO" + invalidPattern = "apimetricskey: A1b2C3d4 E5f6G7h8I9 j0K1l2M3n4O5p6" ) -func TestApiMetrics_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APIMETRICS") - inactiveSecret := testSecrets.MustGetField("APIMETRICS_INACTIVE") +func TestApiMetrics_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_ApiMetrics, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"5po8TFGawiYNCc1ct4ofWkBqzIfA6IeO"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_ApiMetrics, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("ApiMetrics.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("ApiMetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apitemplate/apitemplate_integration_test.go b/pkg/detectors/apitemplate/apitemplate_integration_test.go new file mode 100644 index 000000000000..ff20dbdbd821 --- /dev/null +++ b/pkg/detectors/apitemplate/apitemplate_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package apitemplate + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAPITemplate_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APITEMPLATE") + inactiveSecret := testSecrets.MustGetField("APITEMPLATE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_APITemplate, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_APITemplate, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("APITemplate.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("APITemplate.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apitemplate/apitemplate_test.go b/pkg/detectors/apitemplate/apitemplate_test.go index ff20dbdbd821..e19fce97eb94 100644 --- a/pkg/detectors/apitemplate/apitemplate_test.go +++ b/pkg/detectors/apitemplate/apitemplate_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package apitemplate import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "apitemplatekey: a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8s9T" + invalidPattern = "apitemplatekey: sk_0C2c3D4e5F6@7H8i9J$k1L2m3N4o5P6q7R8s9T" ) -func TestAPITemplate_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APITEMPLATE") - inactiveSecret := testSecrets.MustGetField("APITEMPLATE_INACTIVE") +func TestApiTemplate_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_APITemplate, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8s9T"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_APITemplate, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("APITemplate.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("APITemplate.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apollo/apollo_integration_test.go b/pkg/detectors/apollo/apollo_integration_test.go new file mode 100644 index 000000000000..99ea2c755f97 --- /dev/null +++ b/pkg/detectors/apollo/apollo_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package apollo + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestApollo_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APOLLO") + inactiveSecret := testSecrets.MustGetField("APOLLO_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apollo secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apollo, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apollo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apollo, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Apollo.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Apollo.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apollo/apollo_test.go b/pkg/detectors/apollo/apollo_test.go index 99ea2c755f97..c66ba5417f49 100644 --- a/pkg/detectors/apollo/apollo_test.go +++ b/pkg/detectors/apollo/apollo_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package apollo import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "apollo: A1b2C3d4E5f6G7h8I9j0K1" + invalidPattern = "apollo: A1b2C3d4-5f6G7h8I9j0K1@$" ) -func TestApollo_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APOLLO") - inactiveSecret := testSecrets.MustGetField("APOLLO_INACTIVE") +func TestApollo_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apollo secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apollo, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"A1b2C3d4E5f6G7h8I9j0K1"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apollo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apollo, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Apollo.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Apollo.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/appcues/appcues_integration_test.go b/pkg/detectors/appcues/appcues_integration_test.go new file mode 100644 index 000000000000..c46205d002dc --- /dev/null +++ b/pkg/detectors/appcues/appcues_integration_test.go @@ -0,0 +1,122 @@ +//go:build detectors +// +build detectors + +package appcues + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAppcues_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APPCUES") + user := testSecrets.MustGetField("APPCUES_USER") + id := testSecrets.MustGetField("APPCUES_ID") + inactiveSecret := testSecrets.MustGetField("APPCUES_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s", secret, user, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Appcues, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s but not valid", inactiveSecret, user, id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Appcues, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Appcues.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Appcues.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/appcues/appcues_test.go b/pkg/detectors/appcues/appcues_test.go index c46205d002dc..a239edfac698 100644 --- a/pkg/detectors/appcues/appcues_test.go +++ b/pkg/detectors/appcues/appcues_test.go @@ -1,122 +1,88 @@ -//go:build detectors -// +build detectors - package appcues import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = ` + appcues abcdef12-3456-7890-1234-56789abcdef0 + appcues abcdef123456789012345678901234567890123 + appcues 12345 + ` + invalidPattern = ` + appcues abcdef12#3456-7890-1234-56789abcdef0 + appcues abcdef12345678901&345678901234567890123 + appcues 123456 + ` ) -func TestAppcues_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APPCUES") - user := testSecrets.MustGetField("APPCUES_USER") - id := testSecrets.MustGetField("APPCUES_ID") - inactiveSecret := testSecrets.MustGetField("APPCUES_INACTIVE") +func TestAppCues_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s", secret, user, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Appcues, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"abcdef12-3456-7890-1234-56789abcdef0abcdef123456789012345678901234567890123"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s but not valid", inactiveSecret, user, id)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Appcues, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Appcues.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatal("no raw secret present") - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Appcues.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/appfollow/appfollow_integration_test.go b/pkg/detectors/appfollow/appfollow_integration_test.go new file mode 100644 index 000000000000..fd6e46745cec --- /dev/null +++ b/pkg/detectors/appfollow/appfollow_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package appfollow + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAppfollow_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APPFOLLOW") + inactiveSecret := testSecrets.MustGetField("APPFOLLOW_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a appfollow secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Appfollow, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a appfollow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Appfollow, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Appfollow.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Appfollow.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/appfollow/appfollow_test.go b/pkg/detectors/appfollow/appfollow_test.go index fd6e46745cec..dbae86cdf3fb 100644 --- a/pkg/detectors/appfollow/appfollow_test.go +++ b/pkg/detectors/appfollow/appfollow_test.go @@ -1,120 +1,81 @@ -//go:build detectors -// +build detectors - package appfollow import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.333HbjEo1oxVUFcASR0sQ8cMuIJRLcMd5H9iJWDbovCw6ESjNtuoEMQQGPN9aSoxponxrTPvn1.btADhgNetsaUBuwoyHo5ip0Jab6N6MEBnSaT6CHiO6z" + invalidPattern = "eyQ0eXAiOiJMM1QiLDJhbGciOiJIUXI1NiK7.abcdefgh1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZabcd1234efgh5678ijkl9012.qwerty12345-ASDFG67890_zxcvb_ABCDE" ) -func TestAppfollow_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APPFOLLOW") - inactiveSecret := testSecrets.MustGetField("APPFOLLOW_INACTIVE") +func TestAppFollow_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appfollow secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Appfollow, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("appfollow credentail: %s", validPattern), + want: []string{"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.333HbjEo1oxVUFcASR0sQ8cMuIJRLcMd5H9iJWDbovCw6ESjNtuoEMQQGPN9aSoxponxrTPvn1.btADhgNetsaUBuwoyHo5ip0Jab6N6MEBnSaT6CHiO6z"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appfollow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Appfollow, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("appfollow credentail: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Appfollow.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Appfollow.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/appointedd/appointedd_integration_test.go b/pkg/detectors/appointedd/appointedd_integration_test.go new file mode 100644 index 000000000000..1f2a8dfe66da --- /dev/null +++ b/pkg/detectors/appointedd/appointedd_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package appointedd + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAppointedd_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APPOINTEDD") + inactiveSecret := testSecrets.MustGetField("APPOINTEDD_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a appointedd secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Appointedd, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a appointedd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Appointedd, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Appointedd.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Appointedd.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/appointedd/appointedd_test.go b/pkg/detectors/appointedd/appointedd_test.go index 3323032ef469..08c498c3c240 100644 --- a/pkg/detectors/appointedd/appointedd_test.go +++ b/pkg/detectors/appointedd/appointedd_test.go @@ -4,114 +4,78 @@ import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "Ci0a2bSpRyFcZyEXBEr9RHzf3xXllqO=XVoh+t0L0s8T2s3MFntfWhBlovqLaqEadtuJ9=Jy6yCOXmhbpEZPfY7Y" + invalidPattern = "Ci0a2bSpRyFcZyEXBEr9RHzf3xXllqO^XVoh+t0L0s8T2s3MFntfWhBlovqLaqEadtuJ9$Jy6yCOXmhbpEZPfY7Y" ) -func TestAppointedd_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APPOINTEDD") - inactiveSecret := testSecrets.MustGetField("APPOINTEDD_INACTIVE") +func TestAppFollow_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appointedd secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Appointedd, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appointedd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Appointedd, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("appointedd credentail: %s", validPattern), + want: []string{"Ci0a2bSpRyFcZyEXBEr9RHzf3xXllqO=XVoh+t0L0s8T2s3MFntfWhBlovqLaqEadtuJ9=Jy6yCOXmhbpEZPfY7Y"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("appointedd credentail: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Appointedd.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Appointedd.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/appoptics/appoptics_integration_test.go b/pkg/detectors/appoptics/appoptics_integration_test.go new file mode 100644 index 000000000000..a50a583b039c --- /dev/null +++ b/pkg/detectors/appoptics/appoptics_integration_test.go @@ -0,0 +1,128 @@ +//go:build detectors +// +build detectors + +package appoptics + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAppoptics_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APPOPTICS") + inactiveSecret := testSecrets.MustGetField("APPOPTICS_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a appoptics secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AppOptics, + Verified: true, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a appoptics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AppOptics, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Appoptics.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Appoptics.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/appoptics/appoptics_test.go b/pkg/detectors/appoptics/appoptics_test.go index a50a583b039c..f4c9bd448744 100644 --- a/pkg/detectors/appoptics/appoptics_test.go +++ b/pkg/detectors/appoptics/appoptics_test.go @@ -1,127 +1,80 @@ -//go:build detectors -// +build detectors - package appoptics import ( "context" "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "IABJPR08RmvsGrebJhr1TUdo27-KtTn0mLCKkJJqj5lyba-otXPklygO9DK62o3QSPoIJ4E" + invalidPattern = "IABJPR08Rmvs.GrebJhr1TUdo27_KtTn0mLCKkJJqj5lyba&otXPklygO9DK62o3QSPoIJ4E" ) -func TestAppoptics_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APPOPTICS") - inactiveSecret := testSecrets.MustGetField("APPOPTICS_INACTIVE") +func TestAppOptics_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appoptics secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AppOptics, - Verified: true, - }, - }, - wantErr: false, - wantVerificationErr: false, + name: "valid pattern", + input: fmt.Sprintf("appoptics credentail: %s", validPattern), + want: []string{"IABJPR08RmvsGrebJhr1TUdo27-KtTn0mLCKkJJqj5lyba-otXPklygO9DK62o3QSPoIJ4E"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appoptics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AppOptics, - Verified: false, - }, - }, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - wantVerificationErr: false, + name: "invalid pattern", + input: fmt.Sprintf("appoptics credentail: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Appoptics.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("Appoptics.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/appsynergy/appsynergy_integration_test.go b/pkg/detectors/appsynergy/appsynergy_integration_test.go new file mode 100644 index 000000000000..363af075ed58 --- /dev/null +++ b/pkg/detectors/appsynergy/appsynergy_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package appsynergy + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAppSynergy_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APPSYNERGY") + inactiveSecret := testSecrets.MustGetField("APPSYNERGY_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AppSynergy, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AppSynergy, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AppSynergy.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("AppSynergy.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/appsynergy/appsynergy_test.go b/pkg/detectors/appsynergy/appsynergy_test.go index 363af075ed58..5ca3017f1391 100644 --- a/pkg/detectors/appsynergy/appsynergy_test.go +++ b/pkg/detectors/appsynergy/appsynergy_test.go @@ -1,120 +1,81 @@ -//go:build detectors -// +build detectors - package appsynergy import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "mg1pgwlndtq7rbk8i3kum344aso8ggp02ximdhsp8nsqasd3btxf84lz9ekfdpwo" + invalidPattern = "mG1pgwlndtq7rbk8i3kum344a$o8gGp02ximdhsp8nsqasd3btxf84lz9ekfdpwo" ) -func TestAppSynergy_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APPSYNERGY") - inactiveSecret := testSecrets.MustGetField("APPSYNERGY_INACTIVE") +func TestAppSynergy_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AppSynergy, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("appsynergy credentail: %s", validPattern), + want: []string{"mg1pgwlndtq7rbk8i3kum344aso8ggp02ximdhsp8nsqasd3btxf84lz9ekfdpwo"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AppSynergy, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("appsynergy credentail: %s", invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AppSynergy.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("AppSynergy.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/apptivo/apptivo_integration_test.go b/pkg/detectors/apptivo/apptivo_integration_test.go new file mode 100644 index 000000000000..d86c9872724e --- /dev/null +++ b/pkg/detectors/apptivo/apptivo_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package apptivo + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestApptivo_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("APPTIVO") + id := testSecrets.MustGetField("APPTIVO_KEY") + inactiveSecret := testSecrets.MustGetField("APPTIVO_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apptivo secret %s within apptivo %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apptivo, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a apptivo secret %s within but not valid apptivo %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Apptivo, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Apptivo.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Apptivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/apptivo/apptivo_test.go b/pkg/detectors/apptivo/apptivo_test.go index d86c9872724e..b2c74f4c4166 100644 --- a/pkg/detectors/apptivo/apptivo_test.go +++ b/pkg/detectors/apptivo/apptivo_test.go @@ -1,121 +1,80 @@ -//go:build detectors -// +build detectors - package apptivo import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "apptivokey: 1h3l8yjo3pms2h738k8f9094rkltmgy8md-4 - apptivoID: wV2WGN-KUxROVW0mlq8kur7TbKvCTdNc" + invalidPattern = "apptivokey: 1h3l8yjo3pms2h738k8f9094rkltmgy8md-4" ) -func TestApptivo_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("APPTIVO") - id := testSecrets.MustGetField("APPTIVO_KEY") - inactiveSecret := testSecrets.MustGetField("APPTIVO_INACTIVE") +func TestApptivo_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apptivo secret %s within apptivo %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apptivo, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"1h3l8yjo3pms2h738k8f9094rkltmgy8md-4wV2WGN-KUxROVW0mlq8kur7TbKvCTdNc"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a apptivo secret %s within but not valid apptivo %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Apptivo, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Apptivo.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Apptivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/artifactory/artifactory_integration_test.go b/pkg/detectors/artifactory/artifactory_integration_test.go new file mode 100644 index 000000000000..deab1d778658 --- /dev/null +++ b/pkg/detectors/artifactory/artifactory_integration_test.go @@ -0,0 +1,168 @@ +//go:build detectors +// +build detectors + +package artifactory + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestArtifactory_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("ARTIFACTORY_TOKEN") + inactiveSecret := testSecrets.MustGetField("ARTIFACTORY_INACTIVE") + appURL := testSecrets.MustGetField("ARTIFACTORY_URL") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s", secret, appURL)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, real secrets, verification error due to timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, + Verified: false, + } + r.SetVerificationError(context.DeadlineExceeded) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, real secrets, verification error due to unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, + Verified: false, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a artifactory secret %s but not valid on endpoint %s", inactiveSecret, appURL)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Artifactory.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Artifactory.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/artifactory/artifactory_test.go b/pkg/detectors/artifactory/artifactory_test.go index deab1d778658..b74c91923ea9 100644 --- a/pkg/detectors/artifactory/artifactory_test.go +++ b/pkg/detectors/artifactory/artifactory_test.go @@ -1,167 +1,104 @@ -//go:build detectors -// +build detectors - package artifactory import ( "context" "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl / RVo8ytzB65L.jfrog.io" + // validPattern2 is for cloud endpoints so it does not have any JFrog endpoint + validPattern2 = "5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl" + invalidPattern = "W0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl^&5YcZhIKwxTdxwpZHf9c1Usu8xNtA / rtest#y$zB65L%.jfrog.io" ) -func TestArtifactory_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("ARTIFACTORY_TOKEN") - inactiveSecret := testSecrets.MustGetField("ARTIFACTORY_INACTIVE") - appURL := testSecrets.MustGetField("ARTIFACTORY_URL") +func TestArtifactory_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + cloudEndpoint string + useCloudEndpoint bool + useFoundEndpoint bool + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s", secret, appURL)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, real secrets, verification error due to timeout", - s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)), - verify: true, - }, - want: func() []detectors.Result { - r := detectors.Result{ - DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, - Verified: false, - } - r.SetVerificationError(context.DeadlineExceeded) - return []detectors.Result{r} - }(), - wantErr: false, - }, - { - name: "found, real secrets, verification error due to unexpected api surface", - s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)), - verify: true, - }, - want: func() []detectors.Result { - r := detectors.Result{ - DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, - Verified: false, - } - r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500")) - return []detectors.Result{r} - }(), - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("artifactory credentials: %s", validPattern), + useCloudEndpoint: false, + useFoundEndpoint: true, + want: []string{"5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dlRVo8ytzB65L.jfrog.io"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a artifactory secret %s but not valid on endpoint %s", inactiveSecret, appURL)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern - with cloud endpoints", + input: fmt.Sprintf("artifactory credentials: %s", validPattern2), + cloudEndpoint: "cloudendpoint.jfrog.io", + useCloudEndpoint: true, + useFoundEndpoint: false, + want: []string{"5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dlcloudendpoint.jfrog.io"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: fmt.Sprintf("artifactory credentials: %s", invalidPattern), + useFoundEndpoint: true, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Artifactory.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // this detector use endpoint customizer interface so we need to enable them based on test case + d.UseFoundEndpoints(test.useFoundEndpoint) + d.UseCloudEndpoint(test.useCloudEndpoint) + // if test case provide cloud endpoint use it + if test.useCloudEndpoint && test.cloudEndpoint != "" { + d.SetCloudEndpoint(test.cloudEndpoint) + } + + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - gotErr := "" - if got[i].VerificationError() != nil { - gotErr = got[i].VerificationError().Error() - } - wantErr := "" - if tt.want[i].VerificationError() != nil { - wantErr = tt.want[i].VerificationError().Error() + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if gotErr != wantErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("Artifactory.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/artsy/artsy_integration_test.go b/pkg/detectors/artsy/artsy_integration_test.go new file mode 100644 index 000000000000..bf44ba496710 --- /dev/null +++ b/pkg/detectors/artsy/artsy_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package artsy + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestArtsy_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("ARTSY") + inactiveSecret := testSecrets.MustGetField("ARTSY_INACTIVE") + id := testSecrets.MustGetField("ARTSY_CLIENTID") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Artsy, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Artsy, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Artsy.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Artsy.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/artsy/artsy_test.go b/pkg/detectors/artsy/artsy_test.go index bf44ba496710..3ac7c892db1a 100644 --- a/pkg/detectors/artsy/artsy_test.go +++ b/pkg/detectors/artsy/artsy_test.go @@ -1,121 +1,80 @@ -//go:build detectors -// +build detectors - package artsy import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "artsy_key: rU0K6hwGw9AeANtXrZ8FQJT9jn4sRdlj - artsy_id: hvQ2fMvUPNczDCdmzi0i" + invalidPattern = "artsy_key: rU:K6hwGw9AeANtXrZ8F%JT9jn4sR,lj - artsy_id: hvQ2f-vUPNczD+dmzi0i" ) -func TestArtsy_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("ARTSY") - inactiveSecret := testSecrets.MustGetField("ARTSY_INACTIVE") - id := testSecrets.MustGetField("ARTSY_CLIENTID") +func TestArtsy_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Artsy, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"rU0K6hwGw9AeANtXrZ8FQJT9jn4sRdljhvQ2fMvUPNczDCdmzi0i"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Artsy, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Artsy.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Artsy.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/asanaoauth/asanaoauth_integration_test.go b/pkg/detectors/asanaoauth/asanaoauth_integration_test.go new file mode 100644 index 000000000000..063c55b0cede --- /dev/null +++ b/pkg/detectors/asanaoauth/asanaoauth_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package asanaoauth + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAsanaOauth_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("ASANAOAUTH_TOKEN") + inactiveSecret := testSecrets.MustGetField("ASANAOAUTH_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AsanaOauth, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a asana secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AsanaOauth, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AsanaOauth.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + got[i].AnalysisInfo = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("AsanaOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/asanaoauth/asanaoauth_test.go b/pkg/detectors/asanaoauth/asanaoauth_test.go index 063c55b0cede..ff9e64ddb736 100644 --- a/pkg/detectors/asanaoauth/asanaoauth_test.go +++ b/pkg/detectors/asanaoauth/asanaoauth_test.go @@ -1,121 +1,80 @@ -//go:build detectors -// +build detectors - package asanaoauth import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "asana_key: v1e0uwkmv2ykvsu998lqe4g7ij2gwtg/xn9vh4jvsokdfaic0bn" + invalidPattern = "asana_key: v1e0uwkmv2ykvsu998lqe4g-ij2gwtg/xn9vh4jvsokdfaic0bn" ) -func TestAsanaOauth_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("ASANAOAUTH_TOKEN") - inactiveSecret := testSecrets.MustGetField("ASANAOAUTH_INACTIVE") +func TestAsanaOauth_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AsanaOauth, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"v1e0uwkmv2ykvsu998lqe4g7ij2gwtg/xn9vh4jvsokdfaic0bn"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a asana secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AsanaOauth, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AsanaOauth.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil - got[i].AnalysisInfo = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("AsanaOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_integration_test.go b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_integration_test.go new file mode 100644 index 000000000000..97a61d918f42 --- /dev/null +++ b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package asanapersonalaccesstoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAsanaPersonalAccessToken_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("ASANA_PAT") + inactiveSecret := testSecrets.MustGetField("ASANA_PAT_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a asana secret %s within but unverified", inactiveSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AsanaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("AsanaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go index 97a61d918f42..8cbf64c95d24 100644 --- a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go +++ b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package asanapersonalaccesstoken import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "asana_token: 594776654034514343561917591881414702593902454625364993/1724908107002616220416212965:Yv3DoiSFhtsgUwN3AcnXWjK8zabQHKSHBRHpuNKVjz3oCcpyDIdXRm3GL4SUDkTMFoTbRDCHe8tTBHxdtoXItn" + invalidPattern = "asana_token: 1724908107002616220416212965%594776654034514343561917591881414702593902454625364993:Yv3DoiSFhtsgUwN3AcnXWjK8zabQHKSHBRHpuNKVjz3oCcpyDIdXRm3GL4SUDkTMFoTbRDCHe8tTBHxdtoXItn-ij2gwtg/xn9vh4jvsokdfaic0bn" ) -func TestAsanaPersonalAccessToken_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("ASANA_PAT") - inactiveSecret := testSecrets.MustGetField("ASANA_PAT_INACTIVE") +func TestAsanaPersonalAccessToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"594776654034514343561917591881414702593902454625364993/1724908107002616220416212965:Yv3DoiSFhtsgUwN3AcnXWjK8zabQHKSHBRHpuNKVjz3oCcpyDIdXRm3GL4SUDkTMFoTbRDCHe8tTBHxdtoXItn"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a asana secret %s within but unverified", inactiveSecret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AsanaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("AsanaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/assemblyai/assemblyai_integration_test.go b/pkg/detectors/assemblyai/assemblyai_integration_test.go new file mode 100644 index 000000000000..f042fb413469 --- /dev/null +++ b/pkg/detectors/assemblyai/assemblyai_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package assemblyai + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAssemblyai_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("ASSEMBLYAI") + inactiveSecret := testSecrets.MustGetField("ASSEMBLYAI_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AssemblyAI, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AssemblyAI, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Assemblyai.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Assemblyai.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/assemblyai/assemblyai_test.go b/pkg/detectors/assemblyai/assemblyai_test.go index f042fb413469..5d9dac960a2c 100644 --- a/pkg/detectors/assemblyai/assemblyai_test.go +++ b/pkg/detectors/assemblyai/assemblyai_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package assemblyai import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "assemblyai: mlhekyjhs96mx0r2cxbzky4jzr83fw1q" + invalidPattern = "assemblyai: Mlhekyjzr83fw1qr2cxbzky4jzr83f1q" ) -func TestAssemblyai_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("ASSEMBLYAI") - inactiveSecret := testSecrets.MustGetField("ASSEMBLYAI_INACTIVE") +func TestAssemblyAI_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AssemblyAI, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"mlhekyjhs96mx0r2cxbzky4jzr83fw1q"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AssemblyAI, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Assemblyai.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Assemblyai.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/atera/atera_integration_test.go b/pkg/detectors/atera/atera_integration_test.go new file mode 100644 index 000000000000..ee90eafbc080 --- /dev/null +++ b/pkg/detectors/atera/atera_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package atera + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAtera_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("ATERA") + inactiveSecret := testSecrets.MustGetField("ATERA_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find an atera secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Atera, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find an atera secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Atera, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Atera.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Atera.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/atera/atera_test.go b/pkg/detectors/atera/atera_test.go index ee90eafbc080..e2000a2d6aae 100644 --- a/pkg/detectors/atera/atera_test.go +++ b/pkg/detectors/atera/atera_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package atera import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "atera_key: yoo3d5pu3t4zxd6x1vhk7ykmjqarbsv1" + invalidPattern = "atera_key: yOO3d5pu3t4zxd6x1vhk7ykmjqarbs_1" ) -func TestAtera_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("ATERA") - inactiveSecret := testSecrets.MustGetField("ATERA_INACTIVE") +func TestAtera_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find an atera secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Atera, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"yoo3d5pu3t4zxd6x1vhk7ykmjqarbsv1"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find an atera secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Atera, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Atera.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Atera.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/audd/audd.go b/pkg/detectors/audd/audd.go index 7cfde0d94883..c55cf2115722 100644 --- a/pkg/detectors/audd/audd.go +++ b/pkg/detectors/audd/audd.go @@ -3,11 +3,12 @@ package audd import ( "context" "fmt" - regexp "github.com/wasilibs/go-re2" "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" diff --git a/pkg/detectors/audd/audd_integration_test.go b/pkg/detectors/audd/audd_integration_test.go new file mode 100644 index 000000000000..4f1424150edf --- /dev/null +++ b/pkg/detectors/audd/audd_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package audd + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAudd_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AUDD") + inactiveSecret := testSecrets.MustGetField("AUDD_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a audd secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Audd, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a audd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Audd, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Audd.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Audd.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/audd/audd_test.go b/pkg/detectors/audd/audd_test.go index 4f1424150edf..ff38207c60fb 100644 --- a/pkg/detectors/audd/audd_test.go +++ b/pkg/detectors/audd/audd_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package audd import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestAudd_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AUDD") - inactiveSecret := testSecrets.MustGetField("AUDD_INACTIVE") +var ( + validPattern = "audd_key: 60fzzcspq2balbxn7f3hi2nvg3h07h4z" + invalidPattern = "audd_key: 60fuzcsoq2balxbN7f3hi2nvg3h07h4z" +) + +func TestAudd_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a audd secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Audd, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"60fzzcspq2balbxn7f3hi2nvg3h07h4z"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a audd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Audd, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Audd.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Audd.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_integration_test.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_integration_test.go new file mode 100644 index 000000000000..32a0ac059883 --- /dev/null +++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_integration_test.go @@ -0,0 +1,125 @@ +//go:build detectors +// +build detectors + +package auth0managementapitoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAuth0ManagementApiToken_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + // use 2592000 for 30 days, this is the maximum allowed + managementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN") + inactiveManagementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN_INACTIVE") + domain := testSecrets.MustGetField("AUTH0_MANAGEMENT_DOMAIN") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain %s", managementApiToken, domain)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken, + Redacted: domain, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain https://%s/oauth/token within but not valid", inactiveManagementApiToken, domain)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken, + Redacted: domain, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Auth0ManagementApiToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Auth0ManagementApiToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go index 32a0ac059883..7e098d664105 100644 --- a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go +++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go @@ -1,125 +1,114 @@ -//go:build detectors -// +build detectors - package auth0managementapitoken import ( "context" "fmt" + "math/rand" + "strings" "testing" "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestAuth0ManagementApiToken_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } +var ( + // TODO(kashif): Refactor the fake token generation if possible + validPattern = generateRandomString() // this has the exact token string only which can be used in want too + validDomain = "QHHPu7VPj.sI.auth0.com" + invalidPattern = ` + auth0_credentails: + apiToken: eywT2nGMZwOcbsUVBwfiRPEl8P_wnmo6XfdUoGVwxDfOSjNyqhYqFdi.KojZZOM8Ox + domain: QHHPu7VPj.sI.auth0.com + ` +) - // use 2592000 for 30 days, this is the maximum allowed - managementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN") - inactiveManagementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN_INACTIVE") - domain := testSecrets.MustGetField("AUTH0_MANAGEMENT_DOMAIN") +func TestAuth0ManagementApitToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain %s", managementApiToken, domain)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken, - Redacted: domain, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain https://%s/oauth/token within but not valid", inactiveManagementApiToken, domain)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken, - Redacted: domain, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern", + input: makeFakeTokenString(validPattern, validDomain), + want: []string{validPattern + validDomain}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Auth0ManagementApiToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } - got[i].Raw = nil } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Auth0ManagementApiToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } } -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } - } - }) +// makeFakeTokenString take a string token as parameter and make a string that looks like a token for testing +func makeFakeTokenString(token, domain string) string { + return fmt.Sprintf("auth0:\n apiToken: %s \n domain: %s", token, domain) +} + +// generateRandomString generates exactly 2001 char string for a fake token to by pass the check in detector for testing +func generateRandomString() string { + const length = 2001 + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-" + + random := rand.New(rand.NewSource(time.Now().UnixNano())) + + var builder strings.Builder + builder.Grow(length) + + for i := 0; i < length; i++ { + randomChar := charset[random.Intn(len(charset))] + builder.WriteByte(randomChar) } + + // append ey in start as the token must start with 'ey' + return fmt.Sprintf("ey%s", builder.String()) } diff --git a/pkg/detectors/auth0oauth/auth0oauth_integeration_test.go b/pkg/detectors/auth0oauth/auth0oauth_integeration_test.go new file mode 100644 index 000000000000..5a343487d7ac --- /dev/null +++ b/pkg/detectors/auth0oauth/auth0oauth_integeration_test.go @@ -0,0 +1,125 @@ +//go:build detectors +// +build detectors + +package auth0oauth + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAuth0oauth_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + + domain := testSecrets.MustGetField("AUTH0_DOMAIN") + clientId := testSecrets.MustGetField("AUTH0_CLIENT_ID") // there is AUTH0_CLIENT_ID2 and AUTH0_CLIENT_SECRET2 pair as well + clientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET") + inactiveClientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain %s", clientId, clientSecret, domain)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Auth0oauth, + Redacted: clientId, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain https://%s/oauth/token within but not valid", clientId, inactiveClientSecret, domain)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Auth0oauth, + Redacted: clientId, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Auth0oauth.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Auth0oauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/auth0oauth/auth0oauth_test.go b/pkg/detectors/auth0oauth/auth0oauth_test.go index 5a343487d7ac..e573e5c037d1 100644 --- a/pkg/detectors/auth0oauth/auth0oauth_test.go +++ b/pkg/detectors/auth0oauth/auth0oauth_test.go @@ -1,125 +1,90 @@ -//go:build detectors -// +build detectors - package auth0oauth import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestAuth0oauth_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } +var ( + validPattern = ` + auth0_credentials file: + auth0_clientID: kYWr_tL4eYBtqIIvKfSf2-e4T9Cw1CtwE8ufoESVBB7Hi1U + secret: rXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS-MBqIYgxH0vZaL1s5314lgPDLqHXjZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_tTx1kCXOZY_JUy + domain: 9-KhTIdSopSaMQ2v1YxdFEJN-HNgt7Mn7E8xkfQNqd51AzSGQu2yRaFauth0.com + ` + invalidPattern = ` + auth0_credentials file: + auth0_clientID: e4T9Cw1CtwE8ufoESVBB7Hi1U-e4T9Cw1CtwE8ufoESVBB7Hi1U + secret: MBqIYgxH0vZaL1s5314lgPDLqHX^ZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_+tTx1kCXOZY_JUy-rXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS + domain: 9-KhTIdSopSaMQ2v1YxdFEJN#qd51AzSGQu2yRaFauth1.com + ` +) - domain := testSecrets.MustGetField("AUTH0_DOMAIN") - clientId := testSecrets.MustGetField("AUTH0_CLIENT_ID") // there is AUTH0_CLIENT_ID2 and AUTH0_CLIENT_SECRET2 pair as well - clientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET") - inactiveClientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET_INACTIVE") +func TestAuth0oAuth_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain %s", clientId, clientSecret, domain)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Auth0oauth, - Redacted: clientId, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"kYWr_tL4eYBtqIIvKfSf2-e4T9Cw1CtwE8ufoESVBB7Hi1UrXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS-MBqIYgxH0vZaL1s5314lgPDLqHXjZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_tTx1kCXOZY_JUy"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain https://%s/oauth/token within but not valid", clientId, inactiveClientSecret, domain)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Auth0oauth, - Redacted: clientId, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Auth0oauth.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Auth0oauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/autodesk/autodesk_integration_test.go b/pkg/detectors/autodesk/autodesk_integration_test.go new file mode 100644 index 000000000000..d3d90c83c811 --- /dev/null +++ b/pkg/detectors/autodesk/autodesk_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package autodesk + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAutodesk_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + id := testSecrets.MustGetField("AUTODESK_ID") + secret := testSecrets.MustGetField("AUTODESK_SECRET") + inactiveID := testSecrets.MustGetField("AUTODESK_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Autodesk, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s but not valid", secret, inactiveID)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Autodesk, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Autodesk.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Autodesk.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/autodesk/autodesk_test.go b/pkg/detectors/autodesk/autodesk_test.go index d3d90c83c811..cf469f817ba4 100644 --- a/pkg/detectors/autodesk/autodesk_test.go +++ b/pkg/detectors/autodesk/autodesk_test.go @@ -1,121 +1,80 @@ -//go:build detectors -// +build detectors - package autodesk import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "autodesk_key: 2j8Rl67MjoMruYfyIBgGzy2pxcxIQfet / autodesk_secret rHfzZhsSRruLM3Fn" + invalidPattern = "autodesk: 2mm8Rl67MjoMruYfyIBg5#zy2pxcxIQfet / autodesk_secret RHGklpa" ) -func TestAutodesk_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - id := testSecrets.MustGetField("AUTODESK_ID") - secret := testSecrets.MustGetField("AUTODESK_SECRET") - inactiveID := testSecrets.MustGetField("AUTODESK_INACTIVE") +func TestAutoDesk_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Autodesk, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"2j8Rl67MjoMruYfyIBgGzy2pxcxIQfetrHfzZhsSRruLM3Fn"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s but not valid", secret, inactiveID)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Autodesk, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Autodesk.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Autodesk.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/autoklose/autoklose_integration_test.go b/pkg/detectors/autoklose/autoklose_integration_test.go new file mode 100644 index 000000000000..9675d22cba0f --- /dev/null +++ b/pkg/detectors/autoklose/autoklose_integration_test.go @@ -0,0 +1,123 @@ +//go:build detectors +// +build detectors + +package autoklose + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAutoklose_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AUTOKLOSE") + inactiveSecret := testSecrets.MustGetField("AUTOKLOSE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a autoklose secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Autoklose, + Verified: true, + ExtraData: map[string]string{ + "email": "mladen.stevanovic@vanillasoft.com", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a autoklose secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Autoklose, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Autoklose.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatal("no raw secret present") + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Autoklose.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/autoklose/autoklose_test.go b/pkg/detectors/autoklose/autoklose_test.go index 9675d22cba0f..4ed460f78c3b 100644 --- a/pkg/detectors/autoklose/autoklose_test.go +++ b/pkg/detectors/autoklose/autoklose_test.go @@ -1,123 +1,80 @@ -//go:build detectors -// +build detectors - package autoklose import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "autoklose credentials: KRXaU9GK3f9yHG1FS-mbwhsIXdW22epH" + invalidPattern = "autoklose credentials: KRXaU9GK3f[yHG1FS$]bwhsIXdW22epH" ) -func TestAutoklose_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AUTOKLOSE") - inactiveSecret := testSecrets.MustGetField("AUTOKLOSE_INACTIVE") +func TestAutoKlose_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a autoklose secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Autoklose, - Verified: true, - ExtraData: map[string]string{ - "email": "mladen.stevanovic@vanillasoft.com", - }, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"KRXaU9GK3f9yHG1FS-mbwhsIXdW22epH"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a autoklose secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Autoklose, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Autoklose.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatal("no raw secret present") - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Autoklose.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/autopilot/autopilot_integration_test.go b/pkg/detectors/autopilot/autopilot_integration_test.go new file mode 100644 index 000000000000..b15e26cd8e27 --- /dev/null +++ b/pkg/detectors/autopilot/autopilot_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package autopilot + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAutoPilot_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AUTOPILOT") + inactiveSecret := testSecrets.MustGetField("AUTOPILOT_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a autopilot secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AutoPilot, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a autopilot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AutoPilot, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AutoPilot.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("AutoPilot.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/autopilot/autopilot_test.go b/pkg/detectors/autopilot/autopilot_test.go index b15e26cd8e27..dd5a5f67c301 100644 --- a/pkg/detectors/autopilot/autopilot_test.go +++ b/pkg/detectors/autopilot/autopilot_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package autopilot import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "autopilot credentials: 0fd87cfb1ca6c38c5f1ae5be7b0e395e" + invalidPattern = "autopilot credentials: KRXaU9GK3f[yHG1FS$]bwhsIXdW22epH" ) -func TestAutoPilot_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AUTOPILOT") - inactiveSecret := testSecrets.MustGetField("AUTOPILOT_INACTIVE") +func TestAutoPilot_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a autopilot secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AutoPilot, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"0fd87cfb1ca6c38c5f1ae5be7b0e395e"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a autopilot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AutoPilot, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AutoPilot.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("AutoPilot.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_integration_test.go b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_integration_test.go new file mode 100644 index 000000000000..80664332bfba --- /dev/null +++ b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_integration_test.go @@ -0,0 +1,117 @@ +//go:build detectors +// +build detectors + +package avazapersonalaccesstoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAvazaPersonalAccessToken_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN") + inactiveSecret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a avaza secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a avaza secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AvazaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("AvazaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + s.FromData(ctx, false, data) + } + }) + } +} diff --git a/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go index 80664332bfba..7ef07b611768 100644 --- a/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go +++ b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go @@ -1,116 +1,79 @@ -//go:build detectors -// +build detectors - package avazapersonalaccesstoken import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestAvazaPersonalAccessToken_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN") - inactiveSecret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN_INACTIVE") +var ( + validPattern = "avaza credentials: 01818612883613176996369293-f113ceb9cf4fa63dc367ab4815b0e1edf890745f" + invalidPattern = "avaza credentials: 01818612883613176996369293-fzz3ceb0mf4fp63dh367xb4815b0e1edf890745f" +) + +func TestAvazaPersonalAccessToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a avaza secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"01818612883613176996369293-f113ceb9cf4fa63dc367ab4815b0e1edf890745f"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a avaza secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AvazaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - got[i].Raw = nil + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("AvazaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - s.FromData(ctx, false, data) + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/aviationstack/aviationstack_integration_test.go b/pkg/detectors/aviationstack/aviationstack_integration_test.go new file mode 100644 index 000000000000..b716fcd195e7 --- /dev/null +++ b/pkg/detectors/aviationstack/aviationstack_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package aviationstack + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAviationStack_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AVIATIONSTACK") + inactiveSecret := testSecrets.MustGetField("AVIATIONSTACK_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AviationStack, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AviationStack, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AviationStack.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("AviationStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/aviationstack/aviationstack_test.go b/pkg/detectors/aviationstack/aviationstack_test.go index b716fcd195e7..c76924c9ac32 100644 --- a/pkg/detectors/aviationstack/aviationstack_test.go +++ b/pkg/detectors/aviationstack/aviationstack_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package aviationstack import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "aviationstack credentials: osh0kjinsc2atoaqntoy1hdjppg54449" + invalidPattern = "aviationstack credentials: OSh0lMjinsc2atoaqnto[]1hdjppg5449" ) -func TestAviationStack_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AVIATIONSTACK") - inactiveSecret := testSecrets.MustGetField("AVIATIONSTACK_INACTIVE") +func TestAviationStack_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AviationStack, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"osh0kjinsc2atoaqntoy1hdjppg54449"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AviationStack, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AviationStack.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("AviationStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/aws/aws_integration_test.go b/pkg/detectors/aws/aws_integration_test.go new file mode 100644 index 000000000000..92ffa5c3cab9 --- /dev/null +++ b/pkg/detectors/aws/aws_integration_test.go @@ -0,0 +1,449 @@ +//go:build detectors +// +build detectors + +package aws + +import ( + "context" + "crypto/sha256" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/assert" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +const canaryAccessKeyID = "AKIASP2TPHJSQH3FJRUX" + +var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`) + +// Our AWS detector interacts with AWS in an (expectedly) uncommon way that triggers some odd AWS behavior. (This odd +// behavior doesn't affect "normal" AWS use, so it's not really "broken" - it's just something that we have to work +// around.) The AWS detector code has a long comment explaining this in more detail, but the basic issue is that AWS STS +// is stateful, so the behavior of these tests can vary depending on which of them you run, and in which order. This +// particular test (TestAWS_FromChunk_InvalidValidReuseIDSequence) duplicates some logic in the "big" test table in the +// other test in this file, but extracting it in this way as well makes it fail more consistently when it's supposed to +// fail, which is why it's extracted. +func TestAWS_FromChunk_InvalidValidReuseIDSequence(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AWS") + id := testSecrets.MustGetField("AWS_ID") + inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE") + + d := scanner{} + + ignoreOpts := []cmp.Option{cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError")} + + got, err := d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, inactiveSecret))) + if assert.NoError(t, err) { + want := []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + }, + }, + } + if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" { + t.Errorf("AWS.FromData() (valid ID, invalid secret) diff: (-got +want)\n%s", diff) + } + } + + got, err = d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, secret))) + if assert.NoError(t, err) { + want := []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: true, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester", + "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", + "user_id": "AIDAZAVB57H5V3Q4ACRGM", + }, + }, + } + if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" { + t.Errorf("AWS.FromData() (valid secret after invalid secret using same ID) diff: (-got +want)\n%s", diff) + } + } +} + +func TestAWS_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AWS") + id := testSecrets.MustGetField("AWS_ID") + inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE") + inactiveID := id[:len(id)-3] + "XYZ" + hasher := sha256.New() + hasher.Write([]byte(inactiveSecret)) + hash := string(hasher.Sum(nil)) + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s scanner + args args + want []detectors.Result + wantErr bool + wantVerificationError bool + }{ + { + name: "found, verified", + s: scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: true, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester", + "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", + "user_id": "AIDAZAVB57H5V3Q4ACRGM", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: scanner{verificationClient: unverifiedSecretClient}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + }, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + { + name: "found two, one included for every ID found", + s: scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: "AKIAZAVB57H55F3T4XYZ", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + }, + }, + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: true, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester", + "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", + "user_id": "AIDAZAVB57H5V3Q4ACRGM", + }, + }, + }, + wantErr: false, + }, + { + name: "not found, because unverified secret was a hash", + s: scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", hash, id)), // The secret would satisfy the regex but be filtered out after not passing validation. + verify: true, + }, + want: nil, + wantErr: false, + }, + { + name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash", + s: scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: true, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester", + "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", + "user_id": "AIDAZAVB57H5V3Q4ACRGM", + }, + }, + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: inactiveID, + ExtraData: map[string]string{ + "account": "619888638459", + "resource_type": "Access key", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified, with leading +", + s: scanner{ + verificationClient: unverifiedSecretClient, + }, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + }, + }, + }, + wantErr: false, + }, + { + name: "skipped", + s: scanner{ + skipIDs: map[string]struct{}{ + "AKIAZAVB57H55F3T4BKH": {}, + }, + }, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + wantErr: false, + }, + { + name: "found, would be verified if not for http timeout", + s: scanner{ + verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond), + }, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + }, + }, + }, + wantErr: false, + wantVerificationError: true, + }, + { + name: "found, unverified due to unexpected http response status", + s: scanner{ + verificationClient: common.ConstantResponseHttpClient(500, "internal server error"), + }, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + }, + }, + }, + wantErr: false, + wantVerificationError: true, + }, + { + name: "found, unverified due to invalid aws_secret with valid canary access_key_id", + s: scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", inactiveSecret, canaryAccessKeyID)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: canaryAccessKeyID, + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "171436882533", + "is_canary": "true", + "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries", + }, + }, + }, + wantErr: false, + wantVerificationError: false, + }, + { + name: "found, valid canary token with no verification", + s: scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, canaryAccessKeyID)), + verify: false, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: canaryAccessKeyID, + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "171436882533", + "is_canary": "true", + "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries", + }, + }, + }, + wantErr: false, + wantVerificationError: false, + }, + { + name: "verified secret checked directly after unverified secret with same key id", + s: scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: true, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester", + "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", + "user_id": "AIDAZAVB57H5V3Q4ACRGM", + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.s + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AWS.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationError { + t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError()) + } + } + ignoreOpts := []cmp.Option{ + cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError"), + cmpopts.SortSlices(func(x, y detectors.Result) bool { + return x.Redacted < y.Redacted + }), + } + if diff := cmp.Diff(got, tt.want, ignoreOpts...); diff != "" { + t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.go index 92ffa5c3cab9..576451fedb30 100644 --- a/pkg/detectors/aws/aws_test.go +++ b/pkg/detectors/aws/aws_test.go @@ -1,448 +1,89 @@ -//go:build detectors -// +build detectors - package aws import ( "context" - "crypto/sha256" - "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/stretchr/testify/assert" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -const canaryAccessKeyID = "AKIASP2TPHJSQH3FJRUX" - -var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`) - -// Our AWS detector interacts with AWS in an (expectedly) uncommon way that triggers some odd AWS behavior. (This odd -// behavior doesn't affect "normal" AWS use, so it's not really "broken" - it's just something that we have to work -// around.) The AWS detector code has a long comment explaining this in more detail, but the basic issue is that AWS STS -// is stateful, so the behavior of these tests can vary depending on which of them you run, and in which order. This -// particular test (TestAWS_FromChunk_InvalidValidReuseIDSequence) duplicates some logic in the "big" test table in the -// other test in this file, but extracting it in this way as well makes it fail more consistently when it's supposed to -// fail, which is why it's extracted. -func TestAWS_FromChunk_InvalidValidReuseIDSequence(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AWS") - id := testSecrets.MustGetField("AWS_ID") - inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE") - - d := scanner{} - - ignoreOpts := []cmp.Option{cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError")} - - got, err := d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, inactiveSecret))) - if assert.NoError(t, err) { - want := []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: "AKIAZAVB57H55F3T4BKH", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - }, - }, +var ( + validPattern = ` + aws credentials{ + id: ABIAS9L8MS5IPHTZPPUQ + secret: .v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63; } - if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" { - t.Errorf("AWS.FromData() (valid ID, invalid secret) diff: (-got +want)\n%s", diff) + ` + invalidPattern = ` + aws credentials{ + id: AKIAs9L8MS5iPHTZPPUQ + secret: $YenOG.PKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63; } - } - - got, err = d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, secret))) - if assert.NoError(t, err) { - want := []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: true, - Redacted: "AKIAZAVB57H55F3T4BKH", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester", - "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", - "user_id": "AIDAZAVB57H5V3Q4ACRGM", - }, - }, - } - if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" { - t.Errorf("AWS.FromData() (valid secret after invalid secret using same ID) diff: (-got +want)\n%s", diff) - } - } -} + ` +) -func TestAWS_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AWS") - id := testSecrets.MustGetField("AWS_ID") - inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE") - inactiveID := id[:len(id)-3] + "XYZ" - hasher := sha256.New() - hasher.Write([]byte(inactiveSecret)) - hash := string(hasher.Sum(nil)) +func TestAWS_Pattern(t *testing.T) { + d := scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s scanner - args args - want []detectors.Result - wantErr bool - wantVerificationError bool + name string + input string + want []string }{ { - name: "found, verified", - s: scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: true, - Redacted: "AKIAZAVB57H55F3T4BKH", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester", - "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", - "user_id": "AIDAZAVB57H5V3Q4ACRGM", - }, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: scanner{verificationClient: unverifiedSecretClient}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: "AKIAZAVB57H55F3T4BKH", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - }, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - { - name: "found two, one included for every ID found", - s: scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: "AKIAZAVB57H55F3T4XYZ", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - }, - }, - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: true, - Redacted: "AKIAZAVB57H55F3T4BKH", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester", - "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", - "user_id": "AIDAZAVB57H5V3Q4ACRGM", - }, - }, - }, - wantErr: false, - }, - { - name: "not found, because unverified secret was a hash", - s: scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", hash, id)), // The secret would satisfy the regex but be filtered out after not passing validation. - verify: true, - }, - want: nil, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"ABIAS9L8MS5IPHTZPPUQv2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63"}, }, { - name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash", - s: scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: true, - Redacted: "AKIAZAVB57H55F3T4BKH", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester", - "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", - "user_id": "AIDAZAVB57H5V3Q4ACRGM", - }, - }, - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: inactiveID, - ExtraData: map[string]string{ - "account": "619888638459", - "resource_type": "Access key", - }, - }, - }, - wantErr: false, - }, - { - name: "found, unverified, with leading +", - s: scanner{ - verificationClient: unverifiedSecretClient, - }, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: "AKIAZAVB57H55F3T4BKH", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - }, - }, - }, - wantErr: false, - }, - { - name: "skipped", - s: scanner{ - skipIDs: map[string]struct{}{ - "AKIAZAVB57H55F3T4BKH": {}, - }, - }, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - wantErr: false, - }, - { - name: "found, would be verified if not for http timeout", - s: scanner{ - verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond), - }, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: "AKIAZAVB57H55F3T4BKH", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - }, - }, - }, - wantErr: false, - wantVerificationError: true, - }, - { - name: "found, unverified due to unexpected http response status", - s: scanner{ - verificationClient: common.ConstantResponseHttpClient(500, "internal server error"), - }, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: "AKIAZAVB57H55F3T4BKH", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - }, - }, - }, - wantErr: false, - wantVerificationError: true, - }, - { - name: "found, unverified due to invalid aws_secret with valid canary access_key_id", - s: scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", inactiveSecret, canaryAccessKeyID)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: canaryAccessKeyID, - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "171436882533", - "is_canary": "true", - "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries", - }, - }, - }, - wantErr: false, - wantVerificationError: false, - }, - { - name: "found, valid canary token with no verification", - s: scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, canaryAccessKeyID)), - verify: false, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: canaryAccessKeyID, - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "171436882533", - "is_canary": "true", - "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries", - }, - }, - }, - wantErr: false, - wantVerificationError: false, - }, - { - name: "verified secret checked directly after unverified secret with same key id", - s: scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: true, - Redacted: "AKIAZAVB57H55F3T4BKH", - ExtraData: map[string]string{ - "resource_type": "Access key", - "account": "619888638459", - "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester", - "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/", - "user_id": "AIDAZAVB57H5V3Q4ACRGM", - }, - }, - }, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := tt.s - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AWS.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - if (got[i].VerificationError() != nil) != tt.wantVerificationError { - t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError()) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } + return } - ignoreOpts := []cmp.Option{ - cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError"), - cmpopts.SortSlices(func(x, y detectors.Result) bool { - return x.Redacted < y.Redacted - }), + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } } - if diff := cmp.Diff(got, tt.want, ignoreOpts...); diff != "" { - t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/awssessionkeys/awssessionkeys_test.go b/pkg/detectors/awssessionkeys/awssessionkeys_test.go new file mode 100644 index 000000000000..8ed4028e6ca0 --- /dev/null +++ b/pkg/detectors/awssessionkeys/awssessionkeys_test.go @@ -0,0 +1,91 @@ +package awssessionkey + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = ` + aws credentials{ + id: ASIABBKK02W42Q3IPSPG + secret: fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O + session: >aSqfp/GTZbJP+=tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQ=W5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LR=uf/urSorGbTcqNUvP=i42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNA=dD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa~ + } + ` + invalidPattern = ` + aws credentials{ + id: ASIABBKK02W42Q3IPSPG + secret: $YenOG.PKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63; + } + ` +) + +func TestAWSSessionKey_Pattern(t *testing.T) { + d := scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + { + name: "valid pattern", + input: validPattern, + want: []string{"ASIABBKK02W42Q3IPSPGfkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8OaSqfp/GTZbJP+=tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQ=W5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LR=uf/urSorGbTcqNUvP=i42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNA=dD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa"}, + }, + { + name: "invalid pattern", + input: invalidPattern, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/axonaut/axonaut_integration_test.go b/pkg/detectors/axonaut/axonaut_integration_test.go new file mode 100644 index 000000000000..0d64cbc64e1e --- /dev/null +++ b/pkg/detectors/axonaut/axonaut_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package axonaut + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAxonaut_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AXONAUT") + inactiveSecret := testSecrets.MustGetField("AXONAUT_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a axonaut secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Axonaut, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a axonaut secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Axonaut, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Axonaut.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Axonaut.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/axonaut/axonaut_test.go b/pkg/detectors/axonaut/axonaut_test.go index 0d64cbc64e1e..fdf924afc582 100644 --- a/pkg/detectors/axonaut/axonaut_test.go +++ b/pkg/detectors/axonaut/axonaut_test.go @@ -1,120 +1,88 @@ -//go:build detectors -// +build detectors - package axonaut import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = ` + axonaut credentials{ + key: 4ve4aj6v38uiadaq9hcgpupp2b3lh2k8 + } + ` + invalidPattern = ` + axonaut credentials{ + key: ASIABBKK02W42Q3IPSPG + } + ` ) -func TestAxonaut_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AXONAUT") - inactiveSecret := testSecrets.MustGetField("AXONAUT_INACTIVE") +func TestAxonaut_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a axonaut secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Axonaut, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"4ve4aj6v38uiadaq9hcgpupp2b3lh2k8"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a axonaut secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Axonaut, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Axonaut.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Axonaut.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/aylien/aylien_integration_test.go b/pkg/detectors/aylien/aylien_integration_test.go new file mode 100644 index 000000000000..e1315f5f9b66 --- /dev/null +++ b/pkg/detectors/aylien/aylien_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package aylien + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAylien_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AYLIEN") + id := testSecrets.MustGetField("AYLIEN_ID") + inactiveSecret := testSecrets.MustGetField("AYLIEN_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Aylien, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Aylien, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Aylien.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Aylien.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/aylien/aylien_test.go b/pkg/detectors/aylien/aylien_test.go index e1315f5f9b66..5f01ae03bbde 100644 --- a/pkg/detectors/aylien/aylien_test.go +++ b/pkg/detectors/aylien/aylien_test.go @@ -1,121 +1,88 @@ -//go:build detectors -// +build detectors - package aylien import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = ` + aylien credentials: + aylien key: cr479du2l9pkmhar8gw5hufofvwp86q9 + aylien id: y3ejw028 + ` + invalidPattern = ` + aylien credentials: + aylien key: cr4U9du2l9pkmhar8gw5hufofvWp86q9 + aylien id: y3ejwA8 + ` ) -func TestAylien_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AYLIEN") - id := testSecrets.MustGetField("AYLIEN_ID") - inactiveSecret := testSecrets.MustGetField("AYLIEN_INACTIVE") +func TestAylien_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Aylien, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"cr479du2l9pkmhar8gw5hufofvwp86q9y3ejw028"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Aylien, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Aylien.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Aylien.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/ayrshare/ayrshare_integration_test.go b/pkg/detectors/ayrshare/ayrshare_integration_test.go new file mode 100644 index 000000000000..da11fac95ea7 --- /dev/null +++ b/pkg/detectors/ayrshare/ayrshare_integration_test.go @@ -0,0 +1,120 @@ +//go:build detectors +// +build detectors + +package ayrshare + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAyrshare_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AYRSHARE_TOKEN") + inactiveSecret := testSecrets.MustGetField("AYRSHARE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Ayrshare, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Ayrshare, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Ayrshare.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Ayrshare.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/ayrshare/ayrshare_test.go b/pkg/detectors/ayrshare/ayrshare_test.go index da11fac95ea7..9eb55166fe93 100644 --- a/pkg/detectors/ayrshare/ayrshare_test.go +++ b/pkg/detectors/ayrshare/ayrshare_test.go @@ -1,120 +1,80 @@ -//go:build detectors -// +build detectors - package ayrshare import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = "ayrshare credentials: 2FTJTA1C-BXO0DV4J-HGTP9E62-QHQSILY1" + invalidPattern = "ayrshare credentials: KRXaU9GK3f[yHG1FS$]bwhsIXdW22epH" ) -func TestAyrshare_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AYRSHARE_TOKEN") - inactiveSecret := testSecrets.MustGetField("AYRSHARE_INACTIVE") +func TestAyrShare_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Ayrshare, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"2FTJTA1C-BXO0DV4J-HGTP9E62-QHQSILY1"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Ayrshare, - Verified: false, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Ayrshare.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Ayrshare.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/azure/azure_integration_test.go b/pkg/detectors/azure/azure_integration_test.go new file mode 100644 index 000000000000..9ecc18ab14a5 --- /dev/null +++ b/pkg/detectors/azure/azure_integration_test.go @@ -0,0 +1,113 @@ +//go:build detectors +// +build detectors + +package azure + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAzure_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AZURE_SECRET") + secretInactive := testSecrets.MustGetField("AZURE_INACTIVE") + id := testSecrets.MustGetField("AZURE_ID") + tenantId := testSecrets.MustGetField("AZURE_TENANT_ID") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf(` + tenant_id=%s + client_id=%s + client_secret=%s + client_secret=%s + `, tenantId, id, secretInactive, secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Azure, + Redacted: id, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf(` + tenant_id=%s + client_id=%s + client_secret=%s + `, tenantId, id, secretInactive)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Azure, + Redacted: id, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Azure.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("Azure.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} diff --git a/pkg/detectors/azure/azure_test.go b/pkg/detectors/azure/azure_test.go index 9ecc18ab14a5..74b2fadb50ec 100644 --- a/pkg/detectors/azure/azure_test.go +++ b/pkg/detectors/azure/azure_test.go @@ -1,112 +1,89 @@ -//go:build detectors -// +build detectors - package azure import ( "context" - "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/google/go-cmp/cmp" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestAzure_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AZURE_SECRET") - secretInactive := testSecrets.MustGetField("AZURE_INACTIVE") - id := testSecrets.MustGetField("AZURE_ID") - tenantId := testSecrets.MustGetField("AZURE_TENANT_ID") +var ( + validPattern = ` + azure credentials: + azureClientID: clientid9304d5df4-aac1-6117-552c-7f70c89a40d9 + azureTenant: tenant_idid9304d5df4-aac1-6117-552c-7f70c89a40d9 + azureClientSecret: clientsecretY_0w|[cGpan41k6ng.ol414sp4ccw2v_rkfmbs537i + ` + invalidPattern = ` + azure credentials: + azureClientID: 9304d5df4-aac1-6117-552c-7f70c89a + azureTenant: id9304d5df4-aac1-6117-55-7f70c89a40d9 + azureClientSecret: Y_0w|[cGpan41k6ng. + ` +) + +func TestAzure_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf(` - tenant_id=%s - client_id=%s - client_secret=%s - client_secret=%s - `, tenantId, id, secretInactive, secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Azure, - Redacted: id, - Verified: true, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf(` - tenant_id=%s - client_id=%s - client_secret=%s - `, tenantId, id, secretInactive)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_Azure, - Redacted: id, - Verified: false, - }, - }, - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"304d5df4-aac1-6117-552c-7f70c89a40d9cGpan41k6ng.ol414sp4ccw2v_rkfmbs53304d5df4-aac1-6117-552c-7f70c89a40d9"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Azure.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("Azure.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/azurebatch/azurebatch_integration_test.go b/pkg/detectors/azurebatch/azurebatch_integration_test.go new file mode 100644 index 000000000000..dc92a401b2d0 --- /dev/null +++ b/pkg/detectors/azurebatch/azurebatch_integration_test.go @@ -0,0 +1,130 @@ +//go:build detectors +// +build detectors + +package azurebatch + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAzurebatch_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + url := testSecrets.MustGetField("AZUREBATCH_URL") + secret := testSecrets.MustGetField("AZUREBATCH_KEY") + inactiveSecret := testSecrets.MustGetField("AZUREBATCH_KEY_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within", url, secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureBatch, + Verified: true, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within but not valid", url, inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureBatch, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AzureBatch.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError") + + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("AzureBatch.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/azurebatch/azurebatch_test.go b/pkg/detectors/azurebatch/azurebatch_test.go index dc92a401b2d0..82a5ac19b7b3 100644 --- a/pkg/detectors/azurebatch/azurebatch_test.go +++ b/pkg/detectors/azurebatch/azurebatch_test.go @@ -1,130 +1,88 @@ -//go:build detectors -// +build detectors - package azurebatch import ( "context" - "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = ` + azure batch: + url: https://JrxlYxT+0hW.YSA.batch.azure.com + secret: BXIMbhBlC3=5hIbqCEKvq7opaV2ZfO0XWbcnasZmPm/AJfQqdcnt/AVmKkJ8Qw80Zc1rQDaw+2Ytxc1hDq1m/LB0 + ` + invalidPattern = ` + azure batch: + url: http://invalid.this.batch.azure.com + secret: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/AVmKkJ8Qw80Zc1rQDaw+2Ytxc1hDq1m/ + ` ) -func TestAzurebatch_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - url := testSecrets.MustGetField("AZUREBATCH_URL") - secret := testSecrets.MustGetField("AZUREBATCH_KEY") - inactiveSecret := testSecrets.MustGetField("AZUREBATCH_KEY_INACTIVE") +func TestAzureBatch_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within", url, secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureBatch, - Verified: true, - }, - }, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within but not valid", url, inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureBatch, - Verified: false, - }, - }, - wantErr: false, - wantVerificationErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"https://JrxlYxT+0hW.YSA.batch.azure.comBXIMbhBlC3=5hIbqCEKvq7opaV2ZfO0XWbcnasZmPm/AJfQqdcnt/AVmKkJ8Qw80Zc1rQDaw+2Ytxc1hDq1m/LB0"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - wantVerificationErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AzureBatch.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) - } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("AzureBatch.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } diff --git a/pkg/detectors/azurecontainerregistry/azurecontainerregistry_integration_test.go b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_integration_test.go new file mode 100644 index 000000000000..8ed032f97796 --- /dev/null +++ b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_integration_test.go @@ -0,0 +1,129 @@ +//go:build detectors +// +build detectors + +package azurecontainerregistry + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAzureContainerRegistry_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + azureHost := testSecrets.MustGetField("AZURE_CR_HOST") + password := testSecrets.MustGetField("AZURE_CR_PASSWORD") + passwordInactive := testSecrets.MustGetField("AZURE_CR_PASSWORD_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within", azureHost, password)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureContainerRegistry, + Verified: true, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within but not valid", azureHost, passwordInactive)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureContainerRegistry, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AzureContainerRegistry.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("AzureContainerRegistry.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go index 8ed032f97796..b477796aba98 100644 --- a/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go +++ b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go @@ -1,128 +1,87 @@ -//go:build detectors -// +build detectors - package azurecontainerregistry import ( "context" - "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = ` + azure: + url: 02r0TG8OBkyxWIhSrcCMUQCazXEpVwdrTSVKCstwytZjAa4b2fKJFCy9n.azurecr.io + secret: np5kXaH6Qu9zVE5nudGcsm+3/qSOWt/hQ/4bP0kvk6aw98pJEjE3 + ` + invalidPattern = ` + azure: + url: http://invalid.azurecr.io.azure.com + secret: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/+2Ytxc1hDq1m/ + ` ) -func TestAzureContainerRegistry_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - azureHost := testSecrets.MustGetField("AZURE_CR_HOST") - password := testSecrets.MustGetField("AZURE_CR_PASSWORD") - passwordInactive := testSecrets.MustGetField("AZURE_CR_PASSWORD_INACTIVE") +func TestAzureContainerRegistry_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within", azureHost, password)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureContainerRegistry, - Verified: true, - }, - }, - wantErr: false, - wantVerificationErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"02r0TG8OBkyxWIhSrcCMUQCazXEpVwdrTSVKCstwytZjAa4b2fKJFCy9n.azurecr.ionp5kXaH6Qu9zVE5nudGcsm+3/qSOWt/hQ/4bP0kvk6aw98pJEjE3"}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within but not valid", azureHost, passwordInactive)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureContainerRegistry, - Verified: false, - }, - }, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - wantVerificationErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AzureContainerRegistry.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("AzureContainerRegistry.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go new file mode 100644 index 000000000000..2b7c68f3fd67 --- /dev/null +++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go @@ -0,0 +1,170 @@ +//go:build detectors +// +build detectors + +package azuredevopspersonalaccesstoken + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAzureDevopsPersonalAccessToken_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AZURE_DEVOPS_PAT") + inactiveSecret := testSecrets.MustGetField("AZURE_DEVOPS_PAT_INACTIVE") + org := testSecrets.MustGetField("AZURE_DEVOPS_PAT_ORG") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, + Verified: true, + RawV2: []byte(secret + org), + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within but not valid", inactiveSecret, org)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, + Verified: false, + RawV2: []byte(inactiveSecret + org), + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, + Verified: false, + RawV2: []byte(secret + org), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, + Verified: false, + RawV2: []byte(secret + org), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AzureDevopsPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if len(got[i].RawV2) == 0 { + t.Fatalf("no rawV2 secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("AzureDevopsPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go index 2b7c68f3fd67..b19ef9de3f55 100644 --- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go +++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go @@ -1,169 +1,87 @@ -//go:build detectors -// +build detectors - package azuredevopspersonalaccesstoken import ( "context" - "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = ` + azure: + azure_key: uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un + azure_org_id: WOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG + ` + invalidPattern = ` + azure: + azure_key: uie5tff7m5H5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un + azure_org_id: LOKi + ` ) -func TestAzureDevopsPersonalAccessToken_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AZURE_DEVOPS_PAT") - inactiveSecret := testSecrets.MustGetField("AZURE_DEVOPS_PAT_INACTIVE") - org := testSecrets.MustGetField("AZURE_DEVOPS_PAT_ORG") +func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, - Verified: true, - RawV2: []byte(secret + org), - }, - }, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within but not valid", inactiveSecret, org)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, - Verified: false, - RawV2: []byte(inactiveSecret + org), - }, - }, - wantErr: false, - wantVerificationErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8unWOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "found, would be verified if not for timeout", - s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, - Verified: false, - RawV2: []byte(secret + org), - }, - }, - wantErr: false, - wantVerificationErr: true, - }, - { - name: "found, verified but unexpected api surface", - s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, - Verified: false, - RawV2: []byte(secret + org), - }, - }, - wantErr: false, - wantVerificationErr: true, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AzureDevopsPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - if len(got[i].RawV2) == 0 { - t.Fatalf("no rawV2 secret present: \n %+v", got[i]) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("AzureDevopsPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey_integration_test.go b/pkg/detectors/azurefunctionkey/azurefunctionkey_integration_test.go new file mode 100644 index 000000000000..2d9881ed72fa --- /dev/null +++ b/pkg/detectors/azurefunctionkey/azurefunctionkey_integration_test.go @@ -0,0 +1,170 @@ +//go:build detectors +// +build detectors + +package azurefunctionkey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAzureFunctionKey_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AZURE_FUNCTION_KEY") + inactiveSecret := testSecrets.MustGetField("AZURE_FUNCTION_KEY_INACTIVE") + url := testSecrets.MustGetField("AZURE_FUNCTION_URL") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureFunctionKey, + Verified: true, + RawV2: []byte(secret + url), + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureFunctionKey, + Verified: false, + RawV2: []byte(inactiveSecret + url), + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureFunctionKey, + Verified: false, + RawV2: []byte(secret + url), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureFunctionKey, + Verified: false, + RawV2: []byte(secret + url), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AzureFunctionKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if len(got[i].RawV2) == 0 { + t.Fatalf("no rawV2 secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("AzureFunctionKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go b/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go index 2d9881ed72fa..f284a28d7dd8 100644 --- a/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go +++ b/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go @@ -1,169 +1,90 @@ -//go:build detectors -// +build detectors - package azurefunctionkey import ( "context" - "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = ` + azure: + azureURL: https://z1dUSi5T.azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0E + azureFunctionkey: B8sm0KyfL1y8vPH3IDTdefevHBCGK33-= + ` + invalidPattern = ` + azure: + azureURL: http://invalid.azurecr.io.azure.com + azureFunctionkey: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/+2Ytxc1hDq1m/ + ` ) -func TestAzureFunctionKey_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AZURE_FUNCTION_KEY") - inactiveSecret := testSecrets.MustGetField("AZURE_FUNCTION_KEY_INACTIVE") - url := testSecrets.MustGetField("AZURE_FUNCTION_URL") +func TestAzureFunctionKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureFunctionKey, - Verified: true, - RawV2: []byte(secret + url), - }, - }, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation - verify: true, + name: "valid pattern", + input: validPattern, + want: []string{ + "azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0Ehttps://z1dUSi5T.azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0E", + "B8sm0KyfL1y8vPH3IDTdefevHBCGK33https://z1dUSi5T.azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0E", }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureFunctionKey, - Verified: false, - RawV2: []byte(inactiveSecret + url), - }, - }, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - wantVerificationErr: false, }, { - name: "found, would be verified if not for timeout", - s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureFunctionKey, - Verified: false, - RawV2: []byte(secret + url), - }, - }, - wantErr: false, - wantVerificationErr: true, - }, - { - name: "found, verified but unexpected api surface", - s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureFunctionKey, - Verified: false, - RawV2: []byte(secret + url), - }, - }, - wantErr: false, - wantVerificationErr: true, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AzureFunctionKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - if len(got[i].RawV2) == 0 { - t.Fatalf("no rawV2 secret present: \n %+v", got[i]) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("AzureFunctionKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_integration_test.go b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_integration_test.go new file mode 100644 index 000000000000..c59e16ceede9 --- /dev/null +++ b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_integration_test.go @@ -0,0 +1,170 @@ +//go:build detectors +// +build detectors + +package azuresearchadminkey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAzureSearchAdminKey_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY") + inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_INACTIVE") + service := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_SERVICE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureSearchAdminKey, + Verified: true, + RawV2: []byte(secret + service), + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within but not valid", inactiveSecret, service)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureSearchAdminKey, + Verified: false, + RawV2: []byte(inactiveSecret + service), + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureSearchAdminKey, + Verified: false, + RawV2: []byte(secret + service), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureSearchAdminKey, + Verified: false, + RawV2: []byte(secret + service), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AzureSearchAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if len(got[i].RawV2) == 0 { + t.Fatalf("no rawV2 secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("AzureSearchAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go index c59e16ceede9..aeb5b2094b1f 100644 --- a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go +++ b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go @@ -1,169 +1,87 @@ -//go:build detectors -// +build detectors - package azuresearchadminkey import ( "context" - "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +var ( + validPattern = ` + azure: + azureKey: wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJyma + azureService: TestingService01 + ` + invalidPattern = ` + azure: + Key: wRRPyhjv8m6JGRujUUr-PK#a8d3rJ0mrGAxhmqf3A68OgZmlWUJyma + Service: TS01 + ` ) -func TestAzureSearchAdminKey_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY") - inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_INACTIVE") - service := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_SERVICE") +func TestAzureSearchAdminKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureSearchAdminKey, - Verified: true, - RawV2: []byte(secret + service), - }, - }, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within but not valid", inactiveSecret, service)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureSearchAdminKey, - Verified: false, - RawV2: []byte(inactiveSecret + service), - }, - }, - wantErr: false, - wantVerificationErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJymaTestingService01", "wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJymaazureKey"}, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "found, would be verified if not for timeout", - s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureSearchAdminKey, - Verified: false, - RawV2: []byte(secret + service), - }, - }, - wantErr: false, - wantVerificationErr: true, - }, - { - name: "found, verified but unexpected api surface", - s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureSearchAdminKey, - Verified: false, - RawV2: []byte(secret + service), - }, - }, - wantErr: false, - wantVerificationErr: true, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AzureSearchAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - if len(got[i].RawV2) == 0 { - t.Fatalf("no rawV2 secret present: \n %+v", got[i]) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("AzureSearchAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey_integration_test.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_integration_test.go new file mode 100644 index 000000000000..1967a9b971aa --- /dev/null +++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_integration_test.go @@ -0,0 +1,170 @@ +//go:build detectors +// +build detectors + +package azuresearchquerykey + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAzureSearchQueryKey_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY") + inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_INACTIVE") + url := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_URL") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureSearchQueryKey, + Verified: true, + RawV2: []byte(secret + url), + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureSearchQueryKey, + Verified: false, + RawV2: []byte(inactiveSecret + url), + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureSearchQueryKey, + Verified: false, + RawV2: []byte(secret + url), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureSearchQueryKey, + Verified: false, + RawV2: []byte(secret + url), + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("AzureSearchQueryKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if len(got[i].RawV2) == 0 { + t.Fatalf("no rawV2 secret present: \n %+v", got[i]) + } + if (got[i].VerificationError() != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("AzureSearchQueryKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go index 1967a9b971aa..3af2254ec5de 100644 --- a/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go +++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go @@ -1,169 +1,88 @@ -//go:build detectors -// +build detectors - package azuresearchquerykey import ( "context" - "fmt" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +var ( + validPattern = ` + azure: + azure_url: https://tzyexx2ktdfhha8w1cktqzbrgv37ywtu.search.windows.net/indexes/n81wg81jogjfq93cyxfi67vy2g7vwlcqfgi + azure_key: OKalbM5EBt5hloqU46phTUCZqvNAlZ4S2Jd2gFUCOQ3HG0vQ2uEp - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + ` + invalidPattern = ` + azure: + url: http://invalid.azurecr.io.azure.com + azure_key: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/+2Ytxc1hDq1m/ + ` ) -func TestAzureSearchQueryKey_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY") - inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_INACTIVE") - url := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_URL") +func TestAzureSearchQueryKey_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool - wantVerificationErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureSearchQueryKey, - Verified: true, - RawV2: []byte(secret + url), - }, - }, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureSearchQueryKey, - Verified: false, - RawV2: []byte(inactiveSecret + url), - }, - }, - wantErr: false, - wantVerificationErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - wantVerificationErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"OKalbM5EBt5hloqU46phTUCZqvNAlZ4S2Jd2gFUCOQ3HG0vQ2uEphttps://tzyexx2ktdfhha8w1cktqzbrgv37ywtu.search.windows.net/indexes/n81wg81jogjfq93cyxfi67vy2g7vwlcqfgi"}, }, { - name: "found, would be verified if not for timeout", - s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureSearchQueryKey, - Verified: false, - RawV2: []byte(secret + url), - }, - }, - wantErr: false, - wantVerificationErr: true, - }, - { - name: "found, verified but unexpected api surface", - s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureSearchQueryKey, - Verified: false, - RawV2: []byte(secret + url), - }, - }, - wantErr: false, - wantVerificationErr: true, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("AzureSearchQueryKey.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - if len(got[i].RawV2) == 0 { - t.Fatalf("no rawV2 secret present: \n %+v", got[i]) + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if (got[i].VerificationError() != nil) != tt.wantVerificationErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("AzureSearchQueryKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) } diff --git a/pkg/detectors/azurestorage/azurestorage_integration_test.go b/pkg/detectors/azurestorage/azurestorage_integration_test.go new file mode 100644 index 000000000000..1e45f122ead9 --- /dev/null +++ b/pkg/detectors/azurestorage/azurestorage_integration_test.go @@ -0,0 +1,205 @@ +//go:build detectors +// +build detectors + +package azurestorage + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + regexp "github.com/wasilibs/go-re2" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestAzurestorage_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("AZURE_STORAGE") + inactiveSecret := testSecrets.MustGetField("AZURE_STORAGE_INACTIVE") + + accountNamePat := regexp.MustCompile(`AccountName=(?P[^;]+);AccountKey`) + accountName := accountNamePat.FindStringSubmatch(secret)[1] + validKeyInvalidAccountName := strings.Replace(secret, accountName, "invalid", 1) + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureStorage, + Verified: true, + ExtraData: map[string]string{ + "account_name": "teststoragebytruffle", + }, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureStorage, + Verified: false, + ExtraData: map[string]string{ + "account_name": "teststoragebytruffle", + }, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_AzureStorage, + Verified: false, + ExtraData: map[string]string{ + "account_name": "teststoragebytruffle", + }, + } + r.SetVerificationError(fmt.Errorf("context deadline exceeded"), secret) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)), + verify: true, + }, + want: func() []detectors.Result { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_AzureStorage, + Verified: false, + ExtraData: map[string]string{ + "account_name": "teststoragebytruffle", + }, + } + r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 404"), secret) + return []detectors.Result{r} + }(), + wantErr: false, + }, + { + name: "found secret with invalid account name", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", validKeyInvalidAccountName)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AzureStorage, + Verified: false, + ExtraData: map[string]string{ + "account_name": "invalid", + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Azuretorage.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + gotErr := "" + if got[i].VerificationError() != nil { + gotErr = got[i].VerificationError().Error() + } + wantErr := "" + if tt.want[i].VerificationError() != nil { + wantErr = tt.want[i].VerificationError().Error() + } + if gotErr != wantErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Azurestorage.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/azurestorage/azurestorage_test.go b/pkg/detectors/azurestorage/azurestorage_test.go index 343f65390ab0..676900289051 100644 --- a/pkg/detectors/azurestorage/azurestorage_test.go +++ b/pkg/detectors/azurestorage/azurestorage_test.go @@ -1,203 +1,85 @@ -//go:build detectors -// +build detectors - package azurestorage import ( "context" - "fmt" - regexp "github.com/wasilibs/go-re2" - "strings" "testing" - "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestAzurestorage_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("AZURE_STORAGE") - inactiveSecret := testSecrets.MustGetField("AZURE_STORAGE_INACTIVE") +var ( + validPattern = ` + azure storage: + DefaultEndpointsProtocol=https;AccountName=myStorageAccount;AccountKey=myAccountKey12345==;EndpointSuffix=core.windows.net + ` + invalidPattern = ` + azure storage: + DefaultEndpointsProtocol=https;AccountName=;AccountKey=myAccountKey12345==;EndpointSuffix=core.ubuntu.net + ` +) - accountNamePat := regexp.MustCompile(`AccountName=(?P[^;]+);AccountKey`) - accountName := accountNamePat.FindStringSubmatch(secret)[1] - validKeyInvalidAccountName := strings.Replace(secret, accountName, "invalid", 1) +func TestAzureStorage_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureStorage, - Verified: true, - ExtraData: map[string]string{ - "account_name": "teststoragebytruffle", - }, - }, - }, - wantErr: false, - }, - { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureStorage, - Verified: false, - ExtraData: map[string]string{ - "account_name": "teststoragebytruffle", - }, - }, - }, - wantErr: false, - }, - { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, - }, - { - name: "found, would be verified if not for timeout", - s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)), - verify: true, - }, - want: func() []detectors.Result { - r := detectors.Result{ - DetectorType: detectorspb.DetectorType_AzureStorage, - Verified: false, - ExtraData: map[string]string{ - "account_name": "teststoragebytruffle", - }, - } - r.SetVerificationError(fmt.Errorf("context deadline exceeded"), secret) - return []detectors.Result{r} - }(), - wantErr: false, - }, - { - name: "found, verified but unexpected api surface", - s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)), - verify: true, - }, - want: func() []detectors.Result { - r := detectors.Result{ - DetectorType: detectorspb.DetectorType_AzureStorage, - Verified: false, - ExtraData: map[string]string{ - "account_name": "teststoragebytruffle", - }, - } - r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 404"), secret) - return []detectors.Result{r} - }(), - wantErr: false, + name: "valid pattern", + input: validPattern, + want: []string{"myAccountKey12345=="}, }, { - name: "found secret with invalid account name", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", validKeyInvalidAccountName)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_AzureStorage, - Verified: false, - ExtraData: map[string]string{ - "account_name": "invalid", - }, - }, - }, - wantErr: false, + name: "invalid pattern", + input: invalidPattern, + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("Azuretorage.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - gotErr := "" - if got[i].VerificationError() != nil { - gotErr = got[i].VerificationError().Error() - } - wantErr := "" - if tt.want[i].VerificationError() != nil { - wantErr = tt.want[i].VerificationError().Error() + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) } - if gotErr != wantErr { - t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError()) + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } - ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") - if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { - t.Errorf("Azurestorage.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) - } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) } }) }