From 610cd360ec86ac222581ba30f1ab8026ede83302 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Tue, 8 Oct 2024 17:20:59 +0500 Subject: [PATCH 1/5] Enhanced the easyinsight detector --- pkg/detectors/easyinsight/easyinsight.go | 50 ++++-- .../easyinsight_integration_test.go | 121 +++++++++++++ pkg/detectors/easyinsight/easyinsight_test.go | 163 ++++++++---------- 3 files changed, 227 insertions(+), 107 deletions(-) create mode 100644 pkg/detectors/easyinsight/easyinsight_integration_test.go diff --git a/pkg/detectors/easyinsight/easyinsight.go b/pkg/detectors/easyinsight/easyinsight.go index 60ad7f5640f4..3bc1662a3c41 100644 --- a/pkg/detectors/easyinsight/easyinsight.go +++ b/pkg/detectors/easyinsight/easyinsight.go @@ -4,16 +4,18 @@ import ( "context" b64 "encoding/base64" "fmt" - regexp "github.com/wasilibs/go-re2" + "io" "net/http" "strings" + regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ +type Scanner struct { detectors.DefaultMultiPartCredentialProvider } @@ -24,8 +26,8 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"easyinsight", "easy-insight"}) + `\b([0-9Aa-zA-Z]{20})\b`) - idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"easyinsight", "easy-insight"}) + `\b([a-zA-Z0-9]{20})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"easyinsight", "easy-insight", "key"}) + `\b([0-9a-zA-Z]{20})\b`) + idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"easyinsight", "easy-insight", "id"}) + `\b([a-zA-Z0-9]{20})\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -38,20 +40,22 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) + keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1) - idmatches := idPat.FindAllStringSubmatch(dataStr, -1) + idMatches := idPat.FindAllStringSubmatch(dataStr, -1) - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - for _, idmatch := range idmatches { - if len(idmatch) != 2 { + for _, keyMatch := range keyMatches { + resMatch := strings.TrimSpace(keyMatch[1]) + + for _, idMatch := range idMatches { + resIdMatch := strings.TrimSpace(idMatch[1]) + /* + as key and id regex are same, the strings captured by both regex will be same. + avoid processing when key is same as id. This will allow detector to process only different combinations + */ + if resMatch == resIdMatch { continue } - resIdMatch := strings.TrimSpace(idmatch[1]) s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_EasyInsight, @@ -60,18 +64,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - data := fmt.Sprintf("%s:%s", resIdMatch, resMatch) - sEnc := b64.StdEncoding.EncodeToString([]byte(data)) + auth := fmt.Sprintf("%s:%s", resIdMatch, resMatch) + sEnc := b64.StdEncoding.EncodeToString([]byte(auth)) + req, err := http.NewRequestWithContext(ctx, "GET", "https://www.easy-insight.com/app/api/users.json", nil) if err != nil { continue } + + // add required headers to the request req.Header.Add("Content-Type", "application/json") req.Header.Add("Accept", "application/json") req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) + res, err := client.Do(req) if err == nil { - defer res.Body.Close() + // discard the body content and close it at the end of each iteration. + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true } @@ -79,8 +90,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } results = append(results, s1) + // if key id combination is verified, skip other idMatches for that key + if s1.Verified { + break + } } - } return results, nil diff --git a/pkg/detectors/easyinsight/easyinsight_integration_test.go b/pkg/detectors/easyinsight/easyinsight_integration_test.go new file mode 100644 index 000000000000..098b8a3a3954 --- /dev/null +++ b/pkg/detectors/easyinsight/easyinsight_integration_test.go @@ -0,0 +1,121 @@ +//go:build detectors +// +build detectors + +package easyinsight + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestEasyInsight_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("EASYINSIGHT") + inactiveSecret := testSecrets.MustGetField("EASYINSIGHT_INACTIVE") + id := testSecrets.MustGetField("EASYINSIGHT_ID") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a easyinsight secret %s within easyid %s", secret, id)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_EasyInsight, + Verified: true, + }, + }, + wantErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a easyinsight secret %s within easyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_EasyInsight, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Scanner{} + got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("EasyInsight.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + got[i].Raw = nil + } + if diff := pretty.Compare(got, tt.want); diff != "" { + t.Errorf("EasyInsight.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + b.ResetTimer() + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/easyinsight/easyinsight_test.go b/pkg/detectors/easyinsight/easyinsight_test.go index 098b8a3a3954..00ef59366d40 100644 --- a/pkg/detectors/easyinsight/easyinsight_test.go +++ b/pkg/detectors/easyinsight/easyinsight_test.go @@ -1,121 +1,106 @@ -//go:build detectors -// +build detectors - package easyinsight import ( "context" "fmt" "testing" - "time" - "github.com/kylelemons/godebug/pretty" + "github.com/google/go-cmp/cmp" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" ) -func TestEasyInsight_FromChunk(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1") - if err != nil { - t.Fatalf("could not get test secrets from GCP: %s", err) - } - secret := testSecrets.MustGetField("EASYINSIGHT") - inactiveSecret := testSecrets.MustGetField("EASYINSIGHT_INACTIVE") - id := testSecrets.MustGetField("EASYINSIGHT_ID") +var ( + validKeyPattern = "987ahjjdasgUcaaraAdd" + validIDPattern = "poiuy76RaEf90ertgh0K" + // this should result in 4 combinations + complexPattern = `easyinsight credentials + these credentials are for testing a pattern + key: A876AcaraTsaAKcae09a + id: chECk12345ChecK12345 + ------------------------- + second credentials: + key: B874CDaraTsaAKVBe08A + id: CHECK12345ChecK09876` + invalidPattern = "poiuy76=a_$90ertgh0K" +) + +func TestEasyInsight_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - type args struct { - ctx context.Context - data []byte - verify bool - } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + input string + want []string }{ { - name: "found, verified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a easyinsight secret %s within easyid %s", secret, id)), - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_EasyInsight, - Verified: true, - }, - }, - wantErr: false, + name: "valid pattern", + input: fmt.Sprintf("easyinsight key = '%s' easy-insight id = '%s", validKeyPattern, validIDPattern), + want: []string{validKeyPattern + validIDPattern, validIDPattern + validKeyPattern}, }, { - name: "found, unverified", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a easyinsight secret %s within easyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation - verify: true, - }, - want: []detectors.Result{ - { - DetectorType: detectorspb.DetectorType_EasyInsight, - Verified: false, - }, + name: "valid pattern - complex", + input: fmt.Sprintf("easyinsight token = '%s'", complexPattern), + want: []string{ + "A876AcaraTsaAKcae09achECk12345ChecK12345", + "A876AcaraTsaAKcae09aCHECK12345ChecK09876", + "B874CDaraTsaAKVBe08ACHECK12345ChecK09876", + "B874CDaraTsaAKVBe08AchECk12345ChecK12345", }, - wantErr: false, }, { - name: "not found", - s: Scanner{}, - args: args{ - ctx: context.Background(), - data: []byte("You cannot find the secret within"), - verify: true, - }, - want: nil, - wantErr: false, + name: "valid pattern - out of prefix range", + input: fmt.Sprintf("easyinsight key and id keyword is not close to the real token = '%s|%s'", validKeyPattern, validIDPattern), + want: nil, + }, + { + name: "invalid pattern", + input: fmt.Sprintf("easyinsight = '%s|%s'", invalidPattern, invalidPattern), + want: nil, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) - if (err != nil) != tt.wantErr { - t.Errorf("EasyInsight.FromData() error = %v, wantErr %v", err, tt.wantErr) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) return } - for i := range got { - if len(got[i].Raw) == 0 { - t.Fatalf("no raw secret present: \n %+v", got[i]) - } - got[i].Raw = nil + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("EasyInsight.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return } - }) - } -} -func BenchmarkFromData(benchmark *testing.B) { - ctx := context.Background() - s := Scanner{} - for name, data := range detectors.MustGetBenchmarkData() { - benchmark.Run(name, func(b *testing.B) { - b.ResetTimer() - for n := 0; n < b.N; n++ { - _, err := s.FromData(ctx, false, data) - if err != nil { - b.Fatal(err) + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} } } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } }) } } From 4652e0723623cb37870711de96a7fed16150e4c5 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Tue, 8 Oct 2024 19:41:44 +0500 Subject: [PATCH 2/5] restructured verification code and resolved comments --- pkg/detectors/easyinsight/easyinsight.go | 80 ++++++++++++++---------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/pkg/detectors/easyinsight/easyinsight.go b/pkg/detectors/easyinsight/easyinsight.go index 3bc1662a3c41..b60c965a3b21 100644 --- a/pkg/detectors/easyinsight/easyinsight.go +++ b/pkg/detectors/easyinsight/easyinsight.go @@ -4,11 +4,9 @@ import ( "context" b64 "encoding/base64" "fmt" + regexp "github.com/wasilibs/go-re2" "io" "net/http" - "strings" - - regexp "github.com/wasilibs/go-re2" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -40,52 +38,41 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1) + var keyMatches, idMatches = make(map[string]struct{}), make(map[string]struct{}) - idMatches := idPat.FindAllStringSubmatch(dataStr, -1) + // get unique key and id matches + for _, matches := range keyPat.FindAllStringSubmatch(dataStr, -1) { + keyMatches[matches[1]] = struct{}{} + } - for _, keyMatch := range keyMatches { - resMatch := strings.TrimSpace(keyMatch[1]) + for _, matches := range idPat.FindAllStringSubmatch(dataStr, -1) { + idMatches[matches[1]] = struct{}{} + } - for _, idMatch := range idMatches { - resIdMatch := strings.TrimSpace(idMatch[1]) + for keyMatch := range keyMatches { + for idMatch := range idMatches { /* as key and id regex are same, the strings captured by both regex will be same. avoid processing when key is same as id. This will allow detector to process only different combinations */ - if resMatch == resIdMatch { + if keyMatch == idMatch { continue } s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_EasyInsight, - Raw: []byte(resMatch), - RawV2: []byte(resMatch + resIdMatch), + Raw: []byte(keyMatch), + RawV2: []byte(keyMatch + idMatch), } if verify { - auth := fmt.Sprintf("%s:%s", resIdMatch, resMatch) + auth := fmt.Sprintf("%s:%s", idMatch, keyMatch) sEnc := b64.StdEncoding.EncodeToString([]byte(auth)) - req, err := http.NewRequestWithContext(ctx, "GET", "https://www.easy-insight.com/app/api/users.json", nil) - if err != nil { - continue - } - - // add required headers to the request - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Accept", "application/json") - req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) - - res, err := client.Do(req) - if err == nil { - // discard the body content and close it at the end of each iteration. - _, _ = io.Copy(io.Discard, res.Body) - _ = res.Body.Close() - - if res.StatusCode >= 200 && res.StatusCode < 300 { - s1.Verified = true - } + verified, verificationErr := verifyEasyInsight(ctx, sEnc) + s1.Verified = verified + if verificationErr != nil { + s1.SetVerificationError(verificationErr) } } @@ -107,3 +94,32 @@ func (s Scanner) Type() detectorspb.DetectorType { func (s Scanner) Description() string { return "EasyInsight is a business intelligence tool that provides data visualization and reporting. EasyInsight API keys can be used to access and manage data within the platform." } + +func verifyEasyInsight(ctx context.Context, sEnc string) (bool, error) { + // docs: https://www.easy-insight.com/api/users.html + req, err := http.NewRequestWithContext(ctx, "GET", "https://www.easy-insight.com/app/api/users.json", nil) + if err != nil { + return false, err + } + + // add required headers to the request + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) + + res, reqErr := client.Do(req) + if reqErr != nil { + return false, reqErr + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + if res.StatusCode >= 200 && res.StatusCode < 300 { + return true, nil + } + + // if status code is not handled, return unexpected code error + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) +} From f7e95a900dd1b130a13d607761bf6e9b559edde4 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Tue, 8 Oct 2024 19:48:15 +0500 Subject: [PATCH 3/5] resolved comments --- pkg/detectors/easyinsight/easyinsight.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/detectors/easyinsight/easyinsight.go b/pkg/detectors/easyinsight/easyinsight.go index b60c965a3b21..578245ae2d65 100644 --- a/pkg/detectors/easyinsight/easyinsight.go +++ b/pkg/detectors/easyinsight/easyinsight.go @@ -51,10 +51,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result for keyMatch := range keyMatches { for idMatch := range idMatches { - /* - as key and id regex are same, the strings captured by both regex will be same. - avoid processing when key is same as id. This will allow detector to process only different combinations - */ + //as key and id regex are same, the strings captured by both regex will be same. + //avoid processing when key is same as id. This will allow detector to process only different combinations if keyMatch == idMatch { continue } From 119039f4bee386b74ec172f042e9b9c796ea3f57 Mon Sep 17 00:00:00 2001 From: kashif khan Date: Tue, 8 Oct 2024 20:37:34 +0500 Subject: [PATCH 4/5] added basic auth --- pkg/detectors/easyinsight/easyinsight.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pkg/detectors/easyinsight/easyinsight.go b/pkg/detectors/easyinsight/easyinsight.go index 578245ae2d65..f7eead52c0fd 100644 --- a/pkg/detectors/easyinsight/easyinsight.go +++ b/pkg/detectors/easyinsight/easyinsight.go @@ -2,7 +2,6 @@ package easyinsight import ( "context" - b64 "encoding/base64" "fmt" regexp "github.com/wasilibs/go-re2" "io" @@ -64,10 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - auth := fmt.Sprintf("%s:%s", idMatch, keyMatch) - sEnc := b64.StdEncoding.EncodeToString([]byte(auth)) - - verified, verificationErr := verifyEasyInsight(ctx, sEnc) + verified, verificationErr := verifyEasyInsight(ctx, idMatch, keyMatch) s1.Verified = verified if verificationErr != nil { s1.SetVerificationError(verificationErr) @@ -93,7 +89,7 @@ func (s Scanner) Description() string { return "EasyInsight is a business intelligence tool that provides data visualization and reporting. EasyInsight API keys can be used to access and manage data within the platform." } -func verifyEasyInsight(ctx context.Context, sEnc string) (bool, error) { +func verifyEasyInsight(ctx context.Context, id, key string) (bool, error) { // docs: https://www.easy-insight.com/api/users.html req, err := http.NewRequestWithContext(ctx, "GET", "https://www.easy-insight.com/app/api/users.json", nil) if err != nil { @@ -103,7 +99,8 @@ func verifyEasyInsight(ctx context.Context, sEnc string) (bool, error) { // add required headers to the request req.Header.Add("Content-Type", "application/json") req.Header.Add("Accept", "application/json") - req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc)) + // set basic auth for the request + req.SetBasicAuth(id, key) res, reqErr := client.Do(req) if reqErr != nil { From 3d1acefb24642422e4db26cf3a42375c783e4aaa Mon Sep 17 00:00:00 2001 From: kashif khan Date: Wed, 9 Oct 2024 10:55:06 +0500 Subject: [PATCH 5/5] updated statuscode logic --- pkg/detectors/easyinsight/easyinsight.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/detectors/easyinsight/easyinsight.go b/pkg/detectors/easyinsight/easyinsight.go index f7eead52c0fd..79b40d118771 100644 --- a/pkg/detectors/easyinsight/easyinsight.go +++ b/pkg/detectors/easyinsight/easyinsight.go @@ -111,10 +111,15 @@ func verifyEasyInsight(ctx context.Context, id, key string) (bool, error) { _ = res.Body.Close() }() - if res.StatusCode >= 200 && res.StatusCode < 300 { + switch res.StatusCode { + // id, key verified + case http.StatusOK: return true, nil + // id, key unverified + case http.StatusUnauthorized: + return false, nil + // something invalid + default: + return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) } - - // if status code is not handled, return unexpected code error - return false, fmt.Errorf("unexpected status code: %d", res.StatusCode) }