From 9f5aff4759a073038c4ca709637a92f3cd1e423d Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 11 Sep 2024 21:35:53 +0500 Subject: [PATCH 1/2] [Fixes] - handling of `.privatelink` in account identifier - added unit test for pattern detection. - fixes hard coded account and username in test. --- go.sum | 8 -- pkg/detectors/snowflake/snowflake.go | 19 ++-- pkg/detectors/snowflake/snowflake_test.go | 131 +++++++++++++++++++--- 3 files changed, 124 insertions(+), 34 deletions(-) diff --git a/go.sum b/go.sum index 77823ff15757..087469aa3175 100644 --- a/go.sum +++ b/go.sum @@ -261,8 +261,6 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/felixge/fgprof v0.9.4 h1:ocDNwMFlnA0NU0zSB3I52xkO4sFXk80VK9lXjLClu88= -github.com/felixge/fgprof v0.9.4/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -851,8 +849,6 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -876,8 +872,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1041,8 +1035,6 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/detectors/snowflake/snowflake.go b/pkg/detectors/snowflake/snowflake.go index a53c72635901..93289e4189a2 100644 --- a/pkg/detectors/snowflake/snowflake.go +++ b/pkg/detectors/snowflake/snowflake.go @@ -24,7 +24,7 @@ type Scanner struct { var _ detectors.Detector = (*Scanner)(nil) var ( - accountIdentifierPat = regexp.MustCompile(detectors.PrefixRegex([]string{"account"}) + `\b([a-zA-Z]{7}-[0-9a-zA-Z]{7})\b`) + accountIdentifierPat = regexp.MustCompile(detectors.PrefixRegex([]string{"account"}) + `\b([a-zA-Z]{7}-[0-9a-zA-Z-_]{1,255}(.privatelink)?)\b`) usernameExclusionPat = `!@#$%^&*{}:<>,.;?()/\+=\s\n` ) @@ -71,7 +71,10 @@ func meetsSnowflakePasswordRequirements(password string) (string, bool) { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - accountMatches := accountIdentifierPat.FindAllStringSubmatch(dataStr, -1) + uniqueAccountMatches := make(map[string]struct{}) + for _, match := range accountIdentifierPat.FindAllStringSubmatch(dataStr, -1) { + uniqueAccountMatches[strings.TrimSpace(match[1])] = struct{}{} + } usernameRegexState := common.UsernameRegexCheck(usernameExclusionPat) usernameMatches := usernameRegexState.Matches(data) @@ -79,14 +82,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result passwordRegexState := common.PasswordRegexCheck(" ") // No explicit character exclusions by Snowflake for passwords passwordMatches := passwordRegexState.Matches(data) - for _, accountMatch := range accountMatches { - if len(accountMatch) != 2 { - continue - } - resAccountMatch := strings.TrimSpace(accountMatch[1]) - + for accountMatch := range uniqueAccountMatches { for _, resUsernameMatch := range usernameMatches { - for _, resPasswordMatch := range passwordMatches { _, metPasswordRequirements := meetsSnowflakePasswordRequirements(resPasswordMatch) @@ -95,13 +92,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } // Override default timeout of 60 seconds to 3 seconds to prevent long scan times/improve performance. - uri := fmt.Sprintf("%s:%s@%s/%s?loginTimeout=%d", resUsernameMatch, resPasswordMatch, resAccountMatch, database, timeout) + uri := fmt.Sprintf("%s:%s@%s/%s?loginTimeout=%d", resUsernameMatch, resPasswordMatch, accountMatch, database, timeout) s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Snowflake, Raw: []byte(resPasswordMatch), ExtraData: map[string]string{ - "account": resAccountMatch, + "account": accountMatch, "username": resUsernameMatch, }, } diff --git a/pkg/detectors/snowflake/snowflake_test.go b/pkg/detectors/snowflake/snowflake_test.go index 322d679b1867..f06a872fde00 100644 --- a/pkg/detectors/snowflake/snowflake_test.go +++ b/pkg/detectors/snowflake/snowflake_test.go @@ -9,15 +9,102 @@ import ( "testing" "time" + "github.com/brianvoe/gofakeit/v7" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) +func TestSnowflake_Pattern(t *testing.T) { + username := gofakeit.Username() + password := gofakeit.Password(true, true, true, false, false, 10) + + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + tests := []struct { + name string + input string + want [][]string + }{ + { + name: "Snowflake Credentials", + input: fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", "tuacoip-zt74995", username, password), + want: [][]string{ + []string{"tuacoip-zt74995", username, password}, + }, + }, + { + name: "Private Snowflake Credentials", + input: fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", "tuacoip-zt74995.privatelink", username, password), + want: [][]string{ + []string{"tuacoip-zt74995.privatelink", username, password}, + }, + }, + + { + name: "Snowflake Credentials - Single Character account", + input: fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", "tuacoip-z", username, password), + want: [][]string{ + []string{"tuacoip-z", username, password}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + detectorMatches := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(detectorMatches) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + resultsArray := make([][]string, len(results)) + for i, r := range results { + resultsArray[i] = []string{r.ExtraData["account"], r.ExtraData["username"], string(r.Raw)} + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + actual[r.ExtraData["account"]] = struct{}{} + actual[r.ExtraData["username"]] = struct{}{} + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + for _, value := range v { + expected[value] = struct{}{} + } + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} + func TestSnowflake_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() @@ -26,8 +113,10 @@ func TestSnowflake_FromChunk(t *testing.T) { t.Fatalf("could not get test secrets from GCP: %s", err) } - secret := testSecrets.MustGetField("SNOWFLAKE_PASS") - inactiveSecret := testSecrets.MustGetField("SNOWFLAKE_PASS_INACTIVE") + accountIdentifier := testSecrets.MustGetField("SNOWFLAKE_ACCOUNT") + username := testSecrets.MustGetField("SNOWFLAKE_USERNAME") + password := testSecrets.MustGetField("SNOWFLAKE_PASS") + inactivePassword := testSecrets.MustGetField("SNOWFLAKE_PASS_INACTIVE") // Create a context with a past deadline to simulate DeadlineExceeded error pastTime := time.Now().Add(-time.Second) // Set the deadline in the past @@ -52,7 +141,7 @@ func TestSnowflake_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("snowflake: \n account=tuacoip-zt74995 \n username=zubairkhan14 \n password=%s \n database=SNOWFLAKE", secret)), + data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, password)), verify: true, }, want: []detectors.Result{ @@ -60,9 +149,8 @@ func TestSnowflake_FromChunk(t *testing.T) { DetectorType: detectorspb.DetectorType_Snowflake, Verified: true, ExtraData: map[string]string{ - "account": "tuacoip-zt74995", - "databases": "SNOWFLAKE, SNOWFLAKE_SAMPLE_DATA", - "username": "zubairkhan14", + "account": accountIdentifier, + "username": username, }, }, }, @@ -74,7 +162,7 @@ func TestSnowflake_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("snowflake: \n account=tuacoip-zt74995 \n username=zubairkhan14 \n password=%s \n database=SNOWFLAKE", inactiveSecret)), + data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, inactivePassword)), verify: true, }, want: []detectors.Result{ @@ -82,8 +170,8 @@ func TestSnowflake_FromChunk(t *testing.T) { DetectorType: detectorspb.DetectorType_Snowflake, Verified: false, ExtraData: map[string]string{ - "account": "tuacoip-zt74995", - "username": "zubairkhan14", + "account": accountIdentifier, + "username": username, }, }, }, @@ -107,15 +195,15 @@ func TestSnowflake_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: errorCtx, - data: []byte(fmt.Sprintf("snowflake: \n account=tuacoip-zt74995 \n username=zubairkhan14 \n password=%s \n database=SNOWFLAKE", secret)), + data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, password)), verify: true, }, want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_Snowflake, ExtraData: map[string]string{ - "account": "tuacoip-zt74995", - "username": "zubairkhan14", + "account": accountIdentifier, + "username": username, }, }, }, @@ -130,6 +218,7 @@ func TestSnowflake_FromChunk(t *testing.T) { t.Errorf("Snowflake.FromData() error = %v, wantErr %v", err, tt.wantErr) return } + keysToCopy := []string{"account", "username"} for i := range got { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) @@ -137,6 +226,8 @@ func TestSnowflake_FromChunk(t *testing.T) { if (got[i].VerificationError() != nil) != tt.wantVerificationErr { t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError()) } + + got[i].ExtraData = newMap(got[i].ExtraData, keysToCopy) } ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError") if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { @@ -146,6 +237,16 @@ func TestSnowflake_FromChunk(t *testing.T) { } } +func newMap(extraMap map[string]string, keysToCopy []string) map[string]string { + newExtraDataMap := make(map[string]string) + for _, key := range keysToCopy { + if value, ok := extraMap[key]; ok { + newExtraDataMap[key] = value + } + } + return newExtraDataMap +} + func BenchmarkFromData(benchmark *testing.B) { ctx := context.Background() s := Scanner{} From 04f796e8ee8a069aedc7a43cc2b8b336557d25ff Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 11 Sep 2024 21:44:28 +0500 Subject: [PATCH 2/2] variable name fixes --- pkg/detectors/snowflake/snowflake.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/detectors/snowflake/snowflake.go b/pkg/detectors/snowflake/snowflake.go index 93289e4189a2..00b9e5e4cb7f 100644 --- a/pkg/detectors/snowflake/snowflake.go +++ b/pkg/detectors/snowflake/snowflake.go @@ -82,7 +82,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result passwordRegexState := common.PasswordRegexCheck(" ") // No explicit character exclusions by Snowflake for passwords passwordMatches := passwordRegexState.Matches(data) - for accountMatch := range uniqueAccountMatches { + for resAccountMatch := range uniqueAccountMatches { for _, resUsernameMatch := range usernameMatches { for _, resPasswordMatch := range passwordMatches { _, metPasswordRequirements := meetsSnowflakePasswordRequirements(resPasswordMatch) @@ -92,13 +92,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } // Override default timeout of 60 seconds to 3 seconds to prevent long scan times/improve performance. - uri := fmt.Sprintf("%s:%s@%s/%s?loginTimeout=%d", resUsernameMatch, resPasswordMatch, accountMatch, database, timeout) + uri := fmt.Sprintf("%s:%s@%s/%s?loginTimeout=%d", resUsernameMatch, resPasswordMatch, resAccountMatch, database, timeout) s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_Snowflake, Raw: []byte(resPasswordMatch), ExtraData: map[string]string{ - "account": accountMatch, + "account": resAccountMatch, "username": resUsernameMatch, }, }