fix(secret): skip regular strings contain secret patterns (#7182)

aquasecurity · Jul 25, 2024 · 174b1e3 · 174b1e3
1 parent bff317c
commit 174b1e3
Show file tree

Hide file tree

Showing 5 changed files with 173 additions and 72 deletions.
diff --git a/pkg/fanal/secret/builtin-rules.go b/pkg/fanal/secret/builtin-rules.go
@@ -74,10 +74,10 @@ var (
 
 // Reusable regex patterns
 const (
-	quote       = `["']?`
-	connect     = `\s*(:|=>|=)?\s*`
-	startSecret = `(^|\s+)`
-	endSecret   = `[.,]?(\s+|$)`
+	quote     = `["']?`
+	connect   = `\s*(:|=>|=)?\s*`
+	endSecret = `[.,]?(\s+|$)`
+	startWord = "([^0-9a-zA-Z]|^)"
 
 	aws = `aws_?`
 )
@@ -103,7 +103,7 @@ var builtinRules = []Rule{
 		Category:        CategoryAWS,
 		Severity:        "CRITICAL",
 		Title:           "AWS Access Key ID",
-		Regex:           MustCompile(fmt.Sprintf(`%s(?P<secret>(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})%s%s`, quote, quote, endSecret)),
+		Regex:           MustCompileWithoutWordPrefix(fmt.Sprintf(`(?P<secret>(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})%s%s`, quote, endSecret)),
 		SecretGroupName: "secret",
 		Keywords:        []string{"AKIA", "AGPA", "AIDA", "AROA", "AIPA", "ANPA", "ANVA", "ASIA"},
 	},
@@ -112,41 +112,45 @@ var builtinRules = []Rule{
 		Category:        CategoryAWS,
 		Severity:        "CRITICAL",
 		Title:           "AWS Secret Access Key",
-		Regex:           MustCompile(fmt.Sprintf(`(?i)%s%s%s(sec(ret)?)?_?(access)?_?key%s%s%s(?P<secret>[A-Za-z0-9\/\+=]{40})%s%s`, startSecret, quote, aws, quote, connect, quote, quote, endSecret)),
+		Regex:           MustCompile(fmt.Sprintf(`(?i)%s%s(sec(ret)?)?_?(access)?_?key%s%s%s(?P<secret>[A-Za-z0-9\/\+=]{40})%s%s`, quote, aws, quote, connect, quote, quote, endSecret)),
 		SecretGroupName: "secret",
 		Keywords:        []string{"key"},
 	},
 	{
-		ID:       "github-pat",
-		Category: CategoryGitHub,
-		Title:    "GitHub Personal Access Token",
-		Severity: "CRITICAL",
-		Regex:    MustCompile(`ghp_[0-9a-zA-Z]{36}`),
-		Keywords: []string{"ghp_"},
+		ID:              "github-pat",
+		Category:        CategoryGitHub,
+		Title:           "GitHub Personal Access Token",
+		Severity:        "CRITICAL",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>ghp_[0-9a-zA-Z]{36}`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"ghp_"},
 	},
 	{
-		ID:       "github-oauth",
-		Category: CategoryGitHub,
-		Title:    "GitHub OAuth Access Token",
-		Severity: "CRITICAL",
-		Regex:    MustCompile(`gho_[0-9a-zA-Z]{36}`),
-		Keywords: []string{"gho_"},
+		ID:              "github-oauth",
+		Category:        CategoryGitHub,
+		Title:           "GitHub OAuth Access Token",
+		Severity:        "CRITICAL",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>gho_[0-9a-zA-Z]{36}`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"gho_"},
 	},
 	{
-		ID:       "github-app-token",
-		Category: CategoryGitHub,
-		Title:    "GitHub App Token",
-		Severity: "CRITICAL",
-		Regex:    MustCompile(`(ghu|ghs)_[0-9a-zA-Z]{36}`),
-		Keywords: []string{"ghu_", "ghs_"},
+		ID:              "github-app-token",
+		Category:        CategoryGitHub,
+		Title:           "GitHub App Token",
+		Severity:        "CRITICAL",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>(ghu|ghs)_[0-9a-zA-Z]{36}`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"ghu_", "ghs_"},
 	},
 	{
-		ID:       "github-refresh-token",
-		Category: CategoryGitHub,
-		Title:    "GitHub Refresh Token",
-		Severity: "CRITICAL",
-		Regex:    MustCompile(`ghr_[0-9a-zA-Z]{76}`),
-		Keywords: []string{"ghr_"},
+		ID:              "github-refresh-token",
+		Category:        CategoryGitHub,
+		Title:           "GitHub Refresh Token",
+		Severity:        "CRITICAL",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>ghr_[0-9a-zA-Z]{76}`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"ghr_"},
 	},
 	{
 		ID:       "github-fine-grained-pat",
@@ -157,21 +161,23 @@ var builtinRules = []Rule{
 		Keywords: []string{"github_pat_"},
 	},
 	{
-		ID:       "gitlab-pat",
-		Category: CategoryGitLab,
-		Title:    "GitLab Personal Access Token",
-		Severity: "CRITICAL",
-		Regex:    MustCompile(`glpat-[0-9a-zA-Z\-\_]{20}`),
-		Keywords: []string{"glpat-"},
+		ID:              "gitlab-pat",
+		Category:        CategoryGitLab,
+		Title:           "GitLab Personal Access Token",
+		Severity:        "CRITICAL",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>glpat-[0-9a-zA-Z\-\_]{20}`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"glpat-"},
 	},
 	{
 		// cf. https://huggingface.co/docs/hub/en/security-tokens
-		ID:       "hugging-face-access-token",
-		Category: CategoryHuggingFace,
-		Severity: "CRITICAL",
-		Title:    "Hugging Face Access Token",
-		Regex:    MustCompile(`hf_[A-Za-z0-9]{34,40}`),
-		Keywords: []string{"hf_"},
+		ID:              "hugging-face-access-token",
+		Category:        CategoryHuggingFace,
+		Severity:        "CRITICAL",
+		Title:           "Hugging Face Access Token",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>hf_[A-Za-z0-9]{34,40}`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"hf_"},
 	},
 	{
 		ID:              "private-key",
@@ -191,28 +197,31 @@ var builtinRules = []Rule{
 		Keywords: []string{"shpss_", "shpat_", "shpca_", "shppa_"},
 	},
 	{
-		ID:       "slack-access-token",
-		Category: CategorySlack,
-		Title:    "Slack token",
-		Severity: "HIGH",
-		Regex:    MustCompile(`xox[baprs]-([0-9a-zA-Z]{10,48})`),
-		Keywords: []string{"xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"},
+		ID:              "slack-access-token",
+		Category:        CategorySlack,
+		Title:           "Slack token",
+		Severity:        "HIGH",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>xox[baprs]-([0-9a-zA-Z]{10,48})`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"},
 	},
 	{
-		ID:       "stripe-publishable-token",
-		Category: CategoryStripe,
-		Title:    "Stripe Publishable Key",
-		Severity: "LOW",
-		Regex:    MustCompile(`(?i)pk_(test|live)_[0-9a-z]{10,32}`),
-		Keywords: []string{"pk_test_", "pk_live_"},
+		ID:              "stripe-publishable-token",
+		Category:        CategoryStripe,
+		Title:           "Stripe Publishable Key",
+		Severity:        "LOW",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>(?i)pk_(test|live)_[0-9a-z]{10,32}`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"pk_test_", "pk_live_"},
 	},
 	{
-		ID:       "stripe-secret-token",
-		Category: CategoryStripe,
-		Title:    "Stripe Secret Key",
-		Severity: "CRITICAL",
-		Regex:    MustCompile(`(?i)sk_(test|live)_[0-9a-z]{10,32}`),
-		Keywords: []string{"sk_test_", "sk_live_"},
+		ID:              "stripe-secret-token",
+		Category:        CategoryStripe,
+		Title:           "Stripe Secret Key",
+		Severity:        "CRITICAL",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>(?i)sk_(test|live)_[0-9a-z]{10,32}`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"sk_test_", "sk_live_"},
 	},
 	{
 		ID:       "pypi-upload-token",
@@ -506,20 +515,22 @@ var builtinRules = []Rule{
 		Keywords:        []string{"finicity"},
 	},
 	{
-		ID:       "flutterwave-public-key",
-		Category: CategoryFlutterwave,
-		Title:    "Flutterwave public/secret key",
-		Severity: "MEDIUM",
-		Regex:    MustCompile(`FLW(PUB|SEC)K_TEST-(?i)[a-h0-9]{32}-X`),
-		Keywords: []string{"FLWSECK_TEST-", "FLWPUBK_TEST-"},
+		ID:              "flutterwave-public-key",
+		Category:        CategoryFlutterwave,
+		Title:           "Flutterwave public/secret key",
+		Severity:        "MEDIUM",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>FLW(PUB|SEC)K_TEST-(?i)[a-h0-9]{32}-X`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"FLWSECK_TEST-", "FLWPUBK_TEST-"},
 	},
 	{
-		ID:       "flutterwave-enc-key",
-		Category: CategoryFlutterwave,
-		Title:    "Flutterwave encrypted key",
-		Severity: "MEDIUM",
-		Regex:    MustCompile(`FLWSECK_TEST[a-h0-9]{12}`),
-		Keywords: []string{"FLWSECK_TEST"},
+		ID:              "flutterwave-enc-key",
+		Category:        CategoryFlutterwave,
+		Title:           "Flutterwave encrypted key",
+		Severity:        "MEDIUM",
+		Regex:           MustCompileWithoutWordPrefix(`?P<secret>FLWSECK_TEST[a-h0-9]{12}`),
+		SecretGroupName: "secret",
+		Keywords:        []string{"FLWSECK_TEST"},
 	},
 	{
 		ID:       "frameio-api-token",

diff --git a/pkg/fanal/secret/scanner.go b/pkg/fanal/secret/scanner.go
@@ -3,6 +3,7 @@ package secret
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"os"
 	"regexp"
 	"slices"
@@ -62,6 +63,10 @@ type Regexp struct {
 	*regexp.Regexp
 }
 
+func MustCompileWithoutWordPrefix(str string) *Regexp {
+	return MustCompile(fmt.Sprintf("%s(%s)", startWord, str))
+}
+
 func MustCompile(str string) *Regexp {
 	return &Regexp{regexp.MustCompile(str)}
 }

diff --git a/pkg/fanal/secret/scanner_test.go b/pkg/fanal/secret/scanner_test.go
@@ -315,6 +315,59 @@ func TestSecretScanner(t *testing.T) {
 			},
 		},
 	}
+	wantFindingMyAwsAccessKey := types.SecretFinding{
+		RuleID:    "aws-secret-access-key",
+		Category:  secret.CategoryAWS,
+		Title:     "AWS Secret Access Key",
+		Severity:  "CRITICAL",
+		StartLine: 1,
+		EndLine:   1,
+		Match:     `MyAWS_secret_KEY="****************************************"`,
+		Code: types.Code{
+			Lines: []types.Line{
+				{
+					Number:      1,
+					Content:     "MyAWS_secret_KEY=\"****************************************\"",
+					Highlighted: "MyAWS_secret_KEY=\"****************************************\"",
+					IsCause:     true,
+					FirstCause:  true,
+					LastCause:   true,
+				},
+				{
+					Number:      2,
+					Content:     "our*********************************************************************************************",
+					Highlighted: "our*********************************************************************************************",
+				},
+			},
+		},
+	}
+
+	wantFindingMyGitHubPAT := types.SecretFinding{
+		RuleID:    "github-fine-grained-pat",
+		Category:  secret.CategoryGitHub,
+		Title:     "GitHub Fine-grained personal access tokens",
+		Severity:  "CRITICAL",
+		StartLine: 2,
+		EndLine:   2,
+		Match:     "our*********************************************************************************************",
+		Code: types.Code{
+			Lines: []types.Line{
+				{
+					Number:      1,
+					Content:     "MyAWS_secret_KEY=\"****************************************\"",
+					Highlighted: "MyAWS_secret_KEY=\"****************************************\"",
+				},
+				{
+					Number:      2,
+					Content:     "our*********************************************************************************************",
+					Highlighted: "our*********************************************************************************************",
+					IsCause:     true,
+					FirstCause:  true,
+					LastCause:   true,
+				},
+			},
+		},
+	}
 	wantFindingGHButDisableAWS := types.SecretFinding{
 		RuleID:    "github-pat",
 		Category:  secret.CategoryGitHub,
@@ -419,6 +472,7 @@ func TestSecretScanner(t *testing.T) {
 			},
 		},
 	}
+
 	wantFinding10 := types.SecretFinding{
 		RuleID:    "aws-secret-access-key",
 		Category:  secret.CategoryAWS,
@@ -979,6 +1033,24 @@ func TestSecretScanner(t *testing.T) {
 			inputFilePath: filepath.Join("testdata", "invalid-aws-secrets.txt"),
 			want:          types.Secret{},
 		},
+		{
+			name:          "secret inside another word",
+			configPath:    filepath.Join("testdata", "skip-test.yaml"),
+			inputFilePath: filepath.Join("testdata", "wrapped-secrets.txt"),
+			want:          types.Secret{},
+		},
+		{
+			name:          "sensitive secret inside another word",
+			configPath:    filepath.Join("testdata", "skip-test.yaml"),
+			inputFilePath: filepath.Join("testdata", "wrapped-secrets-sensitive.txt"),
+			want: types.Secret{
+				FilePath: filepath.Join("testdata", "wrapped-secrets-sensitive.txt"),
+				Findings: []types.SecretFinding{
+					wantFindingMyAwsAccessKey,
+					wantFindingMyGitHubPAT,
+				},
+			},
+		},
 		{
 			name:          "asymmetric file",
 			configPath:    filepath.Join("testdata", "skip-test.yaml"),

diff --git a/pkg/fanal/secret/testdata/wrapped-secrets-sensitive.txt b/pkg/fanal/secret/testdata/wrapped-secrets-sensitive.txt
@@ -0,0 +1,2 @@
+MyAWS_secret_KEY="12ASD34qwe56CXZ78tyH10Tna543VBokN85RHCas"
+our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
diff --git a/pkg/fanal/secret/testdata/wrapped-secrets.txt b/pkg/fanal/secret/testdata/wrapped-secrets.txt
@@ -0,0 +1,11 @@
+DISPID_ICANVASRENDERINGCONTEXT2D_CANVAS DISPID_CANVASRENDERCONTEXT2D
+ABCDFLWSECK_TEST-CANVASRENDERCONTEXT2DCANVASRENDA
+ABCFLWSECK_TEST123456789012
+Rought_ICANVASRENDERINGVIACONTEXT2D3D5D7D8D
+Sogho_ICANVASRENDERINGVIACONTEXT2D3D5D7D8D
+Soghu_ICANVASRENDERINGVIACONTEXT2D3D5D7D8D
+Bighr_ICANVASRENDERINGVIACONTEXT2D3D5D7D8DICANVASRENDERINGVIACONTEXT2D3D5D7D8D9D22
+Surhf_ICANVASRENDERINGVIACONTEXT2D3D5D6D7D8D9
+abcdexoxb-1234567890
+APK_TEST_1234567890
+ask_live_superlive1
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		MyAWS_secret_KEY="12ASD34qwe56CXZ78tyH10Tna543VBokN85RHCas"
		our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