Skip to content

Commit

Permalink
feat(azure): skip low-entropy secrets
Browse files Browse the repository at this point in the history
  • Loading branch information
@rgmz
rgmz committed Dec 15, 2024
1 parent e932ea9 commit a9633cc
Show file tree
Hide file tree
Showing 4 changed files with 181 additions and 123 deletions.
45 changes: 27 additions & 18 deletions pkg/detectors/azure_entra/serviceprincipal/v1/spv1.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,13 @@ var _ interface {
detectors.Versioner
} = (*Scanner)(nil)

var (
defaultClient = common.SaneHttpClient()
// TODO: Azure storage access keys and investigate other types of creds.
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
//clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
// TODO: Tighten this regex and replace it with above.
secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]`)
)
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Azure
}

func (s Scanner) Description() string {
return serviceprincipal.Description
}

func (s Scanner) Version() int {
return 1
Expand All @@ -45,6 +43,19 @@ func (s Scanner) Keywords() []string {
return []string{"azure", "az", "entra", "msal", "login.microsoftonline.com", ".onmicrosoft.com"}
}

var (
defaultClient = common.SaneHttpClient()
// TODO: Azure storage access keys and investigate other types of creds.
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
// clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
// TODO: Tighten this regex and replace it with above.
secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})(?:[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]|\z)`)

invalidMatchPat = regexp.MustCompile(`^passwordCredentials":`)
invalidSecretPat = regexp.MustCompile(`^[a-zA-Z]+$`)
)

// FromData will find and optionally verify Azure secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
Expand All @@ -68,20 +79,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Azure
}

func (s Scanner) Description() string {
return serviceprincipal.Description
}

func findSecretMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})
for _, match := range secretPat.FindAllStringSubmatch(data, -1) {
m := match[1]
// Ignore secrets that are handled by the V2 detector.
if v2.SecretPat.MatchString(m) {
// Ignore secrets that are handled by the V2 detector.
continue
} else if detectors.StringShannonEntropy(m) < 3 {
// Ignore low-entropy results.
continue
} else if invalidSecretPat.MatchString(m) || invalidMatchPat.MatchString(match[0]) {
// Ignore patterns that are known to be false.
continue
}
uniqueMatches[m] = struct{}{}
Expand Down
138 changes: 44 additions & 94 deletions pkg/detectors/azure_entra/serviceprincipal/v1/spv1_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,106 +1,36 @@
package v1

import (
"context"
"testing"

"github.com/google/go-cmp/cmp"

"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)

var (
validPattern = `
azure credentials:
azureClientID: clientid9304d5df4-aac1-6117-552c-7f70c89a40d9
azureTenant: tenant_idid9304d5df4-aac1-6117-552c-7f70c89a40d9
azureClientSecret: clientsecretY_0w|[cGpan41k6ng.ol414sp4ccw2v_rkfmbs537i
`
invalidPattern = `
azure credentials:
azureClientID: 9304d5df4-aac1-6117-552c-7f70c89a
azureTenant: id9304d5df4-aac1-6117-55-7f70c89a40d9
azureClientSecret: Y_0w|[cGpan41k6ng.
`
)

func TestAzure_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})

tests := []struct {
name string
input string
want []string
}{
{
name: "valid pattern",
input: validPattern,
want: []string{"304d5df4-aac1-6117-552c-7f70c89a40d9cGpan41k6ng.ol414sp4ccw2v_rkfmbs53304d5df4-aac1-6117-552c-7f70c89a40d9"},
},
{
name: "invalid pattern",
input: invalidPattern,
want: nil,
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
if len(matchedDetectors) == 0 {
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}

results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
t.Errorf("error = %v", err)
return
}

if len(results) != len(test.want) {
if len(results) == 0 {
t.Errorf("did not receive result")
} else {
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
return
}

actual := make(map[string]struct{}, len(results))
for _, r := range results {
if len(r.RawV2) > 0 {
actual[string(r.RawV2)] = struct{}{}
} else {
actual[string(r.Raw)] = struct{}{}
}
}
expected := make(map[string]struct{}, len(test.want))
for _, v := range test.want {
expected[v] = struct{}{}
}

if diff := cmp.Diff(expected, actual); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}

type testCase struct {
Input string
Expected map[string]struct{}
}

func Test_FindClientSecretMatches(t *testing.T) {
cases := map[string]testCase{
// secret
`secret`: {
Input: `"secret": "ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~",`,
Expected: map[string]struct{}{"ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~": {}},
},

// client secret
"client_secret": {
Input: ` "TenantId": "3d7e0652-b03d-4ed2-bf86-f1299cecde17",
"ClientSecret": "gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9",`,
Expected: map[string]struct{}{"gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9": {}},
},
"client secret at end": {
Input: `secret: UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==`,
Expected: map[string]struct{}{
"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==": {},
},
},
"client_secret1": {
Input: ` public static string clientId = "413ff05b-6d54-41a7-9271-9f964bc10624";
public static string clientSecret = "k72~odcN_6TbVh5D~19_1Qkj~87trteArL";
Expand Down Expand Up @@ -155,10 +85,19 @@ configs = {"fs.azure.account.auth.type": "OAuth"`,
Input: ` "AZUREAD-AKS-APPID-SECRET": "8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-",`,
Expected: map[string]struct{}{"8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-": {}},
},
//"client_secret6": {
"client_secret9": {
Input: ` client-id: 49abd816-45d1-479a-b49a-80bcf6d7213a
client-secret: 7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO`,
Expected: map[string]struct{}{"7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO": {}},
},
"client_secret10": {
Input: ` "aadClientSecret": "6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo",`,
Expected: map[string]struct{}{"6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo": {}},
},
// "client_secret6": {
// Input: ``,
// Expected: map[string]struct{}{"": {}},
//},
// },

"password": {
Input: `# Login using Service Principal
Expand All @@ -169,18 +108,29 @@ $Credential = New-Object -TypeName System.Management.Automation.PSCredential -Ar
},

// False positives
"placeholder_secret": {
"invalid - placeholder_secret": {
Input: `- Log in with a service principal using a client secret:
az login --service-principal --username {{http://azure-cli-service-principal}} --password {{secret}} --tenant {{someone.onmicrosoft.com}}`,
Expected: nil,
},
//"client_secret3": {
// Input: ``,
// Expected: map[string]struct{}{
// "": {},
// },
//},
},
"invalid - only alpha characters": {
Input: `"passwordCredentials":[],"preferredTokenSigningKeyThumbprint":null,"publisherName":"Microsoft"`,
},
"invalid - low entropy": {
Input: `clientSecret: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`,
},
"invalid - passwordCredentials1": {
Input: `"passwordCredentials":[{"customKeyIdentifier":"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==","endDate":"2019-07-16T23:01:19.028Z","keyId":`,
},
"invalid - passwordCredentials2": {
Input: `"passwordCredentials":[{"customKeyIdentifier":"TQB5ACAARgBpAHIAcwB0ACAASwBlAHkA"`,
},
"invalid - passwordCredentials3": {
Input: `,"passwordCredentials":[{"customKeyIdentifier":"awBlAHkAZgBvAHIAaQBtAHAAYQBsAGEA",`,
},
"invalid - azure vault path": {
Input: ` public const string MsalArlingtonOBOKeyVaultUri = "https://msidlabs.vault.azure.net:443/secrets/ARLMSIDLAB1-IDLASBS-App-CC-Secret";`,
},
}

for name, test := range cases {
Expand Down
26 changes: 15 additions & 11 deletions pkg/detectors/azure_entra/serviceprincipal/v2/spv2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@ var _ interface {
detectors.Versioner
} = (*Scanner)(nil)

var (
defaultClient = common.SaneHttpClient()
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Azure
}

SecretPat = regexp.MustCompile(`(?:[^a-zA-Z0-9_~.-]|\A)([a-zA-Z0-9_~.-]{3}\dQ~[a-zA-Z0-9_~.-]{31,34})(?:[^a-zA-Z0-9_~.-]|\z)`)
)
func (s Scanner) Description() string {
return serviceprincipal.Description
}

func (s Scanner) Version() int {
return 2
Expand All @@ -42,13 +44,11 @@ func (s Scanner) Keywords() []string {
return []string{"q~"}
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Azure
}
var (
defaultClient = common.SaneHttpClient()

func (s Scanner) Description() string {
return serviceprincipal.Description
}
SecretPat = regexp.MustCompile(`(?:[^a-zA-Z0-9_~.-]|\A)([a-zA-Z0-9_~.-]{3}\dQ~[a-zA-Z0-9_~.-]{31,34})(?:[^a-zA-Z0-9_~.-]|\z)`)
)

// FromData will find and optionally verify Azure secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
Expand Down Expand Up @@ -181,7 +181,11 @@ func createResult(tenantId string, clientId string, clientSecret string, verifie
func findSecretMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})
for _, match := range SecretPat.FindAllStringSubmatch(data, -1) {
uniqueMatches[match[1]] = struct{}{}
m := match[1]
if detectors.StringShannonEntropy(m) < 3 {
continue
}
uniqueMatches[m] = struct{}{}
}
return uniqueMatches
}
Loading

0 comments on commit a9633cc

Please sign in to comment.