fix(azuredevops): detector wasn't working

pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go

@@ -1,13 +1,17 @@
-package azuredevopspersonalaccesstoken
+package azure_devops
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"strings"
 
 	regexp "github.com/wasilibs/go-re2"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -21,81 +25,143 @@ type Scanner struct {
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
-var (
-	defaultClient = common.SaneHttpClient()
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-z]{52})\b`)
-	orgPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z][0-9a-zA-Z-]{5,48}[0-9a-zA-Z])\b`)
-)
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_AzureDevopsPersonalAccessToken
+}
+
+func (s Scanner) Description() string {
+	return "Azure DevOps is a suite of development tools provided by Microsoft. Personal Access Tokens (PATs) are used to authenticate and authorize access to Azure DevOps services and resources."
+}
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"azure"}
+	return []string{"dev.azure.com", "az devops"}
 }
 
+var (
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", "az", "token", "pat"}) + `\b([a-z0-9]{52}|[a-zA-Z0-9]{84})\b`)
+	orgPat = regexp.MustCompile(`dev\.azure\.com/([0-9a-zA-Z][0-9a-zA-Z-]{5,48}[0-9a-zA-Z])\b`)
+
+	invalidOrgCache = simple.NewCache[struct{}]()
+)
+
 // FromData will find and optionally verify AzureDevopsPersonalAccessToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	orgMatches := orgPat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, match := range matches {
-		if len(match) != 2 {
+	// Deduplicate results.
+	keyMatches := make(map[string]struct{})
+	for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+		m := match[1]
+		if detectors.StringShannonEntropy(m) < 3 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
-		for _, orgMatch := range orgMatches {
-			if len(orgMatch) != 2 {
-				continue
-			}
-			resOrgMatch := strings.TrimSpace(orgMatch[1])
+		keyMatches[m] = struct{}{}
+	}
+	orgMatches := make(map[string]struct{})
+	for _, match := range orgPat.FindAllStringSubmatch(dataStr, -1) {
+		m := match[1]
+		if invalidOrgCache.Exists(m) {
+			continue
+		}
+		orgMatches[m] = struct{}{}
+	}
 
-			s1 := detectors.Result{
+	for key := range keyMatches {
+		for org := range orgMatches {
+			r := detectors.Result{
 				DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
-				Raw:          []byte(resMatch),
-				RawV2:        []byte(resMatch + resOrgMatch),
+				Raw:          []byte(key),
+				RawV2:        []byte(fmt.Sprintf(`{"organization":"%s","token":"%s"}`, org, key)),
 			}
 
 			if verify {
-				client := s.client
-				if client == nil {
-					client = defaultClient
-				}
-				req, err := http.NewRequestWithContext(ctx, "GET", "https://dev.azure.com/"+resOrgMatch+"/_apis/projects", nil)
-				if err != nil {
-					continue
+				if s.client == nil {
+					s.client = common.SaneHttpClient()
 				}
-				req.SetBasicAuth("", resMatch)
-				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					hasVerifiedRes, _ := common.ResponseContainsSubstring(res.Body, "lastUpdateTime")
-					if res.StatusCode >= 200 && res.StatusCode < 300 && hasVerifiedRes {
-						s1.Verified = true
-					} else if res.StatusCode == 401 {
-						// The secret is determinately not verified (nothing to do)
-					} else {
-						err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
-						s1.SetVerificationError(err, resMatch)
+
+				isVerified, extraData, verificationErr := verifyMatch(ctx, s.client, org, key)
+				r.Verified = isVerified
+				r.ExtraData = extraData
+				if verificationErr != nil {
+					if errors.Is(verificationErr, errInvalidOrg) {
+						delete(orgMatches, org)
+						invalidOrgCache.Set(org, struct{}{})
+						continue
 					}
-				} else {
-					s1.SetVerificationError(err, resMatch)
+					r.SetVerificationError(verificationErr)
 				}
 			}
 
-			results = append(results, s1)
+			results = append(results, r)
 		}
 	}
 
 	return results, nil
 }
 
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_AzureDevopsPersonalAccessToken
+var errInvalidOrg = errors.New("invalid organization")
+
+func verifyMatch(ctx context.Context, client *http.Client, org string, key string) (bool, map[string]string, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", "https://dev.azure.com/"+org+"/_apis/projects", nil)
+	if err != nil {
+		return false, nil, err
+	}
+
+	req.SetBasicAuth("", key)
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Content-Type", "application/json")
+	res, err := client.Do(req)
+	if err != nil {
+		return false, nil, err
+	}
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		// {"count":1,"value":[{"id":"...","name":"Test","url":"https://dev.azure.com/...","state":"wellFormed","revision":11,"visibility":"private","lastUpdateTime":"2024-12-16T02:23:58.86Z"}]}
+		var projectsRes listProjectsResponse
+		if json.NewDecoder(res.Body).Decode(&projectsRes) != nil {
+			return false, nil, err
+		}
+
+		// Condense a list of organizations + roles.
+		var (
+			extraData map[string]string
+			projects  = make([]string, 0, len(projectsRes.Value))
+		)
+		for _, p := range projectsRes.Value {
+			projects = append(projects, p.Name)
+		}
+		if len(projects) > 0 {
+			extraData = map[string]string{
+				"projects": strings.Join(projects, ","),
+			}
+		}
+		return true, extraData, nil
+	case http.StatusUnauthorized:
+		// The secret is determinately not verified (nothing to do)
+		return false, nil, nil
+	case http.StatusNotFound:
+		// Org doesn't exist.
+		return false, nil, errInvalidOrg
+	default:
+		body, _ := io.ReadAll(res.Body)
+		return false, nil, fmt.Errorf("unexpected HTTP response: status=%d, body=%q", res.StatusCode, string(body))
+	}
 }
 
-func (s Scanner) Description() string {
-	return "Azure DevOps is a suite of development tools provided by Microsoft. Personal Access Tokens (PATs) are used to authenticate and authorize access to Azure DevOps services and resources."
+type listProjectsResponse struct {
+	Count int               `json:"count"`
+	Value []projectResponse `json:"value"`
+}
+
+type projectResponse struct {
+	Id   string `json:"id"`
+	Name string `json:"name"`
 }

pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go

@@ -1,7 +1,7 @@
 //go:build detectors
 // +build detectors
 
-package azuredevopspersonalaccesstoken
+package azure_devops
 
 import (
 	"context"

pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go

@@ -1,4 +1,4 @@
-package azuredevopspersonalaccesstoken
+package azure_devops
 
 import (
 	"context"
@@ -10,19 +10,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
 )
 
-var (
-	validPattern = `
-	azure:
-		azure_key: uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un
-		azure_org_id: WOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG
-	`
-	invalidPattern = `
-	azure:
-		azure_key: uie5tff7m5H5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un
-		azure_org_id: LOKi
-	`
-)
-
 func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) {
 	d := Scanner{}
 	ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
@@ -32,14 +19,67 @@ func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) {
 		input string
 		want  []string
 	}{
+		// old
+		{
+			name: "valid - old token",
+			input: `
+provider "azuredevops" {
+  # Configuration options
+  org_service_url       = "https://dev.azure.com/housemd"
+  personal_access_token = "qkfon5cdjdekin4qnkgfr2nf367h6yjnnqm5upwqepd3rekl4l5a"
+}`,
+			want: []string{"qkfon5cdjdekin4qnkgfr2nf367h6yjnnqm5upwqepd3rekl4l5a:housemd"},
+		},
+
+		// new
 		{
-			name:  "valid pattern",
-			input: validPattern,
-			want:  []string{"uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8unWOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG"},
+			name: "valid - az devops CLI",
+			input: `        echo "Tests failed. Creating a bug in Azure DevOps..."
+        az devops login --organization https://dev.azure.com/TechServicesCorp --token A0us9bS1c6qe5blb6CT4FGRR4JcmPDg7uadVFmw4D65bvtdPcdVdJQQJ99AKACAAAAAPnX9AAAASAhDO4GFB
+        az boards work-item create --title "Automated Bug: Test Failure" --type $(bugType) --description "Tests failed. See results.log for details." --project "Test"`,
+			want: []string{"A0us9bS1c6qe5blb6CT4FGRR4JcmPDg7uadVFmw4D65bvtdPcdVdJQQJ99AKACAAAAAPnX9AAAASAhDO4GFB:TechServicesCorp"},
 		},
 		{
-			name:  "invalid pattern",
-			input: invalidPattern,
+			name: "valid - environment variables",
+			input: `# Base image: Azure CLI with a lightweight Ubuntu distribution-mcr.microsoft.com/azure-cli:2.52.0
+FROM ubuntu:20.04
+
+# Set environment variables for Azure DevOps agent
+ENV AZP_URL=https://dev.azure.com/EBOrg21
+ENV AZP_TOKEN=2ZGS1XLyxTU2wXlrXy71ldl1tBKceXM9kl6mVAeQchvWIErzkwtBJQjJ99AKACAAAAAAAAAAAAASAZDO5BA2
+ENV AZP_POOL=TestParty
+`,
+			want: []string{"2ZGS1XLyxTU2wXlrXy71ldl1tBKceXM9kl6mVAeQchvWIErzkwtBJQjJ99AKACAAAAAAAAAAAAASAZDO5BA2:EBOrg21"},
+		},
+		{
+			name: "valid - jupyter notebook",
+			input: `       "4  https://dev.azure.com/SSGL-SMT/10_BG_AU5...  "
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "token = r\"49QzGd2ZOLTWdoMc0S3M0cZkVVsBMTua01tlMYOkTUnEwxebgYdheQQJ99AKACAAAAAHsyrdAAASAZDOULjm\""
+   ]`,
+			want: []string{"49QzGd2ZOLTWdoMc0S3M0cZkVVsBMTua01tlMYOkTUnEwxebgYdheQQJ99AKACAAAAAHsyrdAAASAZDOULjm:SSGL-SMT"},
+		},
+
+		// Invalid
+		{
+			name:  "invalid",
+			input: `ssh.dev.azure.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H`,
 			want:  nil,
 		},
 	}

pkg/engine/defaults/defaults.go

@@ -894,7 +894,7 @@ func buildDetectorList() []detectors.Detector {
 		&azure_serviceprincipal_v2.Scanner{},
 		&azure_batch.Scanner{},
 		&azurecontainerregistry.Scanner{},
-		&azuredevopspersonalaccesstoken.Scanner{},
+		&azure_devops.Scanner{},
 		// &azurefunctionkey.Scanner{}, // detector is throwing some FPs
 		&azure_openai.Scanner{},
 		&azuresearchadminkey.Scanner{},