Integrate secrets detection library with telemetry (#26399) (#26436)

Co-authored-by: Vincent Dai <[email protected]>
Azure · Oct 23, 2024 · b78aec8 · b78aec8
1 parent bec78c2
commit b78aec8
Show file tree

Hide file tree

Showing 9 changed files with 54 additions and 37 deletions.
diff --git a/src/Accounts/Accounts/CommonModule/AzModule.cs b/src/Accounts/Accounts/CommonModule/AzModule.cs
@@ -385,10 +385,10 @@ public Dictionary<string, string> GetTelemetryInfo(string telemetryId)
             Dictionary<string, string> telemetryInfo = null;
             if (_telemetry.TryGetValue(telemetryId, out var qos))
             {
-                if (qos?.SanitizerInfo?.DetectedProperties?.Count > 0)
+                if (qos?.SanitizerInfo?.DetectedProperties.IsEmpty == false)
                 {
                     var showSecretsWarning = qos.SanitizerInfo.ShowSecretsWarning && qos.SanitizerInfo.SecretsDetected;
-                    var sanitizedProperties = string.Join(", ", qos.SanitizerInfo.DetectedProperties);
+                    var sanitizedProperties = string.Join(", ", qos.SanitizerInfo.DetectedProperties.PropertyNames);
                     var invocationName = qos.InvocationName;
                     telemetryInfo = new Dictionary<string, string>
                     {

diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerCollectionProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerCollectionProvider.cs
@@ -39,13 +39,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti
                         var collItemType = collItem.GetType();
                         if (collItemType == typeof(string))
                         {
-                            if (Service.TrySanitizeData(collItem as string, out string sanitizedData))
+                            if (Service.TrySanitizeData(collItem as string, out var detections, out _))
                             {
                                 telemetry.SecretsDetected = true;
                                 var propertyPath = ResolvePropertyPath(property);
                                 if (!string.IsNullOrEmpty(propertyPath))
                                 {
-                                    telemetry.DetectedProperties.Add(ResolvePropertyPath(property));
+                                    foreach (var detection in detections)
+                                    {
+                                        telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);
+                                    }
                                 }
                             }
                         }

diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerDictionaryProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerDictionaryProvider.cs
@@ -40,14 +40,17 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti
                         var dicItemValueType = dictItemValue.GetType();
                         if (dicItemValueType == typeof(string))
                         {
-                            if (Service.TrySanitizeData(dictItemValue as string, out string sanitizedData))
+                            if (Service.TrySanitizeData(dictItemValue as string, out var detections, out _))
                             {
                                 // Sanitize dictionary item value
                                 telemetry.SecretsDetected = true;
                                 var propertyPath = ResolvePropertyPath(property);
                                 if (!string.IsNullOrEmpty(propertyPath))
                                 {
-                                    telemetry.DetectedProperties.Add(propertyPath);
+                                    foreach (var detection in detections)
+                                    {
+                                        telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);
+                                    }
                                 }
                             }
                         }

diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonArrayProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonArrayProvider.cs
@@ -37,13 +37,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti
                         switch (jItem.Type)
                         {
                             case JTokenType.String:
-                                if (Service.TrySanitizeData(jItem.Value<string>(), out string sanitizedData))
+                                if (Service.TrySanitizeData(jItem.Value<string>(), out var detections, out _))
                                 {
                                     telemetry.SecretsDetected = true;
                                     var propertyPath = ResolvePropertyPath(property);
                                     if (!string.IsNullOrEmpty(propertyPath))
                                     {
-                                        telemetry.DetectedProperties.Add(propertyPath);
+                                        foreach (var detection in detections)
+                                        {
+                                            telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);
+                                        }
                                     }
                                 }
                                 break;

diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonObjectProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonObjectProvider.cs
@@ -37,13 +37,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti
                         switch (propValue.Type)
                         {
                             case JTokenType.String:
-                                if (Service.TrySanitizeData(propValue.Value<string>(), out string sanitizedData))
+                                if (Service.TrySanitizeData(propValue.Value<string>(), out var detections, out _))
                                 {
                                     telemetry.SecretsDetected = true;
                                     var propertyPath = ResolvePropertyPath(property);
                                     if (!string.IsNullOrEmpty(propertyPath))
                                     {
-                                        telemetry.DetectedProperties.Add(propertyPath);
+                                        foreach (var detection in detections)
+                                        {
+                                            telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);
+                                        }
                                     }
                                 }
                                 break;

diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerStringProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerStringProvider.cs
@@ -12,9 +12,9 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
-using System.Collections.Generic;
 using Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Services;
 using Microsoft.WindowsAzure.Commands.Common.Sanitizer;
+using System.Collections.Generic;
 
 namespace Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Providers
 {
@@ -29,13 +29,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti
             var propertyValue = property?.GetValue(sanitizingObject) ?? sanitizingObject;
             if (propertyValue is string data)
             {
-                if (Service.TrySanitizeData(data, out string sanitizedData))
+                if (Service.TrySanitizeData(data, out var detections, out _))
                 {
                     telemetry.SecretsDetected = true;
                     var propertyPath = ResolvePropertyPath(property);
                     if (!string.IsNullOrEmpty(propertyPath))
                     {
-                        telemetry.DetectedProperties.Add(propertyPath);
+                        foreach (var detection in detections)
+                        {
+                            telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);
+                        }
                     }
                 }
             }

diff --git a/src/Accounts/Authentication/Sanitizer/Services/DefaultSanitizerService.cs b/src/Accounts/Authentication/Sanitizer/Services/DefaultSanitizerService.cs
@@ -45,19 +45,20 @@ internal class DefaultSanitizerService : ISanitizerService
             { "Microsoft.Azure.Storage.File.CloudFileDirectory", new[] { "Parent" } },
         };
 
-        private readonly SecretMasker _secretMasker = new SecretMasker(WellKnownRegexPatterns.HighConfidenceMicrosoftSecurityModels, generateCorrelatingIds: true);
+        private readonly SecretMasker _secretMasker = new SecretMasker(WellKnownRegexPatterns.HighConfidenceMicrosoftSecurityModels);
 
-        public bool TrySanitizeData(string data, out string sanitizedData)
+        public bool TrySanitizeData(string data, out IEnumerable<Detection> detections, out string sanitizedData)
         {
             sanitizedData = string.Empty;
 
-            if (!string.IsNullOrWhiteSpace(data))
+            if (string.IsNullOrWhiteSpace(data))
             {
-                var detections = _secretMasker.DetectSecrets(data);
-                return detections.Any();
+                detections = Enumerable.Empty<Detection>();
+                return false;
             }
 
-            return false;
+            detections = _secretMasker.DetectSecrets(data);
+            return detections.Any();
         }
     }
 }
diff --git a/src/Accounts/Authentication/Sanitizer/Services/ISanitizerService.cs b/src/Accounts/Authentication/Sanitizer/Services/ISanitizerService.cs
@@ -12,6 +12,7 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using Microsoft.Security.Utilities;
 using System.Collections.Generic;
 
 namespace Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Services
@@ -20,6 +21,6 @@ public interface ISanitizerService
     {
         IReadOnlyDictionary<string, IEnumerable<string>> IgnoredProperties { get; }
 
-        bool TrySanitizeData(string data, out string sanitizedData);
+        bool TrySanitizeData(string data, out IEnumerable<Detection> detections, out string sanitizedData);
     }
 }
diff --git a/tools/Common.Netcore.Dependencies.targets b/tools/Common.Netcore.Dependencies.targets
@@ -3,22 +3,22 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.24"/>
     <PackageReference Include="Microsoft.Rest.ClientRuntime.Azure" Version="3.3.19"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Aks" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Authentication.Abstractions" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Authorization" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Common" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Compute" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Graph.Rbac" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.KeyVault" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Monitor" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Network" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.PolicyInsights" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.ResourceManager" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Storage" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Storage.Management" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Strategies" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Websites" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Common.Share" Version="1.3.101-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Aks" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Authentication.Abstractions" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Authorization" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Common" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Compute" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Graph.Rbac" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.KeyVault" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Monitor" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Network" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.PolicyInsights" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.ResourceManager" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Storage" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Storage.Management" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Strategies" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Websites" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Common.Share" Version="1.3.102-preview"/>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
   </ItemGroup>
   <ItemGroup>
@@ -37,7 +37,7 @@
     <PackageReference Include="PowerShellStandard.Library" Version="5.1.0" PrivateAssets="All" />
   </ItemGroup>
   <PropertyGroup>
-    <StorageToolsPath>$(NugetPackageRoot)\microsoft.azure.powershell.storage\1.3.101-preview\tools\</StorageToolsPath>
+    <StorageToolsPath>$(NugetPackageRoot)\microsoft.azure.powershell.storage\1.3.102-preview\tools\</StorageToolsPath>
   </PropertyGroup>
   <ItemGroup Condition="'$(OmitJsonPackage)' != 'true'">
     <PackageReference Include="Newtonsoft.Json" Version="13.0.2"/>