From b78aec83a4704e53749338f42f69317c0c5b7ff1 Mon Sep 17 00:00:00 2001 From: Yabo Hu Date: Wed, 23 Oct 2024 15:05:01 +0800 Subject: [PATCH] Integrate secrets detection library with telemetry (#26399) (#26436) Co-authored-by: Vincent Dai <23257217+vidai-msft@users.noreply.github.com> --- .../Accounts/CommonModule/AzModule.cs | 4 +-- .../Providers/SanitizerCollectionProvider.cs | 7 ++-- .../Providers/SanitizerDictionaryProvider.cs | 7 ++-- .../Providers/SanitizerJsonArrayProvider.cs | 7 ++-- .../Providers/SanitizerJsonObjectProvider.cs | 7 ++-- .../Providers/SanitizerStringProvider.cs | 9 +++-- .../Services/DefaultSanitizerService.cs | 13 +++---- .../Sanitizer/Services/ISanitizerService.cs | 3 +- tools/Common.Netcore.Dependencies.targets | 34 +++++++++---------- 9 files changed, 54 insertions(+), 37 deletions(-) diff --git a/src/Accounts/Accounts/CommonModule/AzModule.cs b/src/Accounts/Accounts/CommonModule/AzModule.cs index 086b0d89831c..97170d2f35b2 100644 --- a/src/Accounts/Accounts/CommonModule/AzModule.cs +++ b/src/Accounts/Accounts/CommonModule/AzModule.cs @@ -385,10 +385,10 @@ public Dictionary GetTelemetryInfo(string telemetryId) Dictionary telemetryInfo = null; if (_telemetry.TryGetValue(telemetryId, out var qos)) { - if (qos?.SanitizerInfo?.DetectedProperties?.Count > 0) + if (qos?.SanitizerInfo?.DetectedProperties.IsEmpty == false) { var showSecretsWarning = qos.SanitizerInfo.ShowSecretsWarning && qos.SanitizerInfo.SecretsDetected; - var sanitizedProperties = string.Join(", ", qos.SanitizerInfo.DetectedProperties); + var sanitizedProperties = string.Join(", ", qos.SanitizerInfo.DetectedProperties.PropertyNames); var invocationName = qos.InvocationName; telemetryInfo = new Dictionary { diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerCollectionProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerCollectionProvider.cs index 8e8c7c9025f2..4c9e794b572b 100644 --- a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerCollectionProvider.cs +++ b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerCollectionProvider.cs @@ -39,13 +39,16 @@ public override void SanitizeValue(object sanitizingObject, Stack saniti var collItemType = collItem.GetType(); if (collItemType == typeof(string)) { - if (Service.TrySanitizeData(collItem as string, out string sanitizedData)) + if (Service.TrySanitizeData(collItem as string, out var detections, out _)) { telemetry.SecretsDetected = true; var propertyPath = ResolvePropertyPath(property); if (!string.IsNullOrEmpty(propertyPath)) { - telemetry.DetectedProperties.Add(ResolvePropertyPath(property)); + foreach (var detection in detections) + { + telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker); + } } } } diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerDictionaryProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerDictionaryProvider.cs index ef058f02c79a..81b7ccd33011 100644 --- a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerDictionaryProvider.cs +++ b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerDictionaryProvider.cs @@ -40,14 +40,17 @@ public override void SanitizeValue(object sanitizingObject, Stack saniti var dicItemValueType = dictItemValue.GetType(); if (dicItemValueType == typeof(string)) { - if (Service.TrySanitizeData(dictItemValue as string, out string sanitizedData)) + if (Service.TrySanitizeData(dictItemValue as string, out var detections, out _)) { // Sanitize dictionary item value telemetry.SecretsDetected = true; var propertyPath = ResolvePropertyPath(property); if (!string.IsNullOrEmpty(propertyPath)) { - telemetry.DetectedProperties.Add(propertyPath); + foreach (var detection in detections) + { + telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker); + } } } } diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonArrayProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonArrayProvider.cs index a0773b0c5938..826b6323594e 100644 --- a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonArrayProvider.cs +++ b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonArrayProvider.cs @@ -37,13 +37,16 @@ public override void SanitizeValue(object sanitizingObject, Stack saniti switch (jItem.Type) { case JTokenType.String: - if (Service.TrySanitizeData(jItem.Value(), out string sanitizedData)) + if (Service.TrySanitizeData(jItem.Value(), out var detections, out _)) { telemetry.SecretsDetected = true; var propertyPath = ResolvePropertyPath(property); if (!string.IsNullOrEmpty(propertyPath)) { - telemetry.DetectedProperties.Add(propertyPath); + foreach (var detection in detections) + { + telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker); + } } } break; diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonObjectProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonObjectProvider.cs index 3fc4ee49c505..ef374315506f 100644 --- a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonObjectProvider.cs +++ b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonObjectProvider.cs @@ -37,13 +37,16 @@ public override void SanitizeValue(object sanitizingObject, Stack saniti switch (propValue.Type) { case JTokenType.String: - if (Service.TrySanitizeData(propValue.Value(), out string sanitizedData)) + if (Service.TrySanitizeData(propValue.Value(), out var detections, out _)) { telemetry.SecretsDetected = true; var propertyPath = ResolvePropertyPath(property); if (!string.IsNullOrEmpty(propertyPath)) { - telemetry.DetectedProperties.Add(propertyPath); + foreach (var detection in detections) + { + telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker); + } } } break; diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerStringProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerStringProvider.cs index cac43d3cac83..5c5bd2c2f903 100644 --- a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerStringProvider.cs +++ b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerStringProvider.cs @@ -12,9 +12,9 @@ // limitations under the License. // ---------------------------------------------------------------------------------- -using System.Collections.Generic; using Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Services; using Microsoft.WindowsAzure.Commands.Common.Sanitizer; +using System.Collections.Generic; namespace Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Providers { @@ -29,13 +29,16 @@ public override void SanitizeValue(object sanitizingObject, Stack saniti var propertyValue = property?.GetValue(sanitizingObject) ?? sanitizingObject; if (propertyValue is string data) { - if (Service.TrySanitizeData(data, out string sanitizedData)) + if (Service.TrySanitizeData(data, out var detections, out _)) { telemetry.SecretsDetected = true; var propertyPath = ResolvePropertyPath(property); if (!string.IsNullOrEmpty(propertyPath)) { - telemetry.DetectedProperties.Add(propertyPath); + foreach (var detection in detections) + { + telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker); + } } } } diff --git a/src/Accounts/Authentication/Sanitizer/Services/DefaultSanitizerService.cs b/src/Accounts/Authentication/Sanitizer/Services/DefaultSanitizerService.cs index 470168c40d12..a98f532a855c 100644 --- a/src/Accounts/Authentication/Sanitizer/Services/DefaultSanitizerService.cs +++ b/src/Accounts/Authentication/Sanitizer/Services/DefaultSanitizerService.cs @@ -45,19 +45,20 @@ internal class DefaultSanitizerService : ISanitizerService { "Microsoft.Azure.Storage.File.CloudFileDirectory", new[] { "Parent" } }, }; - private readonly SecretMasker _secretMasker = new SecretMasker(WellKnownRegexPatterns.HighConfidenceMicrosoftSecurityModels, generateCorrelatingIds: true); + private readonly SecretMasker _secretMasker = new SecretMasker(WellKnownRegexPatterns.HighConfidenceMicrosoftSecurityModels); - public bool TrySanitizeData(string data, out string sanitizedData) + public bool TrySanitizeData(string data, out IEnumerable detections, out string sanitizedData) { sanitizedData = string.Empty; - if (!string.IsNullOrWhiteSpace(data)) + if (string.IsNullOrWhiteSpace(data)) { - var detections = _secretMasker.DetectSecrets(data); - return detections.Any(); + detections = Enumerable.Empty(); + return false; } - return false; + detections = _secretMasker.DetectSecrets(data); + return detections.Any(); } } } diff --git a/src/Accounts/Authentication/Sanitizer/Services/ISanitizerService.cs b/src/Accounts/Authentication/Sanitizer/Services/ISanitizerService.cs index 2f24cac0de30..65567fc44634 100644 --- a/src/Accounts/Authentication/Sanitizer/Services/ISanitizerService.cs +++ b/src/Accounts/Authentication/Sanitizer/Services/ISanitizerService.cs @@ -12,6 +12,7 @@ // limitations under the License. // ---------------------------------------------------------------------------------- +using Microsoft.Security.Utilities; using System.Collections.Generic; namespace Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Services @@ -20,6 +21,6 @@ public interface ISanitizerService { IReadOnlyDictionary> IgnoredProperties { get; } - bool TrySanitizeData(string data, out string sanitizedData); + bool TrySanitizeData(string data, out IEnumerable detections, out string sanitizedData); } } diff --git a/tools/Common.Netcore.Dependencies.targets b/tools/Common.Netcore.Dependencies.targets index 71ca0ab640e1..aa2cae1391a0 100644 --- a/tools/Common.Netcore.Dependencies.targets +++ b/tools/Common.Netcore.Dependencies.targets @@ -3,22 +3,22 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -37,7 +37,7 @@ - $(NugetPackageRoot)\microsoft.azure.powershell.storage\1.3.101-preview\tools\ + $(NugetPackageRoot)\microsoft.azure.powershell.storage\1.3.102-preview\tools\