Integrate Secret Scanning from Microsoft.Security.Utilities

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/IntegrationTests/GitStoreIntegrationPushTests.cs

@@ -5,6 +5,7 @@
 using System.Text.Json;
 using System.Threading.Tasks;
 using Azure.Sdk.Tools.TestProxy.Common;
+using Azure.Sdk.Tools.TestProxy.Console;
 using Azure.Sdk.Tools.TestProxy.Store;
 using Microsoft.Extensions.Logging;
 using Xunit;
@@ -581,5 +582,79 @@ public async Task LargePushPerformance(int numberOfFiles, double fileSize)
                 TestHelpers.CleanupIntegrationTestTag(updatedAssets);
             }
         }
+
+        /// <summary>
+        /// 1. Restore from empty tag
+        /// 2. Add/Delete/Update files which will include a fake secret
+        /// 3. Attempt to push
+        /// 4. Assert that the expected exception is thrown, preventing a secret being pushed upstream
+        /// </summary>
+        /// <param name="inputJson"></param>
+        /// <returns></returns>
+        [EnvironmentConditionalSkipTheory]
+        [InlineData(
+        @"{
+              ""AssetsRepo"": ""Azure/azure-sdk-assets-integration"",
+              ""AssetsRepoPrefixPath"": ""pull/scenarios"",
+              ""AssetsRepoId"": """",
+              ""TagPrefix"": ""language/tables"",
+              ""Tag"": """"
+        }")]
+        [Trait("Category", "Integration")]
+        public async Task SecretProtectionPreventsPush(string inputJson)
+        {
+            var folderStructure = new string[]
+            {
+                GitStoretests.AssetsJson
+            };
+            Assets assets = JsonSerializer.Deserialize<Assets>(inputJson);
+            var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure, isPushTest: true);
+            try
+            {
+                ConsoleWrapper consoleWrapper = new ConsoleWrapper();
+                GitStore store = new GitStore(consoleWrapper);
+                var recordingHandler = new RecordingHandler(testFolder, store);
+
+                var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);
+
+                var parsedConfiguration = await store.ParseConfigurationFile(jsonFileLocation);
+                await _defaultStore.Restore(jsonFileLocation);
+
+                // Calling Path.GetFullPath of the Path.Combine will ensure any directory separators are normalized for
+                // the OS the test is running on. The reason being is that AssetsRepoPrefixPath, if there's a separator,
+                // will be a forward one as expected by git but on Windows this won't result in a usable path.
+                string localFilePath = Path.GetFullPath(Path.Combine(parsedConfiguration.AssetsRepoLocation, parsedConfiguration.AssetsRepoPrefixPath));
+
+                // generate a couple strings that LOOKs like secrets to the secret scanner.
+                var secretType1 = TestHelpers.GenerateString(3) + "8Q~" + TestHelpers.GenerateString(34);
+
+                // place an entirely new file with the secret
+                TestHelpers.CreateOrUpdateFileWithContent(localFilePath, "secret_type_1.txt", secretType1);
+
+                // modify an existing file with the secret
+                TestHelpers.CreateOrUpdateFileWithContent(localFilePath, "file2.txt", secretType1);
+
+                // delete a file to ensure that we don't attempt to scan a file that no longer exists
+                File.Delete(Path.Combine(localFilePath, "file5.txt"));
+
+                // Use the built in secretscanner
+                await store.Push(jsonFileLocation);
+
+                // no changes should be committed
+                var pendingChanges = store.DetectPendingChanges(parsedConfiguration);
+                Assert.Equal(2, pendingChanges.Count());
+
+                // now double check the actual scan results to ensure they are where we expect
+                var detectedSecrets = await store.SecretScanner.DiscoverSecrets(parsedConfiguration.AssetsRepoLocation, pendingChanges);
+
+                Assert.Equal(2, detectedSecrets.Count);
+                Assert.Equal("SEC101/156", detectedSecrets[0].Item2.Id);
+                Assert.Equal("SEC101/156", detectedSecrets[1].Item2.Id);
+            }
+            finally
+            {
+                DirectoryHelper.DeleteGitDirectory(testFolder);
+            }
+        }
     }
 }

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/TestHelpers.cs

@@ -11,6 +11,7 @@
 using System.Linq;
 using Xunit;
 using System.Threading.Tasks;
+using System.Security.Cryptography;
 
 namespace Azure.Sdk.Tools.TestProxy.Tests
 {
@@ -347,6 +348,18 @@ public static void CreateFileWithInitialVersion(string testFolder, string fileNa
             File.WriteAllText(fullFileName, "1");
         }
 
+        /// <summary>
+        /// Create a new file with custom text
+        /// </summary>
+        /// <param name="testFolder">The temporary test folder created by TestHelpers.DescribeTestFolder</param>
+        /// <param name="fileName">The file to be created</param>
+        public static void CreateOrUpdateFileWithContent(string testFolder, string fileName, string textContent)
+        {
+            string fullFileName = Path.Combine(testFolder, fileName);
+
+            File.WriteAllText(fullFileName, textContent);
+        }
+
         /// <summary>
         /// This function is used to confirm that the .breadcrumb file under the assets store contains the appropriate
         /// information.
@@ -517,6 +530,23 @@ public static bool CheckExistenceOfTag(Assets assets, string workingDirectory)
             return result.StdOut.Trim().Length > 0;
         }
 
+        public static string GenerateString(int count)
+        {
+            char[] alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789".ToArray();
+
+            StringBuilder builder = new StringBuilder();
+
+            for (int i = 0; i < count; i++)
+            {
+                var bytes = RandomNumberGenerator.GetBytes(1);
+                int index = bytes[0] % alphabet.Length;
+                char ch = alphabet[index];
+                _ = builder.Append(ch);
+            }
+
+            return builder.ToString();
+        }
+
         public static List<T> EnumerateArray<T>(JsonElement element)
         {
             List<T> values = new List<T>();

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Azure.Sdk.Tools.TestProxy.csproj

@@ -22,6 +22,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Security.Utilities.Core" Version="1.4.14" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
   </ItemGroup>
 </Project>

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SecretScanner.cs

@@ -0,0 +1,93 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Azure.Sdk.Tools.TestProxy.Console;
+using Microsoft.Build.Tasks;
+using Microsoft.Security.Utilities;
+
+namespace Azure.Sdk.Tools.TestProxy.Common
+{
+    public class SecretScanner
+    {
+        public SecretMasker SecretMasker = new SecretMasker(
+            WellKnownRegexPatterns.HighConfidenceMicrosoftSecurityModels.Concat(WellKnownRegexPatterns.LowConfidencePotentialSecurityKeys),
+            generateCorrelatingIds: true);
+
+        private IConsoleWrapper Console;
+
+        public SecretScanner(IConsoleWrapper consoleWrapper)
+        {
+            Console = consoleWrapper;
+        }
+
+        public async Task<List<Tuple<string, Detection>>> DiscoverSecrets(string assetRepoRoot, IEnumerable<string> relativePaths)
+        {
+            var detectedSecrets = new List<Tuple<string, Detection>>();
+            var total = relativePaths.Count();
+            var seen = 0;
+            Console.WriteLine(string.Empty);
+            foreach (string filePath in relativePaths)
+            {
+                var content = await ReadFile(Path.Combine(assetRepoRoot, filePath));
+                var fileDetections = DetectSecrets(content);
+
+                if (fileDetections != null && fileDetections.Count > 0)
+                {
+                    foreach (Detection detection in fileDetections)
+                    {
+                        detectedSecrets.Add(Tuple.Create(filePath, detection));
+                    }
+                }
+                seen++;
+                System.Console.Write($"\r\u001b[2KScanned {seen}/{total}.");
+            }
+            Console.WriteLine(string.Empty);
+
+            return detectedSecrets;
+        }
+
+        public async Task<List<Tuple<string, Detection>>> DiscoverSecrets(string inputDirectory)
+        {
+            var files = Directory.GetFiles(inputDirectory, "*", SearchOption.AllDirectories);
+            var detectedSecrets = new List<Tuple<string, Detection>>();
+
+            var seen = 0;
+            Console.WriteLine(string.Empty);
+            foreach (string filePath in files)
+            {
+                var content = await ReadFile(filePath);
+                var fileDetections = DetectSecrets(content);
+
+                if (fileDetections != null && fileDetections.Count > 0)
+                {
+                    foreach(Detection detection in fileDetections)
+                    {
+                        detectedSecrets.Add(Tuple.Create(filePath, detection));
+                    }
+                }
+                seen++;
+                System.Console.Write($"\r\u001b[2KScanned {seen}/{files.Length}.");
+            }
+
+            Console.WriteLine(string.Empty);
+
+            return detectedSecrets;
+        }
+
+        private async Task<string> ReadFile(string filePath)
+        {
+            using (StreamReader reader = new StreamReader(filePath))
+            {
+                return await reader.ReadToEndAsync();
+            }
+        }
+
+        private ICollection<Detection> DetectSecrets(string stringContent)
+        {
+            return SecretMasker.DetectSecrets(stringContent);
+        }
+
+    }
+}

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Console/ConsoleWrapper.cs

@@ -1,4 +1,4 @@
-
+
 using System;
 
 namespace Azure.Sdk.Tools.TestProxy.Console
@@ -20,5 +20,9 @@ public string ReadLine()
         {
             return System.Console.ReadLine();
         }
+        public void ResetCursor()
+        {
+            System.Console.SetCursorPosition(0, System.Console.CursorTop);
+        }
     }
 }

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Console/ConsoleWrapperTester.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 
 namespace Azure.Sdk.Tools.TestProxy.Console
 {
@@ -28,14 +28,22 @@ public void SetReadLineResponse(string readLineResponse)
         {
             _readLineResponse = readLineResponse;
         }
+
         public void Write(string message)
         {
             System.Console.Write(message);
         }
+
         public void WriteLine(string message)
         {
             System.Console.WriteLine(message);
         }
+
+        public void ResetCursor()
+        {
+            // don't need this for testing
+        }
+
         public string ReadLine()
         {
             System.Console.WriteLine($"ReadLine response for test: '{_readLineResponse}'");

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Console/IConsoleWrapper.cs

@@ -1,4 +1,4 @@
-namespace Azure.Sdk.Tools.TestProxy.Console
+namespace Azure.Sdk.Tools.TestProxy.Console
 {
     /// <summary>
     /// IConsoleWrapper is just an interface around Console functions. This is necessary for testing
@@ -9,5 +9,6 @@ public interface IConsoleWrapper
         void Write(string message);
         void WriteLine(string message);
         string ReadLine();
+        public void ResetCursor();
     }
 }