Clean up test discovery (#76663)

* Clean up test discovery Make a few small changes here: 1. Added a README.md to explain the purpose of `TestDiscoveryWorker` 2. Moved to using standard command line arguments so the tool can be run locally more easily * PR feedback
dotnet · Jan 8, 2025 · a3723c4 · a3723c4
1 parent 8096a7c
commit a3723c4
Show file tree

Hide file tree

Showing 5 changed files with 108 additions and 132 deletions.
diff --git a/src/Tools/PrepareTests/MinimizeUtil.cs b/src/Tools/PrepareTests/MinimizeUtil.cs
@@ -53,21 +53,18 @@ internal static void Run(string sourceDirectory, string destinationDirectory, bo
         //  2. Hard link all other files into destination directory
         Dictionary<Guid, List<FilePathInfo>> initialWalk()
         {
-            IEnumerable<string> directories = new[] {
-                Path.Combine(sourceDirectory, "eng"),
-            };
-
-            if (!isUnix)
-            {
-                directories = directories.Concat([Path.Combine(sourceDirectory, "artifacts", "VSSetup")]);
-            }
-
             var artifactsDir = Path.Combine(sourceDirectory, "artifacts/bin");
-            directories = directories.Concat(Directory.EnumerateDirectories(artifactsDir, "*.UnitTests"));
-            directories = directories.Concat(Directory.EnumerateDirectories(artifactsDir, "*.IntegrationTests"));
-            directories = directories.Concat(Directory.EnumerateDirectories(artifactsDir, "RunTests"));
+            List<string> directories =
+            [
+                Path.Combine(sourceDirectory, "eng"),
+                Path.Combine(sourceDirectory, "artifacts", "VSSetup"),
+                .. Directory.EnumerateDirectories(artifactsDir, "*.UnitTests"),
+                .. Directory.EnumerateDirectories(artifactsDir, "*.IntegrationTests"),
+                .. Directory.EnumerateDirectories(artifactsDir, "RunTests")
+            ];
 
             var idToFilePathMap = directories.AsParallel()
+                .Where(x => Directory.Exists(x))
                 .SelectMany(unitDirPath => walkDirectory(unitDirPath, sourceDirectory, destinationDirectory))
                 .GroupBy(pair => pair.mvid)
                 .ToDictionary(
@@ -79,7 +76,7 @@ Dictionary<Guid, List<FilePathInfo>> initialWalk()
 
         static IEnumerable<(Guid mvid, FilePathInfo pathInfo)> walkDirectory(string unitDirPath, string sourceDirectory, string destinationDirectory)
         {
-            Console.WriteLine($"[{DateTime.UtcNow}] Walking {unitDirPath}");
+            Console.WriteLine($"Walking {unitDirPath}");
             string? lastOutputDirectory = null;
             foreach (var sourceFilePath in Directory.EnumerateFiles(unitDirPath, "*", SearchOption.AllDirectories))
             {

diff --git a/src/Tools/PrepareTests/TestDiscovery.cs b/src/Tools/PrepareTests/TestDiscovery.cs
@@ -35,10 +35,11 @@ public static bool RunDiscovery(string repoRootDirectory, string dotnetPath, boo
                 ? dotnetFrameworkWorker
                 : dotnetCoreWorker;
 
-            var result = RunWorker(dotnetPath, workerPath, assembly);
+            var (workerSucceeded, output) = RunWorker(dotnetPath, workerPath, assembly);
             lock (s_lock)
             {
-                success &= result;
+                Console.WriteLine(output);
+                success &= workerSucceeded;
             }
         });
         stopwatch.Stop();
@@ -71,73 +72,35 @@ public static bool RunDiscovery(string repoRootDirectory, string dotnetPath, boo
                 Path.Combine(testDiscoveryWorkerFolder, configuration, "net472", "TestDiscoveryWorker.exe"));
     }
 
-    static bool RunWorker(string dotnetPath, string pathToWorker, string pathToAssembly)
+    static (bool Succeeded, string Output) RunWorker(string dotnetPath, string pathToWorker, string pathToAssembly)
     {
-        var success = true;
-        var pipeClient = new Process();
-        var arguments = new List<string>();
+        var worker = new Process();
+        var arguments = new StringBuilder();
         if (pathToWorker.EndsWith("dll"))
         {
-            arguments.Add(pathToWorker);
-            pipeClient.StartInfo.FileName = dotnetPath;
+            arguments.Append($"exec {pathToWorker}");
+            worker.StartInfo.FileName = dotnetPath;
         }
         else
         {
-            pipeClient.StartInfo.FileName = pathToWorker;
+            worker.StartInfo.FileName = pathToWorker;
         }
 
-        var errorOutput = new StringBuilder();
-
-        using (var pipeServer = new AnonymousPipeServerStream(PipeDirection.Out, HandleInheritability.Inheritable))
-        {
-            // Pass the client process a handle to the server.
-            arguments.Add(pipeServer.GetClientHandleAsString());
-            pipeClient.StartInfo.Arguments = string.Join(" ", arguments);
-            pipeClient.StartInfo.UseShellExecute = false;
-
-            // Errors will be logged to stderr, redirect to us so we can capture it.
-            pipeClient.StartInfo.RedirectStandardError = true;
-            pipeClient.ErrorDataReceived += PipeClient_ErrorDataReceived;
-            pipeClient.Start();
-
-            pipeClient.BeginErrorReadLine();
-
-            pipeServer.DisposeLocalCopyOfClientHandle();
-
-            try
-            {
-                // Read user input and send that to the client process.
-                using var sw = new StreamWriter(pipeServer);
-                sw.AutoFlush = true;
-                // Send a 'sync message' and wait for client to receive it.
-                sw.WriteLine("ASSEMBLY");
-                // Send the console input to the client process.
-                sw.WriteLine(pathToAssembly);
-            }
-            // Catch the IOException that is raised if the pipe is broken
-            // or disconnected.
-            catch (Exception e)
-            {
-                Console.Error.WriteLine($"Error: {e.Message}");
-                success = false;
-            }
-        }
-
-        pipeClient.WaitForExit();
-        success &= pipeClient.ExitCode == 0;
-        pipeClient.Close();
-
-        if (!success)
-        {
-            Console.WriteLine($"Failed to discover tests in {pathToAssembly}:{Environment.NewLine}{errorOutput}");
-        }
-
-        return success;
-
-        void PipeClient_ErrorDataReceived(object sender, DataReceivedEventArgs e)
-        {
-            errorOutput.AppendLine(e.Data);
-        }
+        var pathToOutput = Path.Combine(Path.GetDirectoryName(pathToAssembly)!, "testlist.json");
+        arguments.Append($" --assembly {pathToAssembly} --out {pathToOutput}");
+
+        var output = new StringBuilder();
+        worker.StartInfo.Arguments = arguments.ToString();
+        worker.StartInfo.UseShellExecute = false;
+        worker.StartInfo.RedirectStandardOutput = true;
+        worker.OutputDataReceived += (sender, e) => output.Append(e.Data);
+        worker.Start();
+        worker.BeginOutputReadLine();
+        worker.WaitForExit();
+        var success = worker.ExitCode == 0;
+        worker.Close();
+
+        return (success, output.ToString());
     }
 
     private static List<string> GetAssemblies(string binDirectory, bool isUnix)

diff --git a/src/Tools/TestDiscoveryWorker/Program.cs b/src/Tools/TestDiscoveryWorker/Program.cs
@@ -12,96 +12,102 @@
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Channels;
-
+using Mono.Options;
 using Xunit;
 using Xunit.Abstractions;
 
-int ExitFailure = 1;
-int ExitSuccess = 0;
+const int ExitFailure = 1;
+const int ExitSuccess = 0;
+
+string? assemblyFilePath = null;
+string? outputFilePath = null;
 
-if (args.Length != 1)
+var options = new OptionSet
 {
-    return ExitFailure;
-}
+    { "assembly=", "The assembly file to process.", v => assemblyFilePath = v },
+    { "out=", "The output file name.", v => outputFilePath = v }
+};
 
 try
 {
-    using var pipeClient = new AnonymousPipeClientStream(PipeDirection.In, args[0]);
-    using var sr = new StreamReader(pipeClient);
-    string? output;
+    List<string> extra = options.Parse(args);
 
-    // Wait for 'sync message' from the server.
-    do
+    if (assemblyFilePath is null)
     {
-        output = await sr.ReadLineAsync().ConfigureAwait(false);
+        Console.WriteLine("Must pass an assembly file name.");
+        return ExitFailure;
     }
-    while (!(output?.StartsWith("ASSEMBLY", StringComparison.OrdinalIgnoreCase) == true));
 
-    if ((output = await sr.ReadLineAsync().ConfigureAwait(false)) is not null)
+    if (extra.Count > 0)
     {
-        var assemblyFileName = output;
+        Console.WriteLine($"Unknown arguments: {string.Join(" ", extra)}");
+        return ExitFailure;
+    }
+
+    if (outputFilePath is null)
+    {
+        outputFilePath = Path.Combine(Path.GetDirectoryName(assemblyFilePath)!, "testlist.json");
+    }
 
-#if NET6_0_OR_GREATER
-        var resolver = new System.Runtime.Loader.AssemblyDependencyResolver(assemblyFileName);
-        System.Runtime.Loader.AssemblyLoadContext.Default.Resolving += (context, assemblyName) =>
+#if NET
+    var resolver = new System.Runtime.Loader.AssemblyDependencyResolver(assemblyFilePath);
+    System.Runtime.Loader.AssemblyLoadContext.Default.Resolving += (context, assemblyName) =>
+    {
+        var assemblyPath = resolver.ResolveAssemblyToPath(assemblyName);
+        if (assemblyPath is not null)
         {
-            var assemblyPath = resolver.ResolveAssemblyToPath(assemblyName);
-            if (assemblyPath is not null)
-            {
-                return context.LoadFromAssemblyPath(assemblyPath);
-            }
+            return context.LoadFromAssemblyPath(assemblyPath);
+        }
 
-            return null;
-        };
+        return null;
+    };
 #endif
 
-        string testDescriptor = Path.GetFileName(assemblyFileName);
+    string assemblyFileName = Path.GetFileName(assemblyFilePath);
 #if NET
-        testDescriptor += " (.NET Core)";
+    string tfm = "(.NET Core)";
 #else
-    testDescriptor += " (.NET Framework)";
+    string tfm = "(.NET Framework)";
 #endif
 
-        await Console.Out.WriteLineAsync($"Discovering tests in {testDescriptor}...").ConfigureAwait(false);
+    Console.Write($"Discovering tests in {tfm} {assemblyFileName} ... ");
 
-        using var xunit = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyFileName, shadowCopy: false);
-        var configuration = ConfigReader.Load(assemblyFileName, configFileName: null);
-        var sink = new Sink();
-        xunit.Find(includeSourceInformation: false,
-                   messageSink: sink,
-                   discoveryOptions: TestFrameworkOptions.ForDiscovery(configuration));
+    using var xunit = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyFilePath, shadowCopy: false);
+    var configuration = ConfigReader.Load(assemblyFileName, configFileName: null);
+    var sink = new Sink();
+    xunit.Find(includeSourceInformation: false,
+                messageSink: sink,
+                discoveryOptions: TestFrameworkOptions.ForDiscovery(configuration));
 
-        var testsToWrite = new HashSet<string>();
-        await foreach (var fullyQualifiedName in sink.GetTestCaseNamesAsync())
-        {
-            testsToWrite.Add(fullyQualifiedName);
-        }
-
-        if (sink.AnyWriteFailures)
-        {
-            await Console.Error.WriteLineAsync($"Channel failed to write for '{assemblyFileName}'").ConfigureAwait(false);
-            return ExitFailure;
-        }
-
-#if NET6_0_OR_GREATER
-        await Console.Out.WriteLineAsync($"Discovered {testsToWrite.Count} tests in {testDescriptor}").ConfigureAwait(false);
-#else
-        await Console.Out.WriteLineAsync($"Discovered {testsToWrite.Count} tests in {testDescriptor}").ConfigureAwait(false);
-#endif
+    var testsToWrite = new HashSet<string>();
+    await foreach (var fullyQualifiedName in sink.GetTestCaseNamesAsync())
+    {
+        testsToWrite.Add(fullyQualifiedName);
+    }
 
-        var directory = Path.GetDirectoryName(assemblyFileName);
-        using var fileStream = File.Create(Path.Combine(directory!, "testlist.json"));
-        await JsonSerializer.SerializeAsync(fileStream, testsToWrite).ConfigureAwait(false);
-        return ExitSuccess;
+    if (sink.AnyWriteFailures)
+    {
+        Console.WriteLine($"Channel failed to write for '{assemblyFileName}'");
+        return ExitFailure;
     }
 
+    Console.WriteLine($"{testsToWrite.Count} found");
+
+    using var fileStream = new FileStream(outputFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None);
+    await JsonSerializer.SerializeAsync(fileStream, testsToWrite.OrderBy(x => x)).ConfigureAwait(false);
+    return ExitSuccess;
+}
+catch (OptionException e)
+{
+    Console.WriteLine(e.Message);
+    options.WriteOptionDescriptions(Console.Out);
     return ExitFailure;
 }
 catch (Exception ex)
 {
     // Write the exception details to stderr so the host process can pick it up.
-    await Console.Error.WriteLineAsync(ex.ToString()).ConfigureAwait(false);
-    return 1;
+    Console.WriteLine(ex.ToString());
+    return ExitFailure;
 }
 
 file class Sink : IMessageSink
@@ -151,3 +157,4 @@ private void OnTestDiscovered(ITestCaseDiscoveryMessage testCaseDiscovered)
         }
     }
 }
+
diff --git a/src/Tools/TestDiscoveryWorker/README.md b/src/Tools/TestDiscoveryWorker/README.md
@@ -0,0 +1,7 @@
+# TestDiscoveryWorker
+
+This program runs xUnit discovery on an assembly and writes the results to a file. 
+
+```cmd
+> dotnet exec TestDiscoveryWorker.dll --assembly artifacts\bin\Microsoft.CodeAnalysis.UnitTests\Debug\net9.0\Microsoft.CodeAnalysis.UnitTests.dll --out testlist.json
+```
diff --git a/src/Tools/TestDiscoveryWorker/TestDiscoveryWorker.csproj b/src/Tools/TestDiscoveryWorker/TestDiscoveryWorker.csproj
@@ -4,12 +4,14 @@
     <OutputType>Exe</OutputType>
     <TargetFrameworks>$(NetRoslyn);net472</TargetFrameworks>
     <Nullable>enable</Nullable>
+    <SignAssembly>false</SignAssembly>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="xunit.abstractions" />
     <PackageReference Include="xunit.runner.utility" />
     <PackageReference Include="xunit.extensibility.execution" />
+    <PackageReference Include="Mono.Options" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFramework)'=='net472'">