Add some tests back pressure for msbuild (#2152)

microsoft · Jan 23, 2024 · 00333e1 · 00333e1
1 parent 761b4cc
commit 00333e1
Show file tree

Hide file tree

Showing 8 changed files with 95 additions and 45 deletions.
diff --git a/...grationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestCliTests.cs b/...grationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestCliTests.cs
@@ -32,7 +32,7 @@ public async Task DotnetTest_Should_Execute_Tests(string tfm, BuildConfiguration
             addPublicFeeds: true);
 
         string binlogFile = Path.Combine(generator.TargetAssetPath, "msbuild.binlog");
-        var compilationResult = await DotnetCli.RunAsync($"test -nodeReuse:false {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
+        var compilationResult = await DotnetCli.RunAsync($"test -m:1 -nodeReuse:false {generator.TargetAssetPath}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
         compilationResult.AssertOutputContains("Passed!  - Failed:     0, Passed:     1, Skipped:     0, Total:     1");
     }
 }
diff --git a/...rationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/IgnoreExitCodeTests.cs b/...rationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/IgnoreExitCodeTests.cs
@@ -108,9 +108,9 @@ public async Task If_IgnoreExitCode_Specified_Should_Return_Success_ExitCode(str
         .PatchCodeWithReplace("$TargetFramework$", tfm)
         .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion));
 
-        var compilationResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
+        var compilationResult = await DotnetCli.RunAsync($"restore -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
         compilationResult = await DotnetCli.RunAsync(
-            $"build -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -r {RID}",
+            $"build -m:1 -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -r {RID}",
             _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
         var testHost = TestInfrastructure.TestHost.LocateFrom(generator.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration);
         var testHostResult = await testHost.ExecuteAsync(

diff --git a/...nTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildMSTestRunnerTests.cs b/...nTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildMSTestRunnerTests.cs
@@ -71,9 +71,9 @@ public async Task MSBuildTestTarget_SingleAndMultiTfm_Should_Run_Solution_Tests(
         }
 
         // Build the solution
-        var restoreResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {solution.SolutionFile} --configfile {nugetFile}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
+        var restoreResult = await DotnetCli.RunAsync($"restore -m:1 -nodeReuse:false {solution.SolutionFile} --configfile {nugetFile}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
         restoreResult.AssertOutputNotContains("An approximate best match of");
-        var testResult = await DotnetCli.RunAsync($"{command} -nodeReuse:false {solution.SolutionFile}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
+        var testResult = await DotnetCli.RunAsync($"{command} -m:1 -nodeReuse:false {solution.SolutionFile}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
 
         if (isMultiTfm)
         {

diff --git a/...egrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSTestRunnerTests.cs b/...egrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSTestRunnerTests.cs
@@ -35,9 +35,9 @@ public async Task EnableMSTestRunner_True_Will_Run_Standalone(string tfm, BuildC
                 .PatchCodeWithReplace("$Extra$", string.Empty),
                 addPublicFeeds: true);
             string binlogFile = Path.Combine(generator.TargetAssetPath, "msbuild.binlog");
-            var compilationResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
+            var compilationResult = await DotnetCli.RunAsync($"restore -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
             compilationResult = await DotnetCli.RunAsync(
-                $"{verb} -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}",
+                $"{verb} -m:1 -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}",
                 _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
             var testHost = TestInfrastructure.TestHost.LocateFrom(generator.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration, verb: verb);
             var testHostResult = await testHost.ExecuteAsync();
@@ -79,9 +79,9 @@ public async Task EnableMSTestRunner_True_WithCustomEntryPoint_Will_Run_Standalo
 """),
             addPublicFeeds: true);
             string binlogFile = Path.Combine(generator.TargetAssetPath, "msbuild.binlog");
-            var compilationResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
+            var compilationResult = await DotnetCli.RunAsync($"restore -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
             compilationResult = await DotnetCli.RunAsync(
-                $"{verb} -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}",
+                $"{verb} -m:1 -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}",
                 _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
             var testHost = TestInfrastructure.TestHost.LocateFrom(generator.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration, verb: verb);
             var testHostResult = await testHost.ExecuteAsync();
@@ -110,10 +110,10 @@ public async Task EnableMSTestRunner_False_Will_Run_Empty_Program_EntryPoint_Fro
         .PatchCodeWithReplace("$Extra$", string.Empty),
         addPublicFeeds: true);
             string binlogFile = Path.Combine(generator.TargetAssetPath, "msbuild.binlog");
-            var compilationResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
+            var compilationResult = await DotnetCli.RunAsync($"restore -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
             try
             {
-                compilationResult = await DotnetCli.RunAsync($"{verb} -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
+                compilationResult = await DotnetCli.RunAsync($"{verb} -m:1 -nodeReuse:false {generator.TargetAssetPath} -c {buildConfiguration} -bl:{binlogFile} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path);
                 var testHost = TestInfrastructure.TestHost.LocateFrom(generator.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration, verb: verb);
                 var testHostResult = await testHost.ExecuteAsync();
                 Assert.AreEqual(string.Empty, testHostResult.StandardOutput);

diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Program.cs
@@ -10,6 +10,8 @@
 using Microsoft.Testing.Platform.CommandLine;
 using Microsoft.Testing.Platform.Extensions.TestHost;
 
+CommandLine.MaxOutstandingCommands = Environment.ProcessorCount;
+
 ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args);
 builder.TestHost.AddTestApplicationLifecycleCallbacks(sp => new GlobalTasks(sp.GetCommandLineOptions()));
 

diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs
@@ -23,6 +23,24 @@ public sealed class CommandLine : IDisposable
 
     public string ErrorOutput => string.Join(Environment.NewLine, _errorOutputLines);
 
+    private static int s_maxOutstandingCommand = Environment.ProcessorCount;
+    private static SemaphoreSlim s_maxOutstandingCommands_semaphore = new(s_maxOutstandingCommand, s_maxOutstandingCommand);
+
+    public static int MaxOutstandingCommands
+    {
+        get
+        {
+            return s_maxOutstandingCommand;
+        }
+
+        set
+        {
+            s_maxOutstandingCommand = value;
+            s_maxOutstandingCommands_semaphore.Dispose();
+            s_maxOutstandingCommands_semaphore = new SemaphoreSlim(s_maxOutstandingCommand, s_maxOutstandingCommand);
+        }
+    }
+
     public async Task RunAsync(
         string commandLine,
         IDictionary<string, string>? environmentVariables = null)
@@ -46,44 +64,52 @@ public async Task<int> RunAsyncAndReturnExitCode(
         bool cleanDefaultEnvironmentVariableIfCustomAreProvided = false,
         int timeoutInSeconds = 60)
     {
-        Interlocked.Increment(ref s_totalProcessesAttempt);
-        string[] tokens = commandLine.Split(' ');
-        string command = tokens[0];
-        string arguments = string.Join(" ", tokens.Skip(1));
-        _errorOutputLines.Clear();
-        _standardOutputLines.Clear();
-        var startInfo = new ProcessConfiguration(command)
-        {
-            Arguments = arguments,
-            EnvironmentVariables = environmentVariables,
-            OnErrorOutput = (_, o) => _errorOutputLines.Add(o),
-            OnStandardOutput = (_, o) => _standardOutputLines.Add(o),
-            WorkingDirectory = workingDirectory,
-        };
-        _process = ProcessFactory.Start(startInfo, cleanDefaultEnvironmentVariableIfCustomAreProvided);
-
-        Task<int> exited = _process.WaitForExitAsync();
-        int seconds = timeoutInSeconds;
-        var stopTheTimer = new CancellationTokenSource();
-        var timedOut = Task.Delay(TimeSpan.FromSeconds(seconds), stopTheTimer.Token);
-        if (await Task.WhenAny(exited, timedOut) == exited)
+        await s_maxOutstandingCommands_semaphore.WaitAsync();
+        try
         {
+            Interlocked.Increment(ref s_totalProcessesAttempt);
+            string[] tokens = commandLine.Split(' ');
+            string command = tokens[0];
+            string arguments = string.Join(" ", tokens.Skip(1));
+            _errorOutputLines.Clear();
+            _standardOutputLines.Clear();
+            var startInfo = new ProcessConfiguration(command)
+            {
+                Arguments = arguments,
+                EnvironmentVariables = environmentVariables,
+                OnErrorOutput = (_, o) => _errorOutputLines.Add(o),
+                OnStandardOutput = (_, o) => _standardOutputLines.Add(o),
+                WorkingDirectory = workingDirectory,
+            };
+            _process = ProcessFactory.Start(startInfo, cleanDefaultEnvironmentVariableIfCustomAreProvided);
+
+            Task<int> exited = _process.WaitForExitAsync();
+            int seconds = timeoutInSeconds;
+            var stopTheTimer = new CancellationTokenSource();
+            var timedOut = Task.Delay(TimeSpan.FromSeconds(seconds), stopTheTimer.Token);
+            if (await Task.WhenAny(exited, timedOut) == exited)
+            {
 #if NET8_0_OR_GREATER
-            await stopTheTimer.CancelAsync();
+                await stopTheTimer.CancelAsync();
 #else
-            stopTheTimer.Cancel();
+                stopTheTimer.Cancel();
 #endif
-            return await exited;
-        }
-        else
-        {
-            _process.Kill();
-            throw new TimeoutException(
-                $"""
+                return await exited;
+            }
+            else
+            {
+                _process.Kill();
+                throw new TimeoutException(
+                    $"""
                 Timeout after {seconds}s on command line: '{commandLine}'
                 STD: {StandardOutput}
                 ERR: {ErrorOutput}
                 """);
+            }
+        }
+        finally
+        {
+            s_maxOutstandingCommands_semaphore.Release();
         }
     }
 

diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/RetryHelper.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/RetryHelper.cs
@@ -1,17 +1,39 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using Polly;
+
 namespace Microsoft.Testing.TestInfrastructure;
 
 public class RetryHelper
 {
     public static async Task RetryAsync(Func<Task> action, uint times, TimeSpan every, Func<Exception, bool>? predicate = null)
     {
-        await action();
+        await Policy.Handle<Exception>(exception =>
+        {
+            if (predicate is null)
+            {
+                return true;
+            }
+
+            return predicate(exception);
+        })
+       .WaitAndRetryAsync((int)times, _ => every)
+       .ExecuteAsync(action);
     }
 
     public static async Task<T> RetryAsync<T>(Func<Task<T>> action, uint times, TimeSpan every, Func<Exception, bool>? predicate = null)
     {
-        return await action();
+        return await Policy.Handle<Exception>(exception =>
+        {
+            if (predicate is null)
+            {
+                return true;
+            }
+
+            return predicate(exception);
+        })
+        .WaitAndRetryAsync((int)times, _ => every)
+        .ExecuteAsync(action);
     }
 }
diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TestAssetFixtureBase.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TestAssetFixtureBase.cs
@@ -28,15 +28,15 @@ public async Task InitializeAsync(InitializationContext context)
         => await Parallel.ForEachAsync(GetAssetsToGenerate(), async (asset, _) =>
         {
             var testAsset = await TestAsset.GenerateAssetAsync(asset.Name, asset.Code);
-            var result = await DotnetCli.RunAsync($"build -nodeReuse:false {testAsset.TargetAssetPath} -c Release", _nugetGlobalPackagesDirectory.Path);
+            var result = await DotnetCli.RunAsync($"build -m:1 -nodeReuse:false {testAsset.TargetAssetPath} -c Release", _nugetGlobalPackagesDirectory.Path);
             testAsset.DotnetResult = result;
             _testAssets.TryAdd(asset.ID, testAsset);
         });
 #else
         => await Task.WhenAll(GetAssetsToGenerate().Select(async asset =>
         {
             var testAsset = await TestAsset.GenerateAssetAsync(asset.Name, asset.Code);
-            var result = await DotnetCli.RunAsync($"build -nodeReuse:false {testAsset.TargetAssetPath} -c Release", _nugetGlobalPackagesDirectory.Path);
+            var result = await DotnetCli.RunAsync($"build -m:1 -nodeReuse:false {testAsset.TargetAssetPath} -c Release", _nugetGlobalPackagesDirectory.Path);
             testAsset.DotnetResult = result;
             _testAssets.TryAdd(asset.ID, testAsset);
         }));