Skip to content

Commit

Permalink
prefactor for grouping huge number of tests + fix build break
Browse files Browse the repository at this point in the history
  • Loading branch information
@unrenormalizable
unrenormalizable committed Dec 3, 2024
1 parent 38ff6a0 commit 6dc4b13
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 43 deletions.
6 changes: 6 additions & 0 deletions src/RustAnalyzer.TestAdapter/Common/StringExtensions.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
Expand Down Expand Up @@ -41,4 +42,9 @@ public static string RegexReplace(this string @this, string pattern, string repl
{
return Regex.Replace(@this, pattern, replacement);
}

public static IEnumerable<IEnumerable<string>> PartitionBasedOnMaxCombinedLength(this IEnumerable<string> @this, int maxLength)
{
return new[] { @this };
}
}
88 changes: 46 additions & 42 deletions src/RustAnalyzer.TestAdapter/TestExecutor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,7 @@ public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrame
.Select(async x =>
{
var (c, tcs) = await x;
var ts = c.TestExes.Select(async exe => await RunAndRecordTestResultsFromOneExe(exe, tcs, TestRunParams.FromContainer(c), runContext.IsBeingDebugged, frameworkHandle, tl, ct));
Task.WaitAll(ts.ToArray());
c.TestExes.ForEach(exe => RunAndRecordTestResultsFromOneExe(exe, tcs, TestRunParams.FromContainer(c), runContext.IsBeingDebugged, frameworkHandle, tl, ct));
});

Task.WaitAll(tasks.ToArray());
Expand All @@ -61,7 +60,7 @@ public static async Task RunTestsTestsFromOneSourceAsync(TestContainer container
{
foreach (var (tsi, tcs) in await container.DiscoverTestCasesFromOneSourceAsync(tl, ct))
{
await RunAndRecordTestResultsFromOneExe(tsi.Exe, tcs, TestRunParams.FromContainer(container), runContext.IsBeingDebugged, fh, tl, ct);
RunAndRecordTestResultsFromOneExe(tsi.Exe, tcs, TestRunParams.FromContainer(container), runContext.IsBeingDebugged, fh, tl, ct);
}
}

Expand All @@ -70,15 +69,30 @@ public void Cancel()
_cancelled = true;
}

private static async Task RunAndRecordTestResultsFromOneExe(PathEx exe, IEnumerable<TestCase> testCases, TestRunParams trp, bool isBeingDebugged, IFrameworkHandle fh, TL tl, CancellationToken ct)
private static void RunAndRecordTestResultsFromOneExe(PathEx exe, IEnumerable<TestCase> testCases, TestRunParams trp, bool isBeingDebugged, IFrameworkHandle fh, TL tl, CancellationToken ct)
{
tl.L.WriteLine("RunTestsFromOneExe starting with {0}, {1}, {2}", trp.Source, exe, testCases.Count());
if (!testCases.Any())
{
tl.L.WriteError("RunTestsFromOneSourceAsync: Something has gone wrong. Asking to run empty set of test cases. {0}, {1}", trp.Source, exe);
}

try
{
var testResults = await RunTestsFromOneExe(exe, testCases, trp, tl, isBeingDebugged, fh, ct);
foreach (var testResult in testResults)
{
fh.RecordResult(testResult);
}
var envDict = trp.TestExecutionEnvironment.OverrideProcessEnvironment();
var testCasesMap = testCases.ToImmutableDictionary(x => x.FullyQualifiedNameRustFormat());
var args = testCases.Select(tc => tc.FullyQualifiedNameRustFormat());
var grps = args
.PartitionBasedOnMaxCombinedLength(20000)
.Select(x =>
x
.Concat(new[] { "--exact", "--format", "json", "-Zunstable-options", "--report-time" })
.Concat(trp.AdditionalTestExecutionArguments.FromNullSeparatedArray()));
Parallel.Invoke(
grps
.Select(args => RunTestsFromOneExe(exe, args.ToArray(), testCasesMap, envDict, tl, isBeingDebugged, fh, ct))
.Select(t => (Action)(() => t.Wait()))
.ToArray());
}
catch (Exception e)
{
Expand All @@ -88,21 +102,10 @@ private static async Task RunAndRecordTestResultsFromOneExe(PathEx exe, IEnumera
}
}

private static async Task<IEnumerable<TestResult>> RunTestsFromOneExe(PathEx exe, IEnumerable<TestCase> testCases, TestRunParams trp, TL tl, bool isBeingDebugged, IFrameworkHandle fh, CancellationToken ct)
private static async Task RunTestsFromOneExe(PathEx exe, string[] args, IReadOnlyDictionary<string, TestCase> testCasesMap, IDictionary<string, string> envDict, TL tl, bool isBeingDebugged, IFrameworkHandle fh, CancellationToken ct)
{
tl.L.WriteLine("RunTestsFromOneExe starting with {0}, {1}", trp.Source, exe);
if (!testCases.Any())
{
tl.L.WriteError("RunTestsFromOneSourceAsync: Something has gone wrong. Asking to run empty set of test cases. {0}, {1}", trp.Source, exe);
}

var args = testCases
.Select(tc => tc.FullyQualifiedNameRustFormat())
.Concat(new[] { "--exact", "--format", "json", "-Zunstable-options", "--report-time" })
.Concat(trp.AdditionalTestExecutionArguments.FromNullSeparatedArray())
.ToArray();
var envDict = trp.TestExecutionEnvironment.OverrideProcessEnvironment();
tl.T.TrackEvent("RunTestsFromOneSourceAsync", ("IsBeingDebugged", $"{isBeingDebugged}"), ("Args", string.Join("|", args)), ("Env", trp.TestExecutionEnvironment.ReplaceNullWithBar()));
tl.T.TrackEvent("RunTestsFromOneSourceAsync", ("IsBeingDebugged", $"{isBeingDebugged}"), ("Args", string.Join("|", args)));
var trs = Enumerable.Empty<TestResult>();
if (isBeingDebugged)
{
tl.L.WriteLine("RunTestsFromOneSourceAsync launching test under debugger.");
Expand All @@ -111,30 +114,31 @@ private static async Task<IEnumerable<TestResult>> RunTestsFromOneExe(PathEx exe
{
tl.L.WriteError("RunTestsFromOneSourceAsync launching test under debugger - returned {0}.", rc);
}
}
else
{
using var testExeProc = await ProcessRunner.RunWithLogging(exe, args, exe.GetDirectoryName(), envDict, ct, tl.L, @throw: false);
trs = testExeProc.StandardOutputLines
.Skip(1)
.Take(testExeProc.StandardOutputLines.Count() - 2)
.Select(JsonConvert.DeserializeObject<TestRunInfo>)
.Where(x => x.Event != TestRunInfo.EventType.Started)
.OrderBy(x => x.FQN)
.Select(x => ToTestResult(exe, x, testCasesMap));
var ec = testExeProc.ExitCode ?? 0;
if (ec != 0 && !trs.Any())
{
tl.L.WriteError("RunTestsFromOneSourceAsync test executable exited with code {0}.", ec);
throw new ApplicationException($"Test executable returned {ec}. Check above for the arguments passed to test executable by running it on the command line.");
}

return Enumerable.Empty<TestResult>();
tl.T.TrackEvent("RunTestsFromOneSourceAsync", ("Results", $"{trs.Count()}"));
}

using var testExeProc = await ProcessRunner.RunWithLogging(exe, args, exe.GetDirectoryName(), envDict, ct, tl.L, @throw: false);

var testCasesMap = testCases.ToImmutableDictionary(x => x.FullyQualifiedNameRustFormat());
var tris = testExeProc.StandardOutputLines
.Skip(1)
.Take(testExeProc.StandardOutputLines.Count() - 2)
.Select(JsonConvert.DeserializeObject<TestRunInfo>)
.Where(x => x.Event != TestRunInfo.EventType.Started)
.OrderBy(x => x.FQN)
.Select(x => ToTestResult(exe, x, testCasesMap));
var ec = testExeProc.ExitCode ?? 0;
if (ec != 0 && !tris.Any())
foreach (var tr in trs)
{
tl.L.WriteError("RunTestsFromOneSourceAsync test executable exited with code {0}.", ec);
throw new ApplicationException($"Test executable returned {ec}. Check above for the arguments passed to test executable by running it on the command line.");
fh.RecordResult(tr);
}

tl.T.TrackEvent("RunTestsFromOneSourceAsync", ("Results", $"{tris.Count()}"));

return tris;
}

private static TestResult ToTestResult(PathEx exe, TestRunInfo tri, IReadOnlyDictionary<string, TestCase> testCasesMap)
Expand Down
2 changes: 1 addition & 1 deletion src/TestProjects/workspace_with_tests/integrationtests.approved.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
[Passed] add_one.tests.multiplication_tests.when_operands_are_swapped
[Passed] add_one.tests.should_fail
[Passed] add_one.tests.should_success
[Failed] adder.tests.it_works_failing thread 'tests::it_works_failing' panicked at adder\src/main.rs:34:9:
[Failed] adder.tests.it_works_failing thread 'tests::it_works_failing' panicked at adder\src\main.rs:34:9:
assertion `left == right` failed
left: 3
right: 5
Expand Down

0 comments on commit 6dc4b13

Please sign in to comment.