Skip to content

Commit

Permalink
feat(test): Add runtime tests automatic retry support
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeromelaban
jeromelaban committed Aug 9, 2021
1 parent 35e40d4 commit 8919f17
Show file tree
Hide file tree
Showing 4 changed files with 163 additions and 93 deletions.
2 changes: 2 additions & 0 deletions src/SamplesApp/SamplesApp.UnitTests.Shared/Controls/UnitTest/TestRun.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ private class TestRun
public int Ignored { get; set; }
public int Succeeded { get; set; }
public int Failed { get; set; }

public int CurrentRepeatCount { get; set; }
}
}
}
208 changes: 117 additions & 91 deletions src/SamplesApp/SamplesApp.UnitTests.Shared/Controls/UnitTest/UnitTestsControl.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ namespace Uno.UI.Samples.Tests
{
public sealed partial class UnitTestsControl : UserControl
{
private const int _maxRepeatCount = 3;

private const StringComparison StrComp = StringComparison.InvariantCultureIgnoreCase;
private Task _runner;
private CancellationTokenSource _cts = new CancellationTokenSource();
Expand Down Expand Up @@ -74,7 +76,6 @@ public string NUnitTestResultsDocument
public static readonly DependencyProperty NUnitTestResultsDocumentProperty =
DependencyProperty.Register(nameof(NUnitTestResultsDocument), typeof(string), typeof(UnitTestsControl), new PropertyMetadata(string.Empty));


private void OnRunTests(object sender, RoutedEventArgs e)
{
Interlocked.Exchange(ref _cts, new CancellationTokenSource())?.Cancel(); // cancel any previous CTS
Expand Down Expand Up @@ -202,9 +203,11 @@ void Update()
IsTextSelectionEnabled = true
};

var retriesText = _currentRun.CurrentRepeatCount != 0 ? $" (Retried {_currentRun.CurrentRepeatCount} time(s))" : "";

testResultBlock.Inlines.Add(new Run
{
Text = GetTestResultIcon(testResult) + ' ' + testName,
Text = GetTestResultIcon(testResult) + ' ' + testName + retriesText,
FontSize = 13.5d,
Foreground = new SolidColorBrush(GetTestResultColor(testResult)),
FontWeight = FontWeights.ExtraBold
Expand Down Expand Up @@ -580,131 +583,141 @@ async Task InvokeTestMethod(object[] parameters)
$"{testName}({parameters.Select(p => p?.ToString() ?? "<null>").JoinBy(", ")})";

_currentRun.Run++;
_currentRun.CurrentRepeatCount = 0;

// We await this to make sure the UI is updated before running the test.
// This will help developpers to identify faulty tests when the app is crashing.
await ReportMessage($"Running test {fullTestName}");
ReportTestsResults();

var sw = new Stopwatch();
bool canRetry = true;

try
while (canRetry)
{
if (requiresFullWindow)
canRetry = false;

try
{
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
if (requiresFullWindow)
{
Private.Infrastructure.TestServices.WindowHelper.UseActualWindowRoot = true;
Private.Infrastructure.TestServices.WindowHelper.SaveOriginalWindowContent();
});
}
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
Private.Infrastructure.TestServices.WindowHelper.UseActualWindowRoot = true;
Private.Infrastructure.TestServices.WindowHelper.SaveOriginalWindowContent();
});
}

object returnValue = null;
if (runsOnUIThread)
{
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
object returnValue = null;
if (runsOnUIThread)
{
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
sw.Start();
testClassInfo.Initialize?.Invoke(instance, new object[0]);
returnValue = testMethod.Invoke(instance, parameters);
sw.Stop();
});
}
else
{
sw.Start();
testClassInfo.Initialize?.Invoke(instance, new object[0]);
returnValue = testMethod.Invoke(instance, parameters);
sw.Stop();
});
}
else
{
sw.Start();
testClassInfo.Initialize?.Invoke(instance, new object[0]);
returnValue = testMethod.Invoke(instance, parameters);
sw.Stop();
}
}

if (testMethod.ReturnType == typeof(Task))
{
var task = (Task)returnValue;
var timeoutTask = Task.Delay(DefaultUnitTestTimeout);
if (testMethod.ReturnType == typeof(Task))
{
var task = (Task)returnValue;
var timeoutTask = Task.Delay(DefaultUnitTestTimeout);

var resultingTask = await Task.WhenAny(task, timeoutTask);
var resultingTask = await Task.WhenAny(task, timeoutTask);

if (resultingTask == timeoutTask)
{
throw new TimeoutException(
$"Test execution timed out after {DefaultUnitTestTimeout}");
}
if (resultingTask == timeoutTask)
{
throw new TimeoutException(
$"Test execution timed out after {DefaultUnitTestTimeout}");
}

if (resultingTask.Exception != null)
{
throw resultingTask.Exception;
if (resultingTask.Exception != null)
{
throw resultingTask.Exception;
}
}
}

var console = testConsoleOutput?.GetContentAndReset();
var console = testConsoleOutput?.GetContentAndReset();

if (expectedException == null)
{
_currentRun.Succeeded++;
ReportTestResult(fullTestName, sw.Elapsed, TestResult.Passed, console: console);
if (expectedException == null)
{
_currentRun.Succeeded++;
ReportTestResult(fullTestName, sw.Elapsed, TestResult.Passed, console: console);
}
else
{
_currentRun.Failed++;
ReportTestResult(fullTestName, sw.Elapsed, TestResult.Failed,
message: $"Test did not throw the excepted exception of type {expectedException.ExceptionType.Name}",
console: console);
}
}
else
catch (Exception e)
{
_currentRun.Failed++;
ReportTestResult(fullTestName, sw.Elapsed, TestResult.Failed,
message: $"Test did not throw the excepted exception of type {expectedException.ExceptionType.Name}",
console: console);
}
}
catch (Exception e)
{
sw.Stop();
sw.Stop();

if (e is AggregateException agg)
{
e = agg.InnerExceptions.FirstOrDefault();
}
if (e is AggregateException agg)
{
e = agg.InnerExceptions.FirstOrDefault();
}

if (e is TargetInvocationException tie)
{
e = tie.InnerException;
}
if (e is TargetInvocationException tie)
{
e = tie.InnerException;
}

var console = testConsoleOutput?.GetContentAndReset();
var console = testConsoleOutput?.GetContentAndReset();

if (e is AssertInconclusiveException inconclusiveException)
{
_currentRun.Ignored++;
ReportTestResult(fullTestName, sw.Elapsed, TestResult.Skipped, message: e.Message, console: console);
}
else if (expectedException == null || !expectedException.ExceptionType.IsInstanceOfType(e))
{
_currentRun.Failed++;
ReportTestResult(fullTestName, sw.Elapsed, TestResult.Failed, e, console: console);
}
else
{
_currentRun.Succeeded++;
ReportTestResult(fullTestName, sw.Elapsed, TestResult.Passed, e, console: console);
if (e is AssertInconclusiveException inconclusiveException)
{
_currentRun.Ignored++;
ReportTestResult(fullTestName, sw.Elapsed, TestResult.Skipped, message: e.Message, console: console);
}
else if (expectedException == null || !expectedException.ExceptionType.IsInstanceOfType(e))
{
if (_currentRun.CurrentRepeatCount < _maxRepeatCount)
{
_currentRun.CurrentRepeatCount++;
canRetry = true;

RunCleanup(instance, testClassInfo, testConsoleOutput, testName);
}
else
{
_currentRun.Failed++;
ReportTestResult(fullTestName, sw.Elapsed, TestResult.Failed, e, console: console);
}
}
else
{
_currentRun.Succeeded++;
ReportTestResult(fullTestName, sw.Elapsed, TestResult.Passed, e, console: console);
}
}
}
finally
{
if (requiresFullWindow)
finally
{
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
if (requiresFullWindow)
{
Private.Infrastructure.TestServices.WindowHelper.RestoreOriginalWindowContent();
Private.Infrastructure.TestServices.WindowHelper.UseActualWindowRoot = false;
});
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
Private.Infrastructure.TestServices.WindowHelper.RestoreOriginalWindowContent();
Private.Infrastructure.TestServices.WindowHelper.UseActualWindowRoot = false;
});
}
}
}
}

try
{
testClassInfo.Cleanup?.Invoke(instance, new object[0]);
}
catch (Exception e)
{
_currentRun.Failed++;
ReportTestResult(testName + " Cleanup", TimeSpan.Zero, TestResult.Failed, e, console: testConsoleOutput.GetContentAndReset());
}
RunCleanup(instance, testClassInfo, testConsoleOutput, testName);

if (ct.IsCancellationRequested)
{
Expand All @@ -719,6 +732,19 @@ await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
}
}

private void RunCleanup(object instance, UnitTestClassInfo testClassInfo, CustomConsoleOutput testConsoleOutput, string testName)
{
try
{
testClassInfo.Cleanup?.Invoke(instance, new object[0]);
}
catch (Exception e)
{
_currentRun.Failed++;
ReportTestResult(testName + " Cleanup", TimeSpan.Zero, TestResult.Failed, e, console: testConsoleOutput.GetContentAndReset());
}
}

private bool HasCustomAttribute<T>(MemberInfo testMethod)
=> testMethod.GetCustomAttribute(typeof(T)) != null;

Expand Down
42 changes: 42 additions & 0 deletions src/Uno.UI.RuntimeTests/Tests/UnitTestsTests/Given_UnitTest.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
using System;
using System.Collections.Generic;
using System.Text;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace Uno.UI.RuntimeTests.Tests.UnitTestsTests
{
[TestClass]
class Given_UnitTest
{
static int When_UnhandledException_Count = 0;

[TestMethod]
public void When_UnhandledException()
{
if (When_UnhandledException_Count++ < 2)
{
throw new InvalidOperationException();
}
}
}

[TestClass]
class Given_UnitTest_Initialize
{
static int Initialize_Count = 0;

[TestInitialize]
public void Initialize()
{
if (Initialize_Count++ < 2)
{
throw new InvalidOperationException();
}
}

[TestMethod]
public void When_Success()
{
}
}
}
4 changes: 2 additions & 2 deletions src/Uno.UI.RuntimeTests/Uno.UI.RuntimeTests.Wasm.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
<TargetFrameworks>netstandard2.0</TargetFrameworks>
</PropertyGroup>

<Import Project="../netcore-build.props"/>
<Import Project="../targetframework-override.props"/>
<Import Project="../netcore-build.props" />
<Import Project="../targetframework-override.props" />

<PropertyGroup>
<AssemblyName>Uno.UI.RuntimeTests</AssemblyName>
Expand Down

0 comments on commit 8919f17

Please sign in to comment.