fix(tests): Ensure that UI tests are retried under more conditions

Tests are now retried on Setup/TearDown failures, Timeout and unhandled exceptions

src/SamplesApp/SamplesApp.UITests/SampleControlUITestBase.cs

@@ -46,7 +46,7 @@ static SampleControlUITestBase()
 
 			// Start the app only once, so the tests runs don't restart it
 			// and gain some time for the tests.
-			AppInitializer.ColdStartApp();
+			// AppInitializer.ColdStartApp();
 		}
 
 		public void ValidateAppMode()

src/SamplesApp/SamplesApp.UITests/TestFramework/AutoRetry.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Diagnostics;
 using System.Drawing;
+using System.Threading;
 using NUnit.Framework;
 using NUnit.Framework.Interfaces;
 using NUnit.Framework.Internal;
@@ -12,15 +14,122 @@ namespace SamplesApp.UITests.TestFramework
 	/// maximum number of times.
 	/// </summary>
 	[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false, Inherited = false)]
-	public partial class AutoRetryAttribute : RetryAttribute
+	public partial class AutoRetryAttribute : NUnitAttribute, IRepeatTest, IWrapSetUpTearDown
 	{
+		private readonly int _tryCount;
 
 		/// <summary>
 		/// Construct a <see cref="RetryAttribute" />
 		/// </summary>
 		/// <param name="tryCount">The maximum number of times the test should be run if it fails</param>
-		public AutoRetryAttribute(int tryCount = 3) : base(tryCount)
+		public AutoRetryAttribute(int tryCount = 3)
 		{
+			_tryCount = tryCount;
 		}
+
+		#region IRepeatTest Members
+
+		/// <summary>
+		/// Wrap a command and return the result.
+		/// </summary>
+		/// <param name="command">The command to be wrapped</param>
+		/// <returns>The wrapped command</returns>
+		public TestCommand Wrap(TestCommand command)
+		{
+			return new RetryCommand(command, _tryCount);
+		}
+
+		#endregion
+
+		#region Nested RetryCommand Class
+
+		/// <summary>
+		/// The test command for the <see cref="RetryAttribute"/>
+		/// </summary>
+		public class RetryCommand : DelegatingTestCommand
+		{
+			private readonly int _tryCount;
+
+			/// <summary>
+			/// Initializes a new instance of the <see cref="RetryCommand"/> class.
+			/// </summary>
+			/// <param name="innerCommand">The inner command.</param>
+			/// <param name="tryCount">The maximum number of repetitions</param>
+			public RetryCommand(TestCommand innerCommand, int tryCount)
+				: base(innerCommand)
+			{
+				_tryCount = tryCount;
+			}
+
+			/// <summary>
+			/// Runs the test, saving a TestResult in the supplied TestExecutionContext.
+			/// </summary>
+			/// <param name="context">The context in which the test should run.</param>
+			/// <returns>A TestResult</returns>
+			public override TestResult Execute(TestExecutionContext context)
+			{
+				int count = _tryCount;
+
+				while (count-- > 0)
+				{
+					try
+					{
+						TryResetTimeoutCommand(innerCommand);
+
+						context.CurrentResult = innerCommand.Execute(context);
+					}
+					// Commands are supposed to catch exceptions, but some don't
+					// and we want to look at restructuring the API in the future.
+					catch (Exception ex)
+					{
+						if (context.CurrentResult == null)
+						{
+							context.CurrentResult = context.CurrentTest.MakeTestResult();
+						}
+
+						context.CurrentResult.RecordException(ex);
+					}
+
+					if (context.CurrentResult.ResultState != ResultState.Failure
+						&& context.CurrentResult.ResultState != ResultState.Error
+						&& context.CurrentResult.ResultState != ResultState.SetUpError
+						&& context.CurrentResult.ResultState != ResultState.SetUpFailure)
+					{
+						break;
+					}
+
+					// Clear result for retry
+					if (count > 0)
+					{
+						context.CurrentResult = context.CurrentTest.MakeTestResult();
+						context.CurrentRepeatCount++; // increment Retry count for next iteration. will only happen if we are guaranteed another iteration
+					}
+				}
+
+				return context.CurrentResult;
+			}
+
+			private void TryResetTimeoutCommand(TestCommand innerCommand)
+			{
+				// Apply workaround for https://github.com/nunit/nunit/issues/3284
+				// Resets the command timeout flag on every new test run
+
+				if (innerCommand is TimeoutCommand tc)
+				{
+					if (tc.GetType().GetField(
+						"_commandTimedOut",
+						System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic) is { } timedOut)
+					{
+						timedOut.SetValue(innerCommand, false);
+					}
+					else
+					{
+						throw new Exception("This version of NUnit is not supported");
+					}
+				}
+			}
+		}
+
+		#endregion
 	}
 }

src/SamplesApp/SamplesApp.UITests/TestFramework/TestFrameworkTests.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NUnit.Framework;
+
+namespace SamplesApp.UITests.TestFramework
+{
+	public partial class RetryTests
+	{
+		static int When_Retry_On_Timeout_Count = 0;
+
+		[Test]
+		[AutoRetry]
+		[Timeout(500)]
+		public void When_Retry_On_Timeout()
+		{
+			Console.WriteLine($"Draw_polyline2: {++When_Retry_On_Timeout_Count}");
+			if (When_Retry_On_Timeout_Count < 3)
+			{
+				System.Threading.Thread.Sleep(1000);
+			}
+		}
+
+		static int When_Retry_On_Unhandled_Exception_Count = 0;
+
+		[Test]
+		[AutoRetry]
+		public void When_Retry_On_Unhandled_Exception()
+		{
+			Console.WriteLine($"Draw_polyline2 {++When_Retry_On_Unhandled_Exception_Count}");
+
+			if (When_Retry_On_Unhandled_Exception_Count < 3)
+			{
+				throw new NotImplementedException();
+			}
+		}
+	}
+
+	public partial class RetrySetup
+	{
+		static int Setup_Count = 0;
+
+		[SetUp]
+		public void Setup()
+		{
+			Console.WriteLine($"Setup: {++Setup_Count}");
+
+			if (Setup_Count < 3)
+			{
+				throw new NotImplementedException();
+			}
+		}
+
+		[Test]
+		[AutoRetry]
+		public void When_Success()
+		{
+			Console.WriteLine($"When_Success: {++Setup_Count}");
+		}
+	}
+
+	public partial class RetryTearDown
+	{
+		static int TearDown_Count = 0;
+
+		[TearDown]
+		public void TearDown()
+		{
+			Console.WriteLine($"TearDown: {++TearDown_Count}");
+
+			if (TearDown_Count < 3)
+			{
+				throw new NotImplementedException();
+			}
+		}
+
+		[Test]
+		[AutoRetry]
+		public void When_Success()
+		{
+			Console.WriteLine($"When_Success: {++TearDown_Count}");
+		}
+	}
+}

src/SamplesApp/SamplesApp.UITests/Windows_UI_Xaml_Shapes/Shapes_Tests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Drawing;
 using System.Linq;
 using System.Text;
@@ -15,6 +16,16 @@ namespace SamplesApp.UITests.Windows_UI_Xaml_Shapes
 {
 	public partial class Shapes_Tests : SampleControlUITestBase
 	{
+		[Test]
+		[AutoRetry]
+		[Timeout(5000)]
+		public void Draw_polyline2()
+		{
+			Console.WriteLine("Draw_polyline2");
+			System.Threading.Thread.Sleep(10000);
+		}
+
+
 		[Test]
 		[AutoRetry]
 		public void Draw_polyline()