Merge pull request #7881 from unoplatform/dev/jela/chrome-update

Wasm Runtime test reliability updates

build/ci/.azure-devops-android-tests.yml

@@ -212,7 +212,7 @@ jobs:
 
   ## First tests run (https://github.com/unoplatform/uno/issues/6714)
   - bash: |
-      $(build.sourcesdirectory)/build/android-uitest-run.sh
+      $(build.sourcesdirectory)/build/test-scripts/android-uitest-run.sh
 
     displayName: Run Android Tests
 
@@ -237,7 +237,7 @@ jobs:
 
   ## Second tests run (https://github.com/unoplatform/uno/issues/6714)
   - bash: |
-      $(build.sourcesdirectory)/build/android-uitest-run.sh
+      $(build.sourcesdirectory)/build/test-scripts/android-uitest-run.sh
 
     displayName: Run Android Tests
     condition: eq(variables.ALLOW_RERUN, 'true')

build/ci/.azure-devops-ios-tests-run.yml

@@ -71,7 +71,7 @@ jobs:
 
   ## First UI Tests run (https://github.com/unoplatform/uno/issues/6714)
   - bash: |
-      $(build.sourcesdirectory)/build/ios-uitest-run.sh
+      $(build.sourcesdirectory)/build/test-scripts/ios-uitest-run.sh
 
     displayName: Run iOS Simulator Tests
 
@@ -96,7 +96,7 @@ jobs:
 
   ## Second UI Tests run (https://github.com/unoplatform/uno/issues/6714)
   - bash: |
-      $(build.sourcesdirectory)/build/ios-uitest-run.sh
+      $(build.sourcesdirectory)/build/test-scripts/ios-uitest-run.sh
 
     displayName: Run iOS Simulator Tests (re-run)
     condition: eq(variables.UITEST_ALLOW_RERUN, 'true')

build/ci/.azure-devops-ios-tests.yml

@@ -36,7 +36,7 @@ jobs:
       XamarinSDKVersion: ${{ parameters.XamarinSDKVersionBuild }}
 
   - bash: |
-      $(build.sourcesdirectory)/build/ios-uitest-build.sh
+      $(build.sourcesdirectory)/build/test-scripts/ios-uitest-build.sh
 
     displayName: Build iOS App for UI Tests
 

build/ci/.azure-devops-project-template-tests.yml

@@ -32,7 +32,7 @@ jobs:
   - script: dotnet new -i $(System.ArtifactsDirectory)\NugetPackages\vslatest\Uno.ProjectTemplates.Dotnet*.nupkg
     displayName: Install Project Templates
 
-  - powershell: build\run-template-tests.ps1
+  - powershell: build\test-scripts\run-template-tests.ps1
     displayName: Run Project Templates Tests
     env:
       NUGET_CI_CONFIG: $(Build.SourcesDirectory)\src\nuget.ci.config
@@ -63,7 +63,7 @@ jobs:
   - script: dotnet new -i $(System.ArtifactsDirectory)\NugetPackages\vslatest\Uno.ProjectTemplates.Dotnet*.nupkg
     displayName: Install Project Templates
 
-  - powershell: build\run-net6-template-tests.ps1
+  - powershell: build\test-scripts\run-net6-template-tests.ps1
     displayName: Run Project Templates Tests
     env:
       NUGET_CI_CONFIG: $(Build.SourcesDirectory)\src\nuget.ci.net6.config
@@ -97,7 +97,7 @@ jobs:
   - script: dotnet new -i $(System.ArtifactsDirectory)/NugetPackages/vslatest/Uno.ProjectTemplates.Dotnet*.nupkg
     displayName: Install Project Templates
 
-  - pwsh: build/run-template-tests-linux.ps1
+  - pwsh: build/test-scripts/run-template-tests-linux.ps1
     displayName: Run Project Templates Tests
     env:
       NUGET_CI_CONFIG: $(Build.SourcesDirectory)/src/nuget.ci.config

build/ci/.azure-devops-wasm-uitests.yml

@@ -190,8 +190,8 @@ jobs:
 
    ## First ui tests run
   - bash: |
-      chmod +x $BUILD_SOURCESDIRECTORY/build/run-automated-uitests.sh
-      $BUILD_SOURCESDIRECTORY/build/run-automated-uitests.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/build/test-scripts/wasm-run-automated-uitests.sh
+      $BUILD_SOURCESDIRECTORY/build/test-scripts/wasm-run-automated-uitests.sh
 
     env:
       BUILD_SOURCESDIRECTORY: "$(build.sourcesdirectory)"
@@ -211,8 +211,8 @@ jobs:
 
    ## Second ui tests run
   - bash: |
-      chmod +x $BUILD_SOURCESDIRECTORY/build/run-automated-uitests.sh
-      $BUILD_SOURCESDIRECTORY/build/run-automated-uitests.sh
+      chmod +x $BUILD_SOURCESDIRECTORY/build/test-scripts/wasm-run-automated-uitests.sh
+      $BUILD_SOURCESDIRECTORY/build/test-scripts/wasm-run-automated-uitests.sh
 
     env:
       BUILD_SOURCESDIRECTORY: "$(build.sourcesdirectory)"

build/android-uitest-run.sh → build/test-scripts/android-uitest-run.sh

@@ -59,14 +59,14 @@ then
 	nohup $ANDROID_HOME/emulator/emulator -avd "$AVD_NAME" -skin 1280x800 -memory 2048 -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim > /dev/null 2>&1 &
 
 	# Wait for the emulator to finish booting
-	source $BUILD_SOURCESDIRECTORY/build/android-uitest-wait-systemui.sh
+	source $BUILD_SOURCESDIRECTORY/build/test-scripts/android-uitest-wait-systemui.sh
 
 else
 	# Restart the emulator to avoid running first-time tasks
 	$ANDROID_HOME/platform-tools/adb reboot
 
 	# Wait for the emulator to finish booting
-	source $BUILD_SOURCESDIRECTORY/build/android-uitest-wait-systemui.sh
+	source $BUILD_SOURCESDIRECTORY/build/test-scripts/android-uitest-wait-systemui.sh
 fi
 
 # list active devices

build/local-ios-uitest-run.sh → build/test-scripts/local-ios-uitest-run.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-export BUILD_SOURCESDIRECTORY=`pwd`/..
+export BUILD_SOURCESDIRECTORY=`pwd`/../..
 export BUILD_ARTIFACTSTAGINGDIRECTORY=/tmp/uno-uitests-results
 export UITEST_SNAPSHOTS_ONLY=false
 export UITEST_SNAPSHOTS_GROUP=01

build/run-automated-uitests.sh → build/test-scripts/wasm-run-automated-uitests.sh

@@ -5,8 +5,8 @@ IFS=$'\n\t'
 
 cd $BUILD_SOURCESDIRECTORY/build/wasm-uitest-binaries
 
-npm i chromedriver@86.0.0
-npm i puppeteer@5.3.1
+npm i chromedriver@97.0.0
+npm i puppeteer@13.0.1
 
 # install dotnet serve / Remove as needed
 dotnet tool uninstall dotnet-serve -g || true
@@ -16,7 +16,7 @@ export PATH="$PATH:$BUILD_SOURCESDIRECTORY/build/tools"
 
 export UNO_UITEST_TARGETURI=http://localhost:8000
 export UNO_UITEST_DRIVERPATH_CHROME=$BUILD_SOURCESDIRECTORY/build/wasm-uitest-binaries/node_modules/chromedriver/lib/chromedriver
-export UNO_UITEST_CHROME_BINARY_PATH=$BUILD_SOURCESDIRECTORY/build/wasm-uitest-binaries/node_modules/puppeteer/.local-chromium/linux-800071/chrome-linux/chrome
+export UNO_UITEST_CHROME_BINARY_PATH=$BUILD_SOURCESDIRECTORY/build/wasm-uitest-binaries/node_modules/puppeteer/.local-chromium/linux-938248/chrome-linux/chrome
 export UNO_UITEST_SCREENSHOT_PATH=$BUILD_ARTIFACTSTAGINGDIRECTORY/screenshots/wasm-automated-$SITE_SUFFIX-$UITEST_AUTOMATED_GROUP
 export UNO_UITEST_PLATFORM=Browser
 export UNO_UITEST_BENCHMARKS_PATH=$BUILD_ARTIFACTSTAGINGDIRECTORY/benchmarks/wasm-automated

doc/articles/uno-development/creating-ui-tests.md

@@ -104,7 +104,7 @@ Running UI Tests in iOS Simulators on macOS requires, as of VS4Mac 8.4, to build
 In a terminal, run the following:
 ``` bash
 cd build
-./local-ios-uitest-run.sh
+./test-scripts/local-ios-uitest-run.sh
 ```
 
 The Uno.UI solution will build, and the UI tests will run. You may need to adjust some of the parameters in the script, such as:

src/SamplesApp/SamplesApp.UITests/RuntimeTests.cs

@@ -25,51 +25,23 @@ public partial class RuntimeTests : SampleControlUITestBase
 
 		[Test]
 		[AutoRetry(tryCount: 1)]
-		[Timeout(2400000)] // Adjust this timeout based on average test run duration
+		[Timeout(1800000)] // Adjust this timeout based on average test run duration
 		public async Task RunRuntimeTests()
 		{
 			Run("SamplesApp.Samples.UnitTests.UnitTestsPage");
 
-			IAppQuery AllQuery(IAppQuery query)
+			IAppQuery AllQuery(IAppQuery query) 
 				// .All() is not yet supported for wasm.
 				=> AppInitializer.GetLocalPlatform() == Platform.Browser ? query : query.All();
 
 			var runButton = new QueryEx(q => AllQuery(q).Marked("runButton"));
-			var failedTestsCount = new QueryEx(q => AllQuery(q).Marked("failedTestCount"));
 			var failedTests = new QueryEx(q => AllQuery(q).Marked("failedTests"));
 			var failedTestsDetails = new QueryEx(q => AllQuery(q).Marked("failedTestDetails"));
-			var runningState = new QueryEx(q => AllQuery(q).Marked("runningState"));
-			var runTestCount = new QueryEx(q => AllQuery(q).Marked("runTestCount"));
 			var unitTestsControl = new QueryEx(q => AllQuery(q).Marked("UnitTestsRootControl"));
 
 			async Task<bool> IsTestExecutionDone()
 			{
-				return await GetWithRetry("IsTestExecutionDone", () => runningState.GetDependencyPropertyValue("Text")?.ToString().Equals("Finished", StringComparison.OrdinalIgnoreCase) ?? false);
-			}
-
-			async Task<T> GetWithRetry<T>(string logName, Func<T> getter, int timeoutSeconds = 10)
-			{
-				var sw = Stopwatch.StartNew();
-				Exception lastException = null;
-				do
-				{
-					try
-					{
-						return getter();
-					}
-					catch (Exception e)
-					{
-						lastException = e;
-						Console.WriteLine($"{logName} failed with {e.Message}");
-					}
-
-					await Task.Delay(TimeSpan.FromSeconds(.5));
-
-					Console.WriteLine($"{logName} retrying");
-				}
-				while (sw.Elapsed < TimeSpan.FromSeconds(timeoutSeconds));
-
-				throw lastException;
+				return await GetWithRetry("IsTestExecutionDone", () => unitTestsControl.GetDependencyPropertyValue("RunningStateForUITest")?.ToString().Equals("Finished", StringComparison.OrdinalIgnoreCase) ?? false);
 			}
 
 			_app.WaitForElement(runButton);
@@ -81,7 +53,7 @@ async Task<T> GetWithRetry<T>(string logName, Func<T> getter, int timeoutSeconds
 
 			while(DateTimeOffset.Now - lastChange < TestRunTimeout)
 			{
-				var newValue = await GetWithRetry("GetRunTestCount", () => runTestCount.GetDependencyPropertyValue("Text")?.ToString());
+				var newValue = await GetWithRetry("GetRunTestCount", () => unitTestsControl.GetDependencyPropertyValue("RunTestCountForUITest")?.ToString());
 
 				if (lastValue != newValue)
 				{
@@ -103,7 +75,7 @@ async Task<T> GetWithRetry<T>(string logName, Func<T> getter, int timeoutSeconds
 
 			TestContext.AddTestAttachment(ArchiveResults(unitTestsControl), "runtimetests-results.zip");
 
-			var count = GetValue(nameof(failedTestsCount), failedTestsCount);
+			var count = GetValue(nameof(unitTestsControl), unitTestsControl, "FailedTestCountForUITest");
 			if (count != "0")
 			{
 				var tests = GetValue(nameof(failedTests), failedTests)
@@ -112,12 +84,44 @@ async Task<T> GetWithRetry<T>(string logName, Func<T> getter, int timeoutSeconds
 					.ToArray();
 				var details = GetValue(nameof(failedTestsDetails), failedTestsDetails);
 
-				Assert.Fail($"{tests.Length} unit test(s) failed.\n\tFailing Tests:\n{string.Join("", tests)}\n\n---\n\tDetails:\n{details}");
+				Assert.Fail($"{tests.Length} unit test(s) failed (count={count}).\n\tFailing Tests:\n{string.Join("", tests)}\n\n---\n\tDetails:\n{details}");
 			}
 
 			TakeScreenshot("Runtime Tests Results",	ignoreInSnapshotCompare: true);
 		}
 
+		async Task<T> GetWithRetry<T>(string logName, Func<T> getter, int timeoutSeconds = 15)
+		{
+			var sw = Stopwatch.StartNew();
+			Exception lastException = null;
+			do
+			{
+				try
+				{
+					var result = getter();
+
+					if (sw.Elapsed > TimeSpan.FromSeconds(timeoutSeconds / 2))
+					{
+						Console.WriteLine($"{logName} succeeded after retries");
+					}
+
+					return result;
+				}
+				catch (Exception e)
+				{
+					lastException = e;
+					Console.WriteLine($"{logName} failed with {e.Message}");
+				}
+
+				await Task.Delay(TimeSpan.FromSeconds(.5));
+
+				Console.WriteLine($"{logName} retrying");
+			}
+			while (sw.Elapsed < TimeSpan.FromSeconds(timeoutSeconds));
+
+			throw lastException;
+		}
+
 		private static string ArchiveResults(QueryEx unitTestsControl)
 		{
 			var document = GetValue(nameof(unitTestsControl), unitTestsControl, "NUnitTestResultsDocument");

src/SamplesApp/SamplesApp.UITests/SampleControlUITestBase.cs

@@ -18,6 +18,10 @@ public partial class SampleControlUITestBase
 	{
 		protected IApp _app;
 		private static int _totalTestFixtureCount;
+		private static bool _firstRun = true;
+		private DateTime _startTime;
+		private readonly string _screenShotPath = Environment.GetEnvironmentVariable("UNO_UITEST_SCREENSHOT_PATH");
+
 
 		[OneTimeSetUp]
 		public void SingleSetup()
@@ -67,6 +71,8 @@ public void ValidateAppMode()
 		[AutoRetry]
 		public void BeforeEachTest()
 		{
+			_startTime = DateTime.Now;
+
 			ValidateAutoRetry();
 
 			// Check if the test needs to be ignore or not
@@ -123,6 +129,26 @@ public void AfterEachTest()
 			{
 				TakeScreenshot($"{TestContext.CurrentContext.Test.Name} - Tear down on error", ignoreInSnapshotCompare: true);
 			}
+
+			WriteSystemLogs(GetCurrentStepTitle("log"));
+		}
+
+		private void WriteSystemLogs(string fileName)
+		{
+			if (_app != null && AppInitializer.GetLocalPlatform() == Platform.Browser)
+			{
+				var outputPath = string.IsNullOrEmpty(_screenShotPath)
+					? Environment.CurrentDirectory
+					: _screenShotPath;
+
+				using (var logOutput = new StreamWriter(Path.Combine(outputPath, $"{fileName}_{DateTime.Now:yyyy-MM-dd-HH-mm-ss.fff}.txt")))
+				{
+					foreach (var log in _app.GetSystemLogs(_startTime.ToUniversalTime()))
+					{
+						logOutput.WriteLine($"{log.Timestamp}/{log.Level}: {log.Message}");
+					}
+				}
+			}
 		}
 
 		public ScreenshotInfo TakeScreenshot(string stepName, bool? ignoreInSnapshotCompare = null)
@@ -135,20 +161,13 @@ public ScreenshotInfo TakeScreenshot(string stepName, bool? ignoreInSnapshotComp
 
 		public ScreenshotInfo TakeScreenshot(string stepName, ScreenshotOptions options)
 		{
-			if(_app == null)
+			if (_app == null)
 			{
 				Console.WriteLine($"Skipping TakeScreenshot _app is not available");
 				return null;
 			}
 
-			var title = $"{TestContext.CurrentContext.Test.Name}_{stepName}"
-				.Replace(" ", "_")
-				.Replace(".", "_")
-				.Replace("(", "")
-				.Replace(")", "")
-				.Replace("\"", "")
-				.Replace(",", "_")
-				.Replace("__", "_");
+			var title = GetCurrentStepTitle(stepName);
 
 			var fileInfo = _app.Screenshot(title);
 
@@ -175,14 +194,24 @@ public ScreenshotInfo TakeScreenshot(string stepName, ScreenshotOptions options)
 				TestContext.AddTestAttachment(fileInfo.FullName, stepName);
 			}
 
-			if(options != null)
+			if (options != null)
 			{
 				SetOptions(fileInfo, options);
 			}
 
-			return new ScreenshotInfo(fileInfo, stepName) ;
+			return new ScreenshotInfo(fileInfo, stepName);
 		}
 
+		private static string GetCurrentStepTitle(string stepName) =>
+					$"{TestContext.CurrentContext.Test.Name}_{stepName}"
+						.Replace(" ", "_")
+						.Replace(".", "_")
+						.Replace("(", "")
+						.Replace(")", "")
+						.Replace("\"", "")
+						.Replace(",", "_")
+						.Replace("__", "_");
+
 		public void SetOptions(FileInfo screenshot, ScreenshotOptions options)
 		{
 			var fileName = Path
@@ -274,6 +303,12 @@ protected void Run(string metadataName, bool waitForSampleControl = true, bool s
 					: new QueryEx(q => q.All().Marked("sampleControl"));
 
 				_app.WaitForElement(sampleControlQuery, timeout: TimeSpan.FromSeconds(sampleLoadTimeout));
+
+				if (_firstRun)
+				{
+					_firstRun = false;
+					WriteSystemLogs("AppStartup");
+				}
 			}
 
 			var testRunId = _app.InvokeGeneric("browser:SampleRunner|RunTest", metadataName);