From a9c49ab6a705667a1c7922b15a15c11f11409753 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 13 Mar 2020 11:27:36 +0100 Subject: [PATCH 01/30] Move AppInstallMonitorLog --- tests/xharness/AppInstallMonitorLog.cs | 111 +++++++++++++++++++++++++ tests/xharness/AppRunner.cs | 106 +---------------------- 2 files changed, 112 insertions(+), 105 deletions(-) create mode 100644 tests/xharness/AppInstallMonitorLog.cs diff --git a/tests/xharness/AppInstallMonitorLog.cs b/tests/xharness/AppInstallMonitorLog.cs new file mode 100644 index 000000000000..f93277f49d2f --- /dev/null +++ b/tests/xharness/AppInstallMonitorLog.cs @@ -0,0 +1,111 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Threading; +using Xharness.Logging; + +namespace Xharness { + // Monitor the output from 'mlaunch --installdev' and cancel the installation if there's no output for 1 minute. + class AppInstallMonitorLog : Log { + public override string FullPath => copy_to.FullPath; + + ILog copy_to; + CancellationTokenSource cancellation_source; + + public bool CopyingApp; + public bool CopyingWatchApp; + public TimeSpan AppCopyDuration; + public TimeSpan WatchAppCopyDuration; + public Stopwatch AppCopyStart = new Stopwatch (); + public Stopwatch WatchAppCopyStart = new Stopwatch (); + public int AppPercentComplete; + public int WatchAppPercentComplete; + public long AppBytes; + public long WatchAppBytes; + public long AppTotalBytes; + public long WatchAppTotalBytes; + + public CancellationToken CancellationToken { + get { + return cancellation_source.Token; + } + } + + public AppInstallMonitorLog (ILog copy_to) + : base (copy_to.Logs, $"Watch transfer log for {copy_to.Description}") + { + this.copy_to = copy_to; + cancellation_source = new CancellationTokenSource (); + cancellation_source.Token.Register (() => { + copy_to.WriteLine ("App installation cancelled: it timed out after no output for 1 minute."); + }); + } + + public override Encoding Encoding => copy_to.Encoding; + public override void Flush () + { + copy_to.Flush (); + } + + public override StreamReader GetReader () + { + return copy_to.GetReader (); + } + + protected override void Dispose (bool disposing) + { + base.Dispose (disposing); + copy_to.Dispose (); + cancellation_source.Dispose (); + } + + void ResetTimer () + { + cancellation_source.CancelAfter (TimeSpan.FromMinutes (1)); + } + + public override void WriteLine (string value) + { + var v = value.Trim (); + if (v.StartsWith ("Installing application bundle", StringComparison.Ordinal)) { + if (!CopyingApp) { + CopyingApp = true; + AppCopyStart.Start (); + } else if (!CopyingWatchApp) { + CopyingApp = false; + CopyingWatchApp = true; + AppCopyStart.Stop (); + WatchAppCopyStart.Start (); + } + } else if (v.StartsWith ("PercentComplete: ", StringComparison.Ordinal) && int.TryParse (v.Substring ("PercentComplete: ".Length).Trim (), out var percent)) { + if (CopyingApp) + AppPercentComplete = percent; + else if (CopyingWatchApp) + WatchAppPercentComplete = percent; + } else if (v.StartsWith ("NumBytes: ", StringComparison.Ordinal) && int.TryParse (v.Substring ("NumBytes: ".Length).Trim (), out var num_bytes)) { + if (CopyingApp) { + AppBytes = num_bytes; + AppCopyDuration = AppCopyStart.Elapsed; + } else if (CopyingWatchApp) { + WatchAppBytes = num_bytes; + WatchAppCopyDuration = WatchAppCopyStart.Elapsed; + } + } else if (v.StartsWith ("TotalBytes: ", StringComparison.Ordinal) && int.TryParse (v.Substring ("TotalBytes: ".Length).Trim (), out var total_bytes)) { + if (CopyingApp) + AppTotalBytes = total_bytes; + else if (CopyingWatchApp) + WatchAppTotalBytes = total_bytes; + } + + ResetTimer (); + + copy_to.WriteLine (value); + } + + public override void Write (byte [] buffer, int offset, int count) + { + copy_to.Write (buffer, offset, count); + } + } +} diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index a358aef81b45..6b744a366d0a 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -16,8 +16,7 @@ using Xharness.Logging; using Xharness.Utilities; -namespace Xharness -{ +namespace Xharness { public enum AppRunnerTarget { None, @@ -913,107 +912,4 @@ public static void AddDeviceName (IList args, string device_name) } } } - - // Monitor the output from 'mlaunch --installdev' and cancel the installation if there's no output for 1 minute. - class AppInstallMonitorLog : Log { - public override string FullPath => copy_to.FullPath; - - ILog copy_to; - CancellationTokenSource cancellation_source; - - public bool CopyingApp; - public bool CopyingWatchApp; - public TimeSpan AppCopyDuration; - public TimeSpan WatchAppCopyDuration; - public Stopwatch AppCopyStart = new Stopwatch (); - public Stopwatch WatchAppCopyStart = new Stopwatch (); - public int AppPercentComplete; - public int WatchAppPercentComplete; - public long AppBytes; - public long WatchAppBytes; - public long AppTotalBytes; - public long WatchAppTotalBytes; - - public CancellationToken CancellationToken { - get { - return cancellation_source.Token; - } - } - - public AppInstallMonitorLog (ILog copy_to) - : base (copy_to.Logs, $"Watch transfer log for {copy_to.Description}") - { - this.copy_to = copy_to; - cancellation_source = new CancellationTokenSource (); - cancellation_source.Token.Register (() => { - copy_to.WriteLine ("App installation cancelled: it timed out after no output for 1 minute."); - }); - } - - public override Encoding Encoding => copy_to.Encoding; - public override void Flush () - { - copy_to.Flush (); - } - - public override StreamReader GetReader () - { - return copy_to.GetReader (); - } - - protected override void Dispose (bool disposing) - { - base.Dispose (disposing); - copy_to.Dispose (); - cancellation_source.Dispose (); - } - - void ResetTimer () - { - cancellation_source.CancelAfter (TimeSpan.FromMinutes (1)); - } - - public override void WriteLine (string value) - { - var v = value.Trim (); - if (v.StartsWith ("Installing application bundle", StringComparison.Ordinal)) { - if (!CopyingApp) { - CopyingApp = true; - AppCopyStart.Start (); - } else if (!CopyingWatchApp) { - CopyingApp = false; - CopyingWatchApp = true; - AppCopyStart.Stop (); - WatchAppCopyStart.Start (); - } - } else if (v.StartsWith ("PercentComplete: ", StringComparison.Ordinal) && int.TryParse (v.Substring ("PercentComplete: ".Length).Trim (), out var percent)) { - if (CopyingApp) - AppPercentComplete = percent; - else if (CopyingWatchApp) - WatchAppPercentComplete = percent; - } else if (v.StartsWith ("NumBytes: ", StringComparison.Ordinal) && int.TryParse (v.Substring ("NumBytes: ".Length).Trim (), out var num_bytes)) { - if (CopyingApp) { - AppBytes = num_bytes; - AppCopyDuration = AppCopyStart.Elapsed; - } else if (CopyingWatchApp) { - WatchAppBytes = num_bytes; - WatchAppCopyDuration = WatchAppCopyStart.Elapsed; - } - } else if (v.StartsWith ("TotalBytes: ", StringComparison.Ordinal) && int.TryParse (v.Substring ("TotalBytes: ".Length).Trim (), out var total_bytes)) { - if (CopyingApp) - AppTotalBytes = total_bytes; - else if (CopyingWatchApp) - WatchAppTotalBytes = total_bytes; - } - - ResetTimer (); - - copy_to.WriteLine (value); - } - - public override void Write (byte [] buffer, int offset, int count) - { - copy_to.Write (buffer, offset, count); - } - } } From ba7f37f850043d3ea2e6a4e0a750cc2ae195fa63 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 13 Mar 2020 11:29:35 +0100 Subject: [PATCH 02/30] Move Helpers.GenerateStableGuid --- .../BCLTestImportTargetFactory.cs | 3 +- tests/xharness/Harness.cs | 20 ------------- tests/xharness/Targets/MacTarget.cs | 2 +- .../xharness/Targets/TodayExtensionTarget.cs | 2 +- tests/xharness/Targets/WatchOSTarget.cs | 4 +-- tests/xharness/Utilities/Helpers.cs | 29 +++++++++++++++++++ tests/xharness/xharness.csproj | 13 ++------- 7 files changed, 38 insertions(+), 35 deletions(-) create mode 100644 tests/xharness/Utilities/Helpers.cs diff --git a/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs b/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs index e51a62a54911..58938cf609ae 100644 --- a/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs +++ b/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Collections.Generic; +using Xharness.Utilities; namespace Xharness.BCLTestImporter { // Class that is use as the connection between xharness and the BCLImporter @@ -24,7 +25,7 @@ public BCLTestImportTargetFactory (Harness harness) iOSMonoSDKPath = Harness.MONO_IOS_SDK_DESTDIR, MacMonoSDKPath = Harness.MONO_MAC_SDK_DESTDIR, Override = true, - GuidGenerator = Harness.NewStableGuid, + GuidGenerator = Helpers.GenerateStableGuid, GroupTests = Harness.InCI || Harness.UseGroupedApps, }; } diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 7c7396815db5..1df754624f5b 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -759,26 +759,6 @@ public void Save (string doc, string path) } } - // We want guids that nobody else has, but we also want to generate the same guid - // on subsequent invocations (so that csprojs don't change unnecessarily, which is - // annoying when XS reloads the projects, and also causes unnecessary rebuilds). - // Nothing really breaks when the sequence isn't identical from run to run, so - // this is just a best minimal effort. - static Random guid_generator = new Random (unchecked ((int) 0xdeadf00d)); - public Guid NewStableGuid (string seed = null) - { - var bytes = new byte [16]; - if (seed == null) { - guid_generator.NextBytes (bytes); - } else { - using (var provider = MD5.Create ()) { - var inputBytes = Encoding.UTF8.GetBytes (seed); - bytes = provider.ComputeHash (inputBytes); - } - } - return new Guid (bytes); - } - bool? disable_watchos_on_wrench; public bool DisableWatchOSOnWrench { get { diff --git a/tests/xharness/Targets/MacTarget.cs b/tests/xharness/Targets/MacTarget.cs index fc2b703d3f91..4098a2dc14d6 100644 --- a/tests/xharness/Targets/MacTarget.cs +++ b/tests/xharness/Targets/MacTarget.cs @@ -182,7 +182,7 @@ protected override void PostProcessExecutableProject () { base.PostProcessExecutableProject (); - ProjectGuid = "{" + Harness.NewStableGuid ().ToString ().ToUpper () + "}"; + ProjectGuid = "{" + Helpers.GenerateStableGuid ().ToString ().ToUpper () + "}"; inputProject.SetProjectGuid (ProjectGuid); } } diff --git a/tests/xharness/Targets/TodayExtensionTarget.cs b/tests/xharness/Targets/TodayExtensionTarget.cs index 99e138302dc6..832a79816476 100644 --- a/tests/xharness/Targets/TodayExtensionTarget.cs +++ b/tests/xharness/Targets/TodayExtensionTarget.cs @@ -63,7 +63,7 @@ void CreateTodayContainerProject () csproj.SetProjectReferenceInclude ("TodayExtension.csproj", TodayExtensionProjectPath.Replace ('/', '\\')); csproj.FixCompileInclude ("Main.cs", Path.Combine (Harness.TodayContainerTemplate, "Main.cs").Replace ('/', '\\')); csproj.FixInfoPListInclude (suffix, IsGeneratedBclTest ? GeneratedPath : null); - TodayContainerGuid = "{" + Harness.NewStableGuid ().ToString ().ToUpper () + "}"; + TodayContainerGuid = "{" + Helpers.GenerateStableGuid ().ToString ().ToUpper () + "}"; ProjectGuid = TodayContainerGuid; csproj.SetProjectGuid (TodayContainerGuid); if (MonoNativeInfo != null) { diff --git a/tests/xharness/Targets/WatchOSTarget.cs b/tests/xharness/Targets/WatchOSTarget.cs index 53b816326d02..426f0ca13353 100644 --- a/tests/xharness/Targets/WatchOSTarget.cs +++ b/tests/xharness/Targets/WatchOSTarget.cs @@ -38,7 +38,7 @@ void CreateWatchOSAppProject () csproj.FindAndReplace ("%WATCHEXTENSION_CSPROJ%", Path.GetFileName (WatchOSExtensionProjectPath)); csproj.SetProjectReferenceValue (Path.GetFileName (WatchOSExtensionProjectPath), "Project", WatchOSExtensionGuid); csproj.SetProjectReferenceValue (Path.GetFileName (WatchOSExtensionProjectPath), "Name", Path.GetFileNameWithoutExtension (WatchOSExtensionProjectPath)); - WatchOSAppGuid = "{" + Harness.NewStableGuid ().ToString ().ToUpper () + "}"; + WatchOSAppGuid = "{" + Helpers.GenerateStableGuid ().ToString ().ToUpper () + "}"; csproj.SetProjectGuid (WatchOSAppGuid); csproj.FixInfoPListInclude (suffix); if (MonoNativeInfo != null) { @@ -66,7 +66,7 @@ void CreateWatchOSContainerProject () csproj.FindAndReplace ("%CONTAINER_PATH%", Path.GetFullPath (Harness.WatchOSContainerTemplate).Replace ('/', '\\') + "\\"); csproj.FindAndReplace ("%WATCHAPP_CSPROJ%", Path.GetFileName (WatchOSAppProjectPath)); csproj.SetProjectReferenceValue (Path.GetFileName (WatchOSAppProjectPath), "Name", Path.GetFileNameWithoutExtension (WatchOSAppProjectPath)); - WatchOSGuid = "{" + Harness.NewStableGuid ().ToString ().ToUpper () + "}"; + WatchOSGuid = "{" + Helpers.GenerateStableGuid ().ToString ().ToUpper () + "}"; csproj.SetProjectGuid (WatchOSGuid); csproj.FixInfoPListInclude (Suffix); if (MonoNativeInfo != null) { diff --git a/tests/xharness/Utilities/Helpers.cs b/tests/xharness/Utilities/Helpers.cs new file mode 100644 index 000000000000..b1a31117744f --- /dev/null +++ b/tests/xharness/Utilities/Helpers.cs @@ -0,0 +1,29 @@ +using System; +using System.Security.Cryptography; +using System.Text; + +namespace Xharness.Utilities { + + static class Helpers { + + // We want guids that nobody else has, but we also want to generate the same guid + // on subsequent invocations (so that csprojs don't change unnecessarily, which is + // annoying when XS reloads the projects, and also causes unnecessary rebuilds). + // Nothing really breaks when the sequence isn't identical from run to run, so + // this is just a best minimal effort. + static Random guid_generator = new Random (unchecked ((int) 0xdeadf00d)); + public static Guid GenerateStableGuid (string seed = null) + { + var bytes = new byte [16]; + if (seed == null) { + guid_generator.NextBytes (bytes); + } else { + using (var provider = MD5.Create ()) { + var inputBytes = Encoding.UTF8.GetBytes (seed); + bytes = provider.ComputeHash (inputBytes); + } + } + return new Guid (bytes); + } + } +} diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index f487c2ccbf5d..5ebcda05c082 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -73,6 +73,7 @@ + @@ -148,6 +149,7 @@ + @@ -169,16 +171,7 @@ PreserveNewest - - - - - - - - - - + From 74d7816ecdd779096ac469a0b7072e8a686b9b3e Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 13 Mar 2020 11:30:54 +0100 Subject: [PATCH 03/30] Move DirectoryUtilities --- tests/xharness/TestProject.cs | 2 +- .../Utilities/{TempDirectory.cs => DirectoryUtilities.cs} | 4 ++-- tests/xharness/xharness.csproj | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename tests/xharness/Utilities/{TempDirectory.cs => DirectoryUtilities.cs} (96%) diff --git a/tests/xharness/TestProject.cs b/tests/xharness/TestProject.cs index de59357231fb..ea976ae743f3 100644 --- a/tests/xharness/TestProject.cs +++ b/tests/xharness/TestProject.cs @@ -121,7 +121,7 @@ internal async Task CreateCloneAsync (TestTask test) internal async Task CreateCopyAsync (TestTask test = null) { - var directory = TempDirectory.CreateTemporaryDirectory (test?.TestName ?? System.IO.Path.GetFileNameWithoutExtension (Path)); + var directory = DirectoryUtilities.CreateTemporaryDirectory (test?.TestName ?? System.IO.Path.GetFileNameWithoutExtension (Path)); Directory.CreateDirectory (directory); var original_path = Path; Path = System.IO.Path.Combine (directory, System.IO.Path.GetFileName (Path)); diff --git a/tests/xharness/Utilities/TempDirectory.cs b/tests/xharness/Utilities/DirectoryUtilities.cs similarity index 96% rename from tests/xharness/Utilities/TempDirectory.cs rename to tests/xharness/Utilities/DirectoryUtilities.cs index 54b826644ceb..71a114ba1171 100644 --- a/tests/xharness/Utilities/TempDirectory.cs +++ b/tests/xharness/Utilities/DirectoryUtilities.cs @@ -9,12 +9,12 @@ namespace Xharness.Utilities // * The temporary directories are automatically cleaned on Wrench (unlike /tmp, which isn't) // * The temporary directories stay after a test is run (until a new test run is started), // which makes it easier to re-run (copy-paste) commands that failed. - public static class TempDirectory + public static class DirectoryUtilities { static string root; static int lastNumber; - static TempDirectory () + static DirectoryUtilities () { root = Path.Combine (Path.GetDirectoryName (System.Reflection.Assembly.GetExecutingAssembly ().Location), "tmp-test-dir"); if (Directory.Exists (root)) diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index 5ebcda05c082..1170d5d0e2d9 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -153,7 +153,7 @@ - + From 61646f1ceba7c9dacd2486107055ba72c4eaf3a4 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 13 Mar 2020 11:36:39 +0100 Subject: [PATCH 04/30] Move DirectoryUtilities.RootDirectory --- .../BCLTestImportTargetFactory.cs | 2 +- .../BCLTestProjectGenerator.cs | 3 +- tests/xharness/Harness.cs | 70 ++++++------------- tests/xharness/Jenkins/Jenkins.cs | 58 +++++++-------- .../Jenkins/TestTasks/NUnitExecuteTask.cs | 8 +-- tests/xharness/MakefileGenerator.cs | 5 +- tests/xharness/MonoNativeInfo.cs | 4 +- tests/xharness/Program.cs | 2 +- tests/xharness/SolutionGenerator.cs | 4 +- tests/xharness/Targets/TVOSTarget.cs | 2 +- .../xharness/Utilities/DirectoryUtilities.cs | 36 ++++++++++ 11 files changed, 102 insertions(+), 92 deletions(-) diff --git a/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs b/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs index 58938cf609ae..567d423ffd41 100644 --- a/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs +++ b/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs @@ -16,7 +16,7 @@ public class BCLTestImportTargetFactory { public BCLTestImportTargetFactory (Harness harness) { Harness = harness; - var outputDir = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "bcl-test")); + var outputDir = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "bcl-test")); var projectTemplatePath = outputDir; var registerTypesTemplatePath = Path.Combine (outputDir, "RegisterType.cs.in"); var plistTemplatePath = outputDir; diff --git a/tests/xharness/BCLTestImporter/BCLTestProjectGenerator.cs b/tests/xharness/BCLTestImporter/BCLTestProjectGenerator.cs index a7c838d1bec3..8d8ff4e6e0e5 100644 --- a/tests/xharness/BCLTestImporter/BCLTestProjectGenerator.cs +++ b/tests/xharness/BCLTestImporter/BCLTestProjectGenerator.cs @@ -7,6 +7,7 @@ using Xharness; using Xharness.BCLTestImporter.Templates; using Xharness.BCLTestImporter.Templates.Managed; +using Xharness.Utilities; namespace Xharness.BCLTestImporter { /// @@ -113,7 +114,7 @@ public class BclTestProjectInfo { new BclTestProjectInfo { Name = "mscorlib Part 3", assemblies = new [] { "monotouch_corlib_xunit-test.part3.dll" }, Group = "mscorlib Part 3" }, new BclTestProjectInfo { Name = "SystemCoreXunit Part 1", assemblies = new [] { "monotouch_System.Core_xunit-test.part1.dll" }, Group = "SystemCoreXunit Part 1" }, new BclTestProjectInfo { Name = "SystemCoreXunit Part 2", assemblies = new [] { "monotouch_System.Core_xunit-test.part2.dll" }, Group = "SystemCoreXunit Part 2" }, - new BclTestProjectInfo { Name = "SystemXunit", assemblies = new [] { "monotouch_System_xunit-test.dll" }, ExtraArgs = $"--xml={Path.Combine (Harness.RootDirectory, "bcl-test", "SystemXunitLinker.xml")} --optimize=-custom-attributes-removal", Group = "SystemXunit" }, // special case due to the need of the extra args + new BclTestProjectInfo { Name = "SystemXunit", assemblies = new [] { "monotouch_System_xunit-test.dll" }, ExtraArgs = $"--xml={Path.Combine (DirectoryUtilities.RootDirectory, "bcl-test", "SystemXunitLinker.xml")} --optimize=-custom-attributes-removal", Group = "SystemXunit" }, // special case due to the need of the extra args }; static readonly List CommonIgnoredAssemblies = new List { diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 1df754624f5b..8f8b1b0aa835 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -43,7 +43,7 @@ public class Harness : IHarness public IProcessManager ProcessManager { get; set; } = new ProcessManager (); public string XIBuildPath { - get { return Path.GetFullPath (Path.Combine (RootDirectory, "..", "tools", "xibuild", "xibuild")); } + get { return Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "tools", "xibuild", "xibuild")); } } public static string Timestamp { @@ -52,34 +52,6 @@ public static string Timestamp { } } - // This is the maccore/tests directory. - static string root_directory; - public static string RootDirectory { - get { - if (root_directory == null) { - var testAssemblyDirectory = Path.GetDirectoryName (System.Reflection.Assembly.GetExecutingAssembly ().Location); - var dir = testAssemblyDirectory; - var path = Path.Combine (testAssemblyDirectory, ".git"); - while (!Directory.Exists (path) && path.Length > 3) { - dir = Path.GetDirectoryName (dir); - path = Path.Combine (dir, ".git"); - } - if (!Directory.Exists (path)) - throw new Exception ("Could not find the xamarin-macios repo."); - path = Path.Combine (Path.GetDirectoryName (path), "tests"); - if (!Directory.Exists (path)) - throw new Exception ("Could not find the tests directory."); - root_directory = path; - } - return root_directory; - } - set { - root_directory = value; - if (root_directory != null) - root_directory = Path.GetFullPath (root_directory).TrimEnd ('/'); - } - } - public List IOSTestProjects { get; set; } = new List (); public List MacTestProjects { get; set; } = new List (); @@ -256,7 +228,7 @@ public string MlaunchPath { void LoadConfig () { ParseConfigFiles (); - var src_root = Path.GetDirectoryName (Path.GetFullPath (RootDirectory)); + var src_root = Path.GetDirectoryName (Path.GetFullPath (DirectoryUtilities.RootDirectory)); MONO_PATH = Path.GetFullPath (Path.Combine (src_root, "external", "mono")); TVOS_MONO_PATH = MONO_PATH; INCLUDE_IOS = make_config.ContainsKey ("INCLUDE_IOS") && !string.IsNullOrEmpty (make_config ["INCLUDE_IOS"]); @@ -283,13 +255,13 @@ int AutoConfigureMac (bool generate_projects) new { Directory = "linker/mac/dont link", ProjectFile = "dont link-mac", Name = "dont link", Flavors = MacFlavors.Modern | MacFlavors.Full | MacFlavors.System }, }; foreach (var p in test_suites) { - MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p.Directory, p.ProjectFile + ".csproj"))) { + MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p.Directory, p.ProjectFile + ".csproj"))) { Name = p.Name, TargetFrameworkFlavors = p.Flavors, }); } - MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "introspection", "Mac", "introspection-mac.csproj")), targetFrameworkFlavor: MacFlavors.Modern) { Name = "introspection" }); + MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "introspection", "Mac", "introspection-mac.csproj")), targetFrameworkFlavor: MacFlavors.Modern) { Name = "introspection" }); var hard_coded_test_suites = new [] { new { Directory = "mmptest", ProjectFile = "mmptest", Name = "mmptest", IsNUnit = true, Configurations = (string[]) null, Platform = "x86", Flavors = MacFlavors.Console, }, @@ -299,10 +271,10 @@ int AutoConfigureMac (bool generate_projects) new { Directory = "linker/mac/link sdk", ProjectFile = "link sdk-mac", Name = "link sdk", IsNUnit = false, Configurations = new string [] { "Debug", "Release" }, Platform = "x86", Flavors = MacFlavors.Modern, }, }; foreach (var p in hard_coded_test_suites) { - MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p.Directory, p.ProjectFile + ".csproj")), targetFrameworkFlavor: p.Flavors) { + MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p.Directory, p.ProjectFile + ".csproj")), targetFrameworkFlavor: p.Flavors) { Name = p.Name, IsNUnitProject = p.IsNUnit, - SolutionPath = Path.GetFullPath (Path.Combine (RootDirectory, "tests-mac.sln")), + SolutionPath = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "tests-mac.sln")), Configurations = p.Configurations, Platform = p.Platform, }); @@ -401,18 +373,18 @@ void AutoConfigureIOS () var fsharp_library_projects = new string [] { "fsharplibrary" }; foreach (var p in test_suites) - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".csproj"))) { Name = p }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p + "/" + p + ".csproj"))) { Name = p }); foreach (var p in fsharp_test_suites) - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".fsproj"))) { Name = p }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p + "/" + p + ".fsproj"))) { Name = p }); foreach (var p in library_projects) - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".csproj")), false) { Name = p }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p + "/" + p + ".csproj")), false) { Name = p }); foreach (var p in fsharp_library_projects) - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p + "/" + p + ".fsproj")), false) { Name = p }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p + "/" + p + ".fsproj")), false) { Name = p }); - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "introspection", "iOS", "introspection-ios.csproj"))) { Name = "introspection" }); - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "linker", "ios", "dont link", "dont link.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "linker", "ios", "link all", "link all.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "linker", "ios", "link sdk", "link sdk.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "introspection", "iOS", "introspection-ios.csproj"))) { Name = "introspection" }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "linker", "ios", "dont link", "dont link.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "linker", "ios", "link all", "link all.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "linker", "ios", "link sdk", "link sdk.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); foreach (var flavor in new MonoNativeFlavor[] { MonoNativeFlavor.Compat, MonoNativeFlavor.Unified }) { var monoNativeInfo = new MonoNativeInfo (this, flavor); @@ -429,19 +401,19 @@ void AutoConfigureIOS () var monoImportTestFactory = new BCLTestImportTargetFactory (this); IOSTestProjects.AddRange (monoImportTestFactory.GetiOSBclTargets ()); - WatchOSContainerTemplate = Path.GetFullPath (Path.Combine (RootDirectory, "templates/WatchContainer")); - WatchOSAppTemplate = Path.GetFullPath (Path.Combine (RootDirectory, "templates/WatchApp")); - WatchOSExtensionTemplate = Path.GetFullPath (Path.Combine (RootDirectory, "templates/WatchExtension")); + WatchOSContainerTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "templates/WatchContainer")); + WatchOSAppTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "templates/WatchApp")); + WatchOSExtensionTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "templates/WatchExtension")); - TodayContainerTemplate = Path.GetFullPath (Path.Combine (RootDirectory, "templates", "TodayContainer")); - TodayExtensionTemplate = Path.GetFullPath (Path.Combine (RootDirectory, "templates", "TodayExtension")); - BCLTodayExtensionTemplate = Path.GetFullPath (Path.Combine (RootDirectory, "bcl-test", "templates", "today")); + TodayContainerTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "templates", "TodayContainer")); + TodayExtensionTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "templates", "TodayExtension")); + BCLTodayExtensionTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "bcl-test", "templates", "today")); } Dictionary make_config = new Dictionary (); IEnumerable FindConfigFiles (string name) { - var dir = Path.GetFullPath (RootDirectory); + var dir = Path.GetFullPath (DirectoryUtilities.RootDirectory); while (dir != "/") { var file = Path.Combine (dir, name); if (File.Exists (file)) diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index 74e6f81231df..ad74e9fb5e35 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -923,17 +923,17 @@ Task PopulateTasksAsync () var buildiOSMSBuild_net461 = new MSBuildTask () { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = true, ProjectConfiguration = "Debug-net461", Platform = TestPlatform.iOS, - SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), + SolutionPath = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), SupportsParallelExecution = false, }; var nunitExecutioniOSMSBuild_net461 = new NUnitExecuteTask (buildiOSMSBuild_net461) { - TestLibrary = Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-net461", "net461", "Xamarin.iOS.Tasks.Tests.dll"), + TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-net461", "net461", "Xamarin.iOS.Tasks.Tests.dll"), TestProject = buildiOSMSBuild_net461.TestProject, ProjectConfiguration = "Debug-net461", Platform = TestPlatform.iOS, @@ -947,16 +947,16 @@ Task PopulateTasksAsync () var buildiOSMSBuild_netstandard2 = new MSBuildTask () { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = true, ProjectConfiguration = "Debug-netstandard2.0", Platform = TestPlatform.iOS, - SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), + SolutionPath = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), SupportsParallelExecution = false, }; var nunitExecutioniOSMSBuild_netstandard2 = new NUnitExecuteTask (buildiOSMSBuild_netstandard2) { - TestLibrary = Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-netstandard2.0", "net461", "Xamarin.iOS.Tasks.Tests.dll"), + TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-netstandard2.0", "net461", "Xamarin.iOS.Tasks.Tests.dll"), TestProject = buildiOSMSBuild_netstandard2.TestProject, ProjectConfiguration = "Debug-netstandard2.0", Platform = TestPlatform.iOS, @@ -971,15 +971,15 @@ Task PopulateTasksAsync () var buildInstallSources = new MSBuildTask () { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "InstallSourcesTests.csproj"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "InstallSourcesTests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = false, Platform = TestPlatform.iOS, }; - buildInstallSources.SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "install-source.sln")); // this is required for nuget restore to be executed + buildInstallSources.SolutionPath = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "tools", "install-source", "install-source.sln")); // this is required for nuget restore to be executed var nunitExecutionInstallSource = new NUnitExecuteTask (buildInstallSources) { - TestLibrary = Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "bin", "Release", "InstallSourcesTests.dll"), + TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "bin", "Release", "InstallSourcesTests.dll"), TestProject = buildInstallSources.TestProject, Platform = TestPlatform.iOS, TestName = "Install Sources tests", @@ -1064,17 +1064,17 @@ Task PopulateTasksAsync () var buildMTouch = new MakeTask () { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch", "mtouch.sln"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "mtouch", "mtouch.sln"))), SpecifyPlatform = false, SpecifyConfiguration = false, Platform = TestPlatform.iOS, Target = "dependencies", - WorkingDirectory = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch")), + WorkingDirectory = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "mtouch")), }; var nunitExecutionMTouch = new NUnitExecuteTask (buildMTouch) { - TestLibrary = Path.Combine (Harness.RootDirectory, "mtouch", "bin", "Debug", "mtouch.dll"), - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch", "mtouch.csproj"))), + TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "mtouch", "bin", "Debug", "mtouch.dll"), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "mtouch", "mtouch.csproj"))), Platform = TestPlatform.iOS, TestName = "MTouch tests", Timeout = TimeSpan.FromMinutes (180), @@ -1085,16 +1085,16 @@ Task PopulateTasksAsync () var buildGenerator = new MakeTask { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "src", "generator.sln"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "src", "generator.sln"))), SpecifyPlatform = false, SpecifyConfiguration = false, Platform = TestPlatform.iOS, Target = "build-unit-tests", - WorkingDirectory = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "generator")), + WorkingDirectory = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "generator")), }; var runGenerator = new NUnitExecuteTask (buildGenerator) { - TestLibrary = Path.Combine (Harness.RootDirectory, "generator", "bin", "Debug", "generator-tests.dll"), - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "generator", "generator-tests.csproj"))), + TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "generator", "bin", "Debug", "generator-tests.dll"), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "generator", "generator-tests.csproj"))), Platform = TestPlatform.iOS, TestName = "Generator tests", Mode = "NUnit", @@ -1105,7 +1105,7 @@ Task PopulateTasksAsync () var buildDotNetGenerator = new DotNetBuildTask { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "bgen", "bgen-tests.csproj"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "bgen", "bgen-tests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = false, Platform = TestPlatform.iOS, @@ -1125,7 +1125,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.Mac, TestName = "MMP Regression Tests", Target = "all", // -j" + Environment.ProcessorCount, - WorkingDirectory = Path.Combine (Harness.RootDirectory, "mmptest", "regression"), + WorkingDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "mmptest", "regression"), Ignored = !IncludeMmpTest || !IncludeMac, Timeout = TimeSpan.FromMinutes (30), SupportsParallelExecution = false, // Already doing parallel execution by running "make -jX" @@ -1144,7 +1144,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.Mac, TestName = "Mac Binding Projects", Target = "all", - WorkingDirectory = Path.Combine (Harness.RootDirectory, "mac-binding-project"), + WorkingDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "mac-binding-project"), Ignored = !IncludeMacBindingProject || !IncludeMac, Timeout = TimeSpan.FromMinutes (15), }; @@ -1155,7 +1155,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.All, TestName = "Xtro", Target = "wrench", - WorkingDirectory = Path.Combine (Harness.RootDirectory, "xtro-sharpie"), + WorkingDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "xtro-sharpie"), Ignored = !IncludeXtro, Timeout = TimeSpan.FromMinutes (15), }; @@ -1173,7 +1173,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.All, TestName = "Cecil", Target = "build", - WorkingDirectory = Path.Combine (Harness.RootDirectory, "cecil-tests"), + WorkingDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "cecil-tests"), Ignored = !IncludeCecil, Timeout = TimeSpan.FromMinutes (5), }; @@ -1193,7 +1193,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.All, TestName = "Documentation", Target = "wrench-docs", - WorkingDirectory = Harness.RootDirectory, + WorkingDirectory = DirectoryUtilities.RootDirectory, Ignored = !IncludeDocs, Timeout = TimeSpan.FromMinutes (45), }; @@ -1201,14 +1201,14 @@ Task PopulateTasksAsync () var buildSampleTests = new MSBuildTask { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "sampletester", "sampletester.sln"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "sampletester", "sampletester.sln"))), SpecifyPlatform = false, Platform = TestPlatform.All, ProjectConfiguration = "Debug", }; var runSampleTests = new NUnitExecuteTask (buildSampleTests) { - TestLibrary = Path.Combine (Harness.RootDirectory, "sampletester", "bin", "Debug", "sampletester.dll"), - TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "sampletester", "sampletester.csproj"))), + TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "sampletester", "bin", "Debug", "sampletester.dll"), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "sampletester", "sampletester.csproj"))), Platform = TestPlatform.All, TestName = "Sample tests", Timeout = TimeSpan.FromDays (1), // These can take quite a while to execute. @@ -1306,7 +1306,7 @@ Task BuildTestLibrariesAsync () var sb = new StringBuilder (); var callback_log = new CallbackLog ((v) => sb.Append (v)); var log = Log.CreateAggregatedLog (callback_log, MainLog); - return Harness.ProcessManager.ExecuteCommandAsync ("make", new [] { "all", $"-j{Environment.ProcessorCount}", "-C", Path.Combine (Harness.RootDirectory, "test-libraries") }, log, TimeSpan.FromMinutes (10)).ContinueWith ((v) => { + return Harness.ProcessManager.ExecuteCommandAsync ("make", new [] { "all", $"-j{Environment.ProcessorCount}", "-C", Path.Combine (DirectoryUtilities.RootDirectory, "test-libraries") }, log, TimeSpan.FromMinutes (10)).ContinueWith ((v) => { var per = v.Result; if (!per.Succeeded) { // Only show the log if something went wrong. @@ -1587,7 +1587,7 @@ IEnumerable find_tasks (StreamWriter writer, string ids) server.Stop (); break; case "/favicon.ico": - serveFile = Path.Combine (Harness.RootDirectory, "xharness", "favicon.ico"); + serveFile = Path.Combine (DirectoryUtilities.RootDirectory, "xharness", "favicon.ico"); goto default; case "/index.html": var redirect_to = request.Url.AbsoluteUri.Replace ("/index.html", "/" + Path.GetFileName (LogDirectory) + "/index.html"); @@ -1750,7 +1750,7 @@ public void GenerateReport () foreach (var file in new string [] { "xharness.js", "xharness.css" }) { File.Copy (Path.Combine (dependentFileLocation, file), Path.Combine (LogDirectory, file), true); } - File.Copy (Path.Combine (Harness.RootDirectory, "xharness", "favicon.ico"), Path.Combine (LogDirectory, "favicon.ico"), true); + File.Copy (Path.Combine (DirectoryUtilities.RootDirectory, "xharness", "favicon.ico"), Path.Combine (LogDirectory, "favicon.ico"), true); } } catch (Exception e) { this.MainLog.WriteLine ("Failed to write log: {0}", e); diff --git a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs index 3bc600632cac..c3e9cdacd155 100644 --- a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs @@ -73,15 +73,15 @@ public void FindNUnitConsoleExecutable (ILog log) } if (is_packageref) { - TestExecutable = Path.Combine (Harness.RootDirectory, "..", "tools", $"nunit3-console-{nunit_version}"); + TestExecutable = Path.Combine (DirectoryUtilities.RootDirectory, "..", "tools", $"nunit3-console-{nunit_version}"); if (!File.Exists (TestExecutable)) throw new FileNotFoundException ($"The helper script to execute the unit tests does not exist: {TestExecutable}"); WorkingDirectory = Path.GetDirectoryName (TestProject.Path); } else if (nunit_version [0] == '2') { - TestExecutable = Path.Combine (Harness.RootDirectory, "..", "packages", "NUnit.Runners." + nunit_version, "tools", "nunit-console.exe"); + TestExecutable = Path.Combine (DirectoryUtilities.RootDirectory, "..", "packages", "NUnit.Runners." + nunit_version, "tools", "nunit-console.exe"); WorkingDirectory = Path.Combine (Path.GetDirectoryName (TestExecutable), "lib"); } else { - TestExecutable = Path.Combine (Harness.RootDirectory, "..", "packages", "NUnit.ConsoleRunner." + nunit_version, "tools", "nunit3-console.exe"); + TestExecutable = Path.Combine (DirectoryUtilities.RootDirectory, "..", "packages", "NUnit.ConsoleRunner." + nunit_version, "tools", "nunit3-console.exe"); WorkingDirectory = Path.GetDirectoryName (TestLibrary); } TestExecutable = Path.GetFullPath (TestExecutable); @@ -136,7 +136,7 @@ protected override async Task RunTestAsync () if (ProduceHtmlReport) { try { var output = Logs.Create ($"Log-{Timestamp}.html", "HTML log"); - using (var srt = new StringReader (File.ReadAllText (Path.Combine (Harness.RootDirectory, "HtmlTransform.xslt")))) { + using (var srt = new StringReader (File.ReadAllText (Path.Combine (DirectoryUtilities.RootDirectory, "HtmlTransform.xslt")))) { using (var sri = File.OpenRead (xmlLog)) { using (var xrt = XmlReader.Create (srt)) { using (var xri = XmlReader.Create (sri)) { diff --git a/tests/xharness/MakefileGenerator.cs b/tests/xharness/MakefileGenerator.cs index 0dad09cf1da3..c1d99cc7b853 100644 --- a/tests/xharness/MakefileGenerator.cs +++ b/tests/xharness/MakefileGenerator.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text; using Xharness.Targets; +using Xharness.Utilities; namespace Xharness { @@ -58,7 +59,7 @@ static string CreateRelativePath (string path, string relative_to) public static void CreateMacMakefile (Harness harness, IEnumerable targets) { - var makefile = Path.Combine (Harness.RootDirectory, "Makefile-mac.inc"); + var makefile = Path.Combine (DirectoryUtilities.RootDirectory, "Makefile-mac.inc"); using (var writer = new StreamWriter (makefile, false, new UTF8Encoding (false))) { writer.WriteLine (".stamp-configure-projects-mac: Makefile xharness/xharness.exe"); writer.WriteLine ("\t$(Q) $(SYSTEM_MONO) --debug $(XIBUILD_EXE_PATH) -t -- $(CURDIR)/xharness/xharness.exe $(XHARNESS_VERBOSITY) --configure --autoconf --rootdir $(CURDIR)"); @@ -213,7 +214,7 @@ static string GetMakeName (this Target target, bool escape = true) public static void CreateMakefile (Harness harness, IEnumerable unified_targets, IEnumerable tvos_targets, IEnumerable watchos_targets, IEnumerable today_targets) { var executeSim32 = !harness.InCI; // Waiting for iOS 10.3 simulator to be installed on wrench - var makefile = Path.Combine (Harness.RootDirectory, "Makefile.inc"); + var makefile = Path.Combine (DirectoryUtilities.RootDirectory, "Makefile.inc"); using (var writer = new StreamWriter (makefile, false, new UTF8Encoding (false))) { writer.WriteLine (".stamp-configure-projects: Makefile xharness/xharness.exe"); writer.WriteLine ("\t$(Q) $(SYSTEM_MONO) --debug $(XIBUILD_EXE_PATH) -t -- $(CURDIR)/xharness/xharness.exe $(XHARNESS_VERBOSITY) --configure --autoconf --rootdir $(CURDIR)"); diff --git a/tests/xharness/MonoNativeInfo.cs b/tests/xharness/MonoNativeInfo.cs index 1f241e28b9ce..92176de9041e 100644 --- a/tests/xharness/MonoNativeInfo.cs +++ b/tests/xharness/MonoNativeInfo.cs @@ -157,9 +157,9 @@ public MonoNativeInfo (Harness harness, MonoNativeFlavor flavor) public string FlavorSuffix => Flavor == MonoNativeFlavor.Compat ? "-compat" : "-unified"; public string ProjectName => "mono-native" + FlavorSuffix; - public string ProjectPath => Path.Combine (Harness.RootDirectory, "mono-native", TemplateName + FlavorSuffix + ".csproj"); + public string ProjectPath => Path.Combine (DirectoryUtilities.RootDirectory, "mono-native", TemplateName + FlavorSuffix + ".csproj"); string TemplateName => "mono-native" + TemplateSuffix; - public string TemplatePath => Path.Combine (Harness.RootDirectory, "mono-native", TemplateName + ".csproj.template"); + public string TemplatePath => Path.Combine (DirectoryUtilities.RootDirectory, "mono-native", TemplateName + ".csproj.template"); protected virtual string TemplateSuffix => string.Empty; public void Convert () diff --git a/tests/xharness/Program.cs b/tests/xharness/Program.cs index 4989e3122bf6..5aaecbd4f3e6 100644 --- a/tests/xharness/Program.cs +++ b/tests/xharness/Program.cs @@ -18,7 +18,7 @@ public static int Main (string [] args) { "mac", "Configure for Xamarin.Mac instead of iOS.", (v) => harness.Mac = true }, { "configure", "Creates project files and makefiles.", (v) => harness.Action = HarnessAction.Configure }, { "autoconf", "Automatically decide what to configure.", (v) => harness.AutoConf = true }, - { "rootdir=", "The root directory for the tests.", (v) => Harness.RootDirectory = v }, + { "rootdir=", "The root directory for the tests.", (v) => DirectoryUtilities.RootDirectory = v }, { "project=", "Add a project file to process. This can be specified multiple times.", (v) => harness.IOSTestProjects.Add (new iOSTestProject (v)) }, { "watchos-container-template=", "The directory to use as a template for a watchos container app.", (v) => harness.WatchOSContainerTemplate = v }, { "watchos-app-template=", "The directory to use as a template for a watchos app.", (v) => harness.WatchOSAppTemplate = v }, diff --git a/tests/xharness/SolutionGenerator.cs b/tests/xharness/SolutionGenerator.cs index d24820174640..04c1690d5ac9 100644 --- a/tests/xharness/SolutionGenerator.cs +++ b/tests/xharness/SolutionGenerator.cs @@ -57,8 +57,8 @@ public static void CreateSolution (Harness harness, IEnumerable targets, { var folders = new StringBuilder (); - var srcDirectory = Path.Combine (Harness.RootDirectory, "..", "src"); - var sln_path = exeTarget == null ? Path.Combine (Harness.RootDirectory, "tests-" + infix + ".sln") : Path.Combine (Path.GetDirectoryName (exeTarget.ProjectPath), Path.GetFileNameWithoutExtension (exeTarget.ProjectPath) + ".sln"); + var srcDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "..", "src"); + var sln_path = exeTarget == null ? Path.Combine (DirectoryUtilities.RootDirectory, "tests-" + infix + ".sln") : Path.Combine (Path.GetDirectoryName (exeTarget.ProjectPath), Path.GetFileNameWithoutExtension (exeTarget.ProjectPath) + ".sln"); using (var writer = new StringWriter ()) { writer.WriteLine (); diff --git a/tests/xharness/Targets/TVOSTarget.cs b/tests/xharness/Targets/TVOSTarget.cs index caacb71d6be4..864fb2f22f56 100644 --- a/tests/xharness/Targets/TVOSTarget.cs +++ b/tests/xharness/Targets/TVOSTarget.cs @@ -113,7 +113,7 @@ protected override void ProcessProject () MonoNativeHelper.RemoveSymlinkMode (inputProject); } - var srcDirectory = Path.Combine (Harness.RootDirectory, "..", "src"); + var srcDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "..", "src"); string project_guid; var mt_nunitlite_project_path = Path.GetFullPath (Path.Combine (srcDirectory, "MonoTouch.NUnitLite.tvos.csproj")); diff --git a/tests/xharness/Utilities/DirectoryUtilities.cs b/tests/xharness/Utilities/DirectoryUtilities.cs index 71a114ba1171..a3cba5f3ef75 100644 --- a/tests/xharness/Utilities/DirectoryUtilities.cs +++ b/tests/xharness/Utilities/DirectoryUtilities.cs @@ -52,5 +52,41 @@ public static string CreateTemporaryDirectory (string name = null) throw new Exception ("Could not create temporary directory"); } + + static string root_directory; + public static string RootDirectory { + get { + if (root_directory == null) + root_directory = GetRootDirectory(); + + return root_directory; + } + + set { + root_directory = value; + if (root_directory != null) + root_directory = Path.GetFullPath (root_directory).TrimEnd ('/'); + } + } + + private static string GetRootDirectory() { + var dir = Path.GetDirectoryName (System.Reflection.Assembly.GetExecutingAssembly ().Location); + + var path = Path.Combine (dir, ".git"); + while (!Directory.Exists (path) && path.Length > 3) { + dir = Path.GetDirectoryName (dir); + path = Path.Combine (dir, ".git"); + } + + if (!Directory.Exists (path)) + throw new Exception ("Could not find the xamarin-macios repo."); + + path = Path.Combine (Path.GetDirectoryName (path), "tests"); + + if (!Directory.Exists (path)) + throw new Exception ("Could not find the tests directory."); + + return path; + } } } From 88acdea871001b49d043b806c9aee27a18bf1aa1 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 13 Mar 2020 11:41:41 +0100 Subject: [PATCH 05/30] Add CrashSnapshot ctor --- tests/xharness/AppRunner.cs | 9 +--- tests/xharness/CrashReportSnapshot.cs | 41 +++++++++++-------- .../Jenkins/TestTasks/MacExecuteTask.cs | 2 +- .../xharness/Jenkins/TestTasks/RunXtroTask.cs | 2 +- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 6b744a366d0a..6a0e974bf509 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -490,14 +490,7 @@ public async Task RunAsync () if (!isSimulator) FindDevice (); - crash_reports = new CrashReportSnapshot () { - Device = !isSimulator, - DeviceName = device_name, - Harness = Harness, - Log = main_log, - Logs = Logs, - LogDirectory = LogDirectory, - }; + crash_reports = new CrashReportSnapshot (Harness, main_log, Logs, isDevice: !isSimulator, device_name); var args = new List (); if (!string.IsNullOrEmpty (Harness.XcodeRoot)) { diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs index 8fbec9bde74a..c5bbc41f9ed0 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashReportSnapshot.cs @@ -10,19 +10,27 @@ namespace Xharness { public class CrashReportSnapshot { - public Harness Harness { get; set; } - public ILog Log { get; set; } - public ILogs Logs { get; set; } - public string LogDirectory { get; set; } - public bool Device { get; set; } - public string DeviceName { get; set; } + readonly Harness harness; + readonly bool isDevice; + readonly string deviceName; + + public ILog Log { get; } + public ILogs Logs { get; } public HashSet InitialSet { get; private set; } - public IEnumerable Reports { get; private set; } + + public CrashReportSnapshot (Harness harness, ILog log, ILogs logs, bool isDevice, string deviceName) + { + this.harness = harness ?? throw new ArgumentNullException (nameof (harness)); + this.Log = log ?? throw new ArgumentNullException (nameof (log)); + this.Logs = logs ?? throw new ArgumentNullException (nameof (logs)); + this.isDevice = isDevice; + this.deviceName = deviceName; + } public async Task StartCaptureAsync () { - InitialSet = await Harness.CreateCrashReportsSnapshotAsync (Log, !Device, DeviceName); + InitialSet = await harness.CreateCrashReportsSnapshotAsync (Log, !isDevice, deviceName); } public async Task EndCaptureAsync (TimeSpan timeout) @@ -33,13 +41,12 @@ public async Task EndCaptureAsync (TimeSpan timeout) var watch = new Stopwatch (); watch.Start (); do { - var end_crashes = await Harness.CreateCrashReportsSnapshotAsync (Log, !Device, DeviceName); + var end_crashes = await harness.CreateCrashReportsSnapshotAsync (Log, !isDevice, deviceName); end_crashes.ExceptWith (InitialSet); - Reports = end_crashes; if (end_crashes.Count > 0) { Log.WriteLine ("Found {0} new crash report(s)", end_crashes.Count); List crash_reports; - if (!Device) { + if (!isDevice) { crash_reports = new List (end_crashes.Count); foreach (var path in end_crashes) { Logs.AddFile (path, $"Crash report: {Path.GetFileName (path)}"); @@ -55,15 +62,15 @@ public async Task EndCaptureAsync (TimeSpan timeout) sb.Add ($"--download-crash-report={file}"); sb.Add ($"--download-crash-report-to={crash_report_target.Path}"); sb.Add ("--sdkroot"); - sb.Add (Harness.XcodeRoot); - if (!string.IsNullOrEmpty (DeviceName)) { + sb.Add (harness.XcodeRoot); + if (!string.IsNullOrEmpty (deviceName)) { sb.Add ("--devname"); - sb.Add (DeviceName); + sb.Add (deviceName); } - var result = await Harness.ProcessManager.ExecuteCommandAsync (Harness.MlaunchPath, sb, Log, TimeSpan.FromMinutes (1)); + var result = await harness.ProcessManager.ExecuteCommandAsync (harness.MlaunchPath, sb, Log, TimeSpan.FromMinutes (1)); if (result.Succeeded) { Log.WriteLine ("Downloaded crash report {0} to {1}", file, crash_report_target.Path); - crash_report_target = await Harness.SymbolicateCrashReportAsync (Logs, Log, crash_report_target); + crash_report_target = await harness.SymbolicateCrashReportAsync (Logs, Log, crash_report_target); downloaded_crash_reports.Add (crash_report_target); } else { Log.WriteLine ("Could not download crash report {0}", file); @@ -72,7 +79,7 @@ public async Task EndCaptureAsync (TimeSpan timeout) crash_reports = downloaded_crash_reports; } foreach (var cp in crash_reports) { - Harness.LogWrench ("@MonkeyWrench: AddFile: {0}", cp.Path); + harness.LogWrench ("@MonkeyWrench: AddFile: {0}", cp.Path); Log.WriteLine (" {0}", cp.Path); } crash_report_search_done = true; diff --git a/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs index 491fdbeac32d..def0f09f5dfb 100644 --- a/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs @@ -91,7 +91,7 @@ protected override async Task RunTestAsync () if (!Harness.DryRun) { ExecutionResult = TestExecutingResult.Running; - var snapshot = new CrashReportSnapshot () { Device = false, Harness = Harness, Log = log, Logs = Logs, LogDirectory = LogDirectory }; + var snapshot = new CrashReportSnapshot (Harness, log, Logs, isDevice: false, deviceName: null); await snapshot.StartCaptureAsync (); ProcessExecutionResult result = null; diff --git a/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs b/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs index c0a93467ae68..2ad46335c984 100644 --- a/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs @@ -29,7 +29,7 @@ protected override async Task RunTestAsync () if (!Harness.DryRun) { ExecutionResult = TestExecutingResult.Running; - var snapshot = new CrashReportSnapshot () { Device = false, Harness = Harness, Log = log, Logs = Logs, LogDirectory = LogDirectory }; + var snapshot = new CrashReportSnapshot (Harness, log, Logs, isDevice: false, deviceName: null); await snapshot.StartCaptureAsync (); try { From 140a0cfb6e0583133a14e873ee6ab90b84cbf51a Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 13 Mar 2020 11:45:11 +0100 Subject: [PATCH 06/30] refactor CrashReportSnapshot --- tests/xharness/CrashReportSnapshot.cs | 60 +++++++++++++++++++++++++-- tests/xharness/Harness.cs | 54 ------------------------ 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs index c5bbc41f9ed0..c473d54f5edb 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashReportSnapshot.cs @@ -30,7 +30,7 @@ public CrashReportSnapshot (Harness harness, ILog log, ILogs logs, bool isDevice public async Task StartCaptureAsync () { - InitialSet = await harness.CreateCrashReportsSnapshotAsync (Log, !isDevice, deviceName); + InitialSet = await CreateCrashReportsSnapshotAsync (); } public async Task EndCaptureAsync (TimeSpan timeout) @@ -41,7 +41,7 @@ public async Task EndCaptureAsync (TimeSpan timeout) var watch = new Stopwatch (); watch.Start (); do { - var end_crashes = await harness.CreateCrashReportsSnapshotAsync (Log, !isDevice, deviceName); + var end_crashes = await CreateCrashReportsSnapshotAsync (); end_crashes.ExceptWith (InitialSet); if (end_crashes.Count > 0) { Log.WriteLine ("Found {0} new crash report(s)", end_crashes.Count); @@ -70,7 +70,7 @@ public async Task EndCaptureAsync (TimeSpan timeout) var result = await harness.ProcessManager.ExecuteCommandAsync (harness.MlaunchPath, sb, Log, TimeSpan.FromMinutes (1)); if (result.Succeeded) { Log.WriteLine ("Downloaded crash report {0} to {1}", file, crash_report_target.Path); - crash_report_target = await harness.SymbolicateCrashReportAsync (Logs, Log, crash_report_target); + crash_report_target = await SymbolicateCrashReportAsync (crash_report_target); downloaded_crash_reports.Add (crash_report_target); } else { Log.WriteLine ("Could not download crash report {0}", file); @@ -93,5 +93,59 @@ public async Task EndCaptureAsync (TimeSpan timeout) } } while (!crash_report_search_done); } + + async Task SymbolicateCrashReportAsync (ILogFile report) + { + var symbolicatecrash = Path.Combine (harness.XcodeRoot, "Contents/SharedFrameworks/DTDeviceKitBase.framework/Versions/A/Resources/symbolicatecrash"); + if (!File.Exists (symbolicatecrash)) + symbolicatecrash = Path.Combine (harness.XcodeRoot, "Contents/SharedFrameworks/DVTFoundation.framework/Versions/A/Resources/symbolicatecrash"); + + if (!File.Exists (symbolicatecrash)) { + Log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicatecrash); + return report; + } + + var name = Path.GetFileName (report.Path); + var symbolicated = Logs.Create (Path.ChangeExtension (name, ".symbolicated.log"), $"Symbolicated crash report: {name}", timestamp: false); + var environment = new Dictionary { { "DEVELOPER_DIR", Path.Combine (harness.XcodeRoot, "Contents", "Developer") } }; + var rv = await harness.ProcessManager.ExecuteCommandAsync (symbolicatecrash, new [] { report.Path }, symbolicated, TimeSpan.FromMinutes (1), environment); + if (rv.Succeeded) {; + Log.WriteLine ("Symbolicated {0} successfully.", report.Path); + return symbolicated; + } else { + Log.WriteLine ("Failed to symbolicate {0}.", report.Path); + return report; + } + } + + async Task> CreateCrashReportsSnapshotAsync () + { + var rv = new HashSet (); + + if (!isDevice) { + var dir = Path.Combine (Environment.GetEnvironmentVariable ("HOME"), "Library", "Logs", "DiagnosticReports"); + if (Directory.Exists (dir)) + rv.UnionWith (Directory.EnumerateFiles (dir)); + } else { + var tmp = Path.GetTempFileName (); + try { + var sb = new List (); + sb.Add ($"--list-crash-reports={tmp}"); + sb.Add ("--sdkroot"); + sb.Add (harness.XcodeRoot); + if (!string.IsNullOrEmpty (deviceName)) { + sb.Add ("--devname"); + sb.Add (deviceName); + } + var result = await harness.ProcessManager.ExecuteCommandAsync (harness.MlaunchPath, sb, Log, TimeSpan.FromMinutes (1)); + if (result.Succeeded) + rv.UnionWith (File.ReadAllLines (tmp)); + } finally { + File.Delete (tmp); + } + } + + return rv; + } } } diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 8f8b1b0aa835..0790612750c4 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -750,59 +750,5 @@ public async Task ShowSimulatorList (Log log) await ExecuteXcodeCommandAsync ("simctl", new [] { "list" }, log, TimeSpan.FromSeconds (10)); } - public async Task SymbolicateCrashReportAsync (ILogs logs, ILog log, ILogFile report) - { - var symbolicatecrash = Path.Combine (XcodeRoot, "Contents/SharedFrameworks/DTDeviceKitBase.framework/Versions/A/Resources/symbolicatecrash"); - if (!File.Exists (symbolicatecrash)) - symbolicatecrash = Path.Combine (XcodeRoot, "Contents/SharedFrameworks/DVTFoundation.framework/Versions/A/Resources/symbolicatecrash"); - - if (!File.Exists (symbolicatecrash)) { - log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicatecrash); - return report; - } - - var name = Path.GetFileName (report.Path); - var symbolicated = logs.Create (Path.ChangeExtension (name, ".symbolicated.log"), $"Symbolicated crash report: {name}", timestamp: false); - var environment = new Dictionary { { "DEVELOPER_DIR", Path.Combine (XcodeRoot, "Contents", "Developer") } }; - var rv = await ProcessManager.ExecuteCommandAsync (symbolicatecrash, new [] { report.Path }, symbolicated, TimeSpan.FromMinutes (1), environment); - if (rv.Succeeded) {; - log.WriteLine ("Symbolicated {0} successfully.", report.Path); - return symbolicated; - } else { - log.WriteLine ("Failed to symbolicate {0}.", report.Path); - return report; - } - } - - public async Task> CreateCrashReportsSnapshotAsync (ILog log, bool simulatorOrDesktop, string device) - { - var rv = new HashSet (); - - if (simulatorOrDesktop) { - var dir = Path.Combine (Environment.GetEnvironmentVariable ("HOME"), "Library", "Logs", "DiagnosticReports"); - if (Directory.Exists (dir)) - rv.UnionWith (Directory.EnumerateFiles (dir)); - } else { - var tmp = Path.GetTempFileName (); - try { - var sb = new List (); - sb.Add ($"--list-crash-reports={tmp}"); - sb.Add ("--sdkroot"); - sb.Add (XcodeRoot); - if (!string.IsNullOrEmpty (device)) { - sb.Add ("--devname"); - sb.Add (device); - } - var result = await ProcessManager.ExecuteCommandAsync (MlaunchPath, sb, log, TimeSpan.FromMinutes (1)); - if (result.Succeeded) - rv.UnionWith (File.ReadAllLines (tmp)); - } finally { - File.Delete (tmp); - } - } - - return rv; - } - } } From 9f20fd2801bb96fbfb9854ebc40ed9e78bdb2dee Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 13 Mar 2020 11:54:25 +0100 Subject: [PATCH 07/30] Add WrenchLog --- tests/xharness/AppRunner.cs | 24 ++++++++++++------------ tests/xharness/CrashReportSnapshot.cs | 2 +- tests/xharness/Logging/WrenchLog.cs | 18 ++++++++++++++++++ tests/xharness/xharness.csproj | 1 + 4 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 tests/xharness/Logging/WrenchLog.cs diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 6a0e974bf509..0aa5fd8d21c4 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -415,10 +415,10 @@ public bool EnsureCleanSimulatorState { main_log.WriteLine (new string ('#', 10)); main_log.WriteLine ("End of xml results."); if (timed_out) { - Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} timed out
"); + WrenchLog.WriteLine ($"AddSummary: {mode} timed out
"); return parseResult; } else { - Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} crashed
"); + WrenchLog.WriteLine ($"AddSummary: {mode} crashed
"); main_log.WriteLine ("Test run crashed"); crashed = true; parseResult.crashed = true; @@ -456,19 +456,19 @@ public bool TestsSucceeded (string test_log_path, bool timed_out, out bool crash if (resultLine != null) { var tests_run = resultLine.Replace ("Tests run: ", ""); if (failed) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed: {1}
", mode, tests_run); + WrenchLog.WriteLine ("AddSummary: {0} failed: {1}
", mode, tests_run); main_log.WriteLine ("Test run failed"); return false; } else { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} succeeded: {1}
", mode, tests_run); + WrenchLog.WriteLine ("AddSummary: {0} succeeded: {1}
", mode, tests_run); main_log.WriteLine ("Test run succeeded"); return true; } } else if (timed_out) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} timed out
", mode); + WrenchLog.WriteLine ("AddSummary: {0} timed out
", mode); return false; } else { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed
", mode); + WrenchLog.WriteLine ("AddSummary: {0} crashed
", mode); main_log.WriteLine ("Test run crashed"); crashed = true; return false; @@ -636,7 +636,7 @@ public async Task RunAsync () log.StartCapture (); Logs.Add (log); systemLogs.Add (log); - Harness.LogWrench ("@MonkeyWrench: AddFile: {0}", log.Path); + WrenchLog.WriteLine ("AddFile: {0}", log.Path); } main_log.WriteLine ("*** Executing {0}/{1} in the simulator ***", appName, mode); @@ -766,7 +766,7 @@ public async Task RunAsync () // Upload the system log if (File.Exists (device_system_log.FullPath)) { main_log.WriteLine ("A capture of the device log is: {0}", device_system_log.FullPath); - Harness.LogWrench ("@MonkeyWrench: AddFile: {0}", device_system_log.FullPath); + WrenchLog.WriteLine ("AddFile: {0}", device_system_log.FullPath); } } @@ -776,18 +776,18 @@ public async Task RunAsync () // check the final status var crashed = false; if (File.Exists (listener_log.FullPath)) { - Harness.LogWrench ("@MonkeyWrench: AddFile: {0}", listener_log.FullPath); + WrenchLog.WriteLine ("AddFile: {0}", listener_log.FullPath); success = TestsSucceeded (listener_log.FullPath, timed_out, out crashed); } else if (timed_out) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} never launched
", mode); + WrenchLog.WriteLine ("AddSummary: {0} never launched
", mode); main_log.WriteLine ("Test run never launched"); success = false; } else if (launch_failure) { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} failed to launch
", mode); + WrenchLog.WriteLine ("AddSummary: {0} failed to launch
", mode); main_log.WriteLine ("Test run failed to launch"); success = false; } else { - Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} crashed at startup (no log)
", mode); + WrenchLog.WriteLine ("AddSummary: {0} crashed at startup (no log)
", mode); main_log.WriteLine ("Test run crashed before it started (no log file produced)"); crashed = true; success = false; diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs index c473d54f5edb..c0168db5f521 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashReportSnapshot.cs @@ -79,7 +79,7 @@ public async Task EndCaptureAsync (TimeSpan timeout) crash_reports = downloaded_crash_reports; } foreach (var cp in crash_reports) { - harness.LogWrench ("@MonkeyWrench: AddFile: {0}", cp.Path); + WrenchLog.WriteLine ("AddFile: {0}", cp.Path); Log.WriteLine (" {0}", cp.Path); } crash_report_search_done = true; diff --git a/tests/xharness/Logging/WrenchLog.cs b/tests/xharness/Logging/WrenchLog.cs new file mode 100644 index 000000000000..7cd53bfeed13 --- /dev/null +++ b/tests/xharness/Logging/WrenchLog.cs @@ -0,0 +1,18 @@ +namespace Xharness.Logging { + public static class WrenchLog { + + public static void WriteLine (string message, params object [] args) + { + WriteLine (string.Format (message, args)); + } + + public static void WriteLine (string message) + { + // Disable this for now, since we're not uploading directly to wrench anymore, but instead using the Html Report. + //if (!InWrench) + // return; + + //Console.WriteLine ("@MonkeyWrench: " + message, args); + } + } +} diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index 1170d5d0e2d9..4a56eaa55126 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -133,6 +133,7 @@ + From d3781048b5c3d901de4fcab6ac7fa57ac4f20730 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 13 Mar 2020 11:56:34 +0100 Subject: [PATCH 08/30] Move stuff out of Harness --- tests/xharness/Harness.cs | 114 ++---------------------------- tests/xharness/Jenkins/Jenkins.cs | 4 +- 2 files changed, 9 insertions(+), 109 deletions(-) diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 0790612750c4..86115cae7d17 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -1,10 +1,7 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; -using System.Security.Cryptography; -using System.Text; using System.Threading.Tasks; using System.Xml; using Xharness.BCLTestImporter; @@ -128,13 +125,6 @@ public bool GetIncludeSystemPermissionTests (TestPlatform platform, bool device) } } - public bool IsBetaXcode { - get { - // There's no string.Contains (string, StringComparison) overload, so use IndexOf instead. - return XcodeRoot.IndexOf ("beta", StringComparison.OrdinalIgnoreCase) >= 0; - } - } - static string FindXcode (string path) { var p = path; @@ -166,59 +156,6 @@ public Version XcodeVersion { } } - object mlaunch_lock = new object (); - string DownloadMlaunch () - { - // NOTE: the filename part in the url must be unique so that the caching logic works properly. - var mlaunch_url = "https://dl.xamarin.com/ios/mlaunch-acdb43d346c431b2c40663c938c919dcb0e91bd7.zip"; - var extraction_dir = Path.Combine (Path.GetTempPath (), Path.GetFileNameWithoutExtension (mlaunch_url)); - var mlaunch_path = Path.Combine (extraction_dir, "bin", "mlaunch"); - - lock (mlaunch_lock) { - if (File.Exists (mlaunch_path)) - return mlaunch_path; - - try { - var local_zip = extraction_dir + ".zip"; - Log ("Downloading mlaunch to: {0}", local_zip); - var wc = new System.Net.WebClient (); - wc.DownloadFile (mlaunch_url, local_zip); - Log ("Downloaded mlaunch."); - - var tmp_extraction_dir = extraction_dir + ".tmp"; - if (Directory.Exists (tmp_extraction_dir)) - Directory.Delete (tmp_extraction_dir, true); - if (Directory.Exists (extraction_dir)) - Directory.Delete (extraction_dir, true); - - Log ("Extracting mlaunch..."); - using (var p = new Process ()) { - p.StartInfo.FileName = "unzip"; - p.StartInfo.Arguments = StringUtils.FormatArguments ("-d", tmp_extraction_dir, local_zip); - Log ("{0} {1}", p.StartInfo.FileName, p.StartInfo.Arguments); - p.Start (); - p.WaitForExit (); - if (p.ExitCode != 0) { - Log ("Could not unzip mlaunch, exit code: {0}", p.ExitCode); - return mlaunch_path; - } - } - Directory.Move (tmp_extraction_dir, extraction_dir); - - Log ("Final mlaunch path: {0}", mlaunch_path); - } catch (Exception e) { - Log ("Could not download mlaunch: {0}", e); - } - return mlaunch_path; - } - } - - public string MtouchPath { - get { - return Path.Combine (IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current", "bin", "mtouch"); - } - } - public string MlaunchPath { get { return Path.Combine (IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current", "bin", "mlaunch"); @@ -450,7 +387,7 @@ void ParseConfigFile (string file) } } - public int Configure () + int Configure () { return Mac ? AutoConfigureMac (true) : ConfigureIOS (); } @@ -528,7 +465,7 @@ int ConfigureIOS () return rv; } - public int Install () + int Install () { if (HarnessLog == null) HarnessLog = new ConsoleLog (); @@ -548,7 +485,7 @@ public int Install () return 0; } - public int Uninstall () + int Uninstall () { if (HarnessLog == null) HarnessLog = new ConsoleLog (); @@ -567,7 +504,7 @@ public int Uninstall () return 0; } - public int Run () + int Run () { if (HarnessLog == null) HarnessLog = new ConsoleLog (); @@ -585,7 +522,7 @@ public int Run () return 0; } - public void Log (int min_level, string message) + void Log (int min_level, string message) { if (Verbosity < min_level) return; @@ -610,23 +547,6 @@ public void Log (string message, params object[] args) { Log (0, message, args); } - - public void LogWrench (string message, params object[] args) - { - // Disable this for now, since we're not uploading directly to wrench anymore, but instead using the Html Report. - //if (!InWrench) - // return; - - //Console.WriteLine (message, args); - } - - public void LogWrench (string message) - { - if (!InCI) - return; - - Console.WriteLine (message); - } public bool InCI { get { @@ -661,7 +581,7 @@ public int Execute () } } - public int Jenkins () + int Jenkins () { if (AutoConf) { AutoConfigureIOS (); @@ -715,22 +635,6 @@ public void Save (StringWriter doc, string path) } } - public void Save (string doc, string path) - { - if (!File.Exists (path)) { - File.WriteAllText (path, doc); - Log (1, "Created {0}", path); - } else { - var existing = File.ReadAllText (path); - if (existing == doc) { - Log (1, "Not saved {0}, no change", path); - } else { - File.WriteAllText (path, doc); - Log (1, "Updated {0}", path); - } - } - } - bool? disable_watchos_on_wrench; public bool DisableWatchOSOnWrench { get { @@ -744,11 +648,5 @@ public Task ExecuteXcodeCommandAsync (string executable, { return ProcessManager.ExecuteCommandAsync (Path.Combine (XcodeRoot, "Contents", "Developer", "usr", "bin", executable), args, log, timeout: timeout); } - - public async Task ShowSimulatorList (Log log) - { - await ExecuteXcodeCommandAsync ("simctl", new [] { "list" }, log, TimeSpan.FromSeconds (10)); - } - } } diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index ad74e9fb5e35..4cc9004f39bd 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -872,12 +872,14 @@ void SelectTestsByLabel (int pull_request) // old simulator tests is also a bit special: // - enabled by default if using a beta Xcode, otherwise disabled by default changed = SetEnabled (labels, "old-simulator", ref IncludeOldSimulatorTests); - if (!changed && Harness.IsBetaXcode) { + if (!changed && IsBetaXcode) { IncludeOldSimulatorTests = true; MainLog.WriteLine ("Enabled 'old-simulator' tests because we're using a beta Xcode."); } } + bool IsBetaXcode => Harness.XcodeRoot.IndexOf ("beta", StringComparison.OrdinalIgnoreCase) >= 0; + // Returns true if the value was changed. bool SetEnabled (HashSet labels, string testname, ref bool value) { From c38b3903e9cb7b5a9e0b5d08f3fce15287af9fd5 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 16 Mar 2020 11:22:49 +0100 Subject: [PATCH 09/30] Remove RootDirectory --- tests/xharness/AppRunner.cs | 2 +- tests/xharness/Harness.cs | 9 ++++----- tests/xharness/Program.cs | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 0e487c68d10c..17fe7030571c 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -61,7 +61,7 @@ class AppRunner string configuration; public string Configuration { - get { return configuration ?? Harness.Configuration; } + get { return configuration ?? Harness.BuildConfiguration; } set { configuration = value; } } diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 524682522ed6..393d746f08ed 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -28,9 +28,10 @@ public interface IHarness { Version XcodeVersion { get; } Task ExecuteXcodeCommandAsync (string executable, IList args, ILog log, TimeSpan timeout); } + public class HarnessConfiguration { public bool AutoConf { get; set; } - public string Configuration { get; set; } = "Debug"; + public string BuildConfiguration { get; set; } = "Debug"; public bool DryRun { get; set; } public Dictionary EnvironmentVariables { get; set; } = new Dictionary (); public bool? IncludeSystemPermissionTests { get; set; } @@ -43,7 +44,6 @@ public class HarnessConfiguration { public string PeriodicCommand { get; set; } public string PeriodicCommandArguments { get; set; } public TimeSpan PeriodicCommandInterval { get; set; } - public string RootDirectory { get; set; } public string SdkRoot { get; set; } public AppRunnerTarget Target { get; set; } public double TimeoutInMinutes { get; set; } = 15; @@ -104,7 +104,7 @@ public static string Timestamp { // Run public AppRunnerTarget Target { get; set; } public string SdkRoot { get; set; } - public string Configuration { get; set; } = "Debug"; + public string BuildConfiguration { get; set; } = "Debug"; public string LogFile { get; set; } public string LogDirectory { get; set; } = Environment.CurrentDirectory; public double Timeout { get; set; } = 15; // in minutes @@ -128,7 +128,7 @@ public Harness (IProcessManager processManager, HarnessAction action, HarnessCon throw new ArgumentNullException (nameof (configuration)); autoConf = configuration.AutoConf; - Configuration = configuration.Configuration ?? throw new ArgumentNullException (nameof (configuration)); + BuildConfiguration = configuration.BuildConfiguration ?? throw new ArgumentNullException (nameof (configuration)); DryRun = configuration.DryRun; IncludeSystemPermissionTests = configuration.IncludeSystemPermissionTests; IOSTestProjects = configuration.IOSTestProjects; @@ -139,7 +139,6 @@ public Harness (IProcessManager processManager, HarnessAction action, HarnessCon PeriodicCommand = configuration.PeriodicCommand; PeriodicCommandArguments = configuration.PeriodicCommandArguments; PeriodicCommandInterval = configuration.PeriodicCommandInterval; - DirectoryUtilities.RootDirectory = configuration.RootDirectory; SdkRoot = configuration.SdkRoot; Target = configuration.Target; Timeout = configuration.TimeoutInMinutes; diff --git a/tests/xharness/Program.cs b/tests/xharness/Program.cs index 8839e1025f50..02dade5f0779 100644 --- a/tests/xharness/Program.cs +++ b/tests/xharness/Program.cs @@ -21,7 +21,7 @@ public static int Main (string [] args) { "mac", "Configure for Xamarin.Mac instead of iOS.", (v) => configuration.Mac = true }, { "configure", "Creates project files and makefiles.", (v) => action = HarnessAction.Configure }, { "autoconf", "Automatically decide what to configure.", (v) => configuration.AutoConf = true }, - { "rootdir=", "The root directory for the tests.", (v) => configuration.RootDirectory = v }, + { "rootdir=", "The root directory for the tests.", (v) => DirectoryUtilities.RootDirectory = v }, { "project=", "Add a project file to process. This can be specified multiple times.", (v) => configuration.IOSTestProjects.Add (new iOSTestProject (v)) }, { "watchos-container-template=", "The directory to use as a template for a watchos container app.", (v) => configuration.WatchOSContainerTemplate = v }, { "watchos-app-template=", "The directory to use as a template for a watchos app.", (v) => configuration.WatchOSAppTemplate = v }, @@ -47,7 +47,7 @@ public static int Main (string [] args) { "sdkroot=", "Where Xcode is", (v) => configuration.SdkRoot = v }, { "sdkroot94=", "Where Xcode 9.4 is", (v) => Console.WriteLine ("--sdkroot94 is deprecated"), true }, { "target=", "Where to run the project ([ios|watchos|tvos]-[device|simulator|simulator-32|simulator-64]).", (v) => configuration.Target = v.ParseAsAppRunnerTarget () }, - { "configuration=", "Which configuration to run (defaults to Debug).", (v) => configuration.Configuration = v }, + { "configuration=", "Which configuration to run (defaults to Debug).", (v) => configuration.BuildConfiguration = v }, { "logdirectory=", "Where to store logs.", (v) => configuration.LogDirectory = v }, { "logfile=", "Where to store the log.", (v) => Console.WriteLine("The logfile option is deprecated. Please use logdirectory."), true }, { "timeout=", $"Timeout for a test run (in minutes). Default is {configuration.TimeoutInMinutes} minutes.", (v) => configuration.TimeoutInMinutes = double.Parse (v) }, From 9143073724ecae8a3abb855280b0c9109d18607b Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 16 Mar 2020 11:33:14 +0100 Subject: [PATCH 10/30] Revert RootDirectory --- .../BCLTestImportTargetFactory.cs | 2 +- .../BCLTestProjectGenerator.cs | 2 +- tests/xharness/Harness.cs | 70 +++++++++++++------ tests/xharness/Jenkins/Jenkins.cs | 58 +++++++-------- .../Jenkins/TestTasks/NUnitExecuteTask.cs | 8 +-- tests/xharness/MakefileGenerator.cs | 4 +- tests/xharness/MonoNativeInfo.cs | 4 +- tests/xharness/Program.cs | 2 +- tests/xharness/SolutionGenerator.cs | 4 +- tests/xharness/Targets/TVOSTarget.cs | 2 +- .../xharness/Utilities/DirectoryUtilities.cs | 36 ---------- 11 files changed, 92 insertions(+), 100 deletions(-) diff --git a/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs b/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs index 567d423ffd41..58938cf609ae 100644 --- a/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs +++ b/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs @@ -16,7 +16,7 @@ public class BCLTestImportTargetFactory { public BCLTestImportTargetFactory (Harness harness) { Harness = harness; - var outputDir = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "bcl-test")); + var outputDir = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "bcl-test")); var projectTemplatePath = outputDir; var registerTypesTemplatePath = Path.Combine (outputDir, "RegisterType.cs.in"); var plistTemplatePath = outputDir; diff --git a/tests/xharness/BCLTestImporter/BCLTestProjectGenerator.cs b/tests/xharness/BCLTestImporter/BCLTestProjectGenerator.cs index 8d8ff4e6e0e5..fa7c271fd06b 100644 --- a/tests/xharness/BCLTestImporter/BCLTestProjectGenerator.cs +++ b/tests/xharness/BCLTestImporter/BCLTestProjectGenerator.cs @@ -114,7 +114,7 @@ public class BclTestProjectInfo { new BclTestProjectInfo { Name = "mscorlib Part 3", assemblies = new [] { "monotouch_corlib_xunit-test.part3.dll" }, Group = "mscorlib Part 3" }, new BclTestProjectInfo { Name = "SystemCoreXunit Part 1", assemblies = new [] { "monotouch_System.Core_xunit-test.part1.dll" }, Group = "SystemCoreXunit Part 1" }, new BclTestProjectInfo { Name = "SystemCoreXunit Part 2", assemblies = new [] { "monotouch_System.Core_xunit-test.part2.dll" }, Group = "SystemCoreXunit Part 2" }, - new BclTestProjectInfo { Name = "SystemXunit", assemblies = new [] { "monotouch_System_xunit-test.dll" }, ExtraArgs = $"--xml={Path.Combine (DirectoryUtilities.RootDirectory, "bcl-test", "SystemXunitLinker.xml")} --optimize=-custom-attributes-removal", Group = "SystemXunit" }, // special case due to the need of the extra args + new BclTestProjectInfo { Name = "SystemXunit", assemblies = new [] { "monotouch_System_xunit-test.dll" }, ExtraArgs = $"--xml={Path.Combine (Harness.RootDirectory, "bcl-test", "SystemXunitLinker.xml")} --optimize=-custom-attributes-removal", Group = "SystemXunit" }, // special case due to the need of the extra args }; static readonly List CommonIgnoredAssemblies = new List { diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 393d746f08ed..55e85ac5906d 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -64,7 +64,7 @@ public class Harness : IHarness public IProcessManager ProcessManager { get; } public string XIBuildPath { - get { return Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "tools", "xibuild", "xibuild")); } + get { return Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "tools", "xibuild", "xibuild")); } } public static string Timestamp { @@ -73,6 +73,34 @@ public static string Timestamp { } } + // This is the maccore/tests directory. + static string root_directory; + public static string RootDirectory { + get { + if (root_directory == null) { + var testAssemblyDirectory = Path.GetDirectoryName (System.Reflection.Assembly.GetExecutingAssembly ().Location); + var dir = testAssemblyDirectory; + var path = Path.Combine (testAssemblyDirectory, ".git"); + while (!Directory.Exists (path) && path.Length > 3) { + dir = Path.GetDirectoryName (dir); + path = Path.Combine (dir, ".git"); + } + if (!Directory.Exists (path)) + throw new Exception ("Could not find the xamarin-macios repo."); + path = Path.Combine (Path.GetDirectoryName (path), "tests"); + if (!Directory.Exists (path)) + throw new Exception ("Could not find the tests directory."); + root_directory = path; + } + return root_directory; + } + set { + root_directory = value; + if (root_directory != null) + root_directory = Path.GetFullPath (root_directory).TrimEnd ('/'); + } + } + public List IOSTestProjects { get; } public List MacTestProjects { get; } = new List (); @@ -223,7 +251,7 @@ public string MlaunchPath { void LoadConfig () { ParseConfigFiles (); - var src_root = Path.GetDirectoryName (Path.GetFullPath (DirectoryUtilities.RootDirectory)); + var src_root = Path.GetDirectoryName (Path.GetFullPath (Harness.RootDirectory)); MONO_PATH = Path.GetFullPath (Path.Combine (src_root, "external", "mono")); TVOS_MONO_PATH = MONO_PATH; INCLUDE_IOS = make_config.ContainsKey ("INCLUDE_IOS") && !string.IsNullOrEmpty (make_config ["INCLUDE_IOS"]); @@ -250,13 +278,13 @@ int AutoConfigureMac (bool generate_projects) new { Directory = "linker/mac/dont link", ProjectFile = "dont link-mac", Name = "dont link", Flavors = MacFlavors.Modern | MacFlavors.Full | MacFlavors.System }, }; foreach (var p in test_suites) { - MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p.Directory, p.ProjectFile + ".csproj"))) { + MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, p.Directory, p.ProjectFile + ".csproj"))) { Name = p.Name, TargetFrameworkFlavors = p.Flavors, }); } - MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "introspection", "Mac", "introspection-mac.csproj")), targetFrameworkFlavor: MacFlavors.Modern) { Name = "introspection" }); + MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "introspection", "Mac", "introspection-mac.csproj")), targetFrameworkFlavor: MacFlavors.Modern) { Name = "introspection" }); var hard_coded_test_suites = new [] { new { Directory = "mmptest", ProjectFile = "mmptest", Name = "mmptest", IsNUnit = true, Configurations = (string[]) null, Platform = "x86", Flavors = MacFlavors.Console, }, @@ -266,10 +294,10 @@ int AutoConfigureMac (bool generate_projects) new { Directory = "linker/mac/link sdk", ProjectFile = "link sdk-mac", Name = "link sdk", IsNUnit = false, Configurations = new string [] { "Debug", "Release" }, Platform = "x86", Flavors = MacFlavors.Modern, }, }; foreach (var p in hard_coded_test_suites) { - MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p.Directory, p.ProjectFile + ".csproj")), targetFrameworkFlavor: p.Flavors) { + MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, p.Directory, p.ProjectFile + ".csproj")), targetFrameworkFlavor: p.Flavors) { Name = p.Name, IsNUnitProject = p.IsNUnit, - SolutionPath = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "tests-mac.sln")), + SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "tests-mac.sln")), Configurations = p.Configurations, Platform = p.Platform, }); @@ -368,18 +396,18 @@ void AutoConfigureIOS () var fsharp_library_projects = new string [] { "fsharplibrary" }; foreach (var p in test_suites) - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p + "/" + p + ".csproj"))) { Name = p }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, p + "/" + p + ".csproj"))) { Name = p }); foreach (var p in fsharp_test_suites) - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p + "/" + p + ".fsproj"))) { Name = p }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, p + "/" + p + ".fsproj"))) { Name = p }); foreach (var p in library_projects) - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p + "/" + p + ".csproj")), false) { Name = p }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, p + "/" + p + ".csproj")), false) { Name = p }); foreach (var p in fsharp_library_projects) - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, p + "/" + p + ".fsproj")), false) { Name = p }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, p + "/" + p + ".fsproj")), false) { Name = p }); - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "introspection", "iOS", "introspection-ios.csproj"))) { Name = "introspection" }); - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "linker", "ios", "dont link", "dont link.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "linker", "ios", "link all", "link all.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); - IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "linker", "ios", "link sdk", "link sdk.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "introspection", "iOS", "introspection-ios.csproj"))) { Name = "introspection" }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "linker", "ios", "dont link", "dont link.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "linker", "ios", "link all", "link all.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); + IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "linker", "ios", "link sdk", "link sdk.csproj"))) { Configurations = new string [] { "Debug", "Release" } }); foreach (var flavor in new MonoNativeFlavor[] { MonoNativeFlavor.Compat, MonoNativeFlavor.Unified }) { var monoNativeInfo = new MonoNativeInfo (this, flavor); @@ -396,19 +424,19 @@ void AutoConfigureIOS () var monoImportTestFactory = new BCLTestImportTargetFactory (this); IOSTestProjects.AddRange (monoImportTestFactory.GetiOSBclTargets ()); - WatchOSContainerTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "templates/WatchContainer")); - WatchOSAppTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "templates/WatchApp")); - WatchOSExtensionTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "templates/WatchExtension")); + WatchOSContainerTemplate = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "templates/WatchContainer")); + WatchOSAppTemplate = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "templates/WatchApp")); + WatchOSExtensionTemplate = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "templates/WatchExtension")); - TodayContainerTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "templates", "TodayContainer")); - TodayExtensionTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "templates", "TodayExtension")); - BCLTodayExtensionTemplate = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "bcl-test", "templates", "today")); + TodayContainerTemplate = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "templates", "TodayContainer")); + TodayExtensionTemplate = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "templates", "TodayExtension")); + BCLTodayExtensionTemplate = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "bcl-test", "templates", "today")); } Dictionary make_config = new Dictionary (); IEnumerable FindConfigFiles (string name) { - var dir = Path.GetFullPath (DirectoryUtilities.RootDirectory); + var dir = Path.GetFullPath (Harness.RootDirectory); while (dir != "/") { var file = Path.Combine (dir, name); if (File.Exists (file)) diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index 4cc9004f39bd..20e179dc10a9 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -925,17 +925,17 @@ Task PopulateTasksAsync () var buildiOSMSBuild_net461 = new MSBuildTask () { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = true, ProjectConfiguration = "Debug-net461", Platform = TestPlatform.iOS, - SolutionPath = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), + SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), SupportsParallelExecution = false, }; var nunitExecutioniOSMSBuild_net461 = new NUnitExecuteTask (buildiOSMSBuild_net461) { - TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-net461", "net461", "Xamarin.iOS.Tasks.Tests.dll"), + TestLibrary = Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-net461", "net461", "Xamarin.iOS.Tasks.Tests.dll"), TestProject = buildiOSMSBuild_net461.TestProject, ProjectConfiguration = "Debug-net461", Platform = TestPlatform.iOS, @@ -949,16 +949,16 @@ Task PopulateTasksAsync () var buildiOSMSBuild_netstandard2 = new MSBuildTask () { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "Xamarin.iOS.Tasks.Tests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = true, ProjectConfiguration = "Debug-netstandard2.0", Platform = TestPlatform.iOS, - SolutionPath = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), + SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), SupportsParallelExecution = false, }; var nunitExecutioniOSMSBuild_netstandard2 = new NUnitExecuteTask (buildiOSMSBuild_netstandard2) { - TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-netstandard2.0", "net461", "Xamarin.iOS.Tasks.Tests.dll"), + TestLibrary = Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-netstandard2.0", "net461", "Xamarin.iOS.Tasks.Tests.dll"), TestProject = buildiOSMSBuild_netstandard2.TestProject, ProjectConfiguration = "Debug-netstandard2.0", Platform = TestPlatform.iOS, @@ -973,15 +973,15 @@ Task PopulateTasksAsync () var buildInstallSources = new MSBuildTask () { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "InstallSourcesTests.csproj"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "InstallSourcesTests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = false, Platform = TestPlatform.iOS, }; - buildInstallSources.SolutionPath = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "tools", "install-source", "install-source.sln")); // this is required for nuget restore to be executed + buildInstallSources.SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "install-source.sln")); // this is required for nuget restore to be executed var nunitExecutionInstallSource = new NUnitExecuteTask (buildInstallSources) { - TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "bin", "Release", "InstallSourcesTests.dll"), + TestLibrary = Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "bin", "Release", "InstallSourcesTests.dll"), TestProject = buildInstallSources.TestProject, Platform = TestPlatform.iOS, TestName = "Install Sources tests", @@ -1066,17 +1066,17 @@ Task PopulateTasksAsync () var buildMTouch = new MakeTask () { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "mtouch", "mtouch.sln"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch", "mtouch.sln"))), SpecifyPlatform = false, SpecifyConfiguration = false, Platform = TestPlatform.iOS, Target = "dependencies", - WorkingDirectory = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "mtouch")), + WorkingDirectory = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch")), }; var nunitExecutionMTouch = new NUnitExecuteTask (buildMTouch) { - TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "mtouch", "bin", "Debug", "mtouch.dll"), - TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "mtouch", "mtouch.csproj"))), + TestLibrary = Path.Combine (Harness.RootDirectory, "mtouch", "bin", "Debug", "mtouch.dll"), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch", "mtouch.csproj"))), Platform = TestPlatform.iOS, TestName = "MTouch tests", Timeout = TimeSpan.FromMinutes (180), @@ -1087,16 +1087,16 @@ Task PopulateTasksAsync () var buildGenerator = new MakeTask { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "..", "src", "generator.sln"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "src", "generator.sln"))), SpecifyPlatform = false, SpecifyConfiguration = false, Platform = TestPlatform.iOS, Target = "build-unit-tests", - WorkingDirectory = Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "generator")), + WorkingDirectory = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "generator")), }; var runGenerator = new NUnitExecuteTask (buildGenerator) { - TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "generator", "bin", "Debug", "generator-tests.dll"), - TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "generator", "generator-tests.csproj"))), + TestLibrary = Path.Combine (Harness.RootDirectory, "generator", "bin", "Debug", "generator-tests.dll"), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "generator", "generator-tests.csproj"))), Platform = TestPlatform.iOS, TestName = "Generator tests", Mode = "NUnit", @@ -1107,7 +1107,7 @@ Task PopulateTasksAsync () var buildDotNetGenerator = new DotNetBuildTask { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "bgen", "bgen-tests.csproj"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "bgen", "bgen-tests.csproj"))), SpecifyPlatform = false, SpecifyConfiguration = false, Platform = TestPlatform.iOS, @@ -1127,7 +1127,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.Mac, TestName = "MMP Regression Tests", Target = "all", // -j" + Environment.ProcessorCount, - WorkingDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "mmptest", "regression"), + WorkingDirectory = Path.Combine (Harness.RootDirectory, "mmptest", "regression"), Ignored = !IncludeMmpTest || !IncludeMac, Timeout = TimeSpan.FromMinutes (30), SupportsParallelExecution = false, // Already doing parallel execution by running "make -jX" @@ -1146,7 +1146,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.Mac, TestName = "Mac Binding Projects", Target = "all", - WorkingDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "mac-binding-project"), + WorkingDirectory = Path.Combine (Harness.RootDirectory, "mac-binding-project"), Ignored = !IncludeMacBindingProject || !IncludeMac, Timeout = TimeSpan.FromMinutes (15), }; @@ -1157,7 +1157,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.All, TestName = "Xtro", Target = "wrench", - WorkingDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "xtro-sharpie"), + WorkingDirectory = Path.Combine (Harness.RootDirectory, "xtro-sharpie"), Ignored = !IncludeXtro, Timeout = TimeSpan.FromMinutes (15), }; @@ -1175,7 +1175,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.All, TestName = "Cecil", Target = "build", - WorkingDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "cecil-tests"), + WorkingDirectory = Path.Combine (Harness.RootDirectory, "cecil-tests"), Ignored = !IncludeCecil, Timeout = TimeSpan.FromMinutes (5), }; @@ -1195,7 +1195,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.All, TestName = "Documentation", Target = "wrench-docs", - WorkingDirectory = DirectoryUtilities.RootDirectory, + WorkingDirectory = Harness.RootDirectory, Ignored = !IncludeDocs, Timeout = TimeSpan.FromMinutes (45), }; @@ -1203,14 +1203,14 @@ Task PopulateTasksAsync () var buildSampleTests = new MSBuildTask { Jenkins = this, - TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "sampletester", "sampletester.sln"))), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "sampletester", "sampletester.sln"))), SpecifyPlatform = false, Platform = TestPlatform.All, ProjectConfiguration = "Debug", }; var runSampleTests = new NUnitExecuteTask (buildSampleTests) { - TestLibrary = Path.Combine (DirectoryUtilities.RootDirectory, "sampletester", "bin", "Debug", "sampletester.dll"), - TestProject = new TestProject (Path.GetFullPath (Path.Combine (DirectoryUtilities.RootDirectory, "sampletester", "sampletester.csproj"))), + TestLibrary = Path.Combine (Harness.RootDirectory, "sampletester", "bin", "Debug", "sampletester.dll"), + TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "sampletester", "sampletester.csproj"))), Platform = TestPlatform.All, TestName = "Sample tests", Timeout = TimeSpan.FromDays (1), // These can take quite a while to execute. @@ -1308,7 +1308,7 @@ Task BuildTestLibrariesAsync () var sb = new StringBuilder (); var callback_log = new CallbackLog ((v) => sb.Append (v)); var log = Log.CreateAggregatedLog (callback_log, MainLog); - return Harness.ProcessManager.ExecuteCommandAsync ("make", new [] { "all", $"-j{Environment.ProcessorCount}", "-C", Path.Combine (DirectoryUtilities.RootDirectory, "test-libraries") }, log, TimeSpan.FromMinutes (10)).ContinueWith ((v) => { + return Harness.ProcessManager.ExecuteCommandAsync ("make", new [] { "all", $"-j{Environment.ProcessorCount}", "-C", Path.Combine (Harness.RootDirectory, "test-libraries") }, log, TimeSpan.FromMinutes (10)).ContinueWith ((v) => { var per = v.Result; if (!per.Succeeded) { // Only show the log if something went wrong. @@ -1589,7 +1589,7 @@ IEnumerable find_tasks (StreamWriter writer, string ids) server.Stop (); break; case "/favicon.ico": - serveFile = Path.Combine (DirectoryUtilities.RootDirectory, "xharness", "favicon.ico"); + serveFile = Path.Combine (Harness.RootDirectory, "xharness", "favicon.ico"); goto default; case "/index.html": var redirect_to = request.Url.AbsoluteUri.Replace ("/index.html", "/" + Path.GetFileName (LogDirectory) + "/index.html"); @@ -1752,7 +1752,7 @@ public void GenerateReport () foreach (var file in new string [] { "xharness.js", "xharness.css" }) { File.Copy (Path.Combine (dependentFileLocation, file), Path.Combine (LogDirectory, file), true); } - File.Copy (Path.Combine (DirectoryUtilities.RootDirectory, "xharness", "favicon.ico"), Path.Combine (LogDirectory, "favicon.ico"), true); + File.Copy (Path.Combine (Harness.RootDirectory, "xharness", "favicon.ico"), Path.Combine (LogDirectory, "favicon.ico"), true); } } catch (Exception e) { this.MainLog.WriteLine ("Failed to write log: {0}", e); diff --git a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs index c3e9cdacd155..3bc600632cac 100644 --- a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs @@ -73,15 +73,15 @@ public void FindNUnitConsoleExecutable (ILog log) } if (is_packageref) { - TestExecutable = Path.Combine (DirectoryUtilities.RootDirectory, "..", "tools", $"nunit3-console-{nunit_version}"); + TestExecutable = Path.Combine (Harness.RootDirectory, "..", "tools", $"nunit3-console-{nunit_version}"); if (!File.Exists (TestExecutable)) throw new FileNotFoundException ($"The helper script to execute the unit tests does not exist: {TestExecutable}"); WorkingDirectory = Path.GetDirectoryName (TestProject.Path); } else if (nunit_version [0] == '2') { - TestExecutable = Path.Combine (DirectoryUtilities.RootDirectory, "..", "packages", "NUnit.Runners." + nunit_version, "tools", "nunit-console.exe"); + TestExecutable = Path.Combine (Harness.RootDirectory, "..", "packages", "NUnit.Runners." + nunit_version, "tools", "nunit-console.exe"); WorkingDirectory = Path.Combine (Path.GetDirectoryName (TestExecutable), "lib"); } else { - TestExecutable = Path.Combine (DirectoryUtilities.RootDirectory, "..", "packages", "NUnit.ConsoleRunner." + nunit_version, "tools", "nunit3-console.exe"); + TestExecutable = Path.Combine (Harness.RootDirectory, "..", "packages", "NUnit.ConsoleRunner." + nunit_version, "tools", "nunit3-console.exe"); WorkingDirectory = Path.GetDirectoryName (TestLibrary); } TestExecutable = Path.GetFullPath (TestExecutable); @@ -136,7 +136,7 @@ protected override async Task RunTestAsync () if (ProduceHtmlReport) { try { var output = Logs.Create ($"Log-{Timestamp}.html", "HTML log"); - using (var srt = new StringReader (File.ReadAllText (Path.Combine (DirectoryUtilities.RootDirectory, "HtmlTransform.xslt")))) { + using (var srt = new StringReader (File.ReadAllText (Path.Combine (Harness.RootDirectory, "HtmlTransform.xslt")))) { using (var sri = File.OpenRead (xmlLog)) { using (var xrt = XmlReader.Create (srt)) { using (var xri = XmlReader.Create (sri)) { diff --git a/tests/xharness/MakefileGenerator.cs b/tests/xharness/MakefileGenerator.cs index c1d99cc7b853..8e7c7e64988f 100644 --- a/tests/xharness/MakefileGenerator.cs +++ b/tests/xharness/MakefileGenerator.cs @@ -59,7 +59,7 @@ static string CreateRelativePath (string path, string relative_to) public static void CreateMacMakefile (Harness harness, IEnumerable targets) { - var makefile = Path.Combine (DirectoryUtilities.RootDirectory, "Makefile-mac.inc"); + var makefile = Path.Combine (Harness.RootDirectory, "Makefile-mac.inc"); using (var writer = new StreamWriter (makefile, false, new UTF8Encoding (false))) { writer.WriteLine (".stamp-configure-projects-mac: Makefile xharness/xharness.exe"); writer.WriteLine ("\t$(Q) $(SYSTEM_MONO) --debug $(XIBUILD_EXE_PATH) -t -- $(CURDIR)/xharness/xharness.exe $(XHARNESS_VERBOSITY) --configure --autoconf --rootdir $(CURDIR)"); @@ -214,7 +214,7 @@ static string GetMakeName (this Target target, bool escape = true) public static void CreateMakefile (Harness harness, IEnumerable unified_targets, IEnumerable tvos_targets, IEnumerable watchos_targets, IEnumerable today_targets) { var executeSim32 = !harness.InCI; // Waiting for iOS 10.3 simulator to be installed on wrench - var makefile = Path.Combine (DirectoryUtilities.RootDirectory, "Makefile.inc"); + var makefile = Path.Combine (Harness.RootDirectory, "Makefile.inc"); using (var writer = new StreamWriter (makefile, false, new UTF8Encoding (false))) { writer.WriteLine (".stamp-configure-projects: Makefile xharness/xharness.exe"); writer.WriteLine ("\t$(Q) $(SYSTEM_MONO) --debug $(XIBUILD_EXE_PATH) -t -- $(CURDIR)/xharness/xharness.exe $(XHARNESS_VERBOSITY) --configure --autoconf --rootdir $(CURDIR)"); diff --git a/tests/xharness/MonoNativeInfo.cs b/tests/xharness/MonoNativeInfo.cs index 92176de9041e..1f241e28b9ce 100644 --- a/tests/xharness/MonoNativeInfo.cs +++ b/tests/xharness/MonoNativeInfo.cs @@ -157,9 +157,9 @@ public MonoNativeInfo (Harness harness, MonoNativeFlavor flavor) public string FlavorSuffix => Flavor == MonoNativeFlavor.Compat ? "-compat" : "-unified"; public string ProjectName => "mono-native" + FlavorSuffix; - public string ProjectPath => Path.Combine (DirectoryUtilities.RootDirectory, "mono-native", TemplateName + FlavorSuffix + ".csproj"); + public string ProjectPath => Path.Combine (Harness.RootDirectory, "mono-native", TemplateName + FlavorSuffix + ".csproj"); string TemplateName => "mono-native" + TemplateSuffix; - public string TemplatePath => Path.Combine (DirectoryUtilities.RootDirectory, "mono-native", TemplateName + ".csproj.template"); + public string TemplatePath => Path.Combine (Harness.RootDirectory, "mono-native", TemplateName + ".csproj.template"); protected virtual string TemplateSuffix => string.Empty; public void Convert () diff --git a/tests/xharness/Program.cs b/tests/xharness/Program.cs index 02dade5f0779..01c9a6ce43ea 100644 --- a/tests/xharness/Program.cs +++ b/tests/xharness/Program.cs @@ -21,7 +21,7 @@ public static int Main (string [] args) { "mac", "Configure for Xamarin.Mac instead of iOS.", (v) => configuration.Mac = true }, { "configure", "Creates project files and makefiles.", (v) => action = HarnessAction.Configure }, { "autoconf", "Automatically decide what to configure.", (v) => configuration.AutoConf = true }, - { "rootdir=", "The root directory for the tests.", (v) => DirectoryUtilities.RootDirectory = v }, + { "rootdir=", "The root directory for the tests.", (v) => Harness.RootDirectory = v }, { "project=", "Add a project file to process. This can be specified multiple times.", (v) => configuration.IOSTestProjects.Add (new iOSTestProject (v)) }, { "watchos-container-template=", "The directory to use as a template for a watchos container app.", (v) => configuration.WatchOSContainerTemplate = v }, { "watchos-app-template=", "The directory to use as a template for a watchos app.", (v) => configuration.WatchOSAppTemplate = v }, diff --git a/tests/xharness/SolutionGenerator.cs b/tests/xharness/SolutionGenerator.cs index 04c1690d5ac9..d24820174640 100644 --- a/tests/xharness/SolutionGenerator.cs +++ b/tests/xharness/SolutionGenerator.cs @@ -57,8 +57,8 @@ public static void CreateSolution (Harness harness, IEnumerable targets, { var folders = new StringBuilder (); - var srcDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "..", "src"); - var sln_path = exeTarget == null ? Path.Combine (DirectoryUtilities.RootDirectory, "tests-" + infix + ".sln") : Path.Combine (Path.GetDirectoryName (exeTarget.ProjectPath), Path.GetFileNameWithoutExtension (exeTarget.ProjectPath) + ".sln"); + var srcDirectory = Path.Combine (Harness.RootDirectory, "..", "src"); + var sln_path = exeTarget == null ? Path.Combine (Harness.RootDirectory, "tests-" + infix + ".sln") : Path.Combine (Path.GetDirectoryName (exeTarget.ProjectPath), Path.GetFileNameWithoutExtension (exeTarget.ProjectPath) + ".sln"); using (var writer = new StringWriter ()) { writer.WriteLine (); diff --git a/tests/xharness/Targets/TVOSTarget.cs b/tests/xharness/Targets/TVOSTarget.cs index 864fb2f22f56..caacb71d6be4 100644 --- a/tests/xharness/Targets/TVOSTarget.cs +++ b/tests/xharness/Targets/TVOSTarget.cs @@ -113,7 +113,7 @@ protected override void ProcessProject () MonoNativeHelper.RemoveSymlinkMode (inputProject); } - var srcDirectory = Path.Combine (DirectoryUtilities.RootDirectory, "..", "src"); + var srcDirectory = Path.Combine (Harness.RootDirectory, "..", "src"); string project_guid; var mt_nunitlite_project_path = Path.GetFullPath (Path.Combine (srcDirectory, "MonoTouch.NUnitLite.tvos.csproj")); diff --git a/tests/xharness/Utilities/DirectoryUtilities.cs b/tests/xharness/Utilities/DirectoryUtilities.cs index a3cba5f3ef75..71a114ba1171 100644 --- a/tests/xharness/Utilities/DirectoryUtilities.cs +++ b/tests/xharness/Utilities/DirectoryUtilities.cs @@ -52,41 +52,5 @@ public static string CreateTemporaryDirectory (string name = null) throw new Exception ("Could not create temporary directory"); } - - static string root_directory; - public static string RootDirectory { - get { - if (root_directory == null) - root_directory = GetRootDirectory(); - - return root_directory; - } - - set { - root_directory = value; - if (root_directory != null) - root_directory = Path.GetFullPath (root_directory).TrimEnd ('/'); - } - } - - private static string GetRootDirectory() { - var dir = Path.GetDirectoryName (System.Reflection.Assembly.GetExecutingAssembly ().Location); - - var path = Path.Combine (dir, ".git"); - while (!Directory.Exists (path) && path.Length > 3) { - dir = Path.GetDirectoryName (dir); - path = Path.Combine (dir, ".git"); - } - - if (!Directory.Exists (path)) - throw new Exception ("Could not find the xamarin-macios repo."); - - path = Path.Combine (Path.GetDirectoryName (path), "tests"); - - if (!Directory.Exists (path)) - throw new Exception ("Could not find the tests directory."); - - return path; - } } } From fbcb828c6deaa9ee7ac52c0aafc29234d8f0094a Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 11:01:39 +0100 Subject: [PATCH 11/30] Remove Initialize() calls --- tests/xharness/AppRunner.cs | 54 +++++++++++++++---------------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 52faf2f273eb..440ee714bd3e 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -14,7 +14,6 @@ using Xharness.Listeners; using Xharness.Logging; using Xharness.Utilities; -using System.Net; namespace Xharness { @@ -136,9 +135,6 @@ public AppRunner (IProcessManager processManager, AppInformation Initialize () { - if (AppInformation != null) - return AppInformation; - var csproj = new XmlDocument (); csproj.LoadWithoutNetworkAccess (projectFilePath); string appName = csproj.GetAssemblyName (); @@ -238,8 +234,6 @@ void FindDevice () public async Task InstallAsync (CancellationToken cancellation_token) { - var appInfo = Initialize (); - if (isSimulator) { // We reset the simulator when running, so a separate install step does not make much sense. throw new InvalidOperationException ("Installing to a simulator is not supported."); @@ -256,7 +250,7 @@ public async Task InstallAsync (CancellationToken cancel args.Add ("-v"); args.Add ("--installdev"); - args.Add (appInfo.AppPath); + args.Add (AppInformation.AppPath); AddDeviceName (args, companionDeviceName ?? deviceName); if (mode == RunMode.WatchOS) { @@ -264,16 +258,14 @@ public async Task InstallAsync (CancellationToken cancel args.Add ("ios,watchos"); } - var totalSize = Directory.GetFiles (appInfo.AppPath, "*", SearchOption.AllDirectories).Select ((v) => new FileInfo (v).Length).Sum (); - MainLog.WriteLine ($"Installing '{appInfo.AppPath}' to '{companionDeviceName ?? deviceName}'. Size: {totalSize} bytes = {totalSize / 1024.0 / 1024.0:N2} MB"); + var totalSize = Directory.GetFiles (AppInformation.AppPath, "*", SearchOption.AllDirectories).Select ((v) => new FileInfo (v).Length).Sum (); + MainLog.WriteLine ($"Installing '{AppInformation.AppPath}' to '{companionDeviceName ?? deviceName}'. Size: {totalSize} bytes = {totalSize / 1024.0 / 1024.0:N2} MB"); return await processManager.ExecuteCommandAsync (harness.MlaunchPath, args, MainLog, TimeSpan.FromHours (1), cancellation_token: cancellation_token); } public async Task UninstallAsync () { - var appInfo = Initialize (); - if (isSimulator) throw new InvalidOperationException ("Uninstalling from a simulator is not supported."); @@ -288,7 +280,7 @@ public async Task UninstallAsync () args.Add ("-v"); args.Add ("--uninstalldevbundleid"); - args.Add (appInfo.BundleIdentifier); + args.Add (AppInformation.BundleIdentifier); AddDeviceName (args, companionDeviceName ?? deviceName); return await processManager.ExecuteCommandAsync (harness.MlaunchPath, args, MainLog, TimeSpan.FromMinutes (1)); @@ -436,8 +428,6 @@ public async Task RunAsync () ILog listener_log = null; ILog run_log = MainLog; - var appInfo = Initialize (); - if (!isSimulator) FindDevice (); @@ -532,12 +522,12 @@ public async Task RunAsync () bool launch_failure = false; if (IsExtension) { - switch (appInfo.Extension) { + switch (AppInformation.Extension) { case Extension.TodayExtension: args.Add (isSimulator ? "--launchsimbundleid" : "--launchdevbundleid"); - args.Add ("todayviewforextensions:" + appInfo.BundleIdentifier); + args.Add ("todayviewforextensions:" + AppInformation.BundleIdentifier); args.Add ("--observe-extension"); - args.Add (appInfo.LaunchAppPath); + args.Add (AppInformation.LaunchAppPath); break; case Extension.WatchKit2: default: @@ -545,7 +535,7 @@ public async Task RunAsync () } } else { args.Add (isSimulator ? "--launchsim" : "--launchdev"); - args.Add (appInfo.LaunchAppPath); + args.Add (AppInformation.LaunchAppPath); } if (!isSimulator) args.Add ("--disable-memory-limits"); @@ -584,11 +574,11 @@ public async Task RunAsync () WrenchLog.WriteLine ("AddFile: {0}", log.Path); } - MainLog.WriteLine ("*** Executing {0}/{1} in the simulator ***", appInfo.AppName, mode); + MainLog.WriteLine ("*** Executing {0}/{1} in the simulator ***", AppInformation.AppName, mode); if (EnsureCleanSimulatorState) { foreach (var sim in simulators) - await sim.PrepareSimulatorAsync (MainLog, appInfo.BundleIdentifier); + await sim.PrepareSimulatorAsync (MainLog, AppInformation.BundleIdentifier); } args.Add ($"--device=:v2:udid={simulator.UDID}"); @@ -649,7 +639,7 @@ public async Task RunAsync () log.StopCapture (); } else { - MainLog.WriteLine ("*** Executing {0}/{1} on device '{2}' ***", appInfo.AppName, mode, deviceName); + MainLog.WriteLine ("*** Executing {0}/{1} on device '{2}' ***", AppInformation.AppName, mode, deviceName); if (mode == RunMode.WatchOS) { args.Add ("--attach-native-debugger"); // this prevents the watch from backgrounding the app. @@ -722,7 +712,7 @@ public async Task RunAsync () var crashed = false; if (File.Exists (listener_log.FullPath)) { WrenchLog.WriteLine ("AddFile: {0}", listener_log.FullPath); - success = TestsSucceeded (AppInformation, listener_log.FullPath, timed_out, out crashed); + success = TestsSucceeded (this.AppInformation, listener_log.FullPath, timed_out, out crashed); } else if (timed_out) { WrenchLog.WriteLine ("AddSummary: {0} never launched
", mode); MainLog.WriteLine ("Test run never launched"); @@ -795,9 +785,9 @@ public async Task RunAsync () if (harness.InCI) { XmlResultParser.GenerateFailure (Logs, "crash", - appInfo.AppName, + AppInformation.AppName, variation, - $"App Crash {appInfo.AppName} {variation}", + $"App Crash {AppInformation.AppName} {variation}", $"App crashed {crash_reason}.", crash_reports.Log.FullPath, harness.XmlJargon); @@ -819,9 +809,9 @@ public async Task RunAsync () XmlResultParser.GenerateFailure ( Logs, "crash", - appInfo.AppName, + AppInformation.AppName, variation, - $"App Crash {appInfo.AppName} {variation}", + $"App Crash {AppInformation.AppName} {variation}", $"App crashed: {FailureMessage}", crash_reports.Log.FullPath, harness.XmlJargon); @@ -833,9 +823,9 @@ public async Task RunAsync () XmlResultParser.GenerateFailure ( Logs, "launch", - appInfo.AppName, + AppInformation.AppName, variation, - $"App Launch {appInfo.AppName} {variation} on {deviceName}", + $"App Launch {AppInformation.AppName} {variation} on {deviceName}", $"{FailureMessage} on {deviceName}", MainLog.FullPath, XmlResultJargon.NUnitV3); @@ -858,7 +848,7 @@ public async Task RunAsync () if (isTcp) { XmlResultParser.GenerateFailure (Logs, "tcp-connection", - appInfo.AppName, + AppInformation.AppName, variation, $"TcpConnection on {deviceName}", $"Device {deviceName} could not reach the host over tcp.", @@ -868,10 +858,10 @@ public async Task RunAsync () } else if (timed_out && harness.InCI) { XmlResultParser.GenerateFailure (Logs, "timeout", - appInfo.AppName, + AppInformation.AppName, variation, - $"App Timeout {appInfo.AppName} {variation} on bot {deviceName}", - $"{appInfo.AppName} {variation} Test run timed out after {timeout.TotalMinutes} minute(s) on bot {deviceName}.", + $"App Timeout {AppInformation.AppName} {variation} on bot {deviceName}", + $"{AppInformation.AppName} {variation} Test run timed out after {timeout.TotalMinutes} minute(s) on bot {deviceName}.", MainLog.FullPath, harness.XmlJargon); } From 3435023aff619d6c782123f415ace0c65fcbee5f Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 11:39:34 +0100 Subject: [PATCH 12/30] Make CrashReportSnapshot injectable --- tests/xharness/AppRunner.cs | 9 ++- tests/xharness/CrashReportSnapshot.cs | 77 ++++++++++++------- tests/xharness/Jenkins/Jenkins.cs | 52 +++++++------ .../Jenkins/TestTasks/DotNetTestTask.cs | 5 +- .../Jenkins/TestTasks/MacExecuteTask.cs | 10 ++- tests/xharness/Jenkins/TestTasks/MacTask.cs | 5 +- .../Jenkins/TestTasks/NUnitExecuteTask.cs | 4 +- .../Jenkins/TestTasks/RunDeviceTask.cs | 4 +- .../Jenkins/TestTasks/RunSimulatorTask.cs | 4 +- .../xharness/Jenkins/TestTasks/RunTestTask.cs | 6 +- tests/xharness/Jenkins/TestTasks/RunXITask.cs | 5 +- .../xharness/Jenkins/TestTasks/RunXtroTask.cs | 6 +- .../Tests/CrashReportSnapshotTests.cs | 8 ++ .../Xharness.Tests/Xharness.Tests.csproj | 1 + 14 files changed, 124 insertions(+), 72 deletions(-) create mode 100644 tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 440ee714bd3e..9417ce1930b8 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -14,6 +14,7 @@ using Xharness.Listeners; using Xharness.Logging; using Xharness.Utilities; +using System.Web.UI.HtmlControls; namespace Xharness { @@ -431,7 +432,7 @@ public async Task RunAsync () if (!isSimulator) FindDevice (); - crash_reports = new CrashReportSnapshot (harness, MainLog, Logs, isDevice: !isSimulator, deviceName); + crash_reports = new CrashReportSnapshot (harness, processManager, MainLog, Logs, isDevice: !isSimulator, deviceName); var args = new List (); if (!string.IsNullOrEmpty (harness.XcodeRoot)) { @@ -747,7 +748,7 @@ public async Task RunAsync () if (!success.Value) { int pid = 0; string crash_reason = null; - foreach (var crash in crash_reports.Logs) { + foreach (var crash in Logs) { try { if (pid == 0) { // Find the pid @@ -789,7 +790,7 @@ public async Task RunAsync () variation, $"App Crash {AppInformation.AppName} {variation}", $"App crashed {crash_reason}.", - crash_reports.Log.FullPath, + MainLog.FullPath, harness.XmlJargon); } @@ -813,7 +814,7 @@ public async Task RunAsync () variation, $"App Crash {AppInformation.AppName} {variation}", $"App crashed: {FailureMessage}", - crash_reports.Log.FullPath, + MainLog.FullPath, harness.XmlJargon); } } else if (launch_failure) { diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs index f142793c0685..3fd7811723ef 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashReportSnapshot.cs @@ -4,33 +4,57 @@ using System.IO; using System.Threading; using System.Threading.Tasks; +using Xharness.Execution; using Xharness.Logging; namespace Xharness { - public class CrashReportSnapshot - { + public interface ICrashReportSnapshotFactory { + ICrashReportSnapshot Create (ILog log, ILogs logs, bool isDevice, string deviceName); + } + + public class CrashReportSnapshotFactory : ICrashReportSnapshotFactory { readonly IHarness harness; + readonly IProcessManager processManager; + + public CrashReportSnapshotFactory (IHarness harness, IProcessManager processManager) + { + this.harness = harness ?? throw new ArgumentNullException (nameof (harness)); + this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); + } + + public ICrashReportSnapshot Create (ILog log, ILogs logs, bool isDevice, string deviceName) => + new CrashReportSnapshot (harness, processManager, log, logs, isDevice, deviceName); + } + + public interface ICrashReportSnapshot { + Task EndCaptureAsync (TimeSpan timeout); + Task StartCaptureAsync (); + } + + public class CrashReportSnapshot : ICrashReportSnapshot { + readonly IHarness harness; + readonly IProcessManager processManager; + readonly ILog log; + readonly ILogs logs; readonly bool isDevice; readonly string deviceName; - public ILog Log { get; } - public ILogs Logs { get; } - - public HashSet InitialSet { get; private set; } + HashSet initialSet; - public CrashReportSnapshot (IHarness harness, ILog log, ILogs logs, bool isDevice, string deviceName) + public CrashReportSnapshot (IHarness harness, IProcessManager processManager, ILog log, ILogs logs, bool isDevice, string deviceName) { this.harness = harness ?? throw new ArgumentNullException (nameof (harness)); - this.Log = log ?? throw new ArgumentNullException (nameof (log)); - this.Logs = logs ?? throw new ArgumentNullException (nameof (logs)); + this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); + this.log = log ?? throw new ArgumentNullException (nameof (log)); + this.logs = logs ?? throw new ArgumentNullException (nameof (logs)); this.isDevice = isDevice; this.deviceName = deviceName; } public async Task StartCaptureAsync () { - InitialSet = await CreateCrashReportsSnapshotAsync (); + initialSet = await CreateCrashReportsSnapshotAsync (); } public async Task EndCaptureAsync (TimeSpan timeout) @@ -42,14 +66,14 @@ public async Task EndCaptureAsync (TimeSpan timeout) watch.Start (); do { var end_crashes = await CreateCrashReportsSnapshotAsync (); - end_crashes.ExceptWith (InitialSet); + end_crashes.ExceptWith (initialSet); if (end_crashes.Count > 0) { - Log.WriteLine ("Found {0} new crash report(s)", end_crashes.Count); + log.WriteLine ("Found {0} new crash report(s)", end_crashes.Count); List crash_reports; if (!isDevice) { crash_reports = new List (end_crashes.Count); foreach (var path in end_crashes) { - Logs.AddFile (path, $"Crash report: {Path.GetFileName (path)}"); + logs.AddFile (path, $"Crash report: {Path.GetFileName (path)}"); } } else { // Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench @@ -57,7 +81,7 @@ public async Task EndCaptureAsync (TimeSpan timeout) var downloaded_crash_reports = new List (); foreach (var file in end_crashes) { var name = Path.GetFileName (file); - var crash_report_target = Logs.Create (name, $"Crash report: {name}", timestamp: false); + var crash_report_target = logs.Create (name, $"Crash report: {name}", timestamp: false); var sb = new List (); sb.Add ($"--download-crash-report={file}"); sb.Add ($"--download-crash-report-to={crash_report_target.Path}"); @@ -67,27 +91,27 @@ public async Task EndCaptureAsync (TimeSpan timeout) sb.Add ("--devname"); sb.Add (deviceName); } - var result = await harness.ProcessManager.ExecuteCommandAsync (harness.MlaunchPath, sb, Log, TimeSpan.FromMinutes (1)); + var result = await processManager.ExecuteCommandAsync (harness.MlaunchPath, sb, log, TimeSpan.FromMinutes (1)); if (result.Succeeded) { - Log.WriteLine ("Downloaded crash report {0} to {1}", file, crash_report_target.Path); + log.WriteLine ("Downloaded crash report {0} to {1}", file, crash_report_target.Path); crash_report_target = await SymbolicateCrashReportAsync (crash_report_target); downloaded_crash_reports.Add (crash_report_target); } else { - Log.WriteLine ("Could not download crash report {0}", file); + log.WriteLine ("Could not download crash report {0}", file); } } crash_reports = downloaded_crash_reports; } foreach (var cp in crash_reports) { WrenchLog.WriteLine ("AddFile: {0}", cp.Path); - Log.WriteLine (" {0}", cp.Path); + log.WriteLine (" {0}", cp.Path); } crash_report_search_done = true; } else { if (watch.Elapsed.TotalSeconds > crash_report_search_timeout) { crash_report_search_done = true; } else { - Log.WriteLine ("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int) (crash_report_search_timeout - watch.Elapsed.TotalSeconds)); + log.WriteLine ("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int) (crash_report_search_timeout - watch.Elapsed.TotalSeconds)); Thread.Sleep (TimeSpan.FromSeconds (1)); } } @@ -101,19 +125,20 @@ async Task SymbolicateCrashReportAsync (ILogFile report) symbolicatecrash = Path.Combine (harness.XcodeRoot, "Contents/SharedFrameworks/DVTFoundation.framework/Versions/A/Resources/symbolicatecrash"); if (!File.Exists (symbolicatecrash)) { - Log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicatecrash); + log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicatecrash); return report; } var name = Path.GetFileName (report.Path); - var symbolicated = Logs.Create (Path.ChangeExtension (name, ".symbolicated.log"), $"Symbolicated crash report: {name}", timestamp: false); + var symbolicated = logs.Create (Path.ChangeExtension (name, ".symbolicated.log"), $"Symbolicated crash report: {name}", timestamp: false); var environment = new Dictionary { { "DEVELOPER_DIR", Path.Combine (harness.XcodeRoot, "Contents", "Developer") } }; - var rv = await harness.ProcessManager.ExecuteCommandAsync (symbolicatecrash, new [] { report.Path }, symbolicated, TimeSpan.FromMinutes (1), environment); - if (rv.Succeeded) {; - Log.WriteLine ("Symbolicated {0} successfully.", report.Path); + var rv = await processManager.ExecuteCommandAsync (symbolicatecrash, new [] { report.Path }, symbolicated, TimeSpan.FromMinutes (1), environment); + if (rv.Succeeded) { + ; + log.WriteLine ("Symbolicated {0} successfully.", report.Path); return symbolicated; } else { - Log.WriteLine ("Failed to symbolicate {0}.", report.Path); + log.WriteLine ("Failed to symbolicate {0}.", report.Path); return report; } } @@ -137,7 +162,7 @@ async Task> CreateCrashReportsSnapshotAsync () sb.Add ("--devname"); sb.Add (deviceName); } - var result = await harness.ProcessManager.ExecuteCommandAsync (harness.MlaunchPath, sb, Log, TimeSpan.FromMinutes (1)); + var result = await processManager.ExecuteCommandAsync (harness.MlaunchPath, sb, log, TimeSpan.FromMinutes (1)); if (result.Succeeded) rv.UnionWith (File.ReadAllLines (tmp)); } finally { diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index 27a7ca86aa35..9f8032751d58 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -223,8 +223,13 @@ IEnumerable CreateRunSimulatorTaskAsync (MSBuildTask buildTask throw new NotImplementedException (); } - for (int i = 0; i < targets.Length; i++) - runtasks.Add (new RunSimulatorTask (simulators, buildTask, simulators.SelectDevices (targets [i], SimulatorLoadLog, false)) { Platform = platforms [i], Ignored = ignored[i] || buildTask.Ignored }); + for (int i = 0; i < targets.Length; i++) { + var sims = simulators.SelectDevices (targets [i], SimulatorLoadLog, false); + runtasks.Add (new RunSimulatorTask (simulators, buildTask, processManager, sims) { + Platform = platforms [i], + Ignored = ignored[i] || buildTask.Ignored + }); + } return runtasks; } @@ -545,7 +550,7 @@ async Task> CreateRunSimulatorTasksAsync () } var testVariations = CreateTestVariations (runSimulatorTasks, (buildTask, test, candidates) => - new RunSimulatorTask (simulators, buildTask, candidates?.Cast () ?? test.Candidates)).ToList (); + new RunSimulatorTask (simulators, buildTask, processManager, candidates?.Cast () ?? test.Candidates)).ToList (); foreach (var tv in testVariations) { if (!tv.Ignored) @@ -585,7 +590,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; build64.CloneTestProject (project); - projectTasks.Add (new RunDeviceTask (devices, build64, devices.Connected64BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOS64 }); + projectTasks.Add (new RunDeviceTask (devices, build64, processManager, devices.Connected64BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOS64 }); var build32 = new MSBuildTask { Jenkins = this, @@ -595,7 +600,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; build32.CloneTestProject (project); - projectTasks.Add (new RunDeviceTask (devices, build32, devices.Connected32BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOS32 }); + projectTasks.Add (new RunDeviceTask (devices, build32, processManager, devices.Connected32BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOS32 }); var todayProject = project.AsTodayExtensionProject (); var buildToday = new MSBuildTask { @@ -606,7 +611,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; buildToday.CloneTestProject (todayProject); - projectTasks.Add (new RunDeviceTask (devices, buildToday, devices.Connected64BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOSExtensions, BuildOnly = ForceExtensionBuildOnly }); + projectTasks.Add (new RunDeviceTask (devices, buildToday, processManager, devices.Connected64BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOSExtensions, BuildOnly = ForceExtensionBuildOnly }); } if (!project.SkiptvOSVariation) { @@ -619,7 +624,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; buildTV.CloneTestProject (tvOSProject); - projectTasks.Add (new RunDeviceTask (devices, buildTV, devices.ConnectedTV.Where (d => d.IsSupported (project))) { Ignored = !IncludetvOS }); + projectTasks.Add (new RunDeviceTask (devices, buildTV, processManager, devices.ConnectedTV.Where (d => d.IsSupported (project))) { Ignored = !IncludetvOS }); } if (!project.SkipwatchOSVariation) { @@ -633,7 +638,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; buildWatch32.CloneTestProject (watchOSProject); - projectTasks.Add (new RunDeviceTask (devices, buildWatch32, devices.ConnectedWatch) { Ignored = !IncludewatchOS }); + projectTasks.Add (new RunDeviceTask (devices, buildWatch32, processManager, devices.ConnectedWatch) { Ignored = !IncludewatchOS }); } if (!project.SkipwatchOSARM64_32Variation) { @@ -645,7 +650,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; buildWatch64_32.CloneTestProject (watchOSProject); - projectTasks.Add (new RunDeviceTask (devices, buildWatch64_32, devices.ConnectedWatch32_64.Where (d => d.IsSupported (project))) { Ignored = !IncludewatchOS }); + projectTasks.Add (new RunDeviceTask (devices, buildWatch64_32, processManager, devices.ConnectedWatch32_64.Where (d => d.IsSupported (project))) { Ignored = !IncludewatchOS }); } } foreach (var task in projectTasks) { @@ -656,7 +661,7 @@ Task> CreateRunDeviceTasksAsync () rv.AddRange (projectTasks); } - return Task.FromResult> (CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask (devices, buildTask, candidates?.Cast () ?? test.Candidates))); + return Task.FromResult> (CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask (devices, buildTask, processManager, candidates?.Cast () ?? test.Candidates))); } static string AddSuffixToPath (string path, string suffix) @@ -931,6 +936,8 @@ Task PopulateTasksAsync () //Tasks.AddRange (await CreateRunSimulatorTasksAsync ()); + var crashReportSnapshotFactory = new CrashReportSnapshotFactory (Harness, processManager); + var buildiOSMSBuild_net461 = new MSBuildTask () { Jenkins = this, @@ -942,7 +949,7 @@ Task PopulateTasksAsync () SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), SupportsParallelExecution = false, }; - var nunitExecutioniOSMSBuild_net461 = new NUnitExecuteTask (buildiOSMSBuild_net461) + var nunitExecutioniOSMSBuild_net461 = new NUnitExecuteTask (buildiOSMSBuild_net461, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-net461", "net461", "Xamarin.iOS.Tasks.Tests.dll"), TestProject = buildiOSMSBuild_net461.TestProject, @@ -966,7 +973,7 @@ Task PopulateTasksAsync () SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), SupportsParallelExecution = false, }; - var nunitExecutioniOSMSBuild_netstandard2 = new NUnitExecuteTask (buildiOSMSBuild_netstandard2) { + var nunitExecutioniOSMSBuild_netstandard2 = new NUnitExecuteTask (buildiOSMSBuild_netstandard2, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-netstandard2.0", "net461", "Xamarin.iOS.Tasks.Tests.dll"), TestProject = buildiOSMSBuild_netstandard2.TestProject, ProjectConfiguration = "Debug-netstandard2.0", @@ -988,7 +995,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.iOS, }; buildInstallSources.SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "install-source.sln")); // this is required for nuget restore to be executed - var nunitExecutionInstallSource = new NUnitExecuteTask (buildInstallSources) + var nunitExecutionInstallSource = new NUnitExecuteTask (buildInstallSources, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "bin", "Release", "InstallSourcesTests.dll"), TestProject = buildInstallSources.TestProject, @@ -1045,7 +1052,7 @@ Task PopulateTasksAsync () var ignored_main = ignored; if (project.IsNUnitProject) { var dll = Path.Combine (Path.GetDirectoryName (build.TestProject.Path), project.Xml.GetOutputAssemblyPath (build.ProjectPlatform, build.ProjectConfiguration).Replace ('\\', '/')); - exec = new NUnitExecuteTask (build) { + exec = new NUnitExecuteTask (build, processManager) { Ignored = ignored_main, TestLibrary = dll, TestProject = project, @@ -1056,13 +1063,14 @@ Task PopulateTasksAsync () }; execs = new [] { exec }; } else { - exec = new MacExecuteTask (build) { + exec = new MacExecuteTask (build, processManager, crashReportSnapshotFactory) { Ignored = ignored_main, BCLTest = project.IsBclTest, TestName = project.Name, IsUnitTest = true, }; - execs = CreateTestVariations (new [] { exec }, (buildTask, test, candidates) => new MacExecuteTask (buildTask) { IsUnitTest = true } ); + execs = CreateTestVariations (new [] { exec }, (buildTask, test, candidates) => + new MacExecuteTask (buildTask, processManager, crashReportSnapshotFactory) { IsUnitTest = true } ); } foreach (var e in execs) @@ -1082,7 +1090,7 @@ Task PopulateTasksAsync () Target = "dependencies", WorkingDirectory = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch")), }; - var nunitExecutionMTouch = new NUnitExecuteTask (buildMTouch) + var nunitExecutionMTouch = new NUnitExecuteTask (buildMTouch, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "mtouch", "bin", "Debug", "mtouch.dll"), TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch", "mtouch.csproj"))), @@ -1103,7 +1111,7 @@ Task PopulateTasksAsync () Target = "build-unit-tests", WorkingDirectory = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "generator")), }; - var runGenerator = new NUnitExecuteTask (buildGenerator) { + var runGenerator = new NUnitExecuteTask (buildGenerator, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "generator", "bin", "Debug", "generator-tests.dll"), TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "generator", "generator-tests.csproj"))), Platform = TestPlatform.iOS, @@ -1121,7 +1129,7 @@ Task PopulateTasksAsync () SpecifyConfiguration = false, Platform = TestPlatform.iOS, }; - var runDotNetGenerator = new DotNetTestTask (buildDotNetGenerator) { + var runDotNetGenerator = new DotNetTestTask (buildDotNetGenerator, processManager) { TestProject = buildDotNetGenerator.TestProject, Platform = TestPlatform.iOS, TestName = "Generator tests", @@ -1170,7 +1178,7 @@ Task PopulateTasksAsync () Ignored = !IncludeXtro, Timeout = TimeSpan.FromMinutes (15), }; - var runXtroReporter = new RunXtroTask (buildXtroTests) { + var runXtroReporter = new RunXtroTask (buildXtroTests, processManager, crashReportSnapshotFactory) { Jenkins = this, Platform = TestPlatform.Mac, TestName = buildXtroTests.TestName, @@ -1188,7 +1196,7 @@ Task PopulateTasksAsync () Ignored = !IncludeCecil, Timeout = TimeSpan.FromMinutes (5), }; - var runCecilTests = new NUnitExecuteTask (buildCecilTests) { + var runCecilTests = new NUnitExecuteTask (buildCecilTests, processManager) { TestLibrary = Path.Combine (buildCecilTests.WorkingDirectory, "bin", "Debug", "cecil-tests.dll"), TestProject = new TestProject (Path.Combine (buildCecilTests.WorkingDirectory, "cecil-tests.csproj")), Platform = TestPlatform.iOS, @@ -1217,7 +1225,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.All, ProjectConfiguration = "Debug", }; - var runSampleTests = new NUnitExecuteTask (buildSampleTests) { + var runSampleTests = new NUnitExecuteTask (buildSampleTests, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "sampletester", "bin", "Debug", "sampletester.dll"), TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "sampletester", "sampletester.csproj"))), Platform = TestPlatform.All, diff --git a/tests/xharness/Jenkins/TestTasks/DotNetTestTask.cs b/tests/xharness/Jenkins/TestTasks/DotNetTestTask.cs index e018691e38af..687c11ab3a7a 100644 --- a/tests/xharness/Jenkins/TestTasks/DotNetTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/DotNetTestTask.cs @@ -1,12 +1,13 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using Xharness.Execution; using Xharness.Logging; namespace Xharness.Jenkins.TestTasks { class DotNetTestTask : RunTestTask { - public DotNetTestTask (DotNetBuildTask build_task) - : base (build_task) + public DotNetTestTask (DotNetBuildTask build_task, IProcessManager processManager) + : base (build_task, processManager) { DotNetBuildTask.SetDotNetEnvironmentVariables (Environment); } diff --git a/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs index def0f09f5dfb..9599a47ebdde 100644 --- a/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs @@ -12,14 +12,16 @@ namespace Xharness.Jenkins.TestTasks { class MacExecuteTask : MacTask { + protected ICrashReportSnapshotFactory CrashReportSnapshotFactory { get; } + public string Path; public bool BCLTest; public bool IsUnitTest; - public IProcessManager ProcessManager { get; set; } = new ProcessManager (); - public MacExecuteTask (BuildToolTask build_task) - : base (build_task) + public MacExecuteTask (BuildToolTask build_task, IProcessManager processManager, ICrashReportSnapshotFactory crashReportSnapshotFactory) + : base (build_task, processManager) { + this.CrashReportSnapshotFactory = crashReportSnapshotFactory ?? throw new ArgumentNullException (nameof (crashReportSnapshotFactory)); } public override bool SupportsParallelExecution { @@ -91,7 +93,7 @@ protected override async Task RunTestAsync () if (!Harness.DryRun) { ExecutionResult = TestExecutingResult.Running; - var snapshot = new CrashReportSnapshot (Harness, log, Logs, isDevice: false, deviceName: null); + var snapshot = CrashReportSnapshotFactory.Create (log, Logs, isDevice: false, deviceName: null); await snapshot.StartCaptureAsync (); ProcessExecutionResult result = null; diff --git a/tests/xharness/Jenkins/TestTasks/MacTask.cs b/tests/xharness/Jenkins/TestTasks/MacTask.cs index e7717d1858fc..b5325893fadd 100644 --- a/tests/xharness/Jenkins/TestTasks/MacTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MacTask.cs @@ -1,11 +1,12 @@ using System; +using Xharness.Execution; namespace Xharness.Jenkins.TestTasks { abstract class MacTask : RunTestTask { - public MacTask (BuildToolTask build_task) - : base (build_task) + public MacTask (BuildToolTask build_task, IProcessManager processManager) + : base (build_task, processManager) { } diff --git a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs index 3bc600632cac..6d405cdff611 100644 --- a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs @@ -19,8 +19,8 @@ class NUnitExecuteTask : RunTestTask public bool ProduceHtmlReport = true; public bool InProcess; - public NUnitExecuteTask (BuildToolTask build_task) - : base (build_task) + public NUnitExecuteTask (BuildToolTask build_task, IProcessManager processManager) + : base (build_task, processManager) { } diff --git a/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs b/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs index 21458eb32dc8..71d56acafc5f 100644 --- a/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs @@ -38,8 +38,8 @@ public override string ProgressMessage { } } - public RunDeviceTask (IDeviceLoader devices, MSBuildTask build_task, IEnumerable candidates) - : base (build_task, candidates.OrderBy ((v) => v.DebugSpeed)) + public RunDeviceTask (IDeviceLoader devices, MSBuildTask build_task, IProcessManager processManager, IEnumerable candidates) + : base (build_task, processManager, candidates.OrderBy ((v) => v.DebugSpeed)) { switch (build_task.Platform) { case TestPlatform.iOS: diff --git a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs index f93c72787259..793d2fa15e39 100644 --- a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs @@ -28,8 +28,8 @@ public ISimulatorDevice [] Simulators { } } - public RunSimulatorTask (ISimulatorsLoader simulators, MSBuildTask build_task, IEnumerable candidates = null) - : base (build_task, candidates) + public RunSimulatorTask (ISimulatorsLoader simulators, MSBuildTask build_task, IProcessManager processManager, IEnumerable candidates = null) + : base (build_task, processManager, candidates) { var project = Path.GetFileNameWithoutExtension (ProjectFile); if (project.EndsWith ("-tvos", StringComparison.Ordinal)) { diff --git a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs index a8eadf96c578..574cdf67c786 100644 --- a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs @@ -12,15 +12,17 @@ namespace Xharness.Jenkins.TestTasks { internal abstract class RunTestTask : TestTask { + protected IProcessManager ProcessManager { get; } + public readonly BuildToolTask BuildTask; public TimeSpan Timeout = TimeSpan.FromMinutes (10); public double TimeoutMultiplier { get; set; } = 1; - IProcessManager ProcessManager { get; } = new ProcessManager (); public string WorkingDirectory; - public RunTestTask (BuildToolTask build_task) + public RunTestTask (BuildToolTask build_task, IProcessManager processManager) { this.BuildTask = build_task; + this.ProcessManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); Jenkins = build_task.Jenkins; TestProject = build_task.TestProject; diff --git a/tests/xharness/Jenkins/TestTasks/RunXITask.cs b/tests/xharness/Jenkins/TestTasks/RunXITask.cs index d3263fb68cf5..1b3260d80d3c 100644 --- a/tests/xharness/Jenkins/TestTasks/RunXITask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunXITask.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using Xharness.Collections; +using Xharness.Execution; using Xharness.Hardware; using Xharness.Logging; @@ -23,8 +24,8 @@ abstract class RunXITask : RunTestTask where TDevice : class, IDevice public string BundleIdentifier => runner.AppInformation.BundleIdentifier; - public RunXITask (BuildToolTask build_task, IEnumerable candidates) - : base (build_task) + public RunXITask (BuildToolTask build_task, IProcessManager processManager, IEnumerable candidates) + : base (build_task, processManager) { this.Candidates = candidates; } diff --git a/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs b/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs index 2ad46335c984..61533639a437 100644 --- a/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs @@ -1,13 +1,15 @@ using System; using System.Diagnostics; using System.Threading.Tasks; +using Xharness.Execution; using Xharness.Logging; namespace Xharness.Jenkins.TestTasks { class RunXtroTask : MacExecuteTask { - public RunXtroTask (BuildToolTask build_task) : base (build_task) + public RunXtroTask (BuildToolTask build_task, IProcessManager processManager, ICrashReportSnapshotFactory crashReportSnapshotFactory) + : base (build_task, processManager, crashReportSnapshotFactory) { } @@ -29,7 +31,7 @@ protected override async Task RunTestAsync () if (!Harness.DryRun) { ExecutionResult = TestExecutingResult.Running; - var snapshot = new CrashReportSnapshot (Harness, log, Logs, isDevice: false, deviceName: null); + var snapshot = CrashReportSnapshotFactory.Create (log, Logs, isDevice: false, deviceName: null); await snapshot.StartCaptureAsync (); try { diff --git a/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs b/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs new file mode 100644 index 000000000000..f1359af9ba93 --- /dev/null +++ b/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs @@ -0,0 +1,8 @@ +using NUnit.Framework; + +namespace Xharness.Tests { + [TestFixture] + public class CrashReportSnapshotTests { + + } +} diff --git a/tests/xharness/Xharness.Tests/Xharness.Tests.csproj b/tests/xharness/Xharness.Tests/Xharness.Tests.csproj index 57c349d42850..ad29232c632d 100644 --- a/tests/xharness/Xharness.Tests/Xharness.Tests.csproj +++ b/tests/xharness/Xharness.Tests/Xharness.Tests.csproj @@ -59,6 +59,7 @@ + From b3a7db5422484d3be9633f26c1752f990e4f5f82 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 11:51:06 +0100 Subject: [PATCH 13/30] Remove Harness dependency from CrashReportSnapshot --- tests/xharness/AppRunner.cs | 8 ++- tests/xharness/CrashReportSnapshot.cs | 39 ++++++++----- tests/xharness/Jenkins/Jenkins.cs | 2 +- .../Xharness.Tests/Tests/AppRunnerTests.cs | 58 +++++++++++++++++++ 4 files changed, 90 insertions(+), 17 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 9417ce1930b8..8337af1d5942 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -432,7 +432,13 @@ public async Task RunAsync () if (!isSimulator) FindDevice (); - crash_reports = new CrashReportSnapshot (harness, processManager, MainLog, Logs, isDevice: !isSimulator, deviceName); + crash_reports = new CrashReportSnapshot (processManager, + MainLog, + Logs, + harness.XcodeRoot, + harness.MlaunchPath, + isDevice: !isSimulator, + deviceName); var args = new List (); if (!string.IsNullOrEmpty (harness.XcodeRoot)) { diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs index 3fd7811723ef..2c7b52679192 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashReportSnapshot.cs @@ -14,17 +14,19 @@ public interface ICrashReportSnapshotFactory { } public class CrashReportSnapshotFactory : ICrashReportSnapshotFactory { - readonly IHarness harness; readonly IProcessManager processManager; + readonly string xcodeRoot; + readonly string mlaunchPath; - public CrashReportSnapshotFactory (IHarness harness, IProcessManager processManager) + public CrashReportSnapshotFactory (IProcessManager processManager, string xcodeRoot, string mlaunchPath) { - this.harness = harness ?? throw new ArgumentNullException (nameof (harness)); this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); + this.xcodeRoot = xcodeRoot ?? throw new ArgumentNullException (nameof (xcodeRoot)); + this.mlaunchPath = mlaunchPath ?? throw new ArgumentNullException (nameof (mlaunchPath)); } public ICrashReportSnapshot Create (ILog log, ILogs logs, bool isDevice, string deviceName) => - new CrashReportSnapshot (harness, processManager, log, logs, isDevice, deviceName); + new CrashReportSnapshot (processManager, log, logs, xcodeRoot, mlaunchPath, isDevice, deviceName); } public interface ICrashReportSnapshot { @@ -33,21 +35,29 @@ public interface ICrashReportSnapshot { } public class CrashReportSnapshot : ICrashReportSnapshot { - readonly IHarness harness; readonly IProcessManager processManager; readonly ILog log; readonly ILogs logs; + readonly string xcodeRoot; + readonly string mlaunchPath; readonly bool isDevice; readonly string deviceName; HashSet initialSet; - public CrashReportSnapshot (IHarness harness, IProcessManager processManager, ILog log, ILogs logs, bool isDevice, string deviceName) + public CrashReportSnapshot (IProcessManager processManager, + ILog log, + ILogs logs, + string xcodeRoot, + string mlaunchPath, + bool isDevice, + string deviceName) { - this.harness = harness ?? throw new ArgumentNullException (nameof (harness)); this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); this.log = log ?? throw new ArgumentNullException (nameof (log)); this.logs = logs ?? throw new ArgumentNullException (nameof (logs)); + this.xcodeRoot = xcodeRoot ?? throw new ArgumentNullException (nameof (xcodeRoot)); + this.mlaunchPath = mlaunchPath ?? throw new ArgumentNullException (nameof (mlaunchPath)); this.isDevice = isDevice; this.deviceName = deviceName; } @@ -86,12 +96,12 @@ public async Task EndCaptureAsync (TimeSpan timeout) sb.Add ($"--download-crash-report={file}"); sb.Add ($"--download-crash-report-to={crash_report_target.Path}"); sb.Add ("--sdkroot"); - sb.Add (harness.XcodeRoot); + sb.Add (xcodeRoot); if (!string.IsNullOrEmpty (deviceName)) { sb.Add ("--devname"); sb.Add (deviceName); } - var result = await processManager.ExecuteCommandAsync (harness.MlaunchPath, sb, log, TimeSpan.FromMinutes (1)); + var result = await processManager.ExecuteCommandAsync (mlaunchPath, sb, log, TimeSpan.FromMinutes (1)); if (result.Succeeded) { log.WriteLine ("Downloaded crash report {0} to {1}", file, crash_report_target.Path); crash_report_target = await SymbolicateCrashReportAsync (crash_report_target); @@ -120,9 +130,9 @@ public async Task EndCaptureAsync (TimeSpan timeout) async Task SymbolicateCrashReportAsync (ILogFile report) { - var symbolicatecrash = Path.Combine (harness.XcodeRoot, "Contents/SharedFrameworks/DTDeviceKitBase.framework/Versions/A/Resources/symbolicatecrash"); + var symbolicatecrash = Path.Combine (xcodeRoot, "Contents/SharedFrameworks/DTDeviceKitBase.framework/Versions/A/Resources/symbolicatecrash"); if (!File.Exists (symbolicatecrash)) - symbolicatecrash = Path.Combine (harness.XcodeRoot, "Contents/SharedFrameworks/DVTFoundation.framework/Versions/A/Resources/symbolicatecrash"); + symbolicatecrash = Path.Combine (xcodeRoot, "Contents/SharedFrameworks/DVTFoundation.framework/Versions/A/Resources/symbolicatecrash"); if (!File.Exists (symbolicatecrash)) { log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicatecrash); @@ -131,10 +141,9 @@ async Task SymbolicateCrashReportAsync (ILogFile report) var name = Path.GetFileName (report.Path); var symbolicated = logs.Create (Path.ChangeExtension (name, ".symbolicated.log"), $"Symbolicated crash report: {name}", timestamp: false); - var environment = new Dictionary { { "DEVELOPER_DIR", Path.Combine (harness.XcodeRoot, "Contents", "Developer") } }; + var environment = new Dictionary { { "DEVELOPER_DIR", Path.Combine (xcodeRoot, "Contents", "Developer") } }; var rv = await processManager.ExecuteCommandAsync (symbolicatecrash, new [] { report.Path }, symbolicated, TimeSpan.FromMinutes (1), environment); if (rv.Succeeded) { - ; log.WriteLine ("Symbolicated {0} successfully.", report.Path); return symbolicated; } else { @@ -157,12 +166,12 @@ async Task> CreateCrashReportsSnapshotAsync () var sb = new List (); sb.Add ($"--list-crash-reports={tmp}"); sb.Add ("--sdkroot"); - sb.Add (harness.XcodeRoot); + sb.Add (xcodeRoot); if (!string.IsNullOrEmpty (deviceName)) { sb.Add ("--devname"); sb.Add (deviceName); } - var result = await processManager.ExecuteCommandAsync (harness.MlaunchPath, sb, log, TimeSpan.FromMinutes (1)); + var result = await processManager.ExecuteCommandAsync (mlaunchPath, sb, log, TimeSpan.FromMinutes (1)); if (result.Succeeded) rv.UnionWith (File.ReadAllLines (tmp)); } finally { diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index 9f8032751d58..aa3fc747ef22 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -936,7 +936,7 @@ Task PopulateTasksAsync () //Tasks.AddRange (await CreateRunSimulatorTasksAsync ()); - var crashReportSnapshotFactory = new CrashReportSnapshotFactory (Harness, processManager); + var crashReportSnapshotFactory = new CrashReportSnapshotFactory (processManager, Harness.XcodeRoot, Harness.MlaunchPath); var buildiOSMSBuild_net461 = new MSBuildTask () { diff --git a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs index fa955e6371b6..20bf9d118bf0 100644 --- a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs +++ b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs @@ -274,5 +274,63 @@ public async Task UninstallFromDeviceTest () null, null)); } + + [Test] + public async Task RunOnDeviceTest () + { + Mock harnessMock = new Mock (); + harnessMock.SetupGet (x => x.XcodeRoot).Returns ("/path/to/xcode"); + harnessMock.SetupGet (x => x.MlaunchPath).Returns ("/path/to/mlaunch"); + harnessMock.SetupGet (x => x.Verbosity).Returns (2); + + devices.Setup (x => x.ConnectedDevices).Returns (mockDevices); + + /*var processResult = new ProcessExecutionResult () { ExitCode = 1, TimedOut = false }; + + processManager + .Setup (x => x.ExecuteCommandAsync ( + It.IsAny (), + It.IsAny> (), + It.IsAny (), + It.IsAny (), + It.IsAny> (), + It.IsAny ())) + .ReturnsAsync(processResult); + + var appRunner = new AppRunner (processManager.Object, + simulatorsFactory, + listenerFactory, + devicesFactory, + AppRunnerTarget.Device_iOS, + harnessMock.Object, + mainLog, + Path.Combine (sampleProjectPath, "SystemXunit.csproj"), + "Debug", + Path.Combine (outputPath, "logs")); + + + CancellationToken cancellationToken = new CancellationToken (); + var result = await appRunner.InstallAsync (cancellationToken); + + Assert.AreEqual (1, result.ExitCode); + + processManager.Verify (x => x.ExecuteCommandAsync ( + "/path/to/mlaunch", + new List () { + "--sdkroot", + "/path/to/xcode", + "-v", + "-v", + "-v", + "--installdev", + appPath, + "--devname", + "Test iPad" + }, + mainLog, + TimeSpan.FromHours (1), + null, + cancellationToken));*/ + } } } From a6298c08978b774af2707b69a84b5ba5410fc1af Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 12:13:45 +0100 Subject: [PATCH 14/30] Add device snapshot tests --- tests/xharness/CrashReportSnapshot.cs | 86 ++++++++++--------- tests/xharness/Execution/Mlaunch/Arguments.cs | 12 +++ .../Tests/CrashReportSnapshotTests.cs | 38 +++++++- 3 files changed, 94 insertions(+), 42 deletions(-) diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs index 2c7b52679192..52162eace60d 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashReportSnapshot.cs @@ -77,54 +77,58 @@ public async Task EndCaptureAsync (TimeSpan timeout) do { var end_crashes = await CreateCrashReportsSnapshotAsync (); end_crashes.ExceptWith (initialSet); - if (end_crashes.Count > 0) { - log.WriteLine ("Found {0} new crash report(s)", end_crashes.Count); - List crash_reports; - if (!isDevice) { - crash_reports = new List (end_crashes.Count); - foreach (var path in end_crashes) { - logs.AddFile (path, $"Crash report: {Path.GetFileName (path)}"); - } - } else { - // Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench - // (if we put them in /tmp, they'd never be deleted). - var downloaded_crash_reports = new List (); - foreach (var file in end_crashes) { - var name = Path.GetFileName (file); - var crash_report_target = logs.Create (name, $"Crash report: {name}", timestamp: false); - var sb = new List (); - sb.Add ($"--download-crash-report={file}"); - sb.Add ($"--download-crash-report-to={crash_report_target.Path}"); - sb.Add ("--sdkroot"); - sb.Add (xcodeRoot); - if (!string.IsNullOrEmpty (deviceName)) { - sb.Add ("--devname"); - sb.Add (deviceName); - } - var result = await processManager.ExecuteCommandAsync (mlaunchPath, sb, log, TimeSpan.FromMinutes (1)); - if (result.Succeeded) { - log.WriteLine ("Downloaded crash report {0} to {1}", file, crash_report_target.Path); - crash_report_target = await SymbolicateCrashReportAsync (crash_report_target); - downloaded_crash_reports.Add (crash_report_target); - } else { - log.WriteLine ("Could not download crash report {0}", file); - } - } - crash_reports = downloaded_crash_reports; - } - foreach (var cp in crash_reports) { - WrenchLog.WriteLine ("AddFile: {0}", cp.Path); - log.WriteLine (" {0}", cp.Path); - } - crash_report_search_done = true; - } else { + + if (end_crashes.Count == 0) { if (watch.Elapsed.TotalSeconds > crash_report_search_timeout) { crash_report_search_done = true; } else { log.WriteLine ("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int) (crash_report_search_timeout - watch.Elapsed.TotalSeconds)); Thread.Sleep (TimeSpan.FromSeconds (1)); } + + continue; + } + + log.WriteLine ("Found {0} new crash report(s)", end_crashes.Count); + List crash_reports; + if (!isDevice) { + crash_reports = new List (end_crashes.Count); + foreach (var path in end_crashes) { + logs.AddFile (path, $"Crash report: {Path.GetFileName (path)}"); + } + } else { + // Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench + // (if we put them in /tmp, they'd never be deleted). + var downloaded_crash_reports = new List (); + foreach (var file in end_crashes) { + var name = Path.GetFileName (file); + var crash_report_target = logs.Create (name, $"Crash report: {name}", timestamp: false); + var sb = new List (); + sb.Add ($"--download-crash-report={file}"); + sb.Add ($"--download-crash-report-to={crash_report_target.Path}"); + sb.Add ("--sdkroot"); + sb.Add (xcodeRoot); + if (!string.IsNullOrEmpty (deviceName)) { + sb.Add ("--devname"); + sb.Add (deviceName); + } + var result = await processManager.ExecuteCommandAsync (mlaunchPath, sb, log, TimeSpan.FromMinutes (1)); + if (result.Succeeded) { + log.WriteLine ("Downloaded crash report {0} to {1}", file, crash_report_target.Path); + crash_report_target = await SymbolicateCrashReportAsync (crash_report_target); + downloaded_crash_reports.Add (crash_report_target); + } else { + log.WriteLine ("Could not download crash report {0}", file); + } + } + crash_reports = downloaded_crash_reports; + } + + foreach (var cp in crash_reports) { + WrenchLog.WriteLine ("AddFile: {0}", cp.Path); + log.WriteLine (" {0}", cp.Path); } + crash_report_search_done = true; } while (!crash_report_search_done); } diff --git a/tests/xharness/Execution/Mlaunch/Arguments.cs b/tests/xharness/Execution/Mlaunch/Arguments.cs index fbcc6d3c620e..6bcca17e0b7b 100644 --- a/tests/xharness/Execution/Mlaunch/Arguments.cs +++ b/tests/xharness/Execution/Mlaunch/Arguments.cs @@ -18,6 +18,18 @@ public ListSimulatorsArgument (string outputFile) : base ("listsim", outputFile) } } + public sealed class ListCrashReportsArgument : SingleValueArgument { + public ListCrashReportsArgument (string outputFile) : base ("list-crash-reports", outputFile) + { + } + } + + public sealed class DeviceNameArgument : SingleValueArgument { + public DeviceNameArgument (string deviceName) : base ("devname", deviceName) + { + } + } + public sealed class DefaultOutputFormatArgument : SingleValueArgument { public DefaultOutputFormatArgument () : base ("output-format", "Default") { diff --git a/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs b/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs index f1359af9ba93..11e9e4b65d4b 100644 --- a/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs +++ b/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs @@ -1,8 +1,44 @@ -using NUnit.Framework; +using System; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Xharness.Execution; +using Xharness.Logging; namespace Xharness.Tests { [TestFixture] public class CrashReportSnapshotTests { + const string xcodeRoot = "/path/to/xcode"; + const string mlaunchPath = "/path/to/mlaunch"; + Mock processManager; + Mock log; + Mock logs; + + [SetUp] + public void SetUp () + { + processManager = new Mock (); + log = new Mock (); + logs = new Mock (); + } + + [Test] + public async Task DeviceCaptureTest () + { + var snapshotReport = new CrashReportSnapshot (processManager.Object, + log.Object, + logs.Object, + xcodeRoot, + mlaunchPath, + true, + "Sample iPhone"); + + await snapshotReport.StartCaptureAsync (); + + // mlaunch "--list-crash-reports=C:\\Users\\prvysoky\\AppData\\Local\\Temp\\tmp43C3.tmp", "--sdkroot", "/path/to/xcode", "--devname", "Sample iPhone" + + await snapshotReport.EndCaptureAsync (TimeSpan.FromSeconds (10)); + } } } From 735fadfacdc44196383d6dfbb855c2ecec018830 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 12:50:21 +0100 Subject: [PATCH 15/30] Trim down IHarness and make properties of Harness readonly --- tests/xharness/Harness.cs | 68 ++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 6c6d8999bc2f..30e12139cf99 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -51,8 +51,6 @@ public class HarnessConfiguration { public interface IHarness { HarnessAction Action { get; } - string BCLTodayExtensionTemplate { get; } - string BuildConfiguration { get; } bool DisableWatchOSOnWrench { get; } string DOTNET { get; } bool DryRun { get; } @@ -83,12 +81,9 @@ public interface IHarness { string PeriodicCommandArguments { get; } TimeSpan PeriodicCommandInterval { get; } IProcessManager ProcessManager { get; } - string SdkRoot { get; } - AppRunnerTarget Target { get; } double Timeout { get; } string TodayContainerTemplate { get; } string TodayExtensionTemplate { get; } - string TVOS_MONO_PATH { get; } bool UseGroupedApps { get; } int Verbosity { get; } string WatchOSAppTemplate { get; } @@ -98,7 +93,6 @@ public interface IHarness { Version XcodeVersion { get; } string XIBuildPath { get; } XmlResultJargon XmlJargon { get; } - int Execute (); Task ExecuteXcodeCommandAsync (string executable, IList args, ILog log, TimeSpan timeout); bool GetIncludeSystemPermissionTests (TestPlatform platform, bool device); void Log (int min_level, string message, params object [] args); @@ -108,6 +102,10 @@ public interface IHarness { } public class Harness : IHarness { + readonly AppRunnerTarget target; + readonly string buildConfiguration = "Debug"; + string sdkRoot; + public HarnessAction Action { get; } public int Verbosity { get; } public ILog HarnessLog { get; set; } @@ -115,15 +113,9 @@ public class Harness : IHarness { public XmlResultJargon XmlJargon { get; } public IProcessManager ProcessManager { get; } - public string XIBuildPath { - get { return Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "tools", "xibuild", "xibuild")); } - } + public string XIBuildPath => Path.GetFullPath (Path.Combine (RootDirectory, "..", "tools", "xibuild", "xibuild")); - public static string Timestamp { - get { - return $"{DateTime.Now:yyyyMMdd_HHmmss}"; - } - } + public static string Timestamp => $"{DateTime.Now:yyyyMMdd_HHmmss}"; // This is the maccore/tests directory. static string root_directory; @@ -182,20 +174,16 @@ public static string RootDirectory { public string DOTNET { get; private set; } // Run - public AppRunnerTarget Target { get; set; } - public string SdkRoot { get; set; } - public string BuildConfiguration { get; set; } = "Debug"; - public string LogFile { get; set; } - public string LogDirectory { get; set; } = Environment.CurrentDirectory; - public double Timeout { get; set; } = 15; // in minutes - public double LaunchTimeout { get; set; } // in minutes - public bool DryRun { get; set; } // Most things don't support this. If you need it somewhere, implement it! - public string JenkinsConfiguration { get; set; } - public Dictionary EnvironmentVariables { get; set; } = new Dictionary (); - public string MarkdownSummaryPath { get; set; } - public string PeriodicCommand { get; set; } - public string PeriodicCommandArguments { get; set; } - public TimeSpan PeriodicCommandInterval { get; set; } + public string LogDirectory { get; } = Environment.CurrentDirectory; + public double Timeout { get; } = 15; // in minutes + public double LaunchTimeout { get; } // in minutes + public bool DryRun { get; } // Most things don't support this. If you need it somewhere, implement it! + public string JenkinsConfiguration { get; } + public Dictionary EnvironmentVariables { get; } = new Dictionary (); + public string MarkdownSummaryPath { get; } + public string PeriodicCommand { get; } + public string PeriodicCommandArguments { get; } + public TimeSpan PeriodicCommandInterval { get;} // whether tests that require access to system resources (system contacts, photo library, etc) should be executed or not public bool? IncludeSystemPermissionTests { get; set; } @@ -208,7 +196,7 @@ public Harness (IProcessManager processManager, HarnessAction action, HarnessCon throw new ArgumentNullException (nameof (configuration)); autoConf = configuration.AutoConf; - BuildConfiguration = configuration.BuildConfiguration ?? throw new ArgumentNullException (nameof (configuration)); + buildConfiguration = configuration.BuildConfiguration ?? throw new ArgumentNullException (nameof (configuration)); DryRun = configuration.DryRun; IncludeSystemPermissionTests = configuration.IncludeSystemPermissionTests; IOSTestProjects = configuration.IOSTestProjects; @@ -219,8 +207,8 @@ public Harness (IProcessManager processManager, HarnessAction action, HarnessCon PeriodicCommand = configuration.PeriodicCommand; PeriodicCommandArguments = configuration.PeriodicCommandArguments; PeriodicCommandInterval = configuration.PeriodicCommandInterval; - SdkRoot = configuration.SdkRoot; - Target = configuration.Target; + sdkRoot = configuration.SdkRoot; + target = configuration.Target; Timeout = configuration.TimeoutInMinutes; useSystemXamarinIOSMac = configuration.UseSystemXamarinIOSMac; Verbosity = configuration.Verbosity; @@ -278,7 +266,7 @@ static string FindXcode (string path) public string XcodeRoot { get { - return FindXcode (SdkRoot); + return FindXcode (sdkRoot); } } @@ -313,8 +301,8 @@ void LoadConfig () INCLUDE_MAC = make_config.ContainsKey ("INCLUDE_MAC") && !string.IsNullOrEmpty (make_config ["INCLUDE_MAC"]); MAC_DESTDIR = make_config ["MAC_DESTDIR"]; IOS_DESTDIR = make_config ["IOS_DESTDIR"]; - if (string.IsNullOrEmpty (SdkRoot)) - SdkRoot = make_config ["XCODE_DEVELOPER_ROOT"]; + if (string.IsNullOrEmpty (sdkRoot)) + sdkRoot = make_config ["XCODE_DEVELOPER_ROOT"]; MONO_IOS_SDK_DESTDIR = make_config ["MONO_IOS_SDK_DESTDIR"]; MONO_MAC_SDK_DESTDIR = make_config ["MONO_MAC_SDK_DESTDIR"]; ENABLE_XAMARIN = make_config.ContainsKey ("ENABLE_XAMARIN") && !string.IsNullOrEmpty (make_config ["ENABLE_XAMARIN"]); @@ -613,11 +601,11 @@ int Install () new SimulatorsLoaderFactory (this, ProcessManager), new SimpleListenerFactory (), new DeviceLoaderFactory (this, ProcessManager), - Target, + target, this, HarnessLog, project.Path, - BuildConfiguration); + buildConfiguration); using (var install_log = new AppInstallMonitorLog (runner.MainLog)) { var rv = runner.InstallAsync (install_log.CancellationToken).Result; @@ -638,11 +626,11 @@ int Uninstall () new SimulatorsLoaderFactory (this, ProcessManager), new SimpleListenerFactory (), new DeviceLoaderFactory (this, ProcessManager), - Target, + target, this, HarnessLog, project.Path, - BuildConfiguration); + buildConfiguration); var rv = runner.UninstallAsync ().Result; if (!rv.Succeeded) @@ -661,11 +649,11 @@ int Run () new SimulatorsLoaderFactory (this, ProcessManager), new SimpleListenerFactory (), new DeviceLoaderFactory (this, ProcessManager), - Target, + target, this, HarnessLog, project.Path, - BuildConfiguration); + buildConfiguration); var rv = runner.RunAsync ().Result; if (rv != 0) From eabd707d5920d3eea565b2b27fd7ab87f0f5c659 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 14:08:58 +0100 Subject: [PATCH 16/30] Memoize xcoderoot --- tests/xharness/CrashReportSnapshot.cs | 12 ++++++------ tests/xharness/Harness.cs | 27 ++++++++++++++++----------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs index 52162eace60d..1b0169cd83d6 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashReportSnapshot.cs @@ -134,9 +134,9 @@ public async Task EndCaptureAsync (TimeSpan timeout) async Task SymbolicateCrashReportAsync (ILogFile report) { - var symbolicatecrash = Path.Combine (xcodeRoot, "Contents/SharedFrameworks/DTDeviceKitBase.framework/Versions/A/Resources/symbolicatecrash"); + var symbolicatecrash = Path.Combine (xcodeRoot, "Contents", "SharedFrameworks", "DTDeviceKitBase.framework", "Versions", "A", "Resources", "symbolicatecrash"); if (!File.Exists (symbolicatecrash)) - symbolicatecrash = Path.Combine (xcodeRoot, "Contents/SharedFrameworks/DVTFoundation.framework/Versions/A/Resources/symbolicatecrash"); + symbolicatecrash = Path.Combine (xcodeRoot, "Contents", "SharedFrameworks", "DVTFoundation.framework", "Versions", "A", "Resources", "symbolicatecrash"); if (!File.Exists (symbolicatecrash)) { log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicatecrash); @@ -158,12 +158,12 @@ async Task SymbolicateCrashReportAsync (ILogFile report) async Task> CreateCrashReportsSnapshotAsync () { - var rv = new HashSet (); + var crashes = new HashSet (); if (!isDevice) { var dir = Path.Combine (Environment.GetEnvironmentVariable ("HOME"), "Library", "Logs", "DiagnosticReports"); if (Directory.Exists (dir)) - rv.UnionWith (Directory.EnumerateFiles (dir)); + crashes.UnionWith (Directory.EnumerateFiles (dir)); } else { var tmp = Path.GetTempFileName (); try { @@ -177,13 +177,13 @@ async Task> CreateCrashReportsSnapshotAsync () } var result = await processManager.ExecuteCommandAsync (mlaunchPath, sb, log, TimeSpan.FromMinutes (1)); if (result.Succeeded) - rv.UnionWith (File.ReadAllLines (tmp)); + crashes.UnionWith (File.ReadAllLines (tmp)); } finally { File.Delete (tmp); } } - return rv; + return crashes; } } } diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 30e12139cf99..9a023073f7e6 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -104,7 +104,6 @@ public interface IHarness { public class Harness : IHarness { readonly AppRunnerTarget target; readonly string buildConfiguration = "Debug"; - string sdkRoot; public HarnessAction Action { get; } public int Verbosity { get; } @@ -145,6 +144,15 @@ public static string RootDirectory { } } + string sdkRoot; + string SdkRoot { + get => sdkRoot; + set { + sdkRoot = value; + XcodeRoot = FindXcode (sdkRoot); + } + } + public List IOSTestProjects { get; } public List MacTestProjects { get; } = new List (); @@ -174,6 +182,8 @@ public static string RootDirectory { public string DOTNET { get; private set; } // Run + + public string XcodeRoot { get; private set; } public string LogDirectory { get; } = Environment.CurrentDirectory; public double Timeout { get; } = 15; // in minutes public double LaunchTimeout { get; } // in minutes @@ -183,7 +193,7 @@ public static string RootDirectory { public string MarkdownSummaryPath { get; } public string PeriodicCommand { get; } public string PeriodicCommandArguments { get; } - public TimeSpan PeriodicCommandInterval { get;} + public TimeSpan PeriodicCommandInterval { get; } // whether tests that require access to system resources (system contacts, photo library, etc) should be executed or not public bool? IncludeSystemPermissionTests { get; set; } @@ -207,7 +217,7 @@ public Harness (IProcessManager processManager, HarnessAction action, HarnessCon PeriodicCommand = configuration.PeriodicCommand; PeriodicCommandArguments = configuration.PeriodicCommandArguments; PeriodicCommandInterval = configuration.PeriodicCommandInterval; - sdkRoot = configuration.SdkRoot; + SdkRoot = configuration.SdkRoot; target = configuration.Target; Timeout = configuration.TimeoutInMinutes; useSystemXamarinIOSMac = configuration.UseSystemXamarinIOSMac; @@ -264,12 +274,6 @@ static string FindXcode (string path) } while (true); } - public string XcodeRoot { - get { - return FindXcode (sdkRoot); - } - } - Version xcode_version; public Version XcodeVersion { get { @@ -301,8 +305,8 @@ void LoadConfig () INCLUDE_MAC = make_config.ContainsKey ("INCLUDE_MAC") && !string.IsNullOrEmpty (make_config ["INCLUDE_MAC"]); MAC_DESTDIR = make_config ["MAC_DESTDIR"]; IOS_DESTDIR = make_config ["IOS_DESTDIR"]; - if (string.IsNullOrEmpty (sdkRoot)) - sdkRoot = make_config ["XCODE_DEVELOPER_ROOT"]; + if (string.IsNullOrEmpty (SdkRoot)) + SdkRoot = make_config ["XCODE_DEVELOPER_ROOT"]; MONO_IOS_SDK_DESTDIR = make_config ["MONO_IOS_SDK_DESTDIR"]; MONO_MAC_SDK_DESTDIR = make_config ["MONO_MAC_SDK_DESTDIR"]; ENABLE_XAMARIN = make_config.ContainsKey ("ENABLE_XAMARIN") && !string.IsNullOrEmpty (make_config ["ENABLE_XAMARIN"]); @@ -773,6 +777,7 @@ public void Save (StringWriter doc, string path) } bool? disable_watchos_on_wrench; + public bool DisableWatchOSOnWrench { get { if (!disable_watchos_on_wrench.HasValue) From b68a3233e58cb5ff386ca9efc17dfd02ecf442f1 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 14:15:52 +0100 Subject: [PATCH 17/30] Use MlaunchArguments --- tests/xharness/CrashReportSnapshot.cs | 15 ++++++++------- tests/xharness/Execution/IProcessManager.cs | 2 +- tests/xharness/Execution/ProcessManager.cs | 9 +++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs index 1b0169cd83d6..6052b2560d42 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashReportSnapshot.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Xharness.Execution; +using Xharness.Execution.Mlaunch; using Xharness.Logging; namespace Xharness @@ -167,15 +168,15 @@ async Task> CreateCrashReportsSnapshotAsync () } else { var tmp = Path.GetTempFileName (); try { - var sb = new List (); - sb.Add ($"--list-crash-reports={tmp}"); - sb.Add ("--sdkroot"); - sb.Add (xcodeRoot); + var args = new MlaunchArguments ( + new ListCrashReportsArgument (tmp), + new SdkRootArgument (xcodeRoot)); + if (!string.IsNullOrEmpty (deviceName)) { - sb.Add ("--devname"); - sb.Add (deviceName); + args.Add (new DeviceNameArgument(deviceName)); } - var result = await processManager.ExecuteCommandAsync (mlaunchPath, sb, log, TimeSpan.FromMinutes (1)); + + var result = await processManager.ExecuteCommandAsync (mlaunchPath, args, log, TimeSpan.FromMinutes (1)); if (result.Succeeded) crashes.UnionWith (File.ReadAllLines (tmp)); } finally { diff --git a/tests/xharness/Execution/IProcessManager.cs b/tests/xharness/Execution/IProcessManager.cs index 65c1ab0fca9f..85bad04c7751 100644 --- a/tests/xharness/Execution/IProcessManager.cs +++ b/tests/xharness/Execution/IProcessManager.cs @@ -12,13 +12,13 @@ namespace Xharness.Execution { public class ProcessExecutionResult { public bool TimedOut { get; set; } public int ExitCode { get; set; } - public bool Succeeded => !TimedOut && ExitCode == 0; } // interface that helps to manage the different processes in the app. public interface IProcessManager { Task ExecuteCommandAsync (string filename, IList args, ILog log, TimeSpan timeout, Dictionary environment_variables = null, CancellationToken? cancellation_token = null); + Task ExecuteCommandAsync (string filename, MlaunchArguments args, ILog log, TimeSpan timeout, Dictionary environment_variables = null, CancellationToken? cancellation_token = null); Task RunAsync (Process process, ILog log, CancellationToken? cancellationToken = null, bool? diagnostics = null); Task RunAsync (Process process, ILog log, TimeSpan? timeout = null, Dictionary environment_variables = null, CancellationToken? cancellation_token = null, bool? diagnostics = null); Task RunAsync (Process process, MlaunchArguments args, ILog log, TimeSpan? timeout = null, Dictionary environment_variables = null, CancellationToken? cancellation_token = null, bool? diagnostics = null); diff --git a/tests/xharness/Execution/ProcessManager.cs b/tests/xharness/Execution/ProcessManager.cs index afd69bc1df5c..f11991774d56 100644 --- a/tests/xharness/Execution/ProcessManager.cs +++ b/tests/xharness/Execution/ProcessManager.cs @@ -26,6 +26,15 @@ public async Task ExecuteCommandAsync (string filename, } } + public async Task ExecuteCommandAsync (string filename, MlaunchArguments args, ILog log, TimeSpan timeout, Dictionary environment_variables = null, CancellationToken? cancellation_token = null) + { + using (var p = new Process ()) { + p.StartInfo.FileName = filename; + p.StartInfo.Arguments = args.AsCommandLine (); + return await RunAsync (p, log, timeout, environment_variables, cancellation_token); + } + } + [DllImport ("/usr/lib/libc.dylib")] internal static extern int kill (int pid, int sig); From c407f393bca2e1263f0d19f1c220e0a94e48cfa8 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 14:44:19 +0100 Subject: [PATCH 18/30] Add MlaunchArguments docs --- tests/xharness/Execution/Mlaunch/Arguments.cs | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/xharness/Execution/Mlaunch/Arguments.cs b/tests/xharness/Execution/Mlaunch/Arguments.cs index 6bcca17e0b7b..6696df323ba1 100644 --- a/tests/xharness/Execution/Mlaunch/Arguments.cs +++ b/tests/xharness/Execution/Mlaunch/Arguments.cs @@ -1,47 +1,90 @@ namespace Xharness.Execution.Mlaunch { + /// + /// Specify the location of Apple SDKs, default to 'xcode-select' value. + /// public sealed class SdkRootArgument : SingleValueArgument { public SdkRootArgument (string sdkPath) : base ("sdkroot", sdkPath) { } } + /// + /// List the currently connected devices and their UDIDs. + /// public sealed class ListDevicesArgument : SingleValueArgument { public ListDevicesArgument (string outputFile) : base ("listdev", outputFile) { } } + /// + /// List the available simulators. The output is xml, and written to the specified file. + /// public sealed class ListSimulatorsArgument : SingleValueArgument { public ListSimulatorsArgument (string outputFile) : base ("listsim", outputFile) { } } + /// + /// Lists crash reports on the specified device + /// public sealed class ListCrashReportsArgument : SingleValueArgument { public ListCrashReportsArgument (string outputFile) : base ("list-crash-reports", outputFile) { } } + /// + /// Specify which device (when many are present) the [install|lauch|kill|log]dev command applies + /// public sealed class DeviceNameArgument : SingleValueArgument { public DeviceNameArgument (string deviceName) : base ("devname", deviceName) { } } - + + /// + /// Specify the output format for some commands as Default. + /// public sealed class DefaultOutputFormatArgument : SingleValueArgument { public DefaultOutputFormatArgument () : base ("output-format", "Default") { } } + /// + /// Specify the output format for some commands as XML. + /// public sealed class XmlOutputFormatArgument : SingleValueArgument { public XmlOutputFormatArgument () : base ("output-format", "XML") { } } + /// + /// Download a crash report from the specified device. + /// + public sealed class DownloadCrashReportArgument : SingleValueArgument { + public DownloadCrashReportArgument (string deviceName) : base ("download-crash-report", deviceName) + { + } + } + + /// + /// Specifies the file to save the downloaded crash report. + /// + public sealed class DownloadCrashReportToArgument : SingleValueArgument { + public DownloadCrashReportToArgument (string outputFile) : base ("download-crash-report-to", outputFile) + { + } + } + + /// + /// Include additional data (which can take some time to fetch) when listing the connected devices. + /// Only applicable when output format is xml. + /// public sealed class ListExtraDataArgument : OptionArgument { public ListExtraDataArgument () : base ("list-extra-data") { From 4185aec6c0ea34a4b790859342cfc3bf5bdb6c92 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 16:16:31 +0100 Subject: [PATCH 19/30] Add CrashReportSnapshot docs --- tests/xharness/CrashReportSnapshot.cs | 108 ++++++++++-------- .../Tests/CrashReportSnapshotTests.cs | 90 ++++++++++++++- 2 files changed, 143 insertions(+), 55 deletions(-) diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs index 6052b2560d42..30b5ac6fa7a2 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashReportSnapshot.cs @@ -43,8 +43,10 @@ public class CrashReportSnapshot : ICrashReportSnapshot { readonly string mlaunchPath; readonly bool isDevice; readonly string deviceName; + readonly Func tempFileProvider; + readonly string symbolicateCrashPath; - HashSet initialSet; + HashSet initialCrashes; public CrashReportSnapshot (IProcessManager processManager, ILog log, @@ -52,7 +54,8 @@ public CrashReportSnapshot (IProcessManager processManager, string xcodeRoot, string mlaunchPath, bool isDevice, - string deviceName) + string deviceName, + Func tempFileProvider = null) { this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); this.log = log ?? throw new ArgumentNullException (nameof (log)); @@ -61,94 +64,101 @@ public CrashReportSnapshot (IProcessManager processManager, this.mlaunchPath = mlaunchPath ?? throw new ArgumentNullException (nameof (mlaunchPath)); this.isDevice = isDevice; this.deviceName = deviceName; + this.tempFileProvider = tempFileProvider ?? Path.GetTempFileName; + + symbolicateCrashPath = Path.Combine (xcodeRoot, "Contents", "SharedFrameworks", "DTDeviceKitBase.framework", "Versions", "A", "Resources", "symbolicatecrash"); + if (!File.Exists (symbolicateCrashPath)) + symbolicateCrashPath = Path.Combine (xcodeRoot, "Contents", "SharedFrameworks", "DVTFoundation.framework", "Versions", "A", "Resources", "symbolicatecrash"); + if (!File.Exists (symbolicateCrashPath)) + symbolicateCrashPath = null; } public async Task StartCaptureAsync () { - initialSet = await CreateCrashReportsSnapshotAsync (); + initialCrashes = await CreateCrashReportsSnapshotAsync (); } public async Task EndCaptureAsync (TimeSpan timeout) { // Check for crash reports - var crash_report_search_done = false; - var crash_report_search_timeout = timeout.TotalSeconds; - var watch = new Stopwatch (); - watch.Start (); + var stopwatch = Stopwatch.StartNew (); + do { - var end_crashes = await CreateCrashReportsSnapshotAsync (); - end_crashes.ExceptWith (initialSet); + var newCrashes = await CreateCrashReportsSnapshotAsync (); + newCrashes.ExceptWith (initialCrashes); - if (end_crashes.Count == 0) { - if (watch.Elapsed.TotalSeconds > crash_report_search_timeout) { - crash_report_search_done = true; + if (newCrashes.Count == 0) { + if (stopwatch.Elapsed.TotalSeconds > timeout.TotalSeconds) { + break; } else { - log.WriteLine ("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int) (crash_report_search_timeout - watch.Elapsed.TotalSeconds)); + log.WriteLine ( + "No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", + (int) (timeout.TotalSeconds - stopwatch.Elapsed.TotalSeconds)); + Thread.Sleep (TimeSpan.FromSeconds (1)); } continue; } - log.WriteLine ("Found {0} new crash report(s)", end_crashes.Count); - List crash_reports; + log.WriteLine ("Found {0} new crash report(s)", newCrashes.Count); + + List crashReports; if (!isDevice) { - crash_reports = new List (end_crashes.Count); - foreach (var path in end_crashes) { + crashReports = new List (newCrashes.Count); + foreach (var path in newCrashes) { logs.AddFile (path, $"Crash report: {Path.GetFileName (path)}"); } } else { // Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench // (if we put them in /tmp, they'd never be deleted). - var downloaded_crash_reports = new List (); - foreach (var file in end_crashes) { - var name = Path.GetFileName (file); - var crash_report_target = logs.Create (name, $"Crash report: {name}", timestamp: false); - var sb = new List (); - sb.Add ($"--download-crash-report={file}"); - sb.Add ($"--download-crash-report-to={crash_report_target.Path}"); - sb.Add ("--sdkroot"); - sb.Add (xcodeRoot); + crashReports = new List (); + foreach (var crash in newCrashes) { + var name = Path.GetFileName (crash); + var crashReportFile = logs.Create (name, $"Crash report: {name}", timestamp: false); + var args = new MlaunchArguments ( + new DownloadCrashReportArgument (crash), + new DownloadCrashReportToArgument (crashReportFile.Path), + new SdkRootArgument (xcodeRoot)); + if (!string.IsNullOrEmpty (deviceName)) { - sb.Add ("--devname"); - sb.Add (deviceName); + args.Add (new DeviceNameArgument(deviceName)); } - var result = await processManager.ExecuteCommandAsync (mlaunchPath, sb, log, TimeSpan.FromMinutes (1)); + + var result = await processManager.ExecuteCommandAsync (mlaunchPath, args, log, TimeSpan.FromMinutes (1)); + if (result.Succeeded) { - log.WriteLine ("Downloaded crash report {0} to {1}", file, crash_report_target.Path); - crash_report_target = await SymbolicateCrashReportAsync (crash_report_target); - downloaded_crash_reports.Add (crash_report_target); + log.WriteLine ("Downloaded crash report {0} to {1}", crash, crashReportFile.Path); + crashReportFile = await SymbolicateCrashReportAsync (crashReportFile); + crashReports.Add (crashReportFile); } else { - log.WriteLine ("Could not download crash report {0}", file); + log.WriteLine ("Could not download crash report {0}", crash); } } - crash_reports = downloaded_crash_reports; } - foreach (var cp in crash_reports) { + foreach (var cp in crashReports) { WrenchLog.WriteLine ("AddFile: {0}", cp.Path); log.WriteLine (" {0}", cp.Path); } - crash_report_search_done = true; - } while (!crash_report_search_done); + + break; + + } while (true); } async Task SymbolicateCrashReportAsync (ILogFile report) { - var symbolicatecrash = Path.Combine (xcodeRoot, "Contents", "SharedFrameworks", "DTDeviceKitBase.framework", "Versions", "A", "Resources", "symbolicatecrash"); - if (!File.Exists (symbolicatecrash)) - symbolicatecrash = Path.Combine (xcodeRoot, "Contents", "SharedFrameworks", "DVTFoundation.framework", "Versions", "A", "Resources", "symbolicatecrash"); - - if (!File.Exists (symbolicatecrash)) { - log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicatecrash); + if (symbolicateCrashPath == null) { + log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicateCrashPath); return report; } var name = Path.GetFileName (report.Path); var symbolicated = logs.Create (Path.ChangeExtension (name, ".symbolicated.log"), $"Symbolicated crash report: {name}", timestamp: false); var environment = new Dictionary { { "DEVELOPER_DIR", Path.Combine (xcodeRoot, "Contents", "Developer") } }; - var rv = await processManager.ExecuteCommandAsync (symbolicatecrash, new [] { report.Path }, symbolicated, TimeSpan.FromMinutes (1), environment); - if (rv.Succeeded) { + var result = await processManager.ExecuteCommandAsync (symbolicateCrashPath, new [] { report.Path }, symbolicated, TimeSpan.FromMinutes (1), environment); + if (result.Succeeded) { log.WriteLine ("Symbolicated {0} successfully.", report.Path); return symbolicated; } else { @@ -166,10 +176,10 @@ async Task> CreateCrashReportsSnapshotAsync () if (Directory.Exists (dir)) crashes.UnionWith (Directory.EnumerateFiles (dir)); } else { - var tmp = Path.GetTempFileName (); + var tempFile = tempFileProvider (); try { var args = new MlaunchArguments ( - new ListCrashReportsArgument (tmp), + new ListCrashReportsArgument (tempFile), new SdkRootArgument (xcodeRoot)); if (!string.IsNullOrEmpty (deviceName)) { @@ -178,9 +188,9 @@ async Task> CreateCrashReportsSnapshotAsync () var result = await processManager.ExecuteCommandAsync (mlaunchPath, args, log, TimeSpan.FromMinutes (1)); if (result.Succeeded) - crashes.UnionWith (File.ReadAllLines (tmp)); + crashes.UnionWith (File.ReadAllLines (tempFile)); } finally { - File.Delete (tmp); + File.Delete (tempFile); } } diff --git a/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs b/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs index 11e9e4b65d4b..4c5f0ef7535f 100644 --- a/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs +++ b/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs @@ -1,15 +1,20 @@ using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Threading.Tasks; using Moq; using NUnit.Framework; using Xharness.Execution; +using Xharness.Execution.Mlaunch; using Xharness.Logging; namespace Xharness.Tests { [TestFixture] public class CrashReportSnapshotTests { - const string xcodeRoot = "/path/to/xcode"; - const string mlaunchPath = "/path/to/mlaunch"; + readonly string mlaunchPath = "./mlaunch"; + string tempXcodeRoot; + string symbolicatePath; Mock processManager; Mock log; @@ -21,24 +26,97 @@ public void SetUp () processManager = new Mock (); log = new Mock (); logs = new Mock (); + + tempXcodeRoot = Path.Combine (Path.GetTempPath (), Guid.NewGuid ().ToString()); + symbolicatePath = Path.Combine (tempXcodeRoot, "Contents", "SharedFrameworks", "DTDeviceKitBase.framework", "Versions", "A", "Resources"); + + // Create fake place for device logs + Directory.CreateDirectory (tempXcodeRoot); + + // Create fake symbolicate binary + Directory.CreateDirectory (symbolicatePath); + File.Create (Path.Combine (symbolicatePath, "symbolicatecrash")); + } + + [TearDown] + public void TearDown () { + Directory.Delete (tempXcodeRoot, true); } [Test] public async Task DeviceCaptureTest () { + var tempFilePath = Path.GetTempFileName (); + + const string deviceName = "Sample-iPhone"; + const string crashLogPath = "/path/to/crash.log"; + const string symbolicateLogPath = "/path/to/" + deviceName+ ".symbolicated.log"; + + var crashReport = Mock.Of (x => x.Path == crashLogPath); + var symbolicateReport = Mock.Of (x => x.Path == symbolicateLogPath); + + // Crash report is added + logs.Setup (x => x.Create (deviceName, "Crash report: " + deviceName, It.IsAny ())) + .Returns (crashReport); + + // Symbolicate report is added + logs.Setup (x => x.Create ("crash.symbolicated.log", "Symbolicated crash report: crash.log", It.IsAny ())) + .Returns (symbolicateReport); + + // List of crash reports is retrieved + processManager + .Setup (x => x.ExecuteCommandAsync ( + mlaunchPath, + It.Is (args => args.AsCommandLine () == $"--list-crash-reports={tempFilePath} --sdkroot={tempXcodeRoot} --devname={deviceName}"), + log.Object, + TimeSpan.FromMinutes (1), + null, + null)) + .ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 }); + + // Device crash log is downloaded + processManager + .Setup (x => x.ExecuteCommandAsync ( + mlaunchPath, + It.Is (args => args.AsCommandLine () == $"--download-crash-report={deviceName} --download-crash-report-to={crashLogPath} --sdkroot={tempXcodeRoot} --devname={deviceName}"), + log.Object, + TimeSpan.FromMinutes (1), + null, + null)) + .ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 }); + + // Symbolicate is ran + processManager + .Setup (x => x.ExecuteCommandAsync ( + Path.Combine (symbolicatePath, "symbolicatecrash"), + It.Is> (args => args.First () == crashLogPath), + symbolicateReport, + TimeSpan.FromMinutes (1), + It.IsAny >(), + null)) + .ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 }); + + // Act var snapshotReport = new CrashReportSnapshot (processManager.Object, log.Object, logs.Object, - xcodeRoot, + tempXcodeRoot, mlaunchPath, true, - "Sample iPhone"); + deviceName, + () => tempFilePath); - await snapshotReport.StartCaptureAsync (); + File.WriteAllLines (tempFilePath, new [] { "crash 1", "crash 2" }); - // mlaunch "--list-crash-reports=C:\\Users\\prvysoky\\AppData\\Local\\Temp\\tmp43C3.tmp", "--sdkroot", "/path/to/xcode", "--devname", "Sample iPhone" + await snapshotReport.StartCaptureAsync (); + + File.WriteAllLines (tempFilePath, new [] { "Sample-iPhone" }); await snapshotReport.EndCaptureAsync (TimeSpan.FromSeconds (10)); + + // Verify all calls above + processManager.VerifyAll (); + logs.VerifyAll (); } } } From 8329a72770e7f0da0391cdfa3a1ac9edb60cdef8 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 17:02:27 +0100 Subject: [PATCH 20/30] Rename method --- tests/xharness/CrashReportSnapshot.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs index 30b5ac6fa7a2..577b5177d430 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashReportSnapshot.cs @@ -129,7 +129,7 @@ public async Task EndCaptureAsync (TimeSpan timeout) if (result.Succeeded) { log.WriteLine ("Downloaded crash report {0} to {1}", crash, crashReportFile.Path); - crashReportFile = await SymbolicateCrashReportAsync (crashReportFile); + crashReportFile = await GetSymbolicateCrashReportAsync (crashReportFile); crashReports.Add (crashReportFile); } else { log.WriteLine ("Could not download crash report {0}", crash); @@ -147,7 +147,7 @@ public async Task EndCaptureAsync (TimeSpan timeout) } while (true); } - async Task SymbolicateCrashReportAsync (ILogFile report) + async Task GetSymbolicateCrashReportAsync (ILogFile report) { if (symbolicateCrashPath == null) { log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicateCrashPath); From 67b0093a92d3f37dc767373ac99c8024569472e4 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 17:05:39 +0100 Subject: [PATCH 21/30] Rename class --- tests/xharness/AppRunner.cs | 4 +-- ...rtSnapshot.cs => CrashSnapshotReporter.cs} | 32 +++++++++---------- tests/xharness/Jenkins/Jenkins.cs | 2 +- .../Jenkins/TestTasks/MacExecuteTask.cs | 4 +-- .../xharness/Jenkins/TestTasks/RunXtroTask.cs | 2 +- .../Tests/CrashReportSnapshotTests.cs | 2 +- tests/xharness/xharness.csproj | 2 +- 7 files changed, 24 insertions(+), 24 deletions(-) rename tests/xharness/{CrashReportSnapshot.cs => CrashSnapshotReporter.cs} (87%) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 8337af1d5942..b674c7a8540b 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -424,7 +424,7 @@ public bool TestsSucceeded (AppInformation appInfo, string test_log_path, bool t public async Task RunAsync () { - CrashReportSnapshot crash_reports; + CrashSnapshotReporter crash_reports; ILog device_system_log = null; ILog listener_log = null; ILog run_log = MainLog; @@ -432,7 +432,7 @@ public async Task RunAsync () if (!isSimulator) FindDevice (); - crash_reports = new CrashReportSnapshot (processManager, + crash_reports = new CrashSnapshotReporter (processManager, MainLog, Logs, harness.XcodeRoot, diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashSnapshotReporter.cs similarity index 87% rename from tests/xharness/CrashReportSnapshot.cs rename to tests/xharness/CrashSnapshotReporter.cs index 577b5177d430..10afc438b201 100644 --- a/tests/xharness/CrashReportSnapshot.cs +++ b/tests/xharness/CrashSnapshotReporter.cs @@ -10,32 +10,32 @@ namespace Xharness { - public interface ICrashReportSnapshotFactory { - ICrashReportSnapshot Create (ILog log, ILogs logs, bool isDevice, string deviceName); + public interface ICrashSnapshotReporterFactory { + ICrashSnapshotReporter Create (ILog log, ILogs logs, bool isDevice, string deviceName); } - public class CrashReportSnapshotFactory : ICrashReportSnapshotFactory { + public class CrashSnapshotReporterFactory : ICrashSnapshotReporterFactory { readonly IProcessManager processManager; readonly string xcodeRoot; readonly string mlaunchPath; - public CrashReportSnapshotFactory (IProcessManager processManager, string xcodeRoot, string mlaunchPath) + public CrashSnapshotReporterFactory (IProcessManager processManager, string xcodeRoot, string mlaunchPath) { this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); this.xcodeRoot = xcodeRoot ?? throw new ArgumentNullException (nameof (xcodeRoot)); this.mlaunchPath = mlaunchPath ?? throw new ArgumentNullException (nameof (mlaunchPath)); } - public ICrashReportSnapshot Create (ILog log, ILogs logs, bool isDevice, string deviceName) => - new CrashReportSnapshot (processManager, log, logs, xcodeRoot, mlaunchPath, isDevice, deviceName); + public ICrashSnapshotReporter Create (ILog log, ILogs logs, bool isDevice, string deviceName) => + new CrashSnapshotReporter (processManager, log, logs, xcodeRoot, mlaunchPath, isDevice, deviceName); } - public interface ICrashReportSnapshot { + public interface ICrashSnapshotReporter { Task EndCaptureAsync (TimeSpan timeout); Task StartCaptureAsync (); } - public class CrashReportSnapshot : ICrashReportSnapshot { + public class CrashSnapshotReporter : ICrashSnapshotReporter { readonly IProcessManager processManager; readonly ILog log; readonly ILogs logs; @@ -48,14 +48,14 @@ public class CrashReportSnapshot : ICrashReportSnapshot { HashSet initialCrashes; - public CrashReportSnapshot (IProcessManager processManager, - ILog log, - ILogs logs, - string xcodeRoot, - string mlaunchPath, - bool isDevice, - string deviceName, - Func tempFileProvider = null) + public CrashSnapshotReporter (IProcessManager processManager, + ILog log, + ILogs logs, + string xcodeRoot, + string mlaunchPath, + bool isDevice, + string deviceName, + Func tempFileProvider = null) { this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); this.log = log ?? throw new ArgumentNullException (nameof (log)); diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index aa3fc747ef22..5d91b25bb30e 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -936,7 +936,7 @@ Task PopulateTasksAsync () //Tasks.AddRange (await CreateRunSimulatorTasksAsync ()); - var crashReportSnapshotFactory = new CrashReportSnapshotFactory (processManager, Harness.XcodeRoot, Harness.MlaunchPath); + var crashReportSnapshotFactory = new CrashSnapshotReporterFactory (processManager, Harness.XcodeRoot, Harness.MlaunchPath); var buildiOSMSBuild_net461 = new MSBuildTask () { diff --git a/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs index 9599a47ebdde..a2f90b53ef81 100644 --- a/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs @@ -12,13 +12,13 @@ namespace Xharness.Jenkins.TestTasks { class MacExecuteTask : MacTask { - protected ICrashReportSnapshotFactory CrashReportSnapshotFactory { get; } + protected ICrashSnapshotReporterFactory CrashReportSnapshotFactory { get; } public string Path; public bool BCLTest; public bool IsUnitTest; - public MacExecuteTask (BuildToolTask build_task, IProcessManager processManager, ICrashReportSnapshotFactory crashReportSnapshotFactory) + public MacExecuteTask (BuildToolTask build_task, IProcessManager processManager, ICrashSnapshotReporterFactory crashReportSnapshotFactory) : base (build_task, processManager) { this.CrashReportSnapshotFactory = crashReportSnapshotFactory ?? throw new ArgumentNullException (nameof (crashReportSnapshotFactory)); diff --git a/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs b/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs index 61533639a437..0f59b670b71a 100644 --- a/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs @@ -8,7 +8,7 @@ namespace Xharness.Jenkins.TestTasks { class RunXtroTask : MacExecuteTask { - public RunXtroTask (BuildToolTask build_task, IProcessManager processManager, ICrashReportSnapshotFactory crashReportSnapshotFactory) + public RunXtroTask (BuildToolTask build_task, IProcessManager processManager, ICrashSnapshotReporterFactory crashReportSnapshotFactory) : base (build_task, processManager, crashReportSnapshotFactory) { } diff --git a/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs b/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs index 4c5f0ef7535f..81276bbbf46a 100644 --- a/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs +++ b/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs @@ -97,7 +97,7 @@ public async Task DeviceCaptureTest () .ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 }); // Act - var snapshotReport = new CrashReportSnapshot (processManager.Object, + var snapshotReport = new CrashSnapshotReporter (processManager.Object, log.Object, logs.Object, tempXcodeRoot, diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index 43fd15513bf6..2c56e00047c7 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -85,7 +85,7 @@ - + From 19dff9a669d524a6a16af79e640d46fac12e41bd Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 17:43:09 +0100 Subject: [PATCH 22/30] Refactor CrashSnapshotReporter --- tests/xharness/CrashSnapshotReporter.cs | 66 ++++++++++++++----------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/tests/xharness/CrashSnapshotReporter.cs b/tests/xharness/CrashSnapshotReporter.cs index 10afc438b201..b6ea1726d1f9 100644 --- a/tests/xharness/CrashSnapshotReporter.cs +++ b/tests/xharness/CrashSnapshotReporter.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Xharness.Execution; @@ -84,10 +85,10 @@ public async Task EndCaptureAsync (TimeSpan timeout) var stopwatch = Stopwatch.StartNew (); do { - var newCrashes = await CreateCrashReportsSnapshotAsync (); - newCrashes.ExceptWith (initialCrashes); + var newCrashFiles = await CreateCrashReportsSnapshotAsync (); + newCrashFiles.ExceptWith (initialCrashes); - if (newCrashes.Count == 0) { + if (newCrashFiles.Count == 0) { if (stopwatch.Elapsed.TotalSeconds > timeout.TotalSeconds) { break; } else { @@ -101,40 +102,21 @@ public async Task EndCaptureAsync (TimeSpan timeout) continue; } - log.WriteLine ("Found {0} new crash report(s)", newCrashes.Count); + log.WriteLine ("Found {0} new crash report(s)", newCrashFiles.Count); - List crashReports; + IEnumerable crashReports; if (!isDevice) { - crashReports = new List (newCrashes.Count); - foreach (var path in newCrashes) { + crashReports = new List (newCrashFiles.Count); + foreach (var path in newCrashFiles) { logs.AddFile (path, $"Crash report: {Path.GetFileName (path)}"); } } else { // Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench // (if we put them in /tmp, they'd never be deleted). - crashReports = new List (); - foreach (var crash in newCrashes) { - var name = Path.GetFileName (crash); - var crashReportFile = logs.Create (name, $"Crash report: {name}", timestamp: false); - var args = new MlaunchArguments ( - new DownloadCrashReportArgument (crash), - new DownloadCrashReportToArgument (crashReportFile.Path), - new SdkRootArgument (xcodeRoot)); - - if (!string.IsNullOrEmpty (deviceName)) { - args.Add (new DeviceNameArgument(deviceName)); - } - - var result = await processManager.ExecuteCommandAsync (mlaunchPath, args, log, TimeSpan.FromMinutes (1)); - - if (result.Succeeded) { - log.WriteLine ("Downloaded crash report {0} to {1}", crash, crashReportFile.Path); - crashReportFile = await GetSymbolicateCrashReportAsync (crashReportFile); - crashReports.Add (crashReportFile); - } else { - log.WriteLine ("Could not download crash report {0}", crash); - } - } + crashReports = newCrashFiles + .Select (async crash => await ProcessCrash (crash)) + .Select (t => t.Result) + .Where (c => c != null); } foreach (var cp in crashReports) { @@ -147,6 +129,30 @@ public async Task EndCaptureAsync (TimeSpan timeout) } while (true); } + async Task ProcessCrash (string crashFile) + { + var name = Path.GetFileName (crashFile); + var crashReportFile = logs.Create (name, $"Crash report: {name}", timestamp: false); + var args = new MlaunchArguments ( + new DownloadCrashReportArgument (crashFile), + new DownloadCrashReportToArgument (crashReportFile.Path), + new SdkRootArgument (xcodeRoot)); + + if (!string.IsNullOrEmpty (deviceName)) { + args.Add (new DeviceNameArgument(deviceName)); + } + + var result = await processManager.ExecuteCommandAsync (mlaunchPath, args, log, TimeSpan.FromMinutes (1)); + + if (result.Succeeded) { + log.WriteLine ("Downloaded crash report {0} to {1}", crashFile, crashReportFile.Path); + return await GetSymbolicateCrashReportAsync (crashReportFile); + } else { + log.WriteLine ("Could not download crash report {0}", crashFile); + return null; + } + } + async Task GetSymbolicateCrashReportAsync (ILogFile report) { if (symbolicateCrashPath == null) { From aeac5468249561b2606bea520d48dbeffedcc9af Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 20:15:36 +0100 Subject: [PATCH 23/30] Inject AppRunner --- tests/xharness/AppRunner.cs | 12 ++-- tests/xharness/Harness.cs | 3 + .../Jenkins/TestTasks/RunDeviceTask.cs | 2 + .../Jenkins/TestTasks/RunSimulatorTask.cs | 1 + .../Xharness.Tests/Tests/AppRunnerTests.cs | 71 ++++--------------- 5 files changed, 23 insertions(+), 66 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 3fc5a50a506f..531acb30db2c 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -55,6 +55,7 @@ class AppRunner readonly ISimulatorsLoaderFactory simulatorsLoaderFactory; readonly ISimpleListenerFactory listenerFactory; readonly IDeviceLoaderFactory devicesLoaderFactory; + readonly ICrashSnapshotReporterFactory snapshotReporterFactory; readonly RunMode mode; readonly bool isSimulator; @@ -95,6 +96,7 @@ public AppRunner (IProcessManager processManager, ISimulatorsLoaderFactory simulatorsFactory, ISimpleListenerFactory simpleListenerFactory, IDeviceLoaderFactory devicesFactory, + ICrashSnapshotReporterFactory snapshotReporterFactory, AppRunnerTarget target, IHarness harness, ILog mainLog, @@ -113,6 +115,7 @@ public AppRunner (IProcessManager processManager, this.simulatorsLoaderFactory = simulatorsFactory ?? throw new ArgumentNullException (nameof (simulatorsFactory)); this.listenerFactory = simpleListenerFactory ?? throw new ArgumentNullException (nameof (simpleListenerFactory)); this.devicesLoaderFactory = devicesFactory ?? throw new ArgumentNullException (nameof (devicesFactory)); + this.snapshotReporterFactory = snapshotReporterFactory ?? throw new ArgumentNullException (nameof (snapshotReporterFactory)); this.harness = harness ?? throw new ArgumentNullException (nameof (harness)); this.MainLog = mainLog ?? throw new ArgumentNullException (nameof (mainLog)); this.projectFilePath = projectFilePath ?? throw new ArgumentNullException (nameof (projectFilePath)); @@ -423,7 +426,6 @@ public bool TestsSucceeded (AppInformation appInfo, string test_log_path, bool t public async Task RunAsync () { - CrashSnapshotReporter crash_reports; ILog device_system_log = null; ILog listener_log = null; ILog run_log = MainLog; @@ -431,13 +433,7 @@ public async Task RunAsync () if (!isSimulator) FindDevice (); - crash_reports = new CrashSnapshotReporter (processManager, - MainLog, - Logs, - harness.XcodeRoot, - harness.MlaunchPath, - isDevice: !isSimulator, - deviceName); + ICrashSnapshotReporter crash_reports = snapshotReporterFactory.Create (MainLog, Logs, isDevice: !isSimulator, deviceName); var args = new List (); if (!string.IsNullOrEmpty (harness.XcodeRoot)) { diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 9a023073f7e6..9257c6deeef8 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -605,6 +605,7 @@ int Install () new SimulatorsLoaderFactory (this, ProcessManager), new SimpleListenerFactory (), new DeviceLoaderFactory (this, ProcessManager), + new CrashSnapshotReporterFactory (ProcessManager, XcodeRoot, MlaunchPath), target, this, HarnessLog, @@ -630,6 +631,7 @@ int Uninstall () new SimulatorsLoaderFactory (this, ProcessManager), new SimpleListenerFactory (), new DeviceLoaderFactory (this, ProcessManager), + new CrashSnapshotReporterFactory (ProcessManager, XcodeRoot, MlaunchPath), target, this, HarnessLog, @@ -653,6 +655,7 @@ int Run () new SimulatorsLoaderFactory (this, ProcessManager), new SimpleListenerFactory (), new DeviceLoaderFactory (this, ProcessManager), + new CrashSnapshotReporterFactory (ProcessManager, XcodeRoot, MlaunchPath), target, this, HarnessLog, diff --git a/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs b/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs index 71d56acafc5f..c56efe1b59cf 100644 --- a/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs @@ -83,6 +83,7 @@ protected override async Task RunTestAsync () new SimulatorsLoaderFactory (Harness, processManager), new SimpleListenerFactory (), new DeviceLoaderFactory (Harness, processManager), + new CrashSnapshotReporterFactory (ProcessManager, Harness.XcodeRoot, Harness.MlaunchPath), AppRunnerTarget, Harness, projectFilePath: ProjectFile, @@ -145,6 +146,7 @@ protected override async Task RunTestAsync () new SimulatorsLoaderFactory (Harness, processManager), new SimpleListenerFactory (), new DeviceLoaderFactory (Harness, processManager), + new CrashSnapshotReporterFactory (ProcessManager, Harness.XcodeRoot, Harness.MlaunchPath), AppRunnerTarget, Harness, projectFilePath: ProjectFile, diff --git a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs index 793d2fa15e39..6400db5a2f56 100644 --- a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs @@ -80,6 +80,7 @@ public async Task SelectSimulatorAsync () new SimulatorsLoaderFactory (Harness, processManager), new SimpleListenerFactory (), new DeviceLoaderFactory (Harness, processManager), + new CrashSnapshotReporterFactory (ProcessManager, Harness.XcodeRoot, Harness.MlaunchPath), AppRunnerTarget, Harness, mainLog: Logs.Create ($"run-{Device.UDID}-{Timestamp}.log", "Run log"), diff --git a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs index 20bf9d118bf0..7e80ba435a36 100644 --- a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs +++ b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs @@ -51,11 +51,13 @@ public class AppRunnerTests { Mock simulators; Mock devices; Mock simpleListener; + Mock snapshotReporter; ILog mainLog; ISimulatorsLoaderFactory simulatorsFactory; IDeviceLoaderFactory devicesFactory; ISimpleListenerFactory listenerFactory; + ICrashSnapshotReporterFactory snapshotReporterFactory; [SetUp] public void SetUp () @@ -64,6 +66,7 @@ public void SetUp () simulators = new Mock (); devices = new Mock (); simpleListener = new Mock (); + snapshotReporter = new Mock (); var mock1 = new Mock (); mock1.Setup (m => m.CreateLoader ()).Returns (simulators.Object); @@ -79,6 +82,10 @@ public void SetUp () .Returns ((ListenerTransport.Tcp, simpleListener.Object, null)); listenerFactory = mock3.Object; + var mock4 = new Mock (); + mock4.Setup (m => m.Create (It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns (snapshotReporter.Object); + devicesFactory = mock2.Object; + mainLog = new Mock ().Object; Directory.CreateDirectory (appPath); @@ -91,6 +98,7 @@ public void InitializeTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, AppRunnerTarget.Simulator_iOS64, new Mock ().Object, new Mock().Object, @@ -111,6 +119,7 @@ public void InstallToSimulatorTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, AppRunnerTarget.Simulator_iOS64, new Mock ().Object, new Mock().Object, @@ -130,6 +139,7 @@ public void UninstallToSimulatorTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, AppRunnerTarget.Simulator_iOS64, new Mock ().Object, new Mock().Object, @@ -149,6 +159,7 @@ public void InstallWhenNoDevicesTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, AppRunnerTarget.Device_iOS, new Mock ().Object, new Mock().Object, @@ -187,6 +198,7 @@ public async Task InstallOnDeviceTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, AppRunnerTarget.Device_iOS, harnessMock.Object, mainLog, @@ -244,6 +256,7 @@ public async Task UninstallFromDeviceTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, AppRunnerTarget.Device_iOS, harnessMock.Object, mainLog, @@ -274,63 +287,5 @@ public async Task UninstallFromDeviceTest () null, null)); } - - [Test] - public async Task RunOnDeviceTest () - { - Mock harnessMock = new Mock (); - harnessMock.SetupGet (x => x.XcodeRoot).Returns ("/path/to/xcode"); - harnessMock.SetupGet (x => x.MlaunchPath).Returns ("/path/to/mlaunch"); - harnessMock.SetupGet (x => x.Verbosity).Returns (2); - - devices.Setup (x => x.ConnectedDevices).Returns (mockDevices); - - /*var processResult = new ProcessExecutionResult () { ExitCode = 1, TimedOut = false }; - - processManager - .Setup (x => x.ExecuteCommandAsync ( - It.IsAny (), - It.IsAny> (), - It.IsAny (), - It.IsAny (), - It.IsAny> (), - It.IsAny ())) - .ReturnsAsync(processResult); - - var appRunner = new AppRunner (processManager.Object, - simulatorsFactory, - listenerFactory, - devicesFactory, - AppRunnerTarget.Device_iOS, - harnessMock.Object, - mainLog, - Path.Combine (sampleProjectPath, "SystemXunit.csproj"), - "Debug", - Path.Combine (outputPath, "logs")); - - - CancellationToken cancellationToken = new CancellationToken (); - var result = await appRunner.InstallAsync (cancellationToken); - - Assert.AreEqual (1, result.ExitCode); - - processManager.Verify (x => x.ExecuteCommandAsync ( - "/path/to/mlaunch", - new List () { - "--sdkroot", - "/path/to/xcode", - "-v", - "-v", - "-v", - "--installdev", - appPath, - "--devname", - "Test iPad" - }, - mainLog, - TimeSpan.FromHours (1), - null, - cancellationToken));*/ - } } } From b63618b25f416dd064c8d84bc5b55255486e188b Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 17 Mar 2020 20:19:38 +0100 Subject: [PATCH 24/30] Indent parameters --- tests/xharness/Execution/ProcessManager.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/xharness/Execution/ProcessManager.cs b/tests/xharness/Execution/ProcessManager.cs index f11991774d56..f0d00c967009 100644 --- a/tests/xharness/Execution/ProcessManager.cs +++ b/tests/xharness/Execution/ProcessManager.cs @@ -17,7 +17,12 @@ public ProcessManager () { } - public async Task ExecuteCommandAsync (string filename, IList args, ILog log, TimeSpan timeout, Dictionary environment_variables = null, CancellationToken? cancellation_token = null) + public async Task ExecuteCommandAsync (string filename, + IList args, + ILog log, + TimeSpan timeout, + Dictionary environment_variables = null, + CancellationToken? cancellation_token = null) { using (var p = new Process ()) { p.StartInfo.FileName = filename; @@ -26,7 +31,12 @@ public async Task ExecuteCommandAsync (string filename, } } - public async Task ExecuteCommandAsync (string filename, MlaunchArguments args, ILog log, TimeSpan timeout, Dictionary environment_variables = null, CancellationToken? cancellation_token = null) + public async Task ExecuteCommandAsync (string filename, + MlaunchArguments args, + ILog log, + TimeSpan timeout, + Dictionary environment_variables = null, + CancellationToken? cancellation_token = null) { using (var p = new Process ()) { p.StartInfo.FileName = filename; From 667fdd54377901e08ecaa178bdf36c038cf49285 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Wed, 18 Mar 2020 10:31:02 +0100 Subject: [PATCH 25/30] Indent parameters --- tests/xharness/CrashSnapshotReporter.cs | 16 ++++++++-------- tests/xharness/Execution/ProcessManager.cs | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/xharness/CrashSnapshotReporter.cs b/tests/xharness/CrashSnapshotReporter.cs index b6ea1726d1f9..0a31b65ad1aa 100644 --- a/tests/xharness/CrashSnapshotReporter.cs +++ b/tests/xharness/CrashSnapshotReporter.cs @@ -50,13 +50,13 @@ public class CrashSnapshotReporter : ICrashSnapshotReporter { HashSet initialCrashes; public CrashSnapshotReporter (IProcessManager processManager, - ILog log, - ILogs logs, - string xcodeRoot, - string mlaunchPath, - bool isDevice, - string deviceName, - Func tempFileProvider = null) + ILog log, + ILogs logs, + string xcodeRoot, + string mlaunchPath, + bool isDevice, + string deviceName, + Func tempFileProvider = null) { this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); this.log = log ?? throw new ArgumentNullException (nameof (log)); @@ -66,7 +66,7 @@ public CrashSnapshotReporter (IProcessManager processManager, this.isDevice = isDevice; this.deviceName = deviceName; this.tempFileProvider = tempFileProvider ?? Path.GetTempFileName; - + symbolicateCrashPath = Path.Combine (xcodeRoot, "Contents", "SharedFrameworks", "DTDeviceKitBase.framework", "Versions", "A", "Resources", "symbolicatecrash"); if (!File.Exists (symbolicateCrashPath)) symbolicateCrashPath = Path.Combine (xcodeRoot, "Contents", "SharedFrameworks", "DVTFoundation.framework", "Versions", "A", "Resources", "symbolicatecrash"); diff --git a/tests/xharness/Execution/ProcessManager.cs b/tests/xharness/Execution/ProcessManager.cs index f0d00c967009..6e2031fb5f6b 100644 --- a/tests/xharness/Execution/ProcessManager.cs +++ b/tests/xharness/Execution/ProcessManager.cs @@ -18,11 +18,11 @@ public ProcessManager () } public async Task ExecuteCommandAsync (string filename, - IList args, - ILog log, - TimeSpan timeout, - Dictionary environment_variables = null, - CancellationToken? cancellation_token = null) + IList args, + ILog log, + TimeSpan timeout, + Dictionary environment_variables = null, + CancellationToken? cancellation_token = null) { using (var p = new Process ()) { p.StartInfo.FileName = filename; @@ -32,11 +32,11 @@ public async Task ExecuteCommandAsync (string filename, } public async Task ExecuteCommandAsync (string filename, - MlaunchArguments args, - ILog log, - TimeSpan timeout, - Dictionary environment_variables = null, - CancellationToken? cancellation_token = null) + MlaunchArguments args, + ILog log, + TimeSpan timeout, + Dictionary environment_variables = null, + CancellationToken? cancellation_token = null) { using (var p = new Process ()) { p.StartInfo.FileName = filename; From f1ca7fd8cf1ad6d4e3d602f0c2fd657334fd3494 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Wed, 18 Mar 2020 11:07:53 +0100 Subject: [PATCH 26/30] Fix unit tests --- tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs index 7e80ba435a36..11dda9098188 100644 --- a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs +++ b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs @@ -84,7 +84,7 @@ public void SetUp () var mock4 = new Mock (); mock4.Setup (m => m.Create (It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns (snapshotReporter.Object); - devicesFactory = mock2.Object; + snapshotReporterFactory = mock4.Object; mainLog = new Mock ().Object; From e00cf1f1fc6463877479b1bfa2b5c878c0cc3626 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Wed, 18 Mar 2020 12:57:16 +0100 Subject: [PATCH 27/30] Fix gathering crash logs --- tests/xharness/AppRunner.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 531acb30db2c..458873c90851 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -433,7 +433,9 @@ public async Task RunAsync () if (!isSimulator) FindDevice (); - ICrashSnapshotReporter crash_reports = snapshotReporterFactory.Create (MainLog, Logs, isDevice: !isSimulator, deviceName); + var crashLogs = new Logs (Logs.Directory); + + ICrashSnapshotReporter crash_reports = snapshotReporterFactory.Create (MainLog, crashLogs, isDevice: !isSimulator, deviceName); var args = new List (); if (!string.IsNullOrEmpty (harness.XcodeRoot)) { @@ -749,7 +751,7 @@ public async Task RunAsync () if (!success.Value) { int pid = 0; string crash_reason = null; - foreach (var crash in Logs) { + foreach (var crash in crashLogs) { try { if (pid == 0) { // Find the pid From cda1d758c334e2e82bd0fdd8268a233dfa6ec289 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Wed, 18 Mar 2020 13:50:22 +0100 Subject: [PATCH 28/30] Add crash logs to output --- tests/xharness/AppRunner.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 458873c90851..9d3a457ee0c5 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -751,8 +751,10 @@ public async Task RunAsync () if (!success.Value) { int pid = 0; string crash_reason = null; - foreach (var crash in crashLogs) { + foreach (var crashLog in crashLogs) { try { + Logs.Add (crashLog); + if (pid == 0) { // Find the pid using (var log_reader = MainLog.GetReader ()) { @@ -772,7 +774,7 @@ public async Task RunAsync () } } - using (var crash_reader = crash.GetReader ()) { + using (var crash_reader = crashLog.GetReader ()) { var text = crash_reader.ReadToEnd (); var reader = System.Runtime.Serialization.Json.JsonReaderWriterFactory.CreateJsonReader (Encoding.UTF8.GetBytes (text), new XmlDictionaryReaderQuotas ()); @@ -800,7 +802,7 @@ public async Task RunAsync () break; } } catch (Exception e) { - harness.Log (2, "Failed to process crash report '{1}': {0}", e.Message, crash.Description); + harness.Log (2, "Failed to process crash report '{1}': {0}", e.Message, crashLog.Description); } } if (!string.IsNullOrEmpty (crash_reason)) { From 3b3df84f4c1046df8e9d58377e42eeec8c555add Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Wed, 18 Mar 2020 17:03:14 +0100 Subject: [PATCH 29/30] Fix getting xcode root --- tests/xharness/Harness.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 9257c6deeef8..1adb5e8e309e 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -263,14 +263,17 @@ public bool GetIncludeSystemPermissionTests (TestPlatform platform, bool device) static string FindXcode (string path) { - var p = path; + if (string.IsNullOrEmpty (path)) + return path; + do { - if (p == "/") { + if (path == "/") { throw new Exception (string.Format ("Could not find Xcode.app in {0}", path)); - } else if (File.Exists (Path.Combine (p, "Contents", "MacOS", "Xcode"))) { - return p; + } else if (File.Exists (Path.Combine (path, "Contents", "MacOS", "Xcode"))) { + return path; } - p = Path.GetDirectoryName (p); + + path = Path.GetDirectoryName (path); } while (true); } From 5cf725c3b14dbc8c892fa81daec86dc10bb2dde0 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 19 Mar 2020 16:51:50 +0100 Subject: [PATCH 30/30] Fix alignment --- tests/xharness/CrashSnapshotReporter.cs | 14 +++++++------- tests/xharness/Execution/ProcessManager.cs | 20 ++++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/xharness/CrashSnapshotReporter.cs b/tests/xharness/CrashSnapshotReporter.cs index 0a31b65ad1aa..0bfcc7e8f32a 100644 --- a/tests/xharness/CrashSnapshotReporter.cs +++ b/tests/xharness/CrashSnapshotReporter.cs @@ -50,13 +50,13 @@ public class CrashSnapshotReporter : ICrashSnapshotReporter { HashSet initialCrashes; public CrashSnapshotReporter (IProcessManager processManager, - ILog log, - ILogs logs, - string xcodeRoot, - string mlaunchPath, - bool isDevice, - string deviceName, - Func tempFileProvider = null) + ILog log, + ILogs logs, + string xcodeRoot, + string mlaunchPath, + bool isDevice, + string deviceName, + Func tempFileProvider = null) { this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); this.log = log ?? throw new ArgumentNullException (nameof (log)); diff --git a/tests/xharness/Execution/ProcessManager.cs b/tests/xharness/Execution/ProcessManager.cs index 6e2031fb5f6b..4376b257819b 100644 --- a/tests/xharness/Execution/ProcessManager.cs +++ b/tests/xharness/Execution/ProcessManager.cs @@ -18,11 +18,11 @@ public ProcessManager () } public async Task ExecuteCommandAsync (string filename, - IList args, - ILog log, - TimeSpan timeout, - Dictionary environment_variables = null, - CancellationToken? cancellation_token = null) + IList args, + ILog log, + TimeSpan timeout, + Dictionary environment_variables = null, + CancellationToken? cancellation_token = null) { using (var p = new Process ()) { p.StartInfo.FileName = filename; @@ -32,11 +32,11 @@ public async Task ExecuteCommandAsync (string filename, } public async Task ExecuteCommandAsync (string filename, - MlaunchArguments args, - ILog log, - TimeSpan timeout, - Dictionary environment_variables = null, - CancellationToken? cancellation_token = null) + MlaunchArguments args, + ILog log, + TimeSpan timeout, + Dictionary environment_variables = null, + CancellationToken? cancellation_token = null) { using (var p = new Process ()) { p.StartInfo.FileName = filename;