diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 943e73b6bd63..6ce2bd147d4e 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -54,9 +54,11 @@ class AppRunner readonly ISimulatorsLoaderFactory simulatorsLoaderFactory; readonly ISimpleListenerFactory listenerFactory; readonly IDeviceLoaderFactory devicesLoaderFactory; + readonly ICrashSnapshotReporterFactory snapshotReporterFactory; readonly ICaptureLogFactory captureLogFactory; readonly IDeviceLogCapturerFactory deviceLogCapturerFactory; readonly IResultParser resultParser; + readonly RunMode mode; readonly bool isSimulator; readonly AppRunnerTarget target; @@ -95,6 +97,7 @@ public AppRunner (IProcessManager processManager, ISimulatorsLoaderFactory simulatorsFactory, ISimpleListenerFactory simpleListenerFactory, IDeviceLoaderFactory devicesFactory, + ICrashSnapshotReporterFactory snapshotReporterFactory, ICaptureLogFactory captureLogFactory, IDeviceLogCapturerFactory deviceLogCapturerFactory, IResultParser resultParser, @@ -116,6 +119,7 @@ public AppRunner (IProcessManager processManager, this.simulatorsLoaderFactory = simulatorsFactory ?? throw new ArgumentNullException (nameof (simulatorsFactory)); this.listenerFactory = simpleListenerFactory ?? throw new ArgumentNullException (nameof (simpleListenerFactory)); this.devicesLoaderFactory = devicesFactory ?? throw new ArgumentNullException (nameof (devicesFactory)); + this.snapshotReporterFactory = snapshotReporterFactory ?? throw new ArgumentNullException (nameof (snapshotReporterFactory)); this.captureLogFactory = captureLogFactory ?? throw new ArgumentNullException (nameof (captureLogFactory)); this.deviceLogCapturerFactory = deviceLogCapturerFactory ?? throw new ArgumentNullException (nameof (deviceLogCapturerFactory)); this.resultParser = resultParser ?? throw new ArgumentNullException (nameof (resultParser)); @@ -426,15 +430,15 @@ public bool TestsSucceeded (AppInformation appInfo, string test_log_path, bool t public async Task RunAsync () { - CrashReportSnapshot crash_reports; - ILog deviceSystemLog = null; ILog listener_log = null; ILog run_log = MainLog; if (!isSimulator) FindDevice (); - crash_reports = new CrashReportSnapshot (harness, MainLog, Logs, isDevice: !isSimulator, deviceName); + var crashLogs = new Logs (Logs.Directory); + + ICrashSnapshotReporter crash_reports = snapshotReporterFactory.Create (MainLog, crashLogs, isDevice: !isSimulator, deviceName); var args = new List (); if (!string.IsNullOrEmpty (harness.XcodeRoot)) { @@ -654,7 +658,7 @@ public async Task RunAsync () AddDeviceName (args); - deviceSystemLog = Logs.Create ($"device-{deviceName}-{Helpers.Timestamp}.log", "Device log"); + var deviceSystemLog = Logs.Create ($"device-{deviceName}-{Helpers.Timestamp}.log", "Device log"); var deviceLogCapturer = deviceLogCapturerFactory.Create (harness.HarnessLog,deviceSystemLog, deviceName); deviceLogCapturer.StartCapture (); @@ -754,8 +758,10 @@ public async Task RunAsync () if (!success.Value) { int pid = 0; string crash_reason = null; - foreach (var crash in crash_reports.Logs) { + foreach (var crashLog in crashLogs) { try { + Logs.Add (crashLog); + if (pid == 0) { // Find the pid using (var log_reader = MainLog.GetReader ()) { @@ -775,7 +781,7 @@ public async Task RunAsync () } } - using (var crash_reader = crash.GetReader ()) { + using (var crash_reader = crashLog.GetReader ()) { var text = crash_reader.ReadToEnd (); var reader = System.Runtime.Serialization.Json.JsonReaderWriterFactory.CreateJsonReader (Encoding.UTF8.GetBytes (text), new XmlDictionaryReaderQuotas ()); @@ -796,14 +802,14 @@ public async Task RunAsync () variation, $"App Crash {AppInformation.AppName} {variation}", $"App crashed {crash_reason}.", - crash_reports.Log.FullPath, + MainLog.FullPath, harness.XmlJargon); } break; } } catch (Exception e) { - harness.Log (2, "Failed to process crash report '{1}': {0}", e.Message, crash.Description); + harness.Log (2, "Failed to process crash report '{1}': {0}", e.Message, crashLog.Description); } } if (!string.IsNullOrEmpty (crash_reason)) { @@ -820,7 +826,7 @@ public async Task RunAsync () variation, $"App Crash {AppInformation.AppName} {variation}", $"App crashed: {FailureMessage}", - crash_reports.Log.FullPath, + MainLog.FullPath, harness.XmlJargon); } } else if (launch_failure) { diff --git a/tests/xharness/CrashReportSnapshot.cs b/tests/xharness/CrashReportSnapshot.cs deleted file mode 100644 index f142793c0685..000000000000 --- a/tests/xharness/CrashReportSnapshot.cs +++ /dev/null @@ -1,151 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Xharness.Logging; - -namespace Xharness -{ - public class CrashReportSnapshot - { - readonly IHarness harness; - readonly bool isDevice; - readonly string deviceName; - - public ILog Log { get; } - public ILogs Logs { get; } - - public HashSet InitialSet { get; private set; } - - public CrashReportSnapshot (IHarness harness, ILog log, ILogs logs, bool isDevice, string deviceName) - { - this.harness = harness ?? throw new ArgumentNullException (nameof (harness)); - this.Log = log ?? throw new ArgumentNullException (nameof (log)); - this.Logs = logs ?? throw new ArgumentNullException (nameof (logs)); - this.isDevice = isDevice; - this.deviceName = deviceName; - } - - public async Task StartCaptureAsync () - { - InitialSet = await CreateCrashReportsSnapshotAsync (); - } - - public async Task EndCaptureAsync (TimeSpan timeout) - { - // Check for crash reports - var crash_report_search_done = false; - var crash_report_search_timeout = timeout.TotalSeconds; - var watch = new Stopwatch (); - watch.Start (); - do { - var end_crashes = await CreateCrashReportsSnapshotAsync (); - end_crashes.ExceptWith (InitialSet); - if (end_crashes.Count > 0) { - Log.WriteLine ("Found {0} new crash report(s)", end_crashes.Count); - List crash_reports; - if (!isDevice) { - crash_reports = new List (end_crashes.Count); - foreach (var path in end_crashes) { - Logs.AddFile (path, $"Crash report: {Path.GetFileName (path)}"); - } - } else { - // Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench - // (if we put them in /tmp, they'd never be deleted). - var downloaded_crash_reports = new List (); - foreach (var file in end_crashes) { - var name = Path.GetFileName (file); - var crash_report_target = Logs.Create (name, $"Crash report: {name}", timestamp: false); - var sb = new List (); - sb.Add ($"--download-crash-report={file}"); - sb.Add ($"--download-crash-report-to={crash_report_target.Path}"); - sb.Add ("--sdkroot"); - sb.Add (harness.XcodeRoot); - if (!string.IsNullOrEmpty (deviceName)) { - sb.Add ("--devname"); - sb.Add (deviceName); - } - var result = await harness.ProcessManager.ExecuteCommandAsync (harness.MlaunchPath, sb, Log, TimeSpan.FromMinutes (1)); - if (result.Succeeded) { - Log.WriteLine ("Downloaded crash report {0} to {1}", file, crash_report_target.Path); - crash_report_target = await SymbolicateCrashReportAsync (crash_report_target); - downloaded_crash_reports.Add (crash_report_target); - } else { - Log.WriteLine ("Could not download crash report {0}", file); - } - } - crash_reports = downloaded_crash_reports; - } - foreach (var cp in crash_reports) { - WrenchLog.WriteLine ("AddFile: {0}", cp.Path); - Log.WriteLine (" {0}", cp.Path); - } - crash_report_search_done = true; - } else { - if (watch.Elapsed.TotalSeconds > crash_report_search_timeout) { - crash_report_search_done = true; - } else { - Log.WriteLine ("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int) (crash_report_search_timeout - watch.Elapsed.TotalSeconds)); - Thread.Sleep (TimeSpan.FromSeconds (1)); - } - } - } while (!crash_report_search_done); - } - - async Task SymbolicateCrashReportAsync (ILogFile report) - { - var symbolicatecrash = Path.Combine (harness.XcodeRoot, "Contents/SharedFrameworks/DTDeviceKitBase.framework/Versions/A/Resources/symbolicatecrash"); - if (!File.Exists (symbolicatecrash)) - symbolicatecrash = Path.Combine (harness.XcodeRoot, "Contents/SharedFrameworks/DVTFoundation.framework/Versions/A/Resources/symbolicatecrash"); - - if (!File.Exists (symbolicatecrash)) { - Log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicatecrash); - return report; - } - - var name = Path.GetFileName (report.Path); - var symbolicated = Logs.Create (Path.ChangeExtension (name, ".symbolicated.log"), $"Symbolicated crash report: {name}", timestamp: false); - var environment = new Dictionary { { "DEVELOPER_DIR", Path.Combine (harness.XcodeRoot, "Contents", "Developer") } }; - var rv = await harness.ProcessManager.ExecuteCommandAsync (symbolicatecrash, new [] { report.Path }, symbolicated, TimeSpan.FromMinutes (1), environment); - if (rv.Succeeded) {; - Log.WriteLine ("Symbolicated {0} successfully.", report.Path); - return symbolicated; - } else { - Log.WriteLine ("Failed to symbolicate {0}.", report.Path); - return report; - } - } - - async Task> CreateCrashReportsSnapshotAsync () - { - var rv = new HashSet (); - - if (!isDevice) { - var dir = Path.Combine (Environment.GetEnvironmentVariable ("HOME"), "Library", "Logs", "DiagnosticReports"); - if (Directory.Exists (dir)) - rv.UnionWith (Directory.EnumerateFiles (dir)); - } else { - var tmp = Path.GetTempFileName (); - try { - var sb = new List (); - sb.Add ($"--list-crash-reports={tmp}"); - sb.Add ("--sdkroot"); - sb.Add (harness.XcodeRoot); - if (!string.IsNullOrEmpty (deviceName)) { - sb.Add ("--devname"); - sb.Add (deviceName); - } - var result = await harness.ProcessManager.ExecuteCommandAsync (harness.MlaunchPath, sb, Log, TimeSpan.FromMinutes (1)); - if (result.Succeeded) - rv.UnionWith (File.ReadAllLines (tmp)); - } finally { - File.Delete (tmp); - } - } - - return rv; - } - } -} diff --git a/tests/xharness/CrashSnapshotReporter.cs b/tests/xharness/CrashSnapshotReporter.cs new file mode 100644 index 000000000000..0bfcc7e8f32a --- /dev/null +++ b/tests/xharness/CrashSnapshotReporter.cs @@ -0,0 +1,206 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Xharness.Execution; +using Xharness.Execution.Mlaunch; +using Xharness.Logging; + +namespace Xharness +{ + public interface ICrashSnapshotReporterFactory { + ICrashSnapshotReporter Create (ILog log, ILogs logs, bool isDevice, string deviceName); + } + + public class CrashSnapshotReporterFactory : ICrashSnapshotReporterFactory { + readonly IProcessManager processManager; + readonly string xcodeRoot; + readonly string mlaunchPath; + + public CrashSnapshotReporterFactory (IProcessManager processManager, string xcodeRoot, string mlaunchPath) + { + this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); + this.xcodeRoot = xcodeRoot ?? throw new ArgumentNullException (nameof (xcodeRoot)); + this.mlaunchPath = mlaunchPath ?? throw new ArgumentNullException (nameof (mlaunchPath)); + } + + public ICrashSnapshotReporter Create (ILog log, ILogs logs, bool isDevice, string deviceName) => + new CrashSnapshotReporter (processManager, log, logs, xcodeRoot, mlaunchPath, isDevice, deviceName); + } + + public interface ICrashSnapshotReporter { + Task EndCaptureAsync (TimeSpan timeout); + Task StartCaptureAsync (); + } + + public class CrashSnapshotReporter : ICrashSnapshotReporter { + readonly IProcessManager processManager; + readonly ILog log; + readonly ILogs logs; + readonly string xcodeRoot; + readonly string mlaunchPath; + readonly bool isDevice; + readonly string deviceName; + readonly Func tempFileProvider; + readonly string symbolicateCrashPath; + + HashSet initialCrashes; + + public CrashSnapshotReporter (IProcessManager processManager, + ILog log, + ILogs logs, + string xcodeRoot, + string mlaunchPath, + bool isDevice, + string deviceName, + Func tempFileProvider = null) + { + this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); + this.log = log ?? throw new ArgumentNullException (nameof (log)); + this.logs = logs ?? throw new ArgumentNullException (nameof (logs)); + this.xcodeRoot = xcodeRoot ?? throw new ArgumentNullException (nameof (xcodeRoot)); + this.mlaunchPath = mlaunchPath ?? throw new ArgumentNullException (nameof (mlaunchPath)); + this.isDevice = isDevice; + this.deviceName = deviceName; + this.tempFileProvider = tempFileProvider ?? Path.GetTempFileName; + + symbolicateCrashPath = Path.Combine (xcodeRoot, "Contents", "SharedFrameworks", "DTDeviceKitBase.framework", "Versions", "A", "Resources", "symbolicatecrash"); + if (!File.Exists (symbolicateCrashPath)) + symbolicateCrashPath = Path.Combine (xcodeRoot, "Contents", "SharedFrameworks", "DVTFoundation.framework", "Versions", "A", "Resources", "symbolicatecrash"); + if (!File.Exists (symbolicateCrashPath)) + symbolicateCrashPath = null; + } + + public async Task StartCaptureAsync () + { + initialCrashes = await CreateCrashReportsSnapshotAsync (); + } + + public async Task EndCaptureAsync (TimeSpan timeout) + { + // Check for crash reports + var stopwatch = Stopwatch.StartNew (); + + do { + var newCrashFiles = await CreateCrashReportsSnapshotAsync (); + newCrashFiles.ExceptWith (initialCrashes); + + if (newCrashFiles.Count == 0) { + if (stopwatch.Elapsed.TotalSeconds > timeout.TotalSeconds) { + break; + } else { + log.WriteLine ( + "No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", + (int) (timeout.TotalSeconds - stopwatch.Elapsed.TotalSeconds)); + + Thread.Sleep (TimeSpan.FromSeconds (1)); + } + + continue; + } + + log.WriteLine ("Found {0} new crash report(s)", newCrashFiles.Count); + + IEnumerable crashReports; + if (!isDevice) { + crashReports = new List (newCrashFiles.Count); + foreach (var path in newCrashFiles) { + logs.AddFile (path, $"Crash report: {Path.GetFileName (path)}"); + } + } else { + // Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench + // (if we put them in /tmp, they'd never be deleted). + crashReports = newCrashFiles + .Select (async crash => await ProcessCrash (crash)) + .Select (t => t.Result) + .Where (c => c != null); + } + + foreach (var cp in crashReports) { + WrenchLog.WriteLine ("AddFile: {0}", cp.Path); + log.WriteLine (" {0}", cp.Path); + } + + break; + + } while (true); + } + + async Task ProcessCrash (string crashFile) + { + var name = Path.GetFileName (crashFile); + var crashReportFile = logs.Create (name, $"Crash report: {name}", timestamp: false); + var args = new MlaunchArguments ( + new DownloadCrashReportArgument (crashFile), + new DownloadCrashReportToArgument (crashReportFile.Path), + new SdkRootArgument (xcodeRoot)); + + if (!string.IsNullOrEmpty (deviceName)) { + args.Add (new DeviceNameArgument(deviceName)); + } + + var result = await processManager.ExecuteCommandAsync (mlaunchPath, args, log, TimeSpan.FromMinutes (1)); + + if (result.Succeeded) { + log.WriteLine ("Downloaded crash report {0} to {1}", crashFile, crashReportFile.Path); + return await GetSymbolicateCrashReportAsync (crashReportFile); + } else { + log.WriteLine ("Could not download crash report {0}", crashFile); + return null; + } + } + + async Task GetSymbolicateCrashReportAsync (ILogFile report) + { + if (symbolicateCrashPath == null) { + log.WriteLine ("Can't symbolicate {0} because the symbolicatecrash script {1} does not exist", report.Path, symbolicateCrashPath); + return report; + } + + var name = Path.GetFileName (report.Path); + var symbolicated = logs.Create (Path.ChangeExtension (name, ".symbolicated.log"), $"Symbolicated crash report: {name}", timestamp: false); + var environment = new Dictionary { { "DEVELOPER_DIR", Path.Combine (xcodeRoot, "Contents", "Developer") } }; + var result = await processManager.ExecuteCommandAsync (symbolicateCrashPath, new [] { report.Path }, symbolicated, TimeSpan.FromMinutes (1), environment); + if (result.Succeeded) { + log.WriteLine ("Symbolicated {0} successfully.", report.Path); + return symbolicated; + } else { + log.WriteLine ("Failed to symbolicate {0}.", report.Path); + return report; + } + } + + async Task> CreateCrashReportsSnapshotAsync () + { + var crashes = new HashSet (); + + if (!isDevice) { + var dir = Path.Combine (Environment.GetEnvironmentVariable ("HOME"), "Library", "Logs", "DiagnosticReports"); + if (Directory.Exists (dir)) + crashes.UnionWith (Directory.EnumerateFiles (dir)); + } else { + var tempFile = tempFileProvider (); + try { + var args = new MlaunchArguments ( + new ListCrashReportsArgument (tempFile), + new SdkRootArgument (xcodeRoot)); + + if (!string.IsNullOrEmpty (deviceName)) { + args.Add (new DeviceNameArgument(deviceName)); + } + + var result = await processManager.ExecuteCommandAsync (mlaunchPath, args, log, TimeSpan.FromMinutes (1)); + if (result.Succeeded) + crashes.UnionWith (File.ReadAllLines (tempFile)); + } finally { + File.Delete (tempFile); + } + } + + return crashes; + } + } +} diff --git a/tests/xharness/Execution/IProcessManager.cs b/tests/xharness/Execution/IProcessManager.cs index 65c1ab0fca9f..85bad04c7751 100644 --- a/tests/xharness/Execution/IProcessManager.cs +++ b/tests/xharness/Execution/IProcessManager.cs @@ -12,13 +12,13 @@ namespace Xharness.Execution { public class ProcessExecutionResult { public bool TimedOut { get; set; } public int ExitCode { get; set; } - public bool Succeeded => !TimedOut && ExitCode == 0; } // interface that helps to manage the different processes in the app. public interface IProcessManager { Task ExecuteCommandAsync (string filename, IList args, ILog log, TimeSpan timeout, Dictionary environment_variables = null, CancellationToken? cancellation_token = null); + Task ExecuteCommandAsync (string filename, MlaunchArguments args, ILog log, TimeSpan timeout, Dictionary environment_variables = null, CancellationToken? cancellation_token = null); Task RunAsync (Process process, ILog log, CancellationToken? cancellationToken = null, bool? diagnostics = null); Task RunAsync (Process process, ILog log, TimeSpan? timeout = null, Dictionary environment_variables = null, CancellationToken? cancellation_token = null, bool? diagnostics = null); Task RunAsync (Process process, MlaunchArguments args, ILog log, TimeSpan? timeout = null, Dictionary environment_variables = null, CancellationToken? cancellation_token = null, bool? diagnostics = null); diff --git a/tests/xharness/Execution/Mlaunch/Arguments.cs b/tests/xharness/Execution/Mlaunch/Arguments.cs index fbcc6d3c620e..6696df323ba1 100644 --- a/tests/xharness/Execution/Mlaunch/Arguments.cs +++ b/tests/xharness/Execution/Mlaunch/Arguments.cs @@ -1,35 +1,90 @@ namespace Xharness.Execution.Mlaunch { + /// + /// Specify the location of Apple SDKs, default to 'xcode-select' value. + /// public sealed class SdkRootArgument : SingleValueArgument { public SdkRootArgument (string sdkPath) : base ("sdkroot", sdkPath) { } } + /// + /// List the currently connected devices and their UDIDs. + /// public sealed class ListDevicesArgument : SingleValueArgument { public ListDevicesArgument (string outputFile) : base ("listdev", outputFile) { } } + /// + /// List the available simulators. The output is xml, and written to the specified file. + /// public sealed class ListSimulatorsArgument : SingleValueArgument { public ListSimulatorsArgument (string outputFile) : base ("listsim", outputFile) { } } + /// + /// Lists crash reports on the specified device + /// + public sealed class ListCrashReportsArgument : SingleValueArgument { + public ListCrashReportsArgument (string outputFile) : base ("list-crash-reports", outputFile) + { + } + } + + /// + /// Specify which device (when many are present) the [install|lauch|kill|log]dev command applies + /// + public sealed class DeviceNameArgument : SingleValueArgument { + public DeviceNameArgument (string deviceName) : base ("devname", deviceName) + { + } + } + + /// + /// Specify the output format for some commands as Default. + /// public sealed class DefaultOutputFormatArgument : SingleValueArgument { public DefaultOutputFormatArgument () : base ("output-format", "Default") { } } + /// + /// Specify the output format for some commands as XML. + /// public sealed class XmlOutputFormatArgument : SingleValueArgument { public XmlOutputFormatArgument () : base ("output-format", "XML") { } } + /// + /// Download a crash report from the specified device. + /// + public sealed class DownloadCrashReportArgument : SingleValueArgument { + public DownloadCrashReportArgument (string deviceName) : base ("download-crash-report", deviceName) + { + } + } + + /// + /// Specifies the file to save the downloaded crash report. + /// + public sealed class DownloadCrashReportToArgument : SingleValueArgument { + public DownloadCrashReportToArgument (string outputFile) : base ("download-crash-report-to", outputFile) + { + } + } + + /// + /// Include additional data (which can take some time to fetch) when listing the connected devices. + /// Only applicable when output format is xml. + /// public sealed class ListExtraDataArgument : OptionArgument { public ListExtraDataArgument () : base ("list-extra-data") { diff --git a/tests/xharness/Execution/ProcessManager.cs b/tests/xharness/Execution/ProcessManager.cs index afd69bc1df5c..4376b257819b 100644 --- a/tests/xharness/Execution/ProcessManager.cs +++ b/tests/xharness/Execution/ProcessManager.cs @@ -17,7 +17,12 @@ public ProcessManager () { } - public async Task ExecuteCommandAsync (string filename, IList args, ILog log, TimeSpan timeout, Dictionary environment_variables = null, CancellationToken? cancellation_token = null) + public async Task ExecuteCommandAsync (string filename, + IList args, + ILog log, + TimeSpan timeout, + Dictionary environment_variables = null, + CancellationToken? cancellation_token = null) { using (var p = new Process ()) { p.StartInfo.FileName = filename; @@ -26,6 +31,20 @@ public async Task ExecuteCommandAsync (string filename, } } + public async Task ExecuteCommandAsync (string filename, + MlaunchArguments args, + ILog log, + TimeSpan timeout, + Dictionary environment_variables = null, + CancellationToken? cancellation_token = null) + { + using (var p = new Process ()) { + p.StartInfo.FileName = filename; + p.StartInfo.Arguments = args.AsCommandLine (); + return await RunAsync (p, log, timeout, environment_variables, cancellation_token); + } + } + [DllImport ("/usr/lib/libc.dylib")] internal static extern int kill (int pid, int sig); diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index f1b1fc7232fc..688a935c045b 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -104,7 +104,6 @@ public interface IHarness { public class Harness : IHarness { readonly AppRunnerTarget target; readonly string buildConfiguration = "Debug"; - string sdkRoot; public HarnessAction Action { get; } public int Verbosity { get; } @@ -142,6 +141,15 @@ public static string RootDirectory { } } + string sdkRoot; + string SdkRoot { + get => sdkRoot; + set { + sdkRoot = value; + XcodeRoot = FindXcode (sdkRoot); + } + } + public List IOSTestProjects { get; } public List MacTestProjects { get; } = new List (); @@ -171,6 +179,8 @@ public static string RootDirectory { public string DOTNET { get; private set; } // Run + + public string XcodeRoot { get; private set; } public string LogDirectory { get; } = Environment.CurrentDirectory; public double Timeout { get; } = 15; // in minutes public double LaunchTimeout { get; } // in minutes @@ -180,7 +190,7 @@ public static string RootDirectory { public string MarkdownSummaryPath { get; } public string PeriodicCommand { get; } public string PeriodicCommandArguments { get; } - public TimeSpan PeriodicCommandInterval { get;} + public TimeSpan PeriodicCommandInterval { get; } // whether tests that require access to system resources (system contacts, photo library, etc) should be executed or not public bool? IncludeSystemPermissionTests { get; set; } @@ -207,7 +217,7 @@ public Harness (IProcessManager processManager, IResultParser resultParser, Harn PeriodicCommand = configuration.PeriodicCommand; PeriodicCommandArguments = configuration.PeriodicCommandArguments; PeriodicCommandInterval = configuration.PeriodicCommandInterval; - sdkRoot = configuration.SdkRoot; + SdkRoot = configuration.SdkRoot; target = configuration.Target; Timeout = configuration.TimeoutInMinutes; useSystemXamarinIOSMac = configuration.UseSystemXamarinIOSMac; @@ -253,21 +263,18 @@ public bool GetIncludeSystemPermissionTests (TestPlatform platform, bool device) static string FindXcode (string path) { - var p = path; + if (string.IsNullOrEmpty (path)) + return path; + do { - if (p == "/") { + if (path == "/") { throw new Exception (string.Format ("Could not find Xcode.app in {0}", path)); - } else if (File.Exists (Path.Combine (p, "Contents", "MacOS", "Xcode"))) { - return p; + } else if (File.Exists (Path.Combine (path, "Contents", "MacOS", "Xcode"))) { + return path; } - p = Path.GetDirectoryName (p); - } while (true); - } - public string XcodeRoot { - get { - return FindXcode (sdkRoot); - } + path = Path.GetDirectoryName (path); + } while (true); } Version xcode_version; @@ -301,8 +308,8 @@ void LoadConfig () INCLUDE_MAC = make_config.ContainsKey ("INCLUDE_MAC") && !string.IsNullOrEmpty (make_config ["INCLUDE_MAC"]); MAC_DESTDIR = make_config ["MAC_DESTDIR"]; IOS_DESTDIR = make_config ["IOS_DESTDIR"]; - if (string.IsNullOrEmpty (sdkRoot)) - sdkRoot = make_config ["XCODE_DEVELOPER_ROOT"]; + if (string.IsNullOrEmpty (SdkRoot)) + SdkRoot = make_config ["XCODE_DEVELOPER_ROOT"]; MONO_IOS_SDK_DESTDIR = make_config ["MONO_IOS_SDK_DESTDIR"]; MONO_MAC_SDK_DESTDIR = make_config ["MONO_MAC_SDK_DESTDIR"]; ENABLE_XAMARIN = make_config.ContainsKey ("ENABLE_XAMARIN") && !string.IsNullOrEmpty (make_config ["ENABLE_XAMARIN"]); @@ -601,6 +608,7 @@ int Install () new SimulatorsLoaderFactory (this, ProcessManager), new SimpleListenerFactory (), new DeviceLoaderFactory (this, ProcessManager), + new CrashSnapshotReporterFactory (ProcessManager, XcodeRoot, MlaunchPath), new CaptureLogFactory (), new DeviceLogCapturerFactory (ProcessManager, XcodeRoot, MlaunchPath), new XmlResultParser (), @@ -629,6 +637,7 @@ int Uninstall () new SimulatorsLoaderFactory (this, ProcessManager), new SimpleListenerFactory (), new DeviceLoaderFactory (this, ProcessManager), + new CrashSnapshotReporterFactory (ProcessManager, XcodeRoot, MlaunchPath), new CaptureLogFactory (), new DeviceLogCapturerFactory (ProcessManager, XcodeRoot, MlaunchPath), new XmlResultParser (), @@ -655,6 +664,7 @@ int Run () new SimulatorsLoaderFactory (this, ProcessManager), new SimpleListenerFactory (), new DeviceLoaderFactory (this, ProcessManager), + new CrashSnapshotReporterFactory (ProcessManager, XcodeRoot, MlaunchPath), new CaptureLogFactory (), new DeviceLogCapturerFactory (ProcessManager, XcodeRoot, MlaunchPath), new XmlResultParser (), @@ -782,6 +792,7 @@ public void Save (StringWriter doc, string path) } bool? disable_watchos_on_wrench; + public bool DisableWatchOSOnWrench { get { if (!disable_watchos_on_wrench.HasValue) diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index 6b78373d3680..ea53cab2923c 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -225,8 +225,13 @@ IEnumerable CreateRunSimulatorTaskAsync (MSBuildTask buildTask throw new NotImplementedException (); } - for (int i = 0; i < targets.Length; i++) - runtasks.Add (new RunSimulatorTask (simulators, buildTask, simulators.SelectDevices (targets [i], SimulatorLoadLog, false)) { Platform = platforms [i], Ignored = ignored[i] || buildTask.Ignored }); + for (int i = 0; i < targets.Length; i++) { + var sims = simulators.SelectDevices (targets [i], SimulatorLoadLog, false); + runtasks.Add (new RunSimulatorTask (simulators, buildTask, processManager, sims) { + Platform = platforms [i], + Ignored = ignored[i] || buildTask.Ignored + }); + } return runtasks; } @@ -547,7 +552,7 @@ async Task> CreateRunSimulatorTasksAsync () } var testVariations = CreateTestVariations (runSimulatorTasks, (buildTask, test, candidates) => - new RunSimulatorTask (simulators, buildTask, candidates?.Cast () ?? test.Candidates)).ToList (); + new RunSimulatorTask (simulators, buildTask, processManager, candidates?.Cast () ?? test.Candidates)).ToList (); foreach (var tv in testVariations) { if (!tv.Ignored) @@ -587,7 +592,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; build64.CloneTestProject (project); - projectTasks.Add (new RunDeviceTask (devices, build64, devices.Connected64BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOS64 }); + projectTasks.Add (new RunDeviceTask (devices, build64, processManager, devices.Connected64BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOS64 }); var build32 = new MSBuildTask { Jenkins = this, @@ -597,7 +602,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; build32.CloneTestProject (project); - projectTasks.Add (new RunDeviceTask (devices, build32, devices.Connected32BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOS32 }); + projectTasks.Add (new RunDeviceTask (devices, build32, processManager, devices.Connected32BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOS32 }); var todayProject = project.AsTodayExtensionProject (); var buildToday = new MSBuildTask { @@ -608,7 +613,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; buildToday.CloneTestProject (todayProject); - projectTasks.Add (new RunDeviceTask (devices, buildToday, devices.Connected64BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOSExtensions, BuildOnly = ForceExtensionBuildOnly }); + projectTasks.Add (new RunDeviceTask (devices, buildToday, processManager, devices.Connected64BitIOS.Where (d => d.IsSupported (project))) { Ignored = !IncludeiOSExtensions, BuildOnly = ForceExtensionBuildOnly }); } if (!project.SkiptvOSVariation) { @@ -621,7 +626,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; buildTV.CloneTestProject (tvOSProject); - projectTasks.Add (new RunDeviceTask (devices, buildTV, devices.ConnectedTV.Where (d => d.IsSupported (project))) { Ignored = !IncludetvOS }); + projectTasks.Add (new RunDeviceTask (devices, buildTV, processManager, devices.ConnectedTV.Where (d => d.IsSupported (project))) { Ignored = !IncludetvOS }); } if (!project.SkipwatchOSVariation) { @@ -635,7 +640,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; buildWatch32.CloneTestProject (watchOSProject); - projectTasks.Add (new RunDeviceTask (devices, buildWatch32, devices.ConnectedWatch) { Ignored = !IncludewatchOS }); + projectTasks.Add (new RunDeviceTask (devices, buildWatch32, processManager, devices.ConnectedWatch) { Ignored = !IncludewatchOS }); } if (!project.SkipwatchOSARM64_32Variation) { @@ -647,7 +652,7 @@ Task> CreateRunDeviceTasksAsync () TestName = project.Name, }; buildWatch64_32.CloneTestProject (watchOSProject); - projectTasks.Add (new RunDeviceTask (devices, buildWatch64_32, devices.ConnectedWatch32_64.Where (d => d.IsSupported (project))) { Ignored = !IncludewatchOS }); + projectTasks.Add (new RunDeviceTask (devices, buildWatch64_32, processManager, devices.ConnectedWatch32_64.Where (d => d.IsSupported (project))) { Ignored = !IncludewatchOS }); } } foreach (var task in projectTasks) { @@ -658,7 +663,7 @@ Task> CreateRunDeviceTasksAsync () rv.AddRange (projectTasks); } - return Task.FromResult> (CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask (devices, buildTask, candidates?.Cast () ?? test.Candidates))); + return Task.FromResult> (CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask (devices, buildTask, processManager, candidates?.Cast () ?? test.Candidates))); } static string AddSuffixToPath (string path, string suffix) @@ -933,6 +938,8 @@ Task PopulateTasksAsync () //Tasks.AddRange (await CreateRunSimulatorTasksAsync ()); + var crashReportSnapshotFactory = new CrashSnapshotReporterFactory (processManager, Harness.XcodeRoot, Harness.MlaunchPath); + var buildiOSMSBuild_net461 = new MSBuildTask () { Jenkins = this, @@ -944,7 +951,7 @@ Task PopulateTasksAsync () SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), SupportsParallelExecution = false, }; - var nunitExecutioniOSMSBuild_net461 = new NUnitExecuteTask (buildiOSMSBuild_net461) + var nunitExecutioniOSMSBuild_net461 = new NUnitExecuteTask (buildiOSMSBuild_net461, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-net461", "net461", "Xamarin.iOS.Tasks.Tests.dll"), TestProject = buildiOSMSBuild_net461.TestProject, @@ -968,7 +975,7 @@ Task PopulateTasksAsync () SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")), SupportsParallelExecution = false, }; - var nunitExecutioniOSMSBuild_netstandard2 = new NUnitExecuteTask (buildiOSMSBuild_netstandard2) { + var nunitExecutioniOSMSBuild_netstandard2 = new NUnitExecuteTask (buildiOSMSBuild_netstandard2, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "..", "msbuild", "tests", "Xamarin.iOS.Tasks.Tests", "bin", "Debug-netstandard2.0", "net461", "Xamarin.iOS.Tasks.Tests.dll"), TestProject = buildiOSMSBuild_netstandard2.TestProject, ProjectConfiguration = "Debug-netstandard2.0", @@ -990,7 +997,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.iOS, }; buildInstallSources.SolutionPath = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "install-source.sln")); // this is required for nuget restore to be executed - var nunitExecutionInstallSource = new NUnitExecuteTask (buildInstallSources) + var nunitExecutionInstallSource = new NUnitExecuteTask (buildInstallSources, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "..", "tools", "install-source", "InstallSourcesTests", "bin", "Release", "InstallSourcesTests.dll"), TestProject = buildInstallSources.TestProject, @@ -1047,7 +1054,7 @@ Task PopulateTasksAsync () var ignored_main = ignored; if (project.IsNUnitProject) { var dll = Path.Combine (Path.GetDirectoryName (build.TestProject.Path), project.Xml.GetOutputAssemblyPath (build.ProjectPlatform, build.ProjectConfiguration).Replace ('\\', '/')); - exec = new NUnitExecuteTask (build) { + exec = new NUnitExecuteTask (build, processManager) { Ignored = ignored_main, TestLibrary = dll, TestProject = project, @@ -1058,13 +1065,14 @@ Task PopulateTasksAsync () }; execs = new [] { exec }; } else { - exec = new MacExecuteTask (build) { + exec = new MacExecuteTask (build, processManager, crashReportSnapshotFactory) { Ignored = ignored_main, BCLTest = project.IsBclTest, TestName = project.Name, IsUnitTest = true, }; - execs = CreateTestVariations (new [] { exec }, (buildTask, test, candidates) => new MacExecuteTask (buildTask) { IsUnitTest = true } ); + execs = CreateTestVariations (new [] { exec }, (buildTask, test, candidates) => + new MacExecuteTask (buildTask, processManager, crashReportSnapshotFactory) { IsUnitTest = true } ); } foreach (var e in execs) @@ -1084,7 +1092,7 @@ Task PopulateTasksAsync () Target = "dependencies", WorkingDirectory = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch")), }; - var nunitExecutionMTouch = new NUnitExecuteTask (buildMTouch) + var nunitExecutionMTouch = new NUnitExecuteTask (buildMTouch, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "mtouch", "bin", "Debug", "mtouch.dll"), TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "mtouch", "mtouch.csproj"))), @@ -1105,7 +1113,7 @@ Task PopulateTasksAsync () Target = "build-unit-tests", WorkingDirectory = Path.GetFullPath (Path.Combine (Harness.RootDirectory, "generator")), }; - var runGenerator = new NUnitExecuteTask (buildGenerator) { + var runGenerator = new NUnitExecuteTask (buildGenerator, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "generator", "bin", "Debug", "generator-tests.dll"), TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "generator", "generator-tests.csproj"))), Platform = TestPlatform.iOS, @@ -1123,7 +1131,7 @@ Task PopulateTasksAsync () SpecifyConfiguration = false, Platform = TestPlatform.iOS, }; - var runDotNetGenerator = new DotNetTestTask (buildDotNetGenerator) { + var runDotNetGenerator = new DotNetTestTask (buildDotNetGenerator, processManager) { TestProject = buildDotNetGenerator.TestProject, Platform = TestPlatform.iOS, TestName = "Generator tests", @@ -1172,7 +1180,7 @@ Task PopulateTasksAsync () Ignored = !IncludeXtro, Timeout = TimeSpan.FromMinutes (15), }; - var runXtroReporter = new RunXtroTask (buildXtroTests) { + var runXtroReporter = new RunXtroTask (buildXtroTests, processManager, crashReportSnapshotFactory) { Jenkins = this, Platform = TestPlatform.Mac, TestName = buildXtroTests.TestName, @@ -1190,7 +1198,7 @@ Task PopulateTasksAsync () Ignored = !IncludeCecil, Timeout = TimeSpan.FromMinutes (5), }; - var runCecilTests = new NUnitExecuteTask (buildCecilTests) { + var runCecilTests = new NUnitExecuteTask (buildCecilTests, processManager) { TestLibrary = Path.Combine (buildCecilTests.WorkingDirectory, "bin", "Debug", "cecil-tests.dll"), TestProject = new TestProject (Path.Combine (buildCecilTests.WorkingDirectory, "cecil-tests.csproj")), Platform = TestPlatform.iOS, @@ -1219,7 +1227,7 @@ Task PopulateTasksAsync () Platform = TestPlatform.All, ProjectConfiguration = "Debug", }; - var runSampleTests = new NUnitExecuteTask (buildSampleTests) { + var runSampleTests = new NUnitExecuteTask (buildSampleTests, processManager) { TestLibrary = Path.Combine (Harness.RootDirectory, "sampletester", "bin", "Debug", "sampletester.dll"), TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "sampletester", "sampletester.csproj"))), Platform = TestPlatform.All, diff --git a/tests/xharness/Jenkins/TestTasks/DotNetTestTask.cs b/tests/xharness/Jenkins/TestTasks/DotNetTestTask.cs index e018691e38af..687c11ab3a7a 100644 --- a/tests/xharness/Jenkins/TestTasks/DotNetTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/DotNetTestTask.cs @@ -1,12 +1,13 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using Xharness.Execution; using Xharness.Logging; namespace Xharness.Jenkins.TestTasks { class DotNetTestTask : RunTestTask { - public DotNetTestTask (DotNetBuildTask build_task) - : base (build_task) + public DotNetTestTask (DotNetBuildTask build_task, IProcessManager processManager) + : base (build_task, processManager) { DotNetBuildTask.SetDotNetEnvironmentVariables (Environment); } diff --git a/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs index 241aae11ace6..059c83be52fd 100644 --- a/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs @@ -12,14 +12,16 @@ namespace Xharness.Jenkins.TestTasks { class MacExecuteTask : MacTask { + protected ICrashSnapshotReporterFactory CrashReportSnapshotFactory { get; } + public string Path; public bool BCLTest; public bool IsUnitTest; - public IProcessManager ProcessManager { get; set; } = new ProcessManager (); - public MacExecuteTask (BuildToolTask build_task) - : base (build_task) + public MacExecuteTask (BuildToolTask build_task, IProcessManager processManager, ICrashSnapshotReporterFactory crashReportSnapshotFactory) + : base (build_task, processManager) { + this.CrashReportSnapshotFactory = crashReportSnapshotFactory ?? throw new ArgumentNullException (nameof (crashReportSnapshotFactory)); } public override bool SupportsParallelExecution { @@ -91,7 +93,7 @@ protected override async Task RunTestAsync () if (!Harness.DryRun) { ExecutionResult = TestExecutingResult.Running; - var snapshot = new CrashReportSnapshot (Harness, log, Logs, isDevice: false, deviceName: null); + var snapshot = CrashReportSnapshotFactory.Create (log, Logs, isDevice: false, deviceName: null); await snapshot.StartCaptureAsync (); ProcessExecutionResult result = null; diff --git a/tests/xharness/Jenkins/TestTasks/MacTask.cs b/tests/xharness/Jenkins/TestTasks/MacTask.cs index e7717d1858fc..b5325893fadd 100644 --- a/tests/xharness/Jenkins/TestTasks/MacTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MacTask.cs @@ -1,11 +1,12 @@ using System; +using Xharness.Execution; namespace Xharness.Jenkins.TestTasks { abstract class MacTask : RunTestTask { - public MacTask (BuildToolTask build_task) - : base (build_task) + public MacTask (BuildToolTask build_task, IProcessManager processManager) + : base (build_task, processManager) { } diff --git a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs index 39a6c65ca109..679eeb106553 100644 --- a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs @@ -19,8 +19,8 @@ class NUnitExecuteTask : RunTestTask public bool ProduceHtmlReport = true; public bool InProcess; - public NUnitExecuteTask (BuildToolTask build_task) - : base (build_task) + public NUnitExecuteTask (BuildToolTask build_task, IProcessManager processManager) + : base (build_task, processManager) { } diff --git a/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs b/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs index 22ed6abcd90e..db81733ecfac 100644 --- a/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs @@ -40,8 +40,8 @@ public override string ProgressMessage { } } - public RunDeviceTask (IDeviceLoader devices, MSBuildTask build_task, IEnumerable candidates) - : base (build_task, candidates.OrderBy ((v) => v.DebugSpeed)) + public RunDeviceTask (IDeviceLoader devices, MSBuildTask build_task, IProcessManager processManager, IEnumerable candidates) + : base (build_task, processManager, candidates.OrderBy ((v) => v.DebugSpeed)) { switch (build_task.Platform) { case TestPlatform.iOS: @@ -85,6 +85,7 @@ protected override async Task RunTestAsync () new SimulatorsLoaderFactory (Harness, processManager), new SimpleListenerFactory (), new DeviceLoaderFactory (Harness, processManager), + new CrashSnapshotReporterFactory (ProcessManager, Harness.XcodeRoot, Harness.MlaunchPath), new CaptureLogFactory (), new DeviceLogCapturerFactory (processManager, Harness.XcodeRoot, Harness.MlaunchPath), new XmlResultParser (), @@ -150,6 +151,7 @@ protected override async Task RunTestAsync () new SimulatorsLoaderFactory (Harness, processManager), new SimpleListenerFactory (), new DeviceLoaderFactory (Harness, processManager), + new CrashSnapshotReporterFactory (ProcessManager, Harness.XcodeRoot, Harness.MlaunchPath), new CaptureLogFactory (), new DeviceLogCapturerFactory (processManager, Harness.XcodeRoot, Harness.MlaunchPath), new XmlResultParser (), diff --git a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs index 027a6d6e9b2e..30b9a264c7d4 100644 --- a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs @@ -29,8 +29,8 @@ public ISimulatorDevice [] Simulators { } } - public RunSimulatorTask (ISimulatorsLoader simulators, MSBuildTask build_task, IEnumerable candidates = null) - : base (build_task, candidates) + public RunSimulatorTask (ISimulatorsLoader simulators, MSBuildTask build_task, IProcessManager processManager, IEnumerable candidates = null) + : base (build_task, processManager, candidates) { var project = Path.GetFileNameWithoutExtension (ProjectFile); if (project.EndsWith ("-tvos", StringComparison.Ordinal)) { @@ -81,6 +81,7 @@ public async Task SelectSimulatorAsync () new SimulatorsLoaderFactory (Harness, processManager), new SimpleListenerFactory (), new DeviceLoaderFactory (Harness, processManager), + new CrashSnapshotReporterFactory (ProcessManager, Harness.XcodeRoot, Harness.MlaunchPath), new CaptureLogFactory (), new DeviceLogCapturerFactory (processManager, Harness.XcodeRoot, Harness.MlaunchPath), new XmlResultParser (), diff --git a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs index dd47f707e5a5..ce0272acaed4 100644 --- a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs @@ -12,16 +12,18 @@ namespace Xharness.Jenkins.TestTasks { internal abstract class RunTestTask : TestTask { + protected IProcessManager ProcessManager { get; } + IResultParser ResultParser { get; } = new XmlResultParser (); + public readonly BuildToolTask BuildTask; public TimeSpan Timeout = TimeSpan.FromMinutes (10); public double TimeoutMultiplier { get; set; } = 1; - IProcessManager ProcessManager { get; } = new ProcessManager (); - IResultParser ResultParser { get; } = new XmlResultParser (); public string WorkingDirectory; - public RunTestTask (BuildToolTask build_task) + public RunTestTask (BuildToolTask build_task, IProcessManager processManager) { this.BuildTask = build_task; + this.ProcessManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); Jenkins = build_task.Jenkins; TestProject = build_task.TestProject; diff --git a/tests/xharness/Jenkins/TestTasks/RunXITask.cs b/tests/xharness/Jenkins/TestTasks/RunXITask.cs index a2f1e501b152..bcc8f205bc70 100644 --- a/tests/xharness/Jenkins/TestTasks/RunXITask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunXITask.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using Xharness.Collections; +using Xharness.Execution; using Xharness.Hardware; using Xharness.Logging; @@ -23,8 +24,8 @@ abstract class RunXITask : RunTestTask where TDevice : class, IDevice public string BundleIdentifier => runner.AppInformation.BundleIdentifier; - public RunXITask (BuildToolTask build_task, IEnumerable candidates) - : base (build_task) + public RunXITask (BuildToolTask build_task, IProcessManager processManager, IEnumerable candidates) + : base (build_task, processManager) { this.Candidates = candidates; } diff --git a/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs b/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs index 2ad46335c984..0f59b670b71a 100644 --- a/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunXtroTask.cs @@ -1,13 +1,15 @@ using System; using System.Diagnostics; using System.Threading.Tasks; +using Xharness.Execution; using Xharness.Logging; namespace Xharness.Jenkins.TestTasks { class RunXtroTask : MacExecuteTask { - public RunXtroTask (BuildToolTask build_task) : base (build_task) + public RunXtroTask (BuildToolTask build_task, IProcessManager processManager, ICrashSnapshotReporterFactory crashReportSnapshotFactory) + : base (build_task, processManager, crashReportSnapshotFactory) { } @@ -29,7 +31,7 @@ protected override async Task RunTestAsync () if (!Harness.DryRun) { ExecutionResult = TestExecutingResult.Running; - var snapshot = new CrashReportSnapshot (Harness, log, Logs, isDevice: false, deviceName: null); + var snapshot = CrashReportSnapshotFactory.Create (log, Logs, isDevice: false, deviceName: null); await snapshot.StartCaptureAsync (); try { diff --git a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs index 19c5cda7573b..858f8d58b8e9 100644 --- a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs +++ b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs @@ -51,11 +51,13 @@ public class AppRunnerTests { Mock simulators; Mock devices; Mock simpleListener; + Mock snapshotReporter; ILog mainLog; ISimulatorsLoaderFactory simulatorsFactory; IDeviceLoaderFactory devicesFactory; ISimpleListenerFactory listenerFactory; + ICrashSnapshotReporterFactory snapshotReporterFactory; [SetUp] public void SetUp () @@ -64,6 +66,7 @@ public void SetUp () simulators = new Mock (); devices = new Mock (); simpleListener = new Mock (); + snapshotReporter = new Mock (); var mock1 = new Mock (); mock1.Setup (m => m.CreateLoader ()).Returns (simulators.Object); @@ -79,6 +82,10 @@ public void SetUp () .Returns ((ListenerTransport.Tcp, simpleListener.Object, null)); listenerFactory = mock3.Object; + var mock4 = new Mock (); + mock4.Setup (m => m.Create (It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns (snapshotReporter.Object); + snapshotReporterFactory = mock4.Object; + mainLog = new Mock ().Object; Directory.CreateDirectory (appPath); @@ -91,6 +98,7 @@ public void InitializeTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, Mock.Of (), Mock.Of (), Mock.Of (), @@ -114,6 +122,7 @@ public void InstallToSimulatorTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, Mock.Of (), Mock.Of (), Mock.Of (), @@ -136,6 +145,7 @@ public void UninstallToSimulatorTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, Mock.Of (), Mock.Of (), Mock.Of (), @@ -158,6 +168,7 @@ public void InstallWhenNoDevicesTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, Mock.Of (), Mock.Of (), Mock.Of (), @@ -199,6 +210,7 @@ public async Task InstallOnDeviceTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, Mock.Of (), Mock.Of (), Mock.Of (), @@ -259,6 +271,7 @@ public async Task UninstallFromDeviceTest () simulatorsFactory, listenerFactory, devicesFactory, + snapshotReporterFactory, Mock.Of (), Mock.Of (), Mock.Of (), diff --git a/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs b/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs new file mode 100644 index 000000000000..81276bbbf46a --- /dev/null +++ b/tests/xharness/Xharness.Tests/Tests/CrashReportSnapshotTests.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Xharness.Execution; +using Xharness.Execution.Mlaunch; +using Xharness.Logging; + +namespace Xharness.Tests { + [TestFixture] + public class CrashReportSnapshotTests { + readonly string mlaunchPath = "./mlaunch"; + string tempXcodeRoot; + string symbolicatePath; + + Mock processManager; + Mock log; + Mock logs; + + [SetUp] + public void SetUp () + { + processManager = new Mock (); + log = new Mock (); + logs = new Mock (); + + tempXcodeRoot = Path.Combine (Path.GetTempPath (), Guid.NewGuid ().ToString()); + symbolicatePath = Path.Combine (tempXcodeRoot, "Contents", "SharedFrameworks", "DTDeviceKitBase.framework", "Versions", "A", "Resources"); + + // Create fake place for device logs + Directory.CreateDirectory (tempXcodeRoot); + + // Create fake symbolicate binary + Directory.CreateDirectory (symbolicatePath); + File.Create (Path.Combine (symbolicatePath, "symbolicatecrash")); + } + + [TearDown] + public void TearDown () { + Directory.Delete (tempXcodeRoot, true); + } + + [Test] + public async Task DeviceCaptureTest () + { + var tempFilePath = Path.GetTempFileName (); + + const string deviceName = "Sample-iPhone"; + const string crashLogPath = "/path/to/crash.log"; + const string symbolicateLogPath = "/path/to/" + deviceName+ ".symbolicated.log"; + + var crashReport = Mock.Of (x => x.Path == crashLogPath); + var symbolicateReport = Mock.Of (x => x.Path == symbolicateLogPath); + + // Crash report is added + logs.Setup (x => x.Create (deviceName, "Crash report: " + deviceName, It.IsAny ())) + .Returns (crashReport); + + // Symbolicate report is added + logs.Setup (x => x.Create ("crash.symbolicated.log", "Symbolicated crash report: crash.log", It.IsAny ())) + .Returns (symbolicateReport); + + // List of crash reports is retrieved + processManager + .Setup (x => x.ExecuteCommandAsync ( + mlaunchPath, + It.Is (args => args.AsCommandLine () == $"--list-crash-reports={tempFilePath} --sdkroot={tempXcodeRoot} --devname={deviceName}"), + log.Object, + TimeSpan.FromMinutes (1), + null, + null)) + .ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 }); + + // Device crash log is downloaded + processManager + .Setup (x => x.ExecuteCommandAsync ( + mlaunchPath, + It.Is (args => args.AsCommandLine () == $"--download-crash-report={deviceName} --download-crash-report-to={crashLogPath} --sdkroot={tempXcodeRoot} --devname={deviceName}"), + log.Object, + TimeSpan.FromMinutes (1), + null, + null)) + .ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 }); + + // Symbolicate is ran + processManager + .Setup (x => x.ExecuteCommandAsync ( + Path.Combine (symbolicatePath, "symbolicatecrash"), + It.Is> (args => args.First () == crashLogPath), + symbolicateReport, + TimeSpan.FromMinutes (1), + It.IsAny >(), + null)) + .ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 }); + + // Act + var snapshotReport = new CrashSnapshotReporter (processManager.Object, + log.Object, + logs.Object, + tempXcodeRoot, + mlaunchPath, + true, + deviceName, + () => tempFilePath); + + File.WriteAllLines (tempFilePath, new [] { "crash 1", "crash 2" }); + + await snapshotReport.StartCaptureAsync (); + + File.WriteAllLines (tempFilePath, new [] { "Sample-iPhone" }); + + await snapshotReport.EndCaptureAsync (TimeSpan.FromSeconds (10)); + + // Verify all calls above + processManager.VerifyAll (); + logs.VerifyAll (); + } + } +} diff --git a/tests/xharness/Xharness.Tests/Xharness.Tests.csproj b/tests/xharness/Xharness.Tests/Xharness.Tests.csproj index 8f0d5d5da005..9bd6e1c7d5b4 100644 --- a/tests/xharness/Xharness.Tests/Xharness.Tests.csproj +++ b/tests/xharness/Xharness.Tests/Xharness.Tests.csproj @@ -59,6 +59,7 @@ + diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index 9caa6a4cd2e3..cfb0f0fb2a1a 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -84,7 +84,7 @@ - +