[iOS] Ensure that we do generate xml when we call the CLI. (#102)

* [iOS] Ensure that we do generate xml when we call the CLI.

During the refactor we removed the env vars that are passed to the
runners to decide if xml is used and the jargon to use. The env
paramenters have been added and tests updated. Now the runners will get
notified that xml is used (always) and the jargon (default to xunit).

fixes: #98
fixes: #77

src/Microsoft.DotNet.XHarness.CLI/CommandArguments/iOS/iOSTestCommandArguments.cs

@@ -37,6 +37,11 @@ internal class iOSTestCommandArguments : TestCommandArguments
         /// </summary>
         public TimeSpan LaunchTimeout { get; set; } = TimeSpan.FromMinutes(5);
 
+        /// <summary>
+        /// Allows to specify the xml format to be used in the result files.
+        /// </summary>
+        public XmlResultJargon XmlResultJargon { get; set; } = XmlResultJargon.xUnit; // default by mono
+
         public IReadOnlyCollection<TestTarget> TestTargets { get; set; }
 
         public override IList<string> GetValidationErrors()
@@ -64,7 +69,7 @@ public override IList<string> GetValidationErrors()
                         // let the user know that the target is not known
                         // and all the available ones.
                         var sb = new StringBuilder();
-                        sb.AppendLine($"Failed to parse test target '{targetName}'. Avaliable targets are:");
+                        sb.AppendLine($"Failed to parse test target '{targetName}'. Available targets are:");
                         sb.AppendLine();
                         foreach (var val in Enum.GetValues(typeof(TestTarget)))
                         {
@@ -99,6 +104,17 @@ public override IList<string> GetValidationErrors()
                 errors.Add($"Failed to find Xcode root at {XcodeRoot}");
             }
 
+            if (XmlResultJargon == XmlResultJargon.Missing)
+            {
+                var sb = new StringBuilder();
+                sb.AppendLine($"Failed to parse xml jargon. Available targets are:");
+                sb.AppendLine();
+                foreach (var val in new [] {XmlResultJargon.NUnitV2, XmlResultJargon.NUnitV3, XmlResultJargon.TouchUnit, XmlResultJargon.xUnit})
+                {
+                    sb.AppendLine($"\t* {val.ToString()}");
+                }
+                errors.Add(sb.ToString());
+            }
             return errors;
         }
 

src/Microsoft.DotNet.XHarness.CLI/Commands/iOS/iOSTestCommand.cs

@@ -57,6 +57,12 @@ public iOSTestCommand() : base()
                 { "device-name=", "Name of a specific device, if you wish to target one", v => _arguments.DeviceName = v},
                 { "communication-channel:", $"The communication channel to use to communicate with the default. Can be {CommunicationChannel.Network} and {CommunicationChannel.UsbTunnel}. Default is {CommunicationChannel.UsbTunnel}", v => Enum.TryParse (v, out _channel)},
                 { "launch-timeout=|lt=", "Time span, in seconds, to wait for the iOS app to start.", v => _arguments.LaunchTimeout = TimeSpan.FromSeconds(int.Parse(v))},
+                { "xml-jargon:|xj:", $"The xml format to be used in the unit test results. Can be {XmlResultJargon.TouchUnit} {XmlResultJargon.NUnitV2} {XmlResultJargon.NUnitV3} and {XmlResultJargon.xUnit}", v =>
+                    {
+                        // if we cannot parse it, set it as missing and the error will notify the issue
+                        _arguments.XmlResultJargon = Enum.TryParse(v, out XmlResultJargon jargon) ? jargon : XmlResultJargon.Missing;
+                    }
+                },
             };
 
             foreach (var option in CommonOptions)
@@ -172,6 +178,7 @@ private async Task<ExitCode> RunTest(TestTarget target,
                     mainLog,
                     logs,
                     new Helpers(),
+                    useXmlOutput: true, // the cli ALWAYS will get the output as xml
                     useTcpTunnel: _channel == CommunicationChannel.UsbTunnel);
 
                 (deviceName, success) = await appRunner.RunApp(appBundleInfo,
@@ -180,7 +187,7 @@ private async Task<ExitCode> RunTest(TestTarget target,
                     _arguments.LaunchTimeout,
                     deviceName,
                     verbosity: verbosity,
-                    xmlResultJargon: XmlResultJargon.xUnit,
+                    xmlResultJargon: _arguments.XmlResultJargon,
                     cancellationToken: cancellationToken);
 
                 if (success)

src/Microsoft.DotNet.XHarness.Tests.Runners/AndroidApplicationEntryPoint.cs

@@ -19,7 +19,7 @@ public abstract class AndroidApplicationEntryPoint : ApplicationEntryPoint
         public abstract TextWriter Logger { get; }
 
         /// <summary>
-        /// Implementors should provide a full path in which the final 
+        /// Implementors should provide a full path in which the final
         /// results of the test run will be written. This property must not
         /// return null.
         /// </summary>
@@ -38,16 +38,18 @@ public override async Task RunAsync()
 
             var runner = await InternalRunAsync(logger);
 
-            TestRunner.Jargon jargon = Core.TestRunner.Jargon.NUnitV3;
+            TestRunner.Jargon jargon;
             switch (options.XmlVersion)
             {
                 case XmlVersion.NUnitV2:
                     jargon = Core.TestRunner.Jargon.NUnitV2;
                     break;
-                case XmlVersion.NUnitV3:
-                default: // nunitv3 gives os the most amount of possible details
+                case XmlVersion.NUnitV3: // nunitv3 gives os the most amount of possible details
                     jargon = Core.TestRunner.Jargon.NUnitV3;
                     break;
+                default:
+                    jargon = Core.TestRunner.Jargon.xUnit;
+                    break;
             }
             if (options.EnableXml)
             {

src/Microsoft.DotNet.XHarness.Tests.Runners/ApplicationOptions.cs

@@ -18,6 +18,7 @@ internal enum XmlVersion
     {
         NUnitV2 = 0,
         NUnitV3 = 1,
+        xUnit = 2,
     }
 
     internal class ApplicationOptions
@@ -119,7 +120,7 @@ public ApplicationOptions()
         /// <summary>
         /// The transport to be used to communicate with the host. The default
         /// value is TCP.
-        /// 
+        ///
         /// Supported values are:
         ///
         /// * TCP

src/Microsoft.DotNet.XHarness.iOS.Shared/TestImporter/Templates/Managed/Resources/src/iOSApp/ViewController.cs

@@ -113,6 +113,9 @@ public async override void ViewDidLoad()
                 case XmlVersion.NUnitV3:
                     jargon = Xamarin.iOS.UnitTests.TestRunner.Jargon.NUnitV3;
                     break;
+                case XmlVersion.xUnit:
+                    jargon = Core.TestRunner.Jargon.xUnit;
+                    break;
             }
 
             if (options.EnableXml)

src/Microsoft.DotNet.XHarness.iOS.Shared/TestImporter/Templates/Managed/Resources/src/tvOSApp/ViewController.cs

@@ -105,13 +105,14 @@ public async override void ViewDidLoad()
             Xamarin.iOS.UnitTests.TestRunner.Jargon jargon = Xamarin.iOS.UnitTests.TestRunner.Jargon.NUnitV3;
             switch (options.XmlVersion)
             {
+                case XmlVersion.NUnitV2:
+                    jargon = Xamarin.iOS.UnitTests.TestRunner.Jargon.NUnitV2;
+                    break;
                 case XmlVersion.NUnitV3:
                     jargon = Xamarin.iOS.UnitTests.TestRunner.Jargon.NUnitV3;
                     break;
-
-                default:
-                case XmlVersion.NUnitV2:
-                    jargon = Xamarin.iOS.UnitTests.TestRunner.Jargon.NUnitV2;
+                case XmlVersion.xUnit:
+                    jargon = Core.TestRunner.Jargon.xUnit;
                     break;
             }
 

src/Microsoft.DotNet.XHarness.iOS.Shared/TestImporter/Templates/Managed/Resources/src/watchOS/Extension/InterfaceController.cs

@@ -89,7 +89,7 @@ internal static IEnumerable<TestAssemblyInfo> GetTestAssemblies ()
 				yield return new TestAssemblyInfo (a, name);
 			}
  		}
- 		
+
 		void RunTests ()
 		{
 			var options = ApplicationOptions.Current;
@@ -140,6 +140,9 @@ void RunTests ()
 					case XmlVersion.NUnitV3:
 						jargon = Xamarin.iOS.UnitTests.TestRunner.Jargon.NUnitV3;
 						break;
+					case XmlVersion.xUnit:
+						jargon = Xamarin.iOS.UnitTests.TestRunner.Jargon.xUnit;
+						break;
 					}
 					if (options.EnableXml) {
 						runner.WriteResultsToFile (writer, jargon);
@@ -208,7 +211,7 @@ public override bool Match (NUnit.Framework.Api.ITest test)
 		var method = test as NUnit.Framework.Internal.TestMethod;
 		if (method != null)
 			return Match (method.Parent);
-		
+
 		var name = !string.IsNullOrEmpty (test.Name) ? test.Name : test.FullName;
 		bool rv;
 		if (string.IsNullOrEmpty (name)) {

src/Microsoft.DotNet.XHarness.iOS/AppRunner.cs

@@ -32,6 +32,7 @@ public class AppRunner
         private readonly ILog _mainLog;
         private readonly IHelpers _helpers;
         private readonly bool _useTcpTunnel;
+        private readonly bool _useXmlOutput;
 
         public AppRunner(IProcessManager processManager,
             IHardwareDeviceLoader hardwareDeviceLoader,
@@ -44,6 +45,7 @@ public AppRunner(IProcessManager processManager,
             ILog mainLog,
             ILogs logs,
             IHelpers helpers,
+            bool useXmlOutput,
             bool useTcpTunnel)
         {
             _processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
@@ -57,6 +59,7 @@ public AppRunner(IProcessManager processManager,
             _mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
             _logs = logs ?? throw new ArgumentNullException(nameof(logs));
             _helpers = helpers ?? throw new ArgumentNullException(nameof(helpers));
+            _useXmlOutput = useXmlOutput;
             _useTcpTunnel = useTcpTunnel;
         }
 
@@ -83,7 +86,7 @@ public AppRunner(IProcessManager processManager,
                 new SetEnvVariableArgument("NUNIT_AUTOEXIT", true),
                 new SetAppArgumentArgument("-enablenetwork", true),
                 new SetEnvVariableArgument("NUNIT_ENABLE_NETWORK", true),
-                
+
                 // On macOS we can't edit the TCC database easily
                 // (it requires adding the mac has to be using MDM: https://carlashley.com/2018/09/28/tcc-round-up/)
                 // So by default ignore any tests that would pop up permission dialogs in CI.
@@ -133,6 +136,14 @@ public AppRunner(IProcessManager processManager,
                 args.Add(new SetEnvVariableArgument("USE_TCP_TUNNEL", true));
             }
 
+            if (_useXmlOutput)
+            {
+                // let the runner now via envars that we want to get a xml output, else the runner will default to plain text
+                args.Add (new SetEnvVariableArgument ("NUNIT_ENABLE_XML_OUTPUT", true));
+                args.Add (new SetEnvVariableArgument ("NUNIT_ENABLE_XML_MODE", "wrapped"));
+                args.Add (new SetEnvVariableArgument ("NUNIT_XML_VERSION", $"{xmlResultJargon}"));
+            }
+
             listener.StartAsync();
 
             var crashLogs = new Logs(_logs.Directory);
@@ -338,7 +349,7 @@ public AppRunner(IProcessManager processManager,
                 {
                     await crashReporter.StartCaptureAsync();
 
-                    // create a tunnel to communicate with the device 
+                    // create a tunnel to communicate with the device
                     if (transport == ListenerTransport.Tcp && _useTcpTunnel && listener is SimpleTcpListener tcpListener)
                     {
                         // create a new tunnel using the listener

tests/Microsoft.DotNet.XHarness.iOS.Tests/AppRunnerTests.cs

@@ -125,9 +125,11 @@ public void Dispose()
         }
 
         [Theory]
-        [InlineData(false)]
-        [InlineData(true)]
-        public async Task RunOnSimulatorWithNoAvailableSimulatorTest(bool useTcpTunnel)
+        [InlineData(false, false)] // no tunnel, no xml
+        [InlineData(false, true)] // no tunnel, xml
+        [InlineData(true, false)] // tunnel, no xml
+        [InlineData(true, true)] // tunnel and xml
+        public async Task RunOnSimulatorWithNoAvailableSimulatorTest(bool useTcpTunnel, bool useXmlOutput)
         {
             // Mock finding simulators
             string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
@@ -168,6 +170,7 @@ public async Task RunOnSimulatorWithNoAvailableSimulatorTest(bool useTcpTunnel)
                 _mainLog.Object,
                 _logs.Object,
                 _helpers.Object,
+                useXmlOutput,
                 useTcpTunnel);
 
             var appInformation = new AppBundleInformation(AppName, AppBundleIdentifier, s_appPath, s_appPath, null);
@@ -191,8 +194,12 @@ await Assert.ThrowsAsync<NoDeviceFoundException>(
             _tunnelBore.Verify(t => t.Create(It.IsAny<string>(), It.IsAny<ILog>()), Times.Never); // never create tunnels on simulators
         }
 
-        [Fact]
-        public async Task RunOnSimulatorSuccessfullyTest()
+        [Theory]
+        [InlineData(false, false)] // no tunnel, no xml
+        [InlineData(false, true)] // no tunnel, xml
+        [InlineData(true, false)] // tunnel, no xml
+        [InlineData(true, true)] // tunnel and xml
+        public async Task RunOnSimulatorSuccessfullyTest(bool useTunnel, bool useXml)
         {
             string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
 
@@ -238,7 +245,8 @@ public async Task RunOnSimulatorSuccessfullyTest()
                 _mainLog.Object,
                 _logs.Object,
                 _helpers.Object,
-                false);
+                useXml,
+                useTunnel);
 
             var appInformation = new AppBundleInformation(AppName, AppBundleIdentifier, s_appPath, s_appPath, null);
 
@@ -253,12 +261,14 @@ public async Task RunOnSimulatorSuccessfullyTest()
             Assert.Equal("Test iPhone simulator", deviceName);
             Assert.True(success);
 
+            var xmlParam = useXml ? "-setenv=NUNIT_ENABLE_XML_OUTPUT=true -setenv=NUNIT_ENABLE_XML_MODE=wrapped -setenv=NUNIT_XML_VERSION=xUnit " : "";
+
             var expectedArgs = $"-argument=-connection-mode -argument=none -argument=-app-arg:-autostart " +
                 $"-setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit -setenv=NUNIT_AUTOEXIT=true " +
                 $"-argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true -setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -v -v " +
                 $"-argument=-app-arg:-hostname:127.0.0.1 -setenv=NUNIT_HOSTNAME=127.0.0.1 -argument=-app-arg:-transport:Tcp " +
                 $"-setenv=NUNIT_TRANSPORT=TCP -argument=-app-arg:-hostport:{_listener.Object.Port} " +
-                $"-setenv=NUNIT_HOSTPORT={_listener.Object.Port} --launchsim {StringUtils.FormatArguments(s_appPath)} " +
+                $"-setenv=NUNIT_HOSTPORT={_listener.Object.Port} {xmlParam}--launchsim {StringUtils.FormatArguments(s_appPath)} " +
                 $"--stdout=tty1 --stderr=tty1 --device=:v2:udid={simulator.Object.UDID}";
 
             _processManager
@@ -306,6 +316,7 @@ public async Task RunOnDeviceWithNoAvailableSimulatorTest()
                 _mainLog.Object,
                 _logs.Object,
                 _helpers.Object,
+                true,
                 false);
 
             var appInformation = new AppBundleInformation(AppName, AppBundleIdentifier, s_appPath, s_appPath, null);
@@ -320,9 +331,11 @@ await Assert.ThrowsAsync<NoDeviceFoundException>(
         }
 
         [Theory]
-        [InlineData(false)]
-        [InlineData(true)]
-        public async Task RunOnDeviceSuccessfullyTest(bool useTunnel)
+        [InlineData(false, false)] // no tunnel, no xml
+        [InlineData(false, true)] // no tunnel, xml
+        [InlineData(true, false)] // tunnel, no xml
+        [InlineData(true, true)] // tunnel and xml
+        public async Task RunOnDeviceSuccessfullyTest(bool useTunnel, bool useXml)
         {
             var deviceSystemLog = new Mock<ILog>();
             deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
@@ -363,6 +376,7 @@ public async Task RunOnDeviceSuccessfullyTest(bool useTunnel)
                 _mainLog.Object,
                 _logs.Object,
                 _helpers.Object,
+                useXml,
                 useTunnel);
 
             var appInformation = new AppBundleInformation(AppName, AppBundleIdentifier, s_appPath, s_appPath, null);
@@ -377,16 +391,18 @@ public async Task RunOnDeviceSuccessfullyTest(bool useTunnel)
             Assert.Equal("Test iPhone", deviceName);
             Assert.True(success);
 
-            var ips = string.Join(",", System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList.Select(ip => ip.ToString()));
+            var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList.Select(ip => ip.ToString());
+            var ips = string.Join(",", ipAddresses);
 
             var tunnelParam = useTunnel ? "-setenv=USE_TCP_TUNNEL=true " : "";
+            var xmlParam = useXml ? "-setenv=NUNIT_ENABLE_XML_OUTPUT=true -setenv=NUNIT_ENABLE_XML_MODE=wrapped -setenv=NUNIT_XML_VERSION=xUnit " : "";
 
             var expectedArgs = $"-argument=-connection-mode -argument=none -argument=-app-arg:-autostart " +
                 $"-setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit -setenv=NUNIT_AUTOEXIT=true " +
                 $"-argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true -setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -v -v " +
                 $"-argument=-app-arg:-hostname:{ips} -setenv=NUNIT_HOSTNAME={ips} -argument=-app-arg:-transport:Tcp " +
                 $"-setenv=NUNIT_TRANSPORT=TCP -argument=-app-arg:-hostport:{_listener.Object.Port} " +
-                $"-setenv=NUNIT_HOSTPORT={_listener.Object.Port} {tunnelParam}--launchdev {StringUtils.FormatArguments(s_appPath)} " +
+                $"-setenv=NUNIT_HOSTPORT={_listener.Object.Port} {tunnelParam}{xmlParam}--launchdev {StringUtils.FormatArguments(s_appPath)} " +
                 $"--disable-memory-limits --wait-for-exit --devname \"Test iPhone\"";
 
             _processManager