Add support for custom Apple XHarness commands in Helix SDK (#7380)

Enables to specify `<CustomCommands>` metadata on the `<XHarnessAppBundleToTest>` which will then inject those commands inside of the wrapper scripts we have that set and clean up the iOS simulator/device safely.

Android support will come in a follow-up PR.

azure-pipelines.yml

@@ -178,8 +178,8 @@ stages:
               -ci
               -restore
               -test
-              -projects $(Build.SourcesDirectory)/tests/UnitTests.XHarness.iOS.proj
-              /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/Helix.XHarness.iOS.binlog
+              -projects $(Build.SourcesDirectory)/tests/UnitTests.XHarness.iOS.Simulator.proj
+              /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/UnitTests.XHarness.iOS.Simulator.binlog
               /p:RestoreUsingNuGetTargets=false
             displayName: XHarness iOS Simulator Helix Testing
             env:
@@ -199,19 +199,6 @@ stages:
           #   env:
           #     SYSTEM_ACCESSTOKEN: $(System.AccessToken)
           #     HelixAccessToken: ''
-          - script: eng/common/build.sh
-              -configuration $(_BuildConfig)
-              -prepareMachine
-              -ci
-              -restore
-              -test
-              -projects $(Build.SourcesDirectory)/tests/UnitTests.XHarness.iOS.IncludeCliOnly.proj
-              /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/Helix.XHarness.CLI.binlog
-              /p:RestoreUsingNuGetTargets=false
-            displayName: XHarness CLI pre-install Helix Testing
-            env:
-              SYSTEM_ACCESSTOKEN: $(System.AccessToken)
-              HelixAccessToken: ''
           - script: eng/common/build.sh
               -configuration $(_BuildConfig) 
               -prepareMachine

src/Common/Microsoft.Arcade.Common/IZipArchiveManager.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.IO;
 using System.Threading.Tasks;
 
 namespace Microsoft.Arcade.Common
@@ -16,6 +17,22 @@ public interface IZipArchiveManager
         /// <param name="targetFileName">New name of the file in the archive</param>
         Task AddResourceFileToArchive<TAssembly>(string archivePath, string resourceName, string targetFileName = null);
 
+        /// <summary>
+        /// Creates a file with given content in a given archive.
+        /// </summary>
+        /// <param name="archivePath">Path to the archive</param>
+        /// <param name="targetFilename">New path of the file in the archive</param>
+        /// <param name="content">Content of the file</param>
+        Task AddContentToArchive(string archivePath, string targetFilename, Stream content);
+
+        /// <summary>
+        /// Creates a file with given content in a given archive.
+        /// </summary>
+        /// <param name="archivePath">Path to the archive</param>
+        /// <param name="targetFilename">New path of the file in the archive</param>
+        /// <param name="content">Content of the file</param>
+        Task AddContentToArchive(string archivePath, string targetFilename, string content);
+
         /// <summary>
         /// Compresses a directory into an archive on a given path.
         /// </summary>

src/Common/Microsoft.Arcade.Common/ZipArchiveManager.cs

@@ -4,6 +4,7 @@
 using System.IO;
 using System.IO.Compression;
 using System.Reflection;
+using System.Text;
 using System.Threading.Tasks;
 
 namespace Microsoft.Arcade.Common
@@ -17,9 +18,7 @@ public async Task AddResourceFileToArchive<TAssembly>(string archivePath, string
         }
 
         public void ArchiveDirectory(string directoryPath, string archivePath, bool includeBaseDirectory)
-        {
-            ZipFile.CreateFromDirectory(directoryPath, archivePath, CompressionLevel.Fastest, includeBaseDirectory);
-        }
+            => ZipFile.CreateFromDirectory(directoryPath, archivePath, CompressionLevel.Fastest, includeBaseDirectory);
 
         public void ArchiveFile(string filePath, string archivePath)
         {
@@ -30,7 +29,10 @@ public void ArchiveFile(string filePath, string archivePath)
             }
         }
 
-        private async Task AddContentToArchive(string archivePath, string targetFilename, Stream content)
+        public Task AddContentToArchive(string archivePath, string targetFilename, string content)
+            => AddContentToArchive(archivePath, targetFilename, new MemoryStream(Encoding.UTF8.GetBytes(content)));
+
+        public async Task AddContentToArchive(string archivePath, string targetFilename, Stream content)
         {
             using FileStream archiveStream = new FileStream(archivePath, FileMode.Open);
             using ZipArchive archive = new ZipArchive(archiveStream, ZipArchiveMode.Update);

src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateXHarnessAppleWorkItemsTests.cs

@@ -86,16 +86,19 @@ public void AppleXHarnessWorkItemIsCreated()
             _fileSystem.FileExists(payloadArchive).Should().BeTrue();
 
             var command = workItem.GetMetadata("Command");
-            command.Should().Contain("--command test");
+            command.Should().Contain("System.Foo.app");
+            command.Should().Contain("--targets \"ios-device_13.5\"");
             command.Should().Contain("--timeout \"00:08:55\"");
             command.Should().Contain("--launch-timeout \"00:02:33\"");
 
             _profileProvider
-                .Verify(x => x.AddProfilesToBundles(It.Is<ITaskItem[]>(bundles => bundles.Any(b => b.ItemSpec == "/apps/System.Foo.app"))), Times.AtLeastOnce);
+                .Verify(x => x.AddProfilesToBundles(It.Is<ITaskItem[]>(bundles => bundles.Any(b => b.ItemSpec == "/apps/System.Foo.app"))), Times.Once);
             _zipArchiveManager
-                .Verify(x => x.AddResourceFileToArchive<CreateXHarnessAppleWorkItems>(payloadArchive, It.Is<string>(s => s.Contains("xharness-helix-job.apple.sh")), "xharness-helix-job.apple.sh"), Times.AtLeastOnce);
+                .Verify(x => x.AddResourceFileToArchive<CreateXHarnessAppleWorkItems>(payloadArchive, It.Is<string>(s => s.Contains("xharness-helix-job.apple.sh")), "xharness-helix-job.apple.sh"), Times.Once);
             _zipArchiveManager
-                .Verify(x => x.AddResourceFileToArchive<CreateXHarnessAppleWorkItems>(payloadArchive, It.Is<string>(s => s.Contains("xharness-runner.apple.sh")), "xharness-runner.apple.sh"), Times.AtLeastOnce);
+                .Verify(x => x.AddResourceFileToArchive<CreateXHarnessAppleWorkItems>(payloadArchive, It.Is<string>(s => s.Contains("xharness-runner.apple.sh")), "xharness-runner.apple.sh"), Times.Once);
+            _zipArchiveManager
+                .Verify(x => x.AddContentToArchive(payloadArchive, "command.sh", It.Is<string>(s => s.Contains("xharness apple test"))), Times.Once);
         }
 
         [Fact]
@@ -130,6 +133,34 @@ public void ArchivePayloadIsOverwritten()
             _fileSystem.RemovedFiles.Should().Contain(payloadArchive);
         }
 
+        [Fact]
+        public void CustomCommandsAreExecuted()
+        {
+            var collection = CreateMockServiceCollection();
+            _task.ConfigureServices(collection);
+            _task.AppBundles = new[]
+            {
+                CreateAppBundle("apps/System.Foo.app", "ios-simulator-64_13.5", customCommands: "echo foo"),
+            };
+
+            // Act
+            using var provider = collection.BuildServiceProvider();
+            _task.InvokeExecute(provider).Should().BeTrue();
+
+            // Verify
+            _task.WorkItems.Length.Should().Be(1);
+
+            var workItem = _task.WorkItems.First();
+            workItem.GetMetadata("Identity").Should().Be("System.Foo");
+
+            var payloadArchive = workItem.GetMetadata("PayloadArchive");
+            payloadArchive.Should().NotBeNullOrEmpty();
+            _fileSystem.FileExists(payloadArchive).Should().BeTrue();
+
+            _zipArchiveManager
+                .Verify(x => x.AddContentToArchive(payloadArchive, "command.sh", "echo foo"), Times.Once);
+        }
+
         [Fact]
         public void AreDependenciesRegistered()
         {
@@ -155,31 +186,37 @@ private ITaskItem CreateAppBundle(
             string? testTimeout = null,
             string? launchTimeout = null,
             int expectedExitCode = 0,
-            bool includesTestRunner = true)
+            bool includesTestRunner = true,
+            string? customCommands = null)
         {
             var mockBundle = new Mock<ITaskItem>();
             mockBundle.SetupGet(x => x.ItemSpec).Returns(path);
-            mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAppleWorkItems.TargetPropName)).Returns(targets);
-            mockBundle.Setup(x => x.GetMetadata("IncludesTestRunner")).Returns(includesTestRunner.ToString());
+            mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAppleWorkItems.MetadataNames.Targets)).Returns(targets);
+            mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAppleWorkItems.MetadataNames.IncludesTestRunner)).Returns(includesTestRunner.ToString());
 
             if (workItemTimeout != null)
             {
-                mockBundle.Setup(x => x.GetMetadata("WorkItemTimeout")).Returns(workItemTimeout);
+                mockBundle.Setup(x => x.GetMetadata(XHarnessTaskBase.MetadataName.WorkItemTimeout)).Returns(workItemTimeout);
             }
 
             if (testTimeout != null)
             {
-                mockBundle.Setup(x => x.GetMetadata("TestTimeout")).Returns(testTimeout);
+                mockBundle.Setup(x => x.GetMetadata(XHarnessTaskBase.MetadataName.TestTimeout)).Returns(testTimeout);
             }
 
             if (launchTimeout != null)
             {
-                mockBundle.Setup(x => x.GetMetadata("LaunchTimeout")).Returns(launchTimeout);
+                mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAppleWorkItems.MetadataNames.LaunchTimeout)).Returns(launchTimeout);
             }
 
             if (expectedExitCode != 0)
             {
-                mockBundle.Setup(x => x.GetMetadata("ExpectedExitCode")).Returns(expectedExitCode.ToString());
+                mockBundle.Setup(x => x.GetMetadata(XHarnessTaskBase.MetadataName.ExpectedExitCode)).Returns(expectedExitCode.ToString());
+            }
+
+            if (customCommands != null)
+            {
+                mockBundle.Setup(x => x.GetMetadata(XHarnessTaskBase.MetadataName.CustomCommands)).Returns(customCommands);
             }
 
             _fileSystem.CreateDirectory(path);

src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ProvisioningProfileProviderTests.cs

@@ -133,7 +133,7 @@ private static ITaskItem CreateAppBundle(string path, string targets)
         {
             var mockBundle = new Mock<ITaskItem>();
             mockBundle.SetupGet(x => x.ItemSpec).Returns(path);
-            mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAppleWorkItems.TargetPropName)).Returns(targets);
+            mockBundle.Setup(x => x.GetMetadata(CreateXHarnessAppleWorkItems.MetadataNames.Targets)).Returns(targets);
             return mockBundle.Object;
         }
     }

src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAndroidWorkItems.cs

@@ -66,7 +66,7 @@ private async Task<ITaskItem> PrepareWorkItem(IZipArchiveManager zipArchiveManag
         {
             string workItemName = fileSystem.GetFileNameWithoutExtension(appPackage.ItemSpec);
 
-            var (testTimeout, workItemTimeout, expectedExitCode) = ParseMetadata(appPackage);
+            var (testTimeout, workItemTimeout, expectedExitCode, customCommands) = ParseMetadata(appPackage);
 
             string command = ValidateMetadataAndGetXHarnessAndroidCommand(appPackage, testTimeout, expectedExitCode);
 

src/Microsoft.DotNet.Helix/Sdk/CreateXHarnessAppleWorkItems.cs

@@ -15,12 +15,15 @@ namespace Microsoft.DotNet.Helix.Sdk
     /// </summary>
     public class CreateXHarnessAppleWorkItems : XHarnessTaskBase
     {
-        public const string TargetPropName = "Targets";
         public const string iOSTargetName = "ios-device";
         public const string tvOSTargetName = "tvos-device";
 
-        private const string LaunchTimeoutPropName = "LaunchTimeout";
-        private const string IncludesTestRunnerPropName = "IncludesTestRunner";
+        public static class MetadataNames
+        {
+            public const string Targets = "Targets";
+            public const string LaunchTimeout = "LaunchTimeout";
+            public const string IncludesTestRunner = "IncludesTestRunner";
+        }
 
         private const string EntryPointScript = "xharness-helix-job.apple.sh";
         private const string RunnerScript = "xharness-runner.apple.sh";
@@ -73,7 +76,7 @@ public bool ExecuteTask(
             var tasks = AppBundles.Select(bundle => PrepareWorkItem(zipArchiveManager, fileSystem, bundle));
 
             WorkItems = Task.WhenAll(tasks).GetAwaiter().GetResult().Where(wi => wi != null).ToArray();
-            
+
             return !Log.HasLoggedErrors;
         }
 
@@ -88,79 +91,97 @@ private async Task<ITaskItem> PrepareWorkItem(
             ITaskItem appBundleItem)
         {
             string appFolderPath = appBundleItem.ItemSpec.TrimEnd(Path.DirectorySeparatorChar);
-            
+
             string workItemName = fileSystem.GetFileName(appFolderPath);
             if (workItemName.EndsWith(".app"))
             {
                 workItemName = workItemName.Substring(0, workItemName.Length - 4);
             }
 
-            var (testTimeout, workItemTimeout, expectedExitCode) = ParseMetadata(appBundleItem);
+            var (testTimeout, workItemTimeout, expectedExitCode, customCommands) = ParseMetadata(appBundleItem);
 
             // Validation of any metadata specific to iOS stuff goes here
-            if (!appBundleItem.TryGetMetadata(TargetPropName, out string target))
+            if (!appBundleItem.TryGetMetadata(MetadataNames.Targets, out string targets))
             {
-                Log.LogError($"'{TargetPropName}' metadata must be specified - " +
+                Log.LogError($"'{MetadataNames.Targets}' metadata must be specified - " +
                     "expecting list of target device/simulator platforms to execute tests on (e.g. ios-simulator-64)");
                 return null;
             }
 
-            target = target.ToLowerInvariant();
+            targets = targets.ToLowerInvariant();
 
             // Optional timeout for the how long it takes for the app to be installed, booted and tests start executing
             TimeSpan launchTimeout = s_defaultLaunchTimeout;
-            if (appBundleItem.TryGetMetadata(LaunchTimeoutPropName, out string launchTimeoutProp))
+            if (appBundleItem.TryGetMetadata(MetadataNames.LaunchTimeout, out string launchTimeoutProp))
             {
                 if (!TimeSpan.TryParse(launchTimeoutProp, out launchTimeout) || launchTimeout.Ticks < 0)
                 {
-                    Log.LogError($"Invalid value \"{launchTimeoutProp}\" provided in <{LaunchTimeoutPropName}>");
+                    Log.LogError($"Invalid value \"{launchTimeoutProp}\" provided in <{MetadataNames.LaunchTimeout}>");
                     return null;
                 }
             }
 
             bool includesTestRunner = true;
-            if (appBundleItem.TryGetMetadata(IncludesTestRunnerPropName, out string includesTestRunnerProp))
+            if (appBundleItem.TryGetMetadata(MetadataNames.IncludesTestRunner, out string includesTestRunnerProp))
             {
                 if (includesTestRunnerProp.ToLowerInvariant() == "false")
                 {
                     includesTestRunner = false;
                 }
             }
 
-            if (includesTestRunner && expectedExitCode != 0)
+            if (includesTestRunner && expectedExitCode != 0 && customCommands != null)
             {
                 Log.LogWarning("The ExpectedExitCode property is ignored in the `apple test` scenario");
             }
 
+            if (customCommands == null)
+            {
+                // In case user didn't specify custom commands, we use our default one
+                customCommands = $"xharness apple {(includesTestRunner ? "test" : "run")} " +
+                    "--app \"$app\" " +
+                    "--output-directory \"$output_directory\" " +
+                    "--targets \"$targets\" " +
+                    "--timeout \"$timeout\" " +
+                    (includesTestRunner
+                        ? $"--launch-timeout \"$launch_timeout\" "
+                        : $"--expected-exit-code $expected_exit_code ") +
+                    "--xcode \"$xcode_path\" " +
+                    "-v " +
+                    (!string.IsNullOrEmpty(AppArguments) ? "-- " + AppArguments : string.Empty);
+            }
+
             string appName = fileSystem.GetFileName(appBundleItem.ItemSpec);
-            string command = GetHelixCommand(appName, target, testTimeout, launchTimeout, includesTestRunner, expectedExitCode);
-            string payloadArchivePath = await CreateZipArchiveOfFolder(zipArchiveManager, fileSystem, appFolderPath);
+            string helixCommand = GetHelixCommand(appName, targets, testTimeout, launchTimeout, includesTestRunner, expectedExitCode);
+            string payloadArchivePath = await CreateZipArchiveOfFolder(zipArchiveManager, fileSystem, appFolderPath, customCommands);
 
-            Log.LogMessage($"Creating work item with properties Identity: {workItemName}, Payload: {appFolderPath}, Command: {command}");
+            Log.LogMessage($"Creating work item with properties Identity: {workItemName}, Payload: {appFolderPath}, Command: {helixCommand}");
 
             return new Build.Utilities.TaskItem(workItemName, new Dictionary<string, string>()
             {
                 { "Identity", workItemName },
                 { "PayloadArchive", payloadArchivePath },
-                { "Command", command },
+                { "Command", helixCommand },
                 { "Timeout", workItemTimeout.ToString() },
             });
         }
 
         private string GetHelixCommand(string appName, string targets, TimeSpan testTimeout, TimeSpan launchTimeout, bool includesTestRunner, int expectedExitCode) =>
             $"chmod +x {EntryPointScript} && ./{EntryPointScript} " +
-            $"--app \"$HELIX_WORKITEM_ROOT/{appName}\" " +
-             "--output-directory \"$HELIX_WORKITEM_UPLOAD_ROOT\" " +
+            $"--app \"{appName}\" " +
             $"--targets \"{targets}\" " +
             $"--timeout \"{testTimeout}\" " +
             $"--launch-timeout \"{launchTimeout}\" " +
-             "--xharness-cli-path \"$XHARNESS_CLI_PATH\" " +
-             "--command " + (includesTestRunner ? "test" : "run") +
-            (expectedExitCode != 0 ? $" --expected-exit-code \"{expectedExitCode}\"" : string.Empty) +
+            (includesTestRunner ? "--includes-test-runner " : string.Empty) +
+            $"--expected-exit-code \"{expectedExitCode}\" " +
             (!string.IsNullOrEmpty(XcodeVersion) ? $" --xcode-version \"{XcodeVersion}\"" : string.Empty) +
             (!string.IsNullOrEmpty(AppArguments) ? $" --app-arguments \"{AppArguments}\"" : string.Empty);
 
-        private async Task<string> CreateZipArchiveOfFolder(IZipArchiveManager zipArchiveManager, IFileSystem fileSystem, string folderToZip)
+        private async Task<string> CreateZipArchiveOfFolder(
+            IZipArchiveManager zipArchiveManager,
+            IFileSystem fileSystem,
+            string folderToZip,
+            string injectedCommands)
         {
             if (!fileSystem.DirectoryExists(folderToZip))
             {
@@ -183,6 +204,7 @@ private async Task<string> CreateZipArchiveOfFolder(IZipArchiveManager zipArchiv
             Log.LogMessage($"Adding the XHarness job scripts into the payload archive");
             await zipArchiveManager.AddResourceFileToArchive<CreateXHarnessAppleWorkItems>(outputZipPath, ScriptNamespace + EntryPointScript, EntryPointScript);
             await zipArchiveManager.AddResourceFileToArchive<CreateXHarnessAppleWorkItems>(outputZipPath, ScriptNamespace + RunnerScript, RunnerScript);
+            await zipArchiveManager.AddContentToArchive(outputZipPath, CustomCommandsScript + ".sh", injectedCommands);
 
             return outputZipPath;
         }

src/Microsoft.DotNet.Helix/Sdk/ProvisioningProfileProvider.cs

@@ -62,7 +62,7 @@ public void AddProfilesToBundles(ITaskItem[] appBundles)
 
             foreach (var appBundle in appBundles)
             {
-                if (!appBundle.TryGetMetadata(CreateXHarnessAppleWorkItems.TargetPropName, out string bundleTargets))
+                if (!appBundle.TryGetMetadata(CreateXHarnessAppleWorkItems.MetadataNames.Targets, out string bundleTargets))
                 {
                     _log.LogError("'Targets' metadata must be specified - " +
                         "expecting list of target device/simulator platforms to execute tests on (e.g. ios-simulator-64)");