diff --git a/src/Agent.Worker/TestResults/ResultsCommandExtension.cs b/src/Agent.Worker/TestResults/ResultsCommandExtension.cs index c2a115e4f1..8e6f6ebdee 100644 --- a/src/Agent.Worker/TestResults/ResultsCommandExtension.cs +++ b/src/Agent.Worker/TestResults/ResultsCommandExtension.cs @@ -23,6 +23,8 @@ public sealed class ResultsCommandExtension : AgentService, IWorkerCommandExtens private bool _publishRunLevelAttachments; private int _runCounter = 0; private readonly object _sync = new object(); + private string _testRunSystem; + private const string _testRunSystemCustomFieldName = "TestRunSystem"; public Type ExtensionType => typeof(IWorkerCommandExtension); @@ -198,6 +200,7 @@ private async Task PublishAllTestResultsToSingleTestRunAsync(List result ); testRunData.Attachments = runAttachments.ToArray(); + testRunData.AddCustomField(_testRunSystemCustomFieldName, _testRunSystem); TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken); await publisher.AddResultsAsync(testRun, runResults.ToArray(), _executionContext.CancellationToken); @@ -229,14 +232,16 @@ private async Task PublishToNewTestRunPerTestResultFileAsync(List result .Select(bucket => bucket.Select(pair => pair.file).ToList()) .ToList(); + bool changeTestRunTitle = resultFiles.Count > 1; + foreach (var files in groupedFiles) { // Publish separate test run for each result file that has results. var publishTasks = files.Select(async resultFile => { cancellationToken.ThrowIfCancellationRequested(); - string runName = null; - if (!string.IsNullOrWhiteSpace(_runTitle)) + string runName = _runTitle; + if (!string.IsNullOrWhiteSpace(_runTitle) && changeTestRunTitle) { runName = GetRunTitle(); } @@ -248,6 +253,7 @@ private async Task PublishToNewTestRunPerTestResultFileAsync(List result if (testRunData != null && testRunData.Results != null && testRunData.Results.Length > 0) { + testRunData.AddCustomField(_testRunSystemCustomFieldName, _testRunSystem); TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken); await publisher.AddResultsAsync(testRun, testRunData.Results, _executionContext.CancellationToken); await publisher.EndTestRunAsync(testRunData, testRun.Id, cancellationToken: _executionContext.CancellationToken); @@ -356,6 +362,12 @@ private void LoadPublishTestResultsInputs(IExecutionContext context, Dictionary< _runTitle = string.Empty; } + eventProperties.TryGetValue(PublishTestResultsEventProperties.TestRunSystem, out _testRunSystem); + if (_testRunSystem == null) + { + _testRunSystem = string.Empty; + } + string publishRunAttachmentsInput; eventProperties.TryGetValue(PublishTestResultsEventProperties.PublishRunAttachments, out publishRunAttachmentsInput); if (string.IsNullOrEmpty(publishRunAttachmentsInput) || !bool.TryParse(publishRunAttachmentsInput, out _publishRunLevelAttachments)) @@ -391,5 +403,6 @@ internal static class PublishTestResultsEventProperties public static readonly string RunTitle = "runTitle"; public static readonly string PublishRunAttachments = "publishRunAttachments"; public static readonly string ResultFiles = "resultFiles"; + public static readonly string TestRunSystem = "testRunSystem"; } } \ No newline at end of file diff --git a/src/Agent.Worker/TestResults/TestRunData.cs b/src/Agent.Worker/TestResults/TestRunData.cs index 3d7877d188..73196ec574 100644 --- a/src/Agent.Worker/TestResults/TestRunData.cs +++ b/src/Agent.Worker/TestResults/TestRunData.cs @@ -1,4 +1,5 @@ using Microsoft.TeamFoundation.TestManagement.WebApi; +using System.Collections.Generic; namespace Microsoft.VisualStudio.Services.Agent.Worker.TestResults { @@ -7,7 +8,7 @@ public class TestRunData : RunCreateModel public TestRunData(string name = "", string iteration = "", int[] pointIds = null, ShallowReference plan = null, ShallowReference testSettings = null, int buildId = 0, string state = "", bool? isAutomated = default(bool?), string errorMessage = "", string dueDate = "", string type = "", string controller = "", string buildDropLocation = "", string comment = "", string testEnvironmentId = "", string startedDate = "", string completedDate = "", int[] configIds = null, RunFilter filter = null, ShallowReference dtlTestEnvironment = null, DtlEnvironmentDetails environmentDetails = null, string buildPlatform = null, string buildFlavor = null, string releaseUri = null, string releaseEnvironmentUri = null) : base(name, iteration, pointIds, plan, testSettings, buildId, state, isAutomated, errorMessage, dueDate, type, controller, buildDropLocation, comment, testEnvironmentId, startedDate, completedDate, configIds, filter, dtlTestEnvironment, environmentDetails, null, buildPlatform, buildFlavor, releaseUri, releaseEnvironmentUri) { - + CustomTestFields = new List(); } public string[] Attachments { get; set; } @@ -15,5 +16,14 @@ public class TestRunData : RunCreateModel // Results public TestCaseResultData[] Results { get; set; } + public void AddCustomField(string fieldName, string value) + { + if (string.IsNullOrEmpty(value) || string.IsNullOrEmpty(fieldName)) + { + return; + } + + CustomTestFields.Add(new CustomTestField() { FieldName = fieldName, Value = value }); + } } } \ No newline at end of file diff --git a/src/Test/L0/Worker/TestResults/ResultsCommandExtensionTests.cs b/src/Test/L0/Worker/TestResults/ResultsCommandExtensionTests.cs index 27df016c01..9792275aca 100644 --- a/src/Test/L0/Worker/TestResults/ResultsCommandExtensionTests.cs +++ b/src/Test/L0/Worker/TestResults/ResultsCommandExtensionTests.cs @@ -212,6 +212,176 @@ public void VerifyResultsAreMergedWhenPublishingToSingleTestRun() resultCommand.ProcessCommand(_ec.Object, command); } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "PublishTestResults")] + public void VerifyTestRunSystemPropertyIsSentWhenPublishingToSignleTestRun() + { + SetupMocks(); + var resultCommand = new ResultsCommandExtension(); + resultCommand.Initialize(_hc); + var command = new Command("results", "publish"); + command.Properties.Add("resultFiles", "file1.trx,file2.trx"); + command.Properties.Add("type", "NUnit"); + command.Properties.Add("mergeResults", bool.TrueString); + command.Properties.Add("testRunSystem", "MAVEN"); + var resultsFiles = new List { "file1.trx", "file2.trx" }; + + var testRunData = new TestRunData(); + testRunData.Results = new TestCaseResultData[] { new TestCaseResultData(), new TestCaseResultData() }; + testRunData.Attachments = new string[] { "attachment1", "attachment2" }; + + _mockTestRunPublisher.Setup(q => q.StartTestRunAsync(It.IsAny(), It.IsAny())) + .Callback((TestRunData trd, CancellationToken cancellationToken) => + { + Assert.NotNull(trd.CustomTestFields); + Assert.NotEmpty(trd.CustomTestFields); + Assert.Equal("testRunSystem", trd.CustomTestFields[0].FieldName); + Assert.Equal("MAVEN", trd.CustomTestFields[0].Value); + }); + _mockTestRunPublisher.Setup(q => q.ReadResultsFromFile(It.IsAny(), It.IsAny())) + .Returns(testRunData); + + resultCommand.ProcessCommand(_ec.Object, command); + + // Making sure that the callback is called. + _mockTestRunPublisher.Verify(q => q.StartTestRunAsync(It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "PublishTestResults")] + public void VerifyTestRunTitleIsModifiedWhenPublishingToMultipleTestRun() + { + SetupMocks(); + var resultCommand = new ResultsCommandExtension(); + resultCommand.Initialize(_hc); + var command = new Command("results", "publish"); + command.Properties.Add("resultFiles", "file1.trx,file2.trx"); + command.Properties.Add("type", "NUnit"); + command.Properties.Add("mergeResults", bool.FalseString); + command.Properties.Add("runTitle", "TestRunTitle"); + var resultsFiles = new List { "file1.trx", "file2.trx" }; + + var testRunData = new TestRunData(); + testRunData.Results = new TestCaseResultData[] { new TestCaseResultData(), new TestCaseResultData() }; + testRunData.Attachments = new string[] { "attachment1", "attachment2" }; + int counter = 0; + _mockTestRunPublisher.Setup(q => q.StartTestRunAsync(It.IsAny(), It.IsAny())) + .Callback((TestRunData trd, CancellationToken cancellationToken) => + { + Assert.Equal(StringUtil.Format("{0}_{1}", "TestRunTitle", ++counter), trd.Name); + }); + _mockTestRunPublisher.Setup(q => q.ReadResultsFromFile(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(testRunData); + + resultCommand.ProcessCommand(_ec.Object, command); + + // Making sure that the callback is called. + _mockTestRunPublisher.Verify(q => q.StartTestRunAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "PublishTestResults")] + public void VerifyTestRunTitleShouldNotBeModifiedWhenPublishingToSingleTestRun() + { + SetupMocks(); + var resultCommand = new ResultsCommandExtension(); + resultCommand.Initialize(_hc); + var command = new Command("results", "publish"); + command.Properties.Add("resultFiles", "file1.trx,file2.trx"); + command.Properties.Add("type", "NUnit"); + command.Properties.Add("mergeResults", bool.TrueString); + command.Properties.Add("runTitle", "TestRunTitle"); + var resultsFiles = new List { "file1.trx", "file2.trx" }; + + var testRunData = new TestRunData(); + testRunData.Results = new TestCaseResultData[] { new TestCaseResultData(), new TestCaseResultData() }; + testRunData.Attachments = new string[] { "attachment1", "attachment2" }; + _mockTestRunPublisher.Setup(q => q.StartTestRunAsync(It.IsAny(), It.IsAny())) + .Callback((TestRunData trd, CancellationToken cancellationToken) => + { + Assert.Equal("TestRunTitle", trd.Name); + }); + _mockTestRunPublisher.Setup(q => q.ReadResultsFromFile(It.IsAny(), It.IsAny())) + .Returns(testRunData); + + resultCommand.ProcessCommand(_ec.Object, command); + + // Making sure that the callback is called. + _mockTestRunPublisher.Verify(q => q.StartTestRunAsync(It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "PublishTestResults")] + public void VerifyTestRunTitleShouldNotBeModifiedWhenWhenOnlyOneResultFileIsPublished() + { + SetupMocks(); + var resultCommand = new ResultsCommandExtension(); + resultCommand.Initialize(_hc); + var command = new Command("results", "publish"); + command.Properties.Add("resultFiles", "file1.trx"); + command.Properties.Add("type", "NUnit"); + // Explicitly not merging it to check if the test run title is not modified when there's only one test file. + command.Properties.Add("mergeResults", bool.FalseString); + command.Properties.Add("runTitle", "TestRunTitle"); + var resultsFiles = new List { "file1.trx"}; + + var testRunData = new TestRunData(); + testRunData.Results = new TestCaseResultData[] { new TestCaseResultData()}; + testRunData.Attachments = new string[] { "attachment1" }; + _mockTestRunPublisher.Setup(q => q.StartTestRunAsync(It.IsAny(), It.IsAny())) + .Callback((TestRunData trd, CancellationToken cancellationToken) => + { + Assert.Equal("TestRunTitle", trd.Name); + }); + _mockTestRunPublisher.Setup(q => q.ReadResultsFromFile(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(testRunData); + + resultCommand.ProcessCommand(_ec.Object, command); + + // Making sure that the callback is called. + _mockTestRunPublisher.Verify(q => q.StartTestRunAsync(It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "PublishTestResults")] + public void VerifyTestRunSystemPropertyIsSentWhenPublishingToTestRunPerFile() + { + SetupMocks(); + var resultCommand = new ResultsCommandExtension(); + resultCommand.Initialize(_hc); + var command = new Command("results", "publish"); + command.Properties.Add("resultFiles", "file1.trx,file2.trx"); + command.Properties.Add("type", "NUnit"); + command.Properties.Add("mergeResults", bool.FalseString); + command.Properties.Add("testRunSystem", "MAVEN"); + var resultsFiles = new List { "file1.trx", "file2.trx" }; + + var testRunData = new TestRunData(); + testRunData.Results = new TestCaseResultData[] { new TestCaseResultData(), new TestCaseResultData() }; + testRunData.Attachments = new string[] { "attachment1", "attachment2" }; + + _mockTestRunPublisher.Setup(q => q.StartTestRunAsync(It.IsAny(), It.IsAny())) + .Callback((TestRunData trd, CancellationToken cancellationToken) => + { + Assert.NotNull(trd.CustomTestFields); + Assert.NotEmpty(trd.CustomTestFields); + Assert.Equal("testRunSystem", trd.CustomTestFields[0].FieldName); + Assert.Equal("MAVEN", trd.CustomTestFields[0].Value); + }); + + _mockTestRunPublisher.Setup(q => q.ReadResultsFromFile(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(testRunData); + resultCommand.ProcessCommand(_ec.Object, command); + + // There should be two calls to startestrun + _mockTestRunPublisher.Verify(q=>q.StartTestRunAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "PublishTestResults")]