Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Using TestRunSystem while creating the TestRun #1637

Merged
merged 7 commits into from
Jun 28, 2018
Merged

Using TestRunSystem while creating the TestRun #1637

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7959339
Reading owner attribute from junit xml And not using context owner as…
navin22 May 30, 2018
ab4cd4c
Merge remote-tracking branch 'upstream/master'
navin22 Jun 6, 2018
baee89a
Merge remote-tracking branch 'upstream/master'
navin22 Jun 21, 2018
e3d1221
Using TestRunSystem while creating the TestRun
navin22 Jun 22, 2018
d7a3499
Not modifying the cusotm TestRunTitle if there is only one test resul…
navin22 Jun 22, 2018
00463b6
Adding L0 for TestRunTitle scenarios.
navin22 Jun 22, 2018
f373dd6
Addressed PR Comments
navin22 Jun 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions src/Agent.Worker/TestResults/ResultsCommandExtension.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ public sealed class ResultsCommandExtension : AgentService, IWorkerCommandExtens
private bool _publishRunLevelAttachments;
private int _runCounter = 0;
private readonly object _sync = new object();
private string _testRunSystem;
private const string _testRunSystemCustomFieldName = "TestRunSystem";

public Type ExtensionType => typeof(IWorkerCommandExtension);

Expand Down Expand Up @@ -198,6 +200,7 @@ private async Task PublishAllTestResultsToSingleTestRunAsync(List<string> result
);

testRunData.Attachments = runAttachments.ToArray();
testRunData.AddCustomField(_testRunSystemCustomFieldName, _testRunSystem);

TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, runResults.ToArray(), _executionContext.CancellationToken);
Expand Down Expand Up @@ -229,14 +232,16 @@ private async Task PublishToNewTestRunPerTestResultFileAsync(List<string> result
.Select(bucket => bucket.Select(pair => pair.file).ToList())
.ToList();

bool changeTestRunTitle = resultFiles.Count > 1;

foreach (var files in groupedFiles)
{
// Publish separate test run for each result file that has results.
var publishTasks = files.Select(async resultFile =>
{
cancellationToken.ThrowIfCancellationRequested();
string runName = null;
if (!string.IsNullOrWhiteSpace(_runTitle))
string runName = _runTitle;
if (!string.IsNullOrWhiteSpace(_runTitle) && changeTestRunTitle)
{
runName = GetRunTitle();
}
Expand All @@ -248,6 +253,7 @@ private async Task PublishToNewTestRunPerTestResultFileAsync(List<string> result

if (testRunData != null && testRunData.Results != null && testRunData.Results.Length > 0)
{
testRunData.AddCustomField(_testRunSystemCustomFieldName, _testRunSystem);
TestRun testRun = await publisher.StartTestRunAsync(testRunData, _executionContext.CancellationToken);
await publisher.AddResultsAsync(testRun, testRunData.Results, _executionContext.CancellationToken);
await publisher.EndTestRunAsync(testRunData, testRun.Id, cancellationToken: _executionContext.CancellationToken);
Expand Down Expand Up @@ -356,6 +362,12 @@ private void LoadPublishTestResultsInputs(IExecutionContext context, Dictionary<
_runTitle = string.Empty;
}

eventProperties.TryGetValue(PublishTestResultsEventProperties.TestRunSystem, out _testRunSystem);
if (_testRunSystem == null)
{
_testRunSystem = string.Empty;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

default value should be VSTS instead of empty ? check with PM once.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, but that will come from the caller vsts-task-lib api call. microsoft/azure-pipelines-task-lib#373

}

string publishRunAttachmentsInput;
eventProperties.TryGetValue(PublishTestResultsEventProperties.PublishRunAttachments, out publishRunAttachmentsInput);
if (string.IsNullOrEmpty(publishRunAttachmentsInput) || !bool.TryParse(publishRunAttachmentsInput, out _publishRunLevelAttachments))
Expand Down Expand Up @@ -391,5 +403,6 @@ internal static class PublishTestResultsEventProperties
public static readonly string RunTitle = "runTitle";
public static readonly string PublishRunAttachments = "publishRunAttachments";
public static readonly string ResultFiles = "resultFiles";
public static readonly string TestRunSystem = "testRunSystem";
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TestRunSystem

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought it would be better and consistent if I follow like other properties.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is different from other as in this case this is predefined custom field.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, makes sense. But here it's an even property which will be coming from node. So still having it as is. But when creating the field, having the right case, as you suggested.

}
}
12 changes: 11 additions & 1 deletion src/Agent.Worker/TestResults/TestRunData.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Microsoft.TeamFoundation.TestManagement.WebApi;
using System.Collections.Generic;

namespace Microsoft.VisualStudio.Services.Agent.Worker.TestResults
{
Expand All @@ -7,13 +8,22 @@ public class TestRunData : RunCreateModel
public TestRunData(string name = "", string iteration = "", int[] pointIds = null, ShallowReference plan = null, ShallowReference testSettings = null, int buildId = 0, string state = "", bool? isAutomated = default(bool?), string errorMessage = "", string dueDate = "", string type = "", string controller = "", string buildDropLocation = "", string comment = "", string testEnvironmentId = "", string startedDate = "", string completedDate = "", int[] configIds = null, RunFilter filter = null, ShallowReference dtlTestEnvironment = null, DtlEnvironmentDetails environmentDetails = null, string buildPlatform = null, string buildFlavor = null, string releaseUri = null, string releaseEnvironmentUri = null)
: base(name, iteration, pointIds, plan, testSettings, buildId, state, isAutomated, errorMessage, dueDate, type, controller, buildDropLocation, comment, testEnvironmentId, startedDate, completedDate, configIds, filter, dtlTestEnvironment, environmentDetails, null, buildPlatform, buildFlavor, releaseUri, releaseEnvironmentUri)
{

CustomTestFields = new List<CustomTestField>();
}

public string[] Attachments { get; set; }

// Results
public TestCaseResultData[] Results { get; set; }

public void AddCustomField(string fieldName, string value)
{
if (string.IsNullOrEmpty(value) || string.IsNullOrEmpty(fieldName))
{
return;
}

CustomTestFields.Add(new CustomTestField() { FieldName = fieldName, Value = value });
}
}
}
170 changes: 170 additions & 0 deletions src/Test/L0/Worker/TestResults/ResultsCommandExtensionTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,176 @@ public void VerifyResultsAreMergedWhenPublishingToSingleTestRun()
resultCommand.ProcessCommand(_ec.Object, command);
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "PublishTestResults")]
public void VerifyTestRunSystemPropertyIsSentWhenPublishingToSignleTestRun()
{
SetupMocks();
var resultCommand = new ResultsCommandExtension();
resultCommand.Initialize(_hc);
var command = new Command("results", "publish");
command.Properties.Add("resultFiles", "file1.trx,file2.trx");
command.Properties.Add("type", "NUnit");
command.Properties.Add("mergeResults", bool.TrueString);
command.Properties.Add("testRunSystem", "MAVEN");
var resultsFiles = new List<string> { "file1.trx", "file2.trx" };

var testRunData = new TestRunData();
testRunData.Results = new TestCaseResultData[] { new TestCaseResultData(), new TestCaseResultData() };
testRunData.Attachments = new string[] { "attachment1", "attachment2" };

_mockTestRunPublisher.Setup(q => q.StartTestRunAsync(It.IsAny<TestRunData>(), It.IsAny<CancellationToken>()))
.Callback((TestRunData trd, CancellationToken cancellationToken) =>
{
Assert.NotNull(trd.CustomTestFields);
Assert.NotEmpty(trd.CustomTestFields);
Assert.Equal("testRunSystem", trd.CustomTestFields[0].FieldName);
Assert.Equal("MAVEN", trd.CustomTestFields[0].Value);
});
_mockTestRunPublisher.Setup(q => q.ReadResultsFromFile(It.IsAny<TestRunContext>(), It.IsAny<string>()))
.Returns(testRunData);

resultCommand.ProcessCommand(_ec.Object, command);

// Making sure that the callback is called.
_mockTestRunPublisher.Verify(q => q.StartTestRunAsync(It.IsAny<TestRunData>(), It.IsAny<CancellationToken>()), Times.Once);
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "PublishTestResults")]
public void VerifyTestRunTitleIsModifiedWhenPublishingToMultipleTestRun()
{
SetupMocks();
var resultCommand = new ResultsCommandExtension();
resultCommand.Initialize(_hc);
var command = new Command("results", "publish");
command.Properties.Add("resultFiles", "file1.trx,file2.trx");
command.Properties.Add("type", "NUnit");
command.Properties.Add("mergeResults", bool.FalseString);
command.Properties.Add("runTitle", "TestRunTitle");
var resultsFiles = new List<string> { "file1.trx", "file2.trx" };

var testRunData = new TestRunData();
testRunData.Results = new TestCaseResultData[] { new TestCaseResultData(), new TestCaseResultData() };
testRunData.Attachments = new string[] { "attachment1", "attachment2" };
int counter = 0;
_mockTestRunPublisher.Setup(q => q.StartTestRunAsync(It.IsAny<TestRunData>(), It.IsAny<CancellationToken>()))
.Callback((TestRunData trd, CancellationToken cancellationToken) =>
{
Assert.Equal(StringUtil.Format("{0}_{1}", "TestRunTitle", ++counter), trd.Name);
});
_mockTestRunPublisher.Setup(q => q.ReadResultsFromFile(It.IsAny<TestRunContext>(), It.IsAny<string>(), It.IsAny<string>()))
.Returns(testRunData);

resultCommand.ProcessCommand(_ec.Object, command);

// Making sure that the callback is called.
_mockTestRunPublisher.Verify(q => q.StartTestRunAsync(It.IsAny<TestRunData>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "PublishTestResults")]
public void VerifyTestRunTitleShouldNotBeModifiedWhenPublishingToSingleTestRun()
{
SetupMocks();
var resultCommand = new ResultsCommandExtension();
resultCommand.Initialize(_hc);
var command = new Command("results", "publish");
command.Properties.Add("resultFiles", "file1.trx,file2.trx");
command.Properties.Add("type", "NUnit");
command.Properties.Add("mergeResults", bool.TrueString);
command.Properties.Add("runTitle", "TestRunTitle");
var resultsFiles = new List<string> { "file1.trx", "file2.trx" };

var testRunData = new TestRunData();
testRunData.Results = new TestCaseResultData[] { new TestCaseResultData(), new TestCaseResultData() };
testRunData.Attachments = new string[] { "attachment1", "attachment2" };
_mockTestRunPublisher.Setup(q => q.StartTestRunAsync(It.IsAny<TestRunData>(), It.IsAny<CancellationToken>()))
.Callback((TestRunData trd, CancellationToken cancellationToken) =>
{
Assert.Equal("TestRunTitle", trd.Name);
});
_mockTestRunPublisher.Setup(q => q.ReadResultsFromFile(It.IsAny<TestRunContext>(), It.IsAny<string>()))
.Returns(testRunData);

resultCommand.ProcessCommand(_ec.Object, command);

// Making sure that the callback is called.
_mockTestRunPublisher.Verify(q => q.StartTestRunAsync(It.IsAny<TestRunData>(), It.IsAny<CancellationToken>()), Times.Once);
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "PublishTestResults")]
public void VerifyTestRunTitleShouldNotBeModifiedWhenWhenOnlyOneResultFileIsPublished()
{
SetupMocks();
var resultCommand = new ResultsCommandExtension();
resultCommand.Initialize(_hc);
var command = new Command("results", "publish");
command.Properties.Add("resultFiles", "file1.trx");
command.Properties.Add("type", "NUnit");
// Explicitly not merging it to check if the test run title is not modified when there's only one test file.
command.Properties.Add("mergeResults", bool.FalseString);
command.Properties.Add("runTitle", "TestRunTitle");
var resultsFiles = new List<string> { "file1.trx"};

var testRunData = new TestRunData();
testRunData.Results = new TestCaseResultData[] { new TestCaseResultData()};
testRunData.Attachments = new string[] { "attachment1" };
_mockTestRunPublisher.Setup(q => q.StartTestRunAsync(It.IsAny<TestRunData>(), It.IsAny<CancellationToken>()))
.Callback((TestRunData trd, CancellationToken cancellationToken) =>
{
Assert.Equal("TestRunTitle", trd.Name);
});
_mockTestRunPublisher.Setup(q => q.ReadResultsFromFile(It.IsAny<TestRunContext>(), It.IsAny<string>(), It.IsAny<string>()))
.Returns(testRunData);

resultCommand.ProcessCommand(_ec.Object, command);

// Making sure that the callback is called.
_mockTestRunPublisher.Verify(q => q.StartTestRunAsync(It.IsAny<TestRunData>(), It.IsAny<CancellationToken>()), Times.Once);
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "PublishTestResults")]
public void VerifyTestRunSystemPropertyIsSentWhenPublishingToTestRunPerFile()
{
SetupMocks();
var resultCommand = new ResultsCommandExtension();
resultCommand.Initialize(_hc);
var command = new Command("results", "publish");
command.Properties.Add("resultFiles", "file1.trx,file2.trx");
command.Properties.Add("type", "NUnit");
command.Properties.Add("mergeResults", bool.FalseString);
command.Properties.Add("testRunSystem", "MAVEN");
var resultsFiles = new List<string> { "file1.trx", "file2.trx" };

var testRunData = new TestRunData();
testRunData.Results = new TestCaseResultData[] { new TestCaseResultData(), new TestCaseResultData() };
testRunData.Attachments = new string[] { "attachment1", "attachment2" };

_mockTestRunPublisher.Setup(q => q.StartTestRunAsync(It.IsAny<TestRunData>(), It.IsAny<CancellationToken>()))
.Callback((TestRunData trd, CancellationToken cancellationToken) =>
{
Assert.NotNull(trd.CustomTestFields);
Assert.NotEmpty(trd.CustomTestFields);
Assert.Equal("testRunSystem", trd.CustomTestFields[0].FieldName);
Assert.Equal("MAVEN", trd.CustomTestFields[0].Value);
});

_mockTestRunPublisher.Setup(q => q.ReadResultsFromFile(It.IsAny<TestRunContext>(), It.IsAny<string>(), It.IsAny<string>()))
.Returns(testRunData);
resultCommand.ProcessCommand(_ec.Object, command);

// There should be two calls to startestrun
_mockTestRunPublisher.Verify(q=>q.StartTestRunAsync(It.IsAny<TestRunData>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
}

[Fact]
[Trait("Level", "L0")]
[Trait("Category", "PublishTestResults")]
Expand Down