Skip to content

Commit

Permalink
Blame for vstest (#915)
Browse files Browse the repository at this point in the history
* Feature blame (#1)

This feature gets the faulty test case name in event of a test host crash of reason unknown.
There are various scenarios in which the test host crashes due to a faulty test case and the reason cannot be diagnosed.
This pull request gets the name of the test case which caused the crash and displays it on the console.
To enable the blame mode, use --Blame|/Blame

* Unit tests added and minor changes

* Suggested changes made

* Spaces removed

* Build updated

* Integration test added

* Blame Processor priority changed in test

* BlameUnitTestProject updated

* SDK Previous version updated to 15.3.0-preview-20170628-02
  • Loading branch information
vagisha-nidhi authored and codito committed Jul 14, 2017
1 parent a08c38a commit c62f4d3
Show file tree
Hide file tree
Showing 59 changed files with 2,344 additions and 53 deletions.
47 changes: 46 additions & 1 deletion TestPlatform.sln
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.26326.0
VisualStudioVersion = 15.0.26605.1
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{ED0C35EB-7F31-4841-A24F-8EB708FFA959}"
EndProject
Expand Down Expand Up @@ -151,6 +151,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.TestPlatform.Test
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.TestPlatform.TestHostProvider", "src\Microsoft.TestPlatform.TestHostProvider\Microsoft.TestPlatform.TestHostProvider.csproj", "{11ECCB8B-6958-42A7-BD58-88C09CB149B2}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.TestPlatform.Extensions.BlameDataCollector", "src\Microsoft.TestPlatform.Extensions.BlameDataCollector\Microsoft.TestPlatform.Extensions.BlameDataCollector.csproj", "{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests", "test\Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests\Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj", "{488675EC-C8BB-40E0-AD4F-91F623D548B3}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BlameUnitTestProject", "test\TestAssets\BlameUnitTestProject\BlameUnitTestProject.csproj", "{6B2B841C-CCFF-469A-9939-EB07EA0401AE}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -785,6 +791,42 @@ Global
{11ECCB8B-6958-42A7-BD58-88C09CB149B2}.Release|x64.Build.0 = Release|Any CPU
{11ECCB8B-6958-42A7-BD58-88C09CB149B2}.Release|x86.ActiveCfg = Release|Any CPU
{11ECCB8B-6958-42A7-BD58-88C09CB149B2}.Release|x86.Build.0 = Release|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Debug|x64.ActiveCfg = Debug|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Debug|x64.Build.0 = Debug|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Debug|x86.ActiveCfg = Debug|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Debug|x86.Build.0 = Debug|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Release|Any CPU.Build.0 = Release|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Release|x64.ActiveCfg = Release|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Release|x64.Build.0 = Release|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Release|x86.ActiveCfg = Release|Any CPU
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A}.Release|x86.Build.0 = Release|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Debug|x64.ActiveCfg = Debug|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Debug|x64.Build.0 = Debug|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Debug|x86.ActiveCfg = Debug|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Debug|x86.Build.0 = Debug|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Release|Any CPU.Build.0 = Release|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Release|x64.ActiveCfg = Release|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Release|x64.Build.0 = Release|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Release|x86.ActiveCfg = Release|Any CPU
{488675EC-C8BB-40E0-AD4F-91F623D548B3}.Release|x86.Build.0 = Release|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Debug|x64.ActiveCfg = Debug|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Debug|x64.Build.0 = Debug|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Debug|x86.ActiveCfg = Debug|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Debug|x86.Build.0 = Debug|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Release|Any CPU.Build.0 = Release|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Release|x64.ActiveCfg = Release|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Release|x64.Build.0 = Release|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Release|x86.ActiveCfg = Release|Any CPU
{6B2B841C-CCFF-469A-9939-EB07EA0401AE}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -851,5 +893,8 @@ Global
{7B48115A-B766-4B55-93A8-C08A42C37710} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6}
{FBF74C8F-695C-4967-84AC-358EEFB1376D} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6}
{11ECCB8B-6958-42A7-BD58-88C09CB149B2} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959}
{76D4BB7E-D981-42D5-BE96-6FAD8DEF9A4A} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959}
{488675EC-C8BB-40E0-AD4F-91F623D548B3} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6}
{6B2B841C-CCFF-469A-9939-EB07EA0401AE} = {8DA7CBD9-F17E-41B6-90C4-CFF55848A25A}
EndGlobalSection
EndGlobal
8 changes: 8 additions & 0 deletions scripts/build.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,14 @@ function Publish-Package
Write-Verbose "Move-Item $coreCLR20PackageDir\$file $coreCLRExtensionsDir -Force"
Move-Item $coreCLR20PackageDir\$file $coreCLRExtensionsDir -Force
}

# Publish Datacollector
$TPB_TargetFrameworkStandard = "netstandard1.5"
$blameDataCollector = Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.Extensions.BlameDataCollector\bin\$TPB_Configuration"
$blameDataCollectorNet46 = Join-Path $blameDataCollector $TPB_TargetFramework
$blameDataCollectorNetStandard = Join-Path $blameDataCollector $TPB_TargetFrameworkStandard
Copy-Item $blameDataCollectorNet46\* $fullCLRExtensionsDir -Force
Copy-Item $blameDataCollectorNetStandard\* $coreCLRExtensionsDir -Force

# Note Note: If there are some dependencies for the TestHostRuntimeProvider assemblies, those need to be moved too.
$runtimeproviders = @("Microsoft.TestPlatform.TestHostRuntimeProvider.dll", "Microsoft.TestPlatform.TestHostRuntimeProvider.pdb")
Expand Down
2 changes: 1 addition & 1 deletion scripts/build/TestPlatform.Dependencies.props
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<VSSdkBuildToolsVersion>15.0.26201</VSSdkBuildToolsVersion>

<!-- Name of the elements must be in sync with test\Microsoft.TestPlatform.TestUtilities\IntegrationTestBase.cs -->
<NETTestSdkPreviousVersion>15.0.0</NETTestSdkPreviousVersion>
<NETTestSdkPreviousVersion>15.3.0-preview-20170628-02</NETTestSdkPreviousVersion>

<MSTestFrameworkVersion>1.2.0-beta</MSTestFrameworkVersion>
<MSTestAdapterVersion>1.2.0-beta</MSTestAdapterVersion>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,6 @@ private void ProcessRequests(ITestRequestManager testRequestManager)

case MessageType.StartDiscovery:
{

var discoveryPayload = this.dataSerializer.DeserializePayload<DiscoveryRequestPayload>(message);
this.StartDiscovery(discoveryPayload, testRequestManager);
break;
Expand Down
1 change: 1 addition & 0 deletions src/Microsoft.TestPlatform.Common/Friends.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,6 @@
[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]
[assembly: InternalsVisibleTo("Microsoft.TestPlatform.CommunicationUtilities.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
[assembly: InternalsVisibleTo("Microsoft.TestPlatform.Client.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
[assembly: InternalsVisibleTo("Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]

#endregion
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Logging
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;

using CommonResources = Microsoft.VisualStudio.TestPlatform.Common.Resources.Resources;
using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework;

/// <summary>
/// Responsible for managing logger extensions and broadcasting results
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.TestPlatform.Extensions.BlameDataCollector
{
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Xml;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection;

/// <summary>
/// The blame collector.
/// </summary>
[DataCollectorFriendlyName("Blame")]
[DataCollectorTypeUri("datacollector://Microsoft/TestPlatform/Extensions/Blame/v1")]
public class BlameCollector : DataCollector, ITestExecutionEnvironmentSpecifier
{
private DataCollectionSink dataCollectionSink;
private DataCollectionEnvironmentContext context;
private DataCollectionEvents events;
private List<TestCase> testSequence;
private IBlameReaderWriter blameReaderWriter;
private XmlElement configurationElement;
private int testStartCount;
private int testEndCount;

/// <summary>
/// Initializes a new instance of the <see cref="BlameCollector"/> class.
/// Using XmlReaderWriter by default
/// </summary>
public BlameCollector()
: this(new XmlReaderWriter())
{
}

/// <summary>
/// Initializes a new instance of the <see cref="BlameCollector"/> class.
/// </summary>
/// <param name="blameReaderWriter">
/// BlameReaderWriter instance.
/// </param>
protected BlameCollector(IBlameReaderWriter blameReaderWriter)
{
this.blameReaderWriter = blameReaderWriter;
}

/// <summary>
/// Gets environment variables that should be set in the test execution environment
/// </summary>
/// <returns>Environment variables that should be set in the test execution environment</returns>
public IEnumerable<KeyValuePair<string, string>> GetTestExecutionEnvironmentVariables()
{
return Enumerable.Empty<KeyValuePair<string, string>>();
}

/// <summary>
/// Initializes parameters for the new instance of the class <see cref="BlameDataCollector"/>
/// </summary>
/// <param name="configurationElement">The Xml Element to save to</param>
/// <param name="events">Data collection events to which methods subscribe</param>
/// <param name="dataSink">A data collection sink for data transfer</param>
/// <param name="logger">Data Collection Logger to send messages to the client </param>
/// <param name="environmentContext">Context of data collector environment</param>
public override void Initialize(
XmlElement configurationElement,
DataCollectionEvents events,
DataCollectionSink dataSink,
DataCollectionLogger logger,
DataCollectionEnvironmentContext environmentContext)
{
ValidateArg.NotNull(logger, nameof(logger));

this.events = events;
this.dataCollectionSink = dataSink;
this.context = environmentContext;
this.configurationElement = configurationElement;
this.testSequence = new List<TestCase>();

// Subscribing to events
this.events.SessionEnd += this.SessionEnded_Handler;
this.events.TestCaseStart += this.EventsTestCaseStart;
this.events.TestCaseEnd += this.EventsTestCaseEnd;
}

/// <summary>
/// Called when Test Case Start event is invoked
/// </summary>
/// <param name="sender">Sender</param>
/// <param name="e">TestCaseStartEventArgs</param>
private void EventsTestCaseStart(object sender, TestCaseStartEventArgs e)
{
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("Blame Collector : Test Case Start");
}

this.testSequence.Add(e.TestElement);
this.testStartCount++;
}

/// <summary>
/// Called when Test Case End event is invoked
/// </summary>
/// <param name="sender">Sender</param>
/// <param name="e">TestCaseEndEventArgs</param>
private void EventsTestCaseEnd(object sender, TestCaseEndEventArgs e)
{
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("Blame Collector : Test Case End");
}

this.testEndCount++;
}

/// <summary>
/// Called when Session End event is invoked
/// </summary>
/// <param name="sender">Sender</param>
/// <param name="args">SessionEndEventArgs</param>
private void SessionEnded_Handler(object sender, SessionEndEventArgs args)
{
if (EqtTrace.IsInfoEnabled)
{
EqtTrace.Info("Blame Collector : Session End");
}

// If the last test crashes, it will not invoke a test case end and therefore
// In case of crash testStartCount will be greater than testEndCount and we need to write the sequence
// And send the attachment
if (this.testStartCount > this.testEndCount)
{
var filepath = Path.Combine(this.GetResultsDirectory(), Constants.AttachmentFileName);
filepath = this.blameReaderWriter.WriteTestSequence(this.testSequence, filepath);
var fileTranferInformation = new FileTransferInformation(this.context.SessionDataCollectionContext, filepath, true);
this.dataCollectionSink.SendFileAsync(fileTranferInformation);
}

this.DeregisterEvents();
}

/// <summary>
/// Method to deregister handlers and cleanup
/// </summary>
private void DeregisterEvents()
{
this.events.SessionEnd -= this.SessionEnded_Handler;
this.events.TestCaseStart -= this.EventsTestCaseStart;
this.events.TestCaseEnd -= this.EventsTestCaseEnd;
}

private string GetResultsDirectory()
{
try
{
XmlElement resultsDirectoryElement = this.configurationElement["ResultsDirectory"];
string resultsDirectory = resultsDirectoryElement != null ? resultsDirectoryElement.InnerText : string.Empty;
return resultsDirectory;
}
catch (NullReferenceException exception)
{
if (EqtTrace.IsErrorEnabled)
{
EqtTrace.Error("Blame Collector : " + exception);
}

return string.Empty;
}
}
}
}
Loading

0 comments on commit c62f4d3

Please sign in to comment.