Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[wasm] workaround corrupted test results file #769

Merged
merged 1 commit into from
Nov 19, 2021
Merged

[wasm] workaround corrupted test results file #769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 23 additions & 3 deletions src/Microsoft.DotNet.XHarness.CLI/Commands/WASM/WasmTestMessagesProcessor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@
using System.IO;
using System.Text.Json;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;

#nullable enable
namespace Microsoft.DotNet.XHarness.CLI.Commands.Wasm
{
public class WasmTestMessagesProcessor
{
private MemoryStream? _xmlResultsMemoryStream;
private StreamWriter? _xmlResultsFileWriter;
private readonly StreamWriter _stdoutFileWriter;
private readonly string _xmlResultsFilePath;
Expand Down Expand Up @@ -80,7 +82,8 @@ private void InvokeInternal(string message)
if (line.Contains("STARTRESULTXML"))
{
_logger.LogDebug("Reached start of testResults.xml");
_xmlResultsFileWriter = File.CreateText(_xmlResultsFilePath);
_xmlResultsMemoryStream = new MemoryStream();
_xmlResultsFileWriter = new StreamWriter(_xmlResultsMemoryStream);
return;
}
else if (line.StartsWith("[PASS]") || line.StartsWith("[SKIP]"))
Expand Down Expand Up @@ -114,10 +117,27 @@ private void InvokeInternal(string message)
{
if (line.Contains("ENDRESULTXML"))
{
_logger.LogDebug("Reached end of testResults.xml");
_logger.LogDebug($"Reached end of {_xmlResultsFilePath}");
_xmlResultsFileWriter.Flush();
_xmlResultsMemoryStream!.Seek(0, SeekOrigin.Begin);
try
{
// we validate it, to make sure it's not corrupted.
var testResults = XElement.Load(_xmlResultsMemoryStream);
using (var testResultsFile = File.CreateText(_xmlResultsFilePath))
{
testResults.Save(testResultsFile);
_logger.LogInformation($"Written {_xmlResultsMemoryStream.Length} original bytes of {_xmlResultsFilePath} as {testResultsFile.BaseStream.Length} formatted bytes");
};
}
catch (Exception ex)
{
_logger.LogError($"Error while saving testResults.xml {ex}");
}
_xmlResultsFileWriter.Dispose();
_xmlResultsFileWriter = null;
_xmlResultsMemoryStream.Dispose();
_xmlResultsMemoryStream = null;
return;
}

Expand All @@ -129,7 +149,7 @@ private void InvokeInternal(string message)
// after the tests have run, and the xml results file
// has been written to the console
if (line.StartsWith("WASM EXIT"))
{
{
_logger.LogDebug("Reached wasm exit");
WasmExitReceivedTcs.SetResult(true);
}
Expand Down
136 changes: 81 additions & 55 deletions src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
Expand All @@ -21,62 +23,86 @@ internal class ThreadlessXunitTestRunner
{
public static async Task<int> Run(string assemblyFileName, bool printXml, XunitFilters filters)
{
var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false };
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);

discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);

Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);

discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)");

var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);

if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null)
{
testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); };
try
{
var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false };
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);

discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);

Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);

discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)");

var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);

if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null)
{
testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); };
}
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };

testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); };

controller.RunTests(testCasesToRun, resultsSink, testOptions);

while (!resultsSink.Finished.WaitOne(0))
{
await Task.Delay(1);
}

if (printXml)
{
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
using (var ms = new MemoryStream())
{
using (var tw = new StreamWriter(ms, Encoding.UTF8, 100*1024, true))
{
// no new line. make it single line, single WS message
resultsXml.Save(tw, SaveOptions.DisableFormatting);
}
ms.Seek(0, SeekOrigin.Begin);
using (var stdout = Console.OpenStandardOutput())
{
using (var twc = new StreamWriter(stdout))
{
twc.WriteLine($"STARTRESULTXML {ms.Length}");
twc.Flush();
ms.CopyTo(stdout, 100 * 1024);
twc.WriteLine();
twc.WriteLine("ENDRESULTXML");
}
}
Console.WriteLine($"Finished writing {ms.Length} bytes of RESULTXML");
}
}

var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return failed ? 1 : 0;
}
catch (Exception ex)
{
Console.Error.WriteLine($"ThreadlessXunitTestRunner failed: {ex}");
return 2;
}
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };

testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); };

controller.RunTests(testCasesToRun, resultsSink, testOptions);

while (!resultsSink.Finished.WaitOne(0))
{
await Task.Delay(1);
}

if (printXml)
{
Console.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(Console.Out);
Console.WriteLine();
Console.WriteLine($"ENDRESULTXML");
}

var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
Console.Out.Flush();
return failed ? 1 : 0;
}
}

Expand Down