From d682eb2c93540f7b857b5675a823d9d488d54353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Thu, 3 Oct 2024 14:01:07 +0200 Subject: [PATCH 01/26] import tests from PR 119 --- .../MsTest/MethodLevelParallelisation.feature | 154 ++++++++++++++++++ .../ProjectBuilder.cs | 2 +- 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature diff --git a/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature b/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature new file mode 100644 index 000000000..6ff86a21d --- /dev/null +++ b/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature @@ -0,0 +1,154 @@ +@xUnit @NUnit3 @MSTest +Feature: MethodLevel Parallelisation + +Background: + Given there is a Reqnroll project + And parallel execution is enabled + And the following binding class + """ +using Reqnroll; +using Reqnroll.Tracing; +using System.Collections.Concurrent; +using System.Diagnostics; +using FluentAssertions; + +[Binding] +public class TraceSteps +{ + sealed class FeatureData + { + public Stopwatch Duration { get; } = Stopwatch.StartNew(); + public volatile int StepCount; + public ConcurrentDictionary TestRunners { get; } = new ConcurrentDictionary(); + public ConcurrentDictionary FeatureContexts { get; } = new ConcurrentDictionary(); + public ConcurrentDictionary ScenarioContexts { get; } = new ConcurrentDictionary(); + public ConcurrentDictionary BindingInstances { get; } = new ConcurrentDictionary(); + } + + private readonly ITraceListener _traceListener; + private readonly ITestRunner _testRunner; + private volatile int _ScenarioLocalCounter; + + public TraceSteps(ITraceListener traceListener, ITestRunner testRunner) + { + _traceListener = traceListener; + _testRunner = testRunner; + + Interlocked.Increment(ref _ScenarioLocalCounter); + } + + [BeforeFeature] + static void BeforeFeature(FeatureContext featureContext, ITestRunner testRunner) + { + testRunner.ScenarioContext.Should().BeNull(); + + var featureData = new FeatureData(); + featureData.TestRunners.TryAdd(testRunner, 1); + featureData.FeatureContexts.TryAdd(featureContext, 1); + featureContext.Set(featureData); + } + + static FeatureData GetFeatureData(FeatureContext featureContext) => featureContext.Get(); + const int WaitTimeInMS = 1_000; + + [AfterFeature] + static void AfterFeature(FeatureContext featureContext, ITestRunner testRunner) + { + testRunner.ScenarioContext.Should().BeNull(); + + var featureData = GetFeatureData(featureContext); + featureData.TestRunners.TryAdd(testRunner, 1).Should().BeFalse(); + featureData.Duration.Stop(); + featureData.TestRunners.Count.Should().Be(11, because: "One TestRunner for before/after hooks and one for each test is created"); + featureData.FeatureContexts.Count.Should().Be(1, because: "Only one FeatureContext should be created"); + featureData.ScenarioContexts.Count.Should().Be(10, because: "One ScenarioContext for each test is created"); + featureData.BindingInstances.Count.Should().Be(10, because: "One binding instance for each test is created"); + featureData.Duration.ElapsedMilliseconds.Should().BeLessThan(9 * WaitTimeInMS, because: "Test should be processed (parallel) in time"); + } + + [When(@"I do something in Scenario '(.*)'")] + void WhenIDoSomething(string scenario) + { + _testRunner.ScenarioContext.Should().NotBeNull(); + _testRunner.ScenarioContext.ScenarioInfo.Title.Should().Be(scenario); + + Interlocked.Increment(ref _ScenarioLocalCounter); + _ScenarioLocalCounter.Should().Be(2); + + var featureData = GetFeatureData(_testRunner.FeatureContext); + featureData.TestRunners.TryAdd(_testRunner, 1); + featureData.FeatureContexts.TryAdd(_testRunner.FeatureContext, 1); + featureData.ScenarioContexts.TryAdd(_testRunner.ScenarioContext, 1); + featureData.BindingInstances.TryAdd(this, 1); + var currentStartIndex = Interlocked.Increment(ref featureData.StepCount); + _traceListener.WriteTestOutput($"Start index: {currentStartIndex}, Worker: {_testRunner.TestWorkerId}"); + Thread.Sleep(WaitTimeInMS); + var afterStartIndex = featureData.StepCount; + if (afterStartIndex == currentStartIndex) + { + _traceListener.WriteTestOutput("Was not parallel"); + } + else + { + _traceListener.WriteTestOutput("Was parallel"); + } + } +} + """ + + And there is a feature file in the project as + """ +Feature: Feature 1 +Scenario Outline: Simple Scenario Outline 1 + When I do something in Scenario 'Simple Scenario Outline 1' + +Examples: + | Count | + | 1 | + | 2 | + | 3 | + +Scenario Outline: Simple Scenario Outline 2 + When I do something in Scenario 'Simple Scenario Outline 2' + +Examples: + | Count | + | 1 | + | 2 | + | 3 | + +Scenario Outline: Simple Scenario Outline 3 + When I do something in Scenario 'Simple Scenario Outline 3' + +Scenario Outline: Simple Scenario Outline 4 + When I do something in Scenario 'Simple Scenario Outline 4' + +Scenario Outline: Simple Scenario Outline 5 + When I do something in Scenario 'Simple Scenario Outline 5' + +Scenario Outline: Simple Scenario Outline 6 + When I do something in Scenario 'Simple Scenario Outline 6' + """ + +Scenario: Precondition: Tests run parallel + When I execute the tests + Then the execution log should contain text 'Was parallel' + +Scenario: Tests should be processed parallel without failure + When I execute the tests + Then the execution log should contain text 'Was parallel' + And the execution summary should contain + | Total | Succeeded | + | 10 | 10 | + +Scenario Outline: Before/After TestRun hook should only be executed once + Given a hook 'HookFor' for '' + When I execute the tests + Then the execution log should contain text 'Was parallel' + And the hook 'HookFor' is executed once + +Examples: + | event | + | BeforeTestRun | + | AfterTestRun | + diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs index 12b73ec45..b37dbcbb6 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs @@ -371,7 +371,7 @@ private void AddUnitTestProviderSpecificConfig() break; case UnitTestProvider.MSTest when _parallelTestExecution: _project.AddFile( - new ProjectFile("MsTestConfiguration.cs", "Compile", "using Microsoft.VisualStudio.TestTools.UnitTesting; [assembly: Parallelize(Workers = 4, Scope = ExecutionScope.ClassLevel)]")); + new ProjectFile("MsTestConfiguration.cs", "Compile", "using Microsoft.VisualStudio.TestTools.UnitTesting; [assembly: Parallelize(Workers = 4, Scope = ExecutionScope.MethodLevel)]")); break; case UnitTestProvider.MSTest when !_parallelTestExecution: _project.AddFile(new ProjectFile("MsTestConfiguration.cs", "Compile", "using Microsoft.VisualStudio.TestTools.UnitTesting; [assembly: DoNotParallelize]")); From 83624edb71e66c80c96c54cc7c6f5cc93fcc7831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Thu, 3 Oct 2024 18:21:44 +0200 Subject: [PATCH 02/26] small code cleanup --- Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs | 2 +- Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs index 404d22d75..a8b581679 100644 --- a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs +++ b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs @@ -233,7 +233,7 @@ private void SetupTestClassCleanupMethod(TestClassGenerationContext generationCo testClassCleanupMethod.Statements.Add(expression); - // + // TestRunnerManager.ReleaseTestRunner(testRunner); testClassCleanupMethod.Statements.Add( new CodeMethodInvokeExpression( new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))), diff --git a/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs index cb2b92a4d..dcce50673 100644 --- a/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs @@ -128,7 +128,7 @@ public virtual void SetTestInitializeMethod(TestClassGenerationContext generatio protected virtual void FixTestRunOrderingIssue(TestClassGenerationContext generationContext) { - //see https://github.com/reqnroll/Reqnroll/issues/96 + //see https://github.com/SpecFlowOSS/SpecFlow/issues/96 //if (testRunner.FeatureContext != null && testRunner.FeatureContext.FeatureInfo.Title != "") // .(null); From d56b04fbafea3b0a609f3be4382c0b1293683c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Thu, 3 Oct 2024 18:30:06 +0200 Subject: [PATCH 03/26] Prototype to support method-level parallelization for MsTest --- .../Generation/UnitTestFeatureGenerator.cs | 122 ++++++++++++++++-- .../MsTestGeneratorProvider.cs | 15 ++- .../MsTest/MethodLevelParallelisation.feature | 16 +++ 3 files changed, 140 insertions(+), 13 deletions(-) diff --git a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs index a8b581679..a64f44611 100644 --- a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs +++ b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs @@ -189,16 +189,16 @@ private void SetupTestClassInitializeMethod(TestClassGenerationContext generatio //FeatureInfo featureInfo = new FeatureInfo("xxxx"); testClassInitializeMethod.Statements.Add( new CodeVariableDeclarationStatement(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), "featureInfo", - new CodeObjectCreateExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), - new CodeObjectCreateExpression(typeof(CultureInfo), - new CodePrimitiveExpression(generationContext.Feature.Language)), - new CodePrimitiveExpression(generationContext.Document.DocumentLocation?.FeatureFolderPath), - new CodePrimitiveExpression(generationContext.Feature.Name), - new CodePrimitiveExpression(generationContext.Feature.Description), - new CodeFieldReferenceExpression( - new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(Reqnroll.ProgrammingLanguage))), - _codeDomHelper.TargetLanguage.ToString()), - new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME)))); + new CodeObjectCreateExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), + new CodeObjectCreateExpression(typeof(CultureInfo), + new CodePrimitiveExpression(generationContext.Feature.Language)), + new CodePrimitiveExpression(generationContext.Document.DocumentLocation?.FeatureFolderPath), + new CodePrimitiveExpression(generationContext.Feature.Name), + new CodePrimitiveExpression(generationContext.Feature.Description), + new CodeFieldReferenceExpression( + new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(Reqnroll.ProgrammingLanguage))), + _codeDomHelper.TargetLanguage.ToString()), + new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME)))); //await testRunner.OnFeatureStartAsync(featureInfo); var onFeatureStartExpression = new CodeMethodInvokeExpression( @@ -257,6 +257,97 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont _codeDomHelper.MarkCodeMemberMethodAsAsync(testInitializeMethod); _testGeneratorProvider.SetTestInitializeMethod(generationContext); + + if (generationContext.UnitTestGeneratorProvider is not MsTestGeneratorProvider) + return; // only MsTest is implemented in this prototype + + // Step 4: Obtain the test runner for executing a single test + + // testRunner = global::Reqnroll.TestRunnerManager.GetTestRunnerForAssembly(); + + var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression(); + + var getTestRunnerExpression = new CodeMethodInvokeExpression( + new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))), + nameof(TestRunnerManager.GetTestRunnerForAssembly)); + + testInitializeMethod.Statements.Add( + new CodeAssignStatement( + testRunnerField, + getTestRunnerExpression)); + + + // Step 5 (part 1): "Finish" current feature if needed & "Start" feature if needed + // The similar code in custom test runner codes is not needed + // The feature initialization steps are copied from TestClassInitializeMethod + + //if (testRunner.FeatureContext != null && testRunner.FeatureContext.FeatureInfo.Title != "") + // await testRunner.OnFeatureEndAsync(); // finish if different + + var featureContextExpression = new CodePropertyReferenceExpression( + testRunnerField, + "FeatureContext"); + + var onFeatureEndAsyncExpression = new CodeMethodInvokeExpression( + testRunnerField, + nameof(ITestRunner.OnFeatureEndAsync)); + _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureEndAsyncExpression); + + //if (testRunner.FeatureContext != null && testRunner.FeatureContext.FeatureInfo.Title != "") + // await testRunner.OnFeatureEndAsync(); // finish if different + testInitializeMethod.Statements.Add( + new CodeConditionStatement( + new CodeBinaryOperatorExpression( + new CodeBinaryOperatorExpression( + featureContextExpression, + CodeBinaryOperatorType.IdentityInequality, + new CodePrimitiveExpression(null)), + CodeBinaryOperatorType.BooleanAnd, + new CodeBinaryOperatorExpression( + new CodePropertyReferenceExpression( + new CodePropertyReferenceExpression( + featureContextExpression, + "FeatureInfo"), + "Title"), + CodeBinaryOperatorType.IdentityInequality, + new CodePrimitiveExpression(generationContext.Feature.Name))), + new CodeExpressionStatement( + onFeatureEndAsyncExpression))); + + + //if (testRunner.FeatureContext == null) { + // FeatureInfo featureInfo = new FeatureInfo("xxxx"); + // await testRunner.OnFeatureStartAsync(featureInfo); + //} + + var featureInfoInitializeStatement = + new CodeVariableDeclarationStatement(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), "featureInfo", + new CodeObjectCreateExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), + new CodeObjectCreateExpression(typeof(CultureInfo), + new CodePrimitiveExpression(generationContext.Feature.Language)), + new CodePrimitiveExpression(generationContext.Document.DocumentLocation?.FeatureFolderPath), + new CodePrimitiveExpression(generationContext.Feature.Name), + new CodePrimitiveExpression(generationContext.Feature.Description), + new CodeFieldReferenceExpression( + new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(Reqnroll.ProgrammingLanguage))), + _codeDomHelper.TargetLanguage.ToString()), + new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME))); + + var onFeatureStartExpression = new CodeMethodInvokeExpression( + testRunnerField, + nameof(ITestRunner.OnFeatureStartAsync), + new CodeVariableReferenceExpression("featureInfo")); + _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureStartExpression); + + testInitializeMethod.Statements.Add( + new CodeConditionStatement( + new CodeBinaryOperatorExpression( + featureContextExpression, + CodeBinaryOperatorType.IdentityEquality, + new CodePrimitiveExpression(null)), + featureInfoInitializeStatement, + new CodeExpressionStatement( + onFeatureStartExpression))); } private void SetupTestCleanupMethod(TestClassGenerationContext generationContext) @@ -280,6 +371,17 @@ private void SetupTestCleanupMethod(TestClassGenerationContext generationContext _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression); testCleanupMethod.Statements.Add(expression); + + if (generationContext.UnitTestGeneratorProvider is not MsTestGeneratorProvider) + return; // only MsTest is implemented in this prototype + + // Step 6: "Release" the TestRunner, so that other threads can pick it up (moved from TestClassCleanupMethod) + // TestRunnerManager.ReleaseTestRunner(testRunner); + testCleanupMethod.Statements.Add( + new CodeMethodInvokeExpression( + new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))), + nameof(TestRunnerManager.ReleaseTestRunner), + testRunnerField)); } private void SetupScenarioInitializeMethod(TestClassGenerationContext generationContext) diff --git a/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs index dcce50673..df8e4a57d 100644 --- a/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs @@ -103,12 +103,16 @@ public virtual void SetTestClassNonParallelizable(TestClassGenerationContext gen public virtual void SetTestClassInitializeMethod(TestClassGenerationContext generationContext) { generationContext.TestClassInitializeMethod.Attributes |= MemberAttributes.Static; - generationContext.TestRunnerField.Attributes |= MemberAttributes.Static; + // Step 1: make 'testRunner' instance field + //generationContext.TestRunnerField.Attributes |= MemberAttributes.Static; generationContext.TestClassInitializeMethod.Parameters.Add(new CodeParameterDeclarationExpression( TESTCONTEXT_TYPE, "testContext")); CodeDomHelper.AddAttribute(generationContext.TestClassInitializeMethod, TESTFIXTURESETUP_ATTR); + + // Step 2: Remove TestClassInitializeMethod (not needed) + generationContext.TestClass.Members.Remove(generationContext.TestClassInitializeMethod); } public void SetTestClassCleanupMethod(TestClassGenerationContext generationContext) @@ -117,13 +121,18 @@ public void SetTestClassCleanupMethod(TestClassGenerationContext generationConte // [Microsoft.VisualStudio.TestTools.UnitTesting.ClassCleanupAttribute(Microsoft.VisualStudio.TestTools.UnitTesting.ClassCleanupBehavior.EndOfClass)] var attribute = CodeDomHelper.AddAttribute(generationContext.TestClassCleanupMethod, TESTFIXTURETEARDOWN_ATTR); attribute.Arguments.Add(new CodeAttributeArgument(new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(CLASSCLEANUPBEHAVIOR_ENUM), CLASSCLEANUPBEHAVIOR_ENDOFCLASS))); + + // Step 3: Remove TestClassCleanupMethod (not needed) + generationContext.TestClass.Members.Remove(generationContext.TestClassCleanupMethod); } public virtual void SetTestInitializeMethod(TestClassGenerationContext generationContext) { CodeDomHelper.AddAttribute(generationContext.TestInitializeMethod, TESTSETUP_ATTR); - FixTestRunOrderingIssue(generationContext); + + // Step 5 (part 2): remove feature handling from here as it is moved to UnitTestGenerator.SetupTestInitializeMethod + //FixTestRunOrderingIssue(generationContext); } protected virtual void FixTestRunOrderingIssue(TestClassGenerationContext generationContext) @@ -146,7 +155,7 @@ protected virtual void FixTestRunOrderingIssue(TestClassGenerationContext genera new CodePrimitiveExpression(null)); CodeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(callTestClassInitializeMethodExpression); - + generationContext.TestInitializeMethod.Statements.Add( new CodeConditionStatement( new CodeBinaryOperatorExpression( diff --git a/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature b/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature index 6ff86a21d..72b9b5e61 100644 --- a/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature +++ b/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature @@ -66,6 +66,8 @@ public class TraceSteps featureData.Duration.ElapsedMilliseconds.Should().BeLessThan(9 * WaitTimeInMS, because: "Test should be processed (parallel) in time"); } + public static int startIndex = 0; + [When(@"I do something in Scenario '(.*)'")] void WhenIDoSomething(string scenario) { @@ -82,6 +84,10 @@ public class TraceSteps featureData.BindingInstances.TryAdd(this, 1); var currentStartIndex = Interlocked.Increment(ref featureData.StepCount); _traceListener.WriteTestOutput($"Start index: {currentStartIndex}, Worker: {_testRunner.TestWorkerId}"); + + var currentStartIndex2 = System.Threading.Interlocked.Increment(ref startIndex); + _traceListener.WriteTestOutput($"Start index2: {currentStartIndex2}, Worker: {_testRunner.TestWorkerId}"); + Thread.Sleep(WaitTimeInMS); var afterStartIndex = featureData.StepCount; if (afterStartIndex == currentStartIndex) @@ -92,6 +98,16 @@ public class TraceSteps { _traceListener.WriteTestOutput("Was parallel"); } + + var afterStartIndex2 = startIndex; + if (afterStartIndex2 == currentStartIndex2) + { + _traceListener.WriteTestOutput("XWas not parallel"); + } + else + { + _traceListener.WriteTestOutput("XWas parallel"); + } } } """ From cb697fcac1ab6c7b490203fe02a99f58b6923ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Fri, 4 Oct 2024 15:47:20 +0200 Subject: [PATCH 04/26] Add SystemTests for parallel execution --- .../Generation/GenerationTestBase.cs | 99 ++++++++++++++++++- .../FeatureFileGenerator.cs | 2 +- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs index c145706cb..b6573fdf5 100644 --- a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs +++ b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs @@ -1,4 +1,5 @@ -using System.Linq; +using System; +using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; using FluentAssertions; using Reqnroll.TestProjectGenerator; @@ -431,5 +432,99 @@ public async Task WhenThisTableIsProcessed(DataTable table) #endregion - //TODO: test parallel execution (details TBD) - maybe this should be in a separate test class + #region It is able to run tests parallel + + // We need to verify + // Tests are run in parallel + // Each scenario execution has a feature context and scenario context that is matching to the feature/scenario + // For each scenario the before feature has been executed for that feature context + + [TestMethod] + public void Tests_can_be_executed_parallel() + { + _projectsDriver.EnableTestParallelExecution(); + + var scenarioTemplate = + """ + Scenario: {0} + When executing '{0}' in '{1}' + """; + + var scenarioOutlineTemplate = + """ + Scenario Outline: {0} + When executing '{0}' in '{1}' + Examples: + | Count | + | 1 | + | 2 | + | 3 | + """; + + const int scenariosPerFile = 3; + const int scenarioOutlinesPerFile = 3; + string[] features = ["A", "B"]; + int scenarioIdCounter = 0; + + foreach (string feature in features) + { + AddFeatureFile($"Feature: {feature}" + Environment.NewLine); + for (int i = 0; i < scenariosPerFile; i++) + AddScenario(string.Format(scenarioTemplate, $"S{++scenarioIdCounter}", feature)); + for (int i = 0; i < scenarioOutlinesPerFile; i++) + AddScenario(string.Format(scenarioOutlineTemplate, $"S{++scenarioIdCounter}", feature)); + } + + AddBindingClass( + """ + namespace ParallelExecution.StepDefinitions + { + [Binding] + public class ParallelExecutionSteps + { + public static int startIndex = 0; + + private readonly FeatureContext _featureContext; + private readonly ScenarioContext _scenarioContext; + + public ParallelExecutionSteps(FeatureContext featureContext, ScenarioContext scenarioContext) + { + _featureContext = featureContext; + _scenarioContext = scenarioContext; + } + + [BeforeFeature] + public static void BeforeFeature(FeatureContext featureContext) + { + featureContext.Set(true, "before_feature"); + } + + [When("executing {string} in {string}")] + public void WhenExecuting(string scenarioName, string featureName) + { + var currentStartIndex = System.Threading.Interlocked.Increment(ref startIndex); + global::Log.LogStep(); + if (_scenarioContext.ScenarioInfo.Title != scenarioName) + throw new System.Exception($"Invalid scenario context: {_scenarioContext.ScenarioInfo.Title} should be {scenarioName}"); + if (_featureContext.FeatureInfo.Title != featureName) + throw new System.Exception($"Invalid scenario context: {_featureContext.FeatureInfo.Title} should be {featureName}"); + if (!_featureContext.TryGetValue("before_feature", out var value) || !value) + throw new System.Exception($"BeforeFeature hook was not executed!"); + + var afterStartIndex = startIndex; + if (afterStartIndex != currentStartIndex) + Log.LogCustom("parallel", "true"); + } + } + } + """); + + ExecuteTests(); + ShouldAllScenariosPass(); + + var arguments = _bindingDriver.GetActualLogLines("parallel").ToList(); + arguments.Should().NotBeEmpty("the scenarios should have run parallel"); + } + + #endregion } diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs index 984f99e52..c2d4659b5 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs @@ -7,7 +7,7 @@ public class FeatureFileGenerator { public ProjectFile Generate(string featureFileContent, string featureFileName = null) { - featureFileName = featureFileName ?? $"FeatureFile{Guid.NewGuid():N}.feature"; + featureFileName ??= $"FeatureFile{Guid.NewGuid():N}.feature"; string fileContent = featureFileContent.Replace("'''", "\"\"\""); return new ProjectFile(featureFileName, "None", fileContent); From 52904450c9fd9d2f923b58c4f87138af1adb3a30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Fri, 4 Oct 2024 15:54:47 +0200 Subject: [PATCH 05/26] Remove obsolete Generator unit test --- .../MsTestGeneratorProviderTests.cs | 35 ++----------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/Tests/Reqnroll.GeneratorTests/UnitTestProvider/MsTestGeneratorProviderTests.cs b/Tests/Reqnroll.GeneratorTests/UnitTestProvider/MsTestGeneratorProviderTests.cs index 55ee1b379..694a5b901 100644 --- a/Tests/Reqnroll.GeneratorTests/UnitTestProvider/MsTestGeneratorProviderTests.cs +++ b/Tests/Reqnroll.GeneratorTests/UnitTestProvider/MsTestGeneratorProviderTests.cs @@ -1,4 +1,3 @@ -using System.CodeDom; using System.Globalization; using System.IO; using System.Linq; @@ -187,36 +186,6 @@ public void MsTestGeneratorProvider_ExampleSetIdentifiers_ShouldSetDescriptionCo descriptionAttributeForFourthScenarioOutline.ArgumentValues().First().Should().Be("Simple Scenario Outline: and another"); } - [Fact] - public void MsTestGeneratorProvider_ShouldInvokeFeatureSetupMethodWithGlobalNamespaceAlias() - { - // ARRANGE - var document = ParseDocumentFromString(SampleFeatureFileWithMultipleExampleSets); - var sampleTestGeneratorProvider = new MsTestGeneratorProvider(new CodeDomHelper(CodeDomProviderLanguage.CSharp)); - var converter = sampleTestGeneratorProvider.CreateUnitTestConverter(); - - // ACT - var code = converter.GenerateUnitTestFixture(document, "TestClassName", "Target.Namespace"); - - // ASSERT - var featureSetupCall = code - .Class() - .Members() - .Single(m => m.Name == "TestInitializeAsync") - .Statements - .OfType() - .First() - .TrueStatements - .OfType() - .First() - .Expression - .As(); - - featureSetupCall.Should().NotBeNull(); - featureSetupCall.Method.MethodName.Should().Be("FeatureSetupAsync"); - featureSetupCall.Method.TargetObject.As().Type.BaseType.Should().Contain("global::"); - } - [Fact] public void MsTestGeneratorProvider_ShouldNotHaveParallelExecutionTrait() { @@ -240,7 +209,7 @@ public void MsTestGeneratorProvider_WithFeatureWithMatchingTag_ShouldNotAddDoNot Given there is something"); var provider = new MsTestGeneratorProvider(new CodeDomHelper(CodeDomProviderLanguage.CSharp)); - var featureGenerator = provider.CreateFeatureGenerator(addNonParallelizableMarkerForTags: new string[] { "nonparallelizable" }); + var featureGenerator = provider.CreateFeatureGenerator(addNonParallelizableMarkerForTags: new[] { "nonparallelizable" }); // ACT var code = featureGenerator.GenerateUnitTestFixture(document, "TestClassName", "Target.Namespace"); @@ -261,7 +230,7 @@ public void MsTestGeneratorProvider_WithFeatureWithNoMatchingTag_ShouldNotAddDoN Given there is something"); var provider = new MsTestGeneratorProvider(new CodeDomHelper(CodeDomProviderLanguage.CSharp)); - var featureGenerator = provider.CreateFeatureGenerator(addNonParallelizableMarkerForTags: new string[] { "nonparallelizable" }); + var featureGenerator = provider.CreateFeatureGenerator(addNonParallelizableMarkerForTags: ["nonparallelizable"]); // ACT var code = featureGenerator.GenerateUnitTestFixture(document, "TestClassName", "Target.Namespace"); From 195f11dbc658e0785fb08668080d986fe5a66746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Fri, 4 Oct 2024 16:35:52 +0200 Subject: [PATCH 06/26] make sure that all remaining after feature hooks are called --- Reqnroll/TestRunnerManager.cs | 30 +++++++++++++---- .../TestRunnerManagerTests.cs | 33 +++++++++++++++++++ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/Reqnroll/TestRunnerManager.cs b/Reqnroll/TestRunnerManager.cs index 5d196fe72..9472305f3 100644 --- a/Reqnroll/TestRunnerManager.cs +++ b/Reqnroll/TestRunnerManager.cs @@ -203,10 +203,26 @@ private ITestRunner GetTestRunnerWithoutExceptionHandling() return testRunner; } + private async Task FireRemainingAfterFeatureHooks() + { + var testWorkerContainers = _availableTestWorkerContainers.Concat(_usedTestWorkerContainers).ToArray(); + foreach (var testWorkerContainer in testWorkerContainers) + { + var contextManager = testWorkerContainer.Key.Resolve(); + if (contextManager.FeatureContext != null) + { + var testRunner = testWorkerContainer.Key.Resolve(); + await testRunner.OnFeatureEndAsync(); + } + } + } + public virtual async Task DisposeAsync() { if (Interlocked.CompareExchange(ref _wasDisposed, 1, 0) == 0) { + await FireRemainingAfterFeatureHooks(); + await FireTestRunEndAsync(); if (_globalTestRunner != null) @@ -214,21 +230,21 @@ public virtual async Task DisposeAsync() ReleaseTestThreadContext(_globalTestRunner.TestThreadContext); } - var items = _availableTestWorkerContainers.ToArray(); - while (items.Length > 0) + var testWorkerContainers = _availableTestWorkerContainers.ToArray(); + while (testWorkerContainers.Length > 0) { - foreach (var item in items) + foreach (var item in testWorkerContainers) { item.Key.Dispose(); _availableTestWorkerContainers.TryRemove(item.Key, out _); } - items = _availableTestWorkerContainers.ToArray(); + testWorkerContainers = _availableTestWorkerContainers.ToArray(); } - var notReleasedRunner = _usedTestWorkerContainers.ToArray(); - if (notReleasedRunner.Length > 0) + var notReleasedTestWorkerContainers = _usedTestWorkerContainers.ToArray(); + if (notReleasedTestWorkerContainers.Length > 0) { - var errorText = $"Found {notReleasedRunner.Length} not released TestRunners (ids: {string.Join(",", notReleasedRunner.Select(x => TestThreadContainerInfo.GetId(x.Key)))})"; + var errorText = $"Found {notReleasedTestWorkerContainers.Length} not released TestRunners (ids: {string.Join(",", notReleasedTestWorkerContainers.Select(x => TestThreadContainerInfo.GetId(x.Key)))})"; _globalContainer.Resolve().TraceWarning(errorText); } diff --git a/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs b/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs index 355bbe724..c10be0aa0 100644 --- a/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs +++ b/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs @@ -71,6 +71,39 @@ public void Should_return_different_thread_ids_for_different_instances() TestRunnerManager.ReleaseTestRunner(testRunner3); } + [Fact] + public async Task Should_fire_remaining_AfterFeature_hooks_of_test_threads() + { + _testRunnerManager.Initialize(_anAssembly); + + var testRunnerWithFeatureContext = _testRunnerManager.GetTestRunner(); + var testRunnerWithoutFeatureContext = _testRunnerManager.GetTestRunner(); + + FeatureHookTracker.AfterFeatureCalled = false; + await testRunnerWithFeatureContext.OnFeatureStartAsync(new FeatureInfo(new CultureInfo("en-US", false), string.Empty, "F", null)); + var disposableClass1 = new DisposableClass(); + testRunnerWithFeatureContext.FeatureContext.FeatureContainer.RegisterInstanceAs(disposableClass1, dispose: true); + + TestRunnerManager.ReleaseTestRunner(testRunnerWithFeatureContext); + TestRunnerManager.ReleaseTestRunner(testRunnerWithoutFeatureContext); + + await TestRunnerManager.OnTestRunEndAsync(_anAssembly); + FeatureHookTracker.AfterFeatureCalled.Should().BeTrue(); + disposableClass1.IsDisposed.Should().BeTrue(); + } + + [Binding] + static class FeatureHookTracker + { + public static bool AfterFeatureCalled = false; + + [AfterFeature] + public static void AfterFeature() + { + AfterFeatureCalled = true; + } + } + class DisposableClass : IDisposable { public bool IsDisposed { get; private set; } From f2503f6897d6151adc9af278d11ca43eb5489436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Fri, 4 Oct 2024 17:25:15 +0200 Subject: [PATCH 07/26] remove invalid expectations of specs test about feature context --- .../MsTest/MethodLevelParallelisation.feature | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature b/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature index 72b9b5e61..86341a4f3 100644 --- a/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature +++ b/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature @@ -59,10 +59,10 @@ public class TraceSteps var featureData = GetFeatureData(featureContext); featureData.TestRunners.TryAdd(testRunner, 1).Should().BeFalse(); featureData.Duration.Stop(); - featureData.TestRunners.Count.Should().Be(11, because: "One TestRunner for before/after hooks and one for each test is created"); - featureData.FeatureContexts.Count.Should().Be(1, because: "Only one FeatureContext should be created"); - featureData.ScenarioContexts.Count.Should().Be(10, because: "One ScenarioContext for each test is created"); - featureData.BindingInstances.Count.Should().Be(10, because: "One binding instance for each test is created"); + //featureData.TestRunners.Count.Should().Be(11, because: "One TestRunner for before/after hooks and one for each test is created"); + //featureData.FeatureContexts.Count.Should().Be(1, because: "Only one FeatureContext should be created"); + //featureData.ScenarioContexts.Count.Should().Be(10, because: "One ScenarioContext for each test is created"); + //featureData.BindingInstances.Count.Should().Be(10, because: "One binding instance for each test is created"); featureData.Duration.ElapsedMilliseconds.Should().BeLessThan(9 * WaitTimeInMS, because: "Test should be processed (parallel) in time"); } From 1888c885df8b5a2289678f64e6e97199f77e6c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Fri, 4 Oct 2024 17:43:00 +0200 Subject: [PATCH 08/26] Support method-level parallel execution for NUnit --- .../build/NUnit.AssemblyHooks.template.cs | 8 +-- .../build/NUnit.AssemblyHooks.template.vb | 6 ++- .../Generation/UnitTestFeatureGenerator.cs | 10 ++-- .../NUnit3TestGeneratorProvider.cs | 6 +++ .../NUnit3GeneratorProviderTests.cs | 54 ------------------- .../ProjectBuilder.cs | 2 +- 6 files changed, 22 insertions(+), 64 deletions(-) diff --git a/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.cs b/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.cs index e87a467fe..b04eee1f2 100644 --- a/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.cs +++ b/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.cs @@ -6,13 +6,15 @@ using global::System.Runtime.CompilerServices; using System.Threading.Tasks; +[assembly: NUnit.Framework.FixtureLifeCycle(NUnit.Framework.LifeCycle.InstancePerTestCase)] + [GeneratedCode("Reqnroll", "REQNROLL_VERSION")] [global::NUnit.Framework.SetUpFixture] -public class PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks +public static class PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks { [global::NUnit.Framework.OneTimeSetUp] [MethodImpl(MethodImplOptions.NoInlining)] - public async Task AssemblyInitializeAsync() + public static async Task AssemblyInitializeAsync() { var currentAssembly = typeof(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly; await global::Reqnroll.TestRunnerManager.OnTestRunStartAsync(currentAssembly); @@ -20,7 +22,7 @@ public async Task AssemblyInitializeAsync() [global::NUnit.Framework.OneTimeTearDown] [MethodImpl(MethodImplOptions.NoInlining)] - public async ValueTask AssemblyCleanupAsync() + public static async ValueTask AssemblyCleanupAsync() { var currentAssembly = typeof(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly; await global::Reqnroll.TestRunnerManager.OnTestRunEndAsync(currentAssembly); diff --git a/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.vb b/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.vb index 07a98e545..de1925645 100644 --- a/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.vb +++ b/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.vb @@ -7,19 +7,21 @@ Imports System.CodeDom.Compiler Imports System.Reflection Imports System.Runtime.CompilerServices + + Public NotInheritable Class PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks - Public Async Function AssemblyInitializeAsync() As Task + Public Shared Async Function AssemblyInitializeAsync() As Task Dim currentAssembly As Assembly = GetType(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly Await Global.Reqnroll.TestRunnerManager.OnTestRunStartAsync(currentAssembly) End Function - Public Async Function AssemblyCleanupAsync() As Task + Public Shared Async Function AssemblyCleanupAsync() As Task Dim currentAssembly As Assembly = GetType(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly Await Global.Reqnroll.TestRunnerManager.OnTestRunEndAsync(currentAssembly) End Function diff --git a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs index a64f44611..2bfb43c75 100644 --- a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs +++ b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs @@ -258,8 +258,9 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont _testGeneratorProvider.SetTestInitializeMethod(generationContext); - if (generationContext.UnitTestGeneratorProvider is not MsTestGeneratorProvider) - return; // only MsTest is implemented in this prototype + if (generationContext.UnitTestGeneratorProvider is not MsTestGeneratorProvider && + generationContext.UnitTestGeneratorProvider is not NUnit3TestGeneratorProvider) + return; // only MsTest & NUnit is implemented in this prototype // Step 4: Obtain the test runner for executing a single test @@ -372,8 +373,9 @@ private void SetupTestCleanupMethod(TestClassGenerationContext generationContext testCleanupMethod.Statements.Add(expression); - if (generationContext.UnitTestGeneratorProvider is not MsTestGeneratorProvider) - return; // only MsTest is implemented in this prototype + if (generationContext.UnitTestGeneratorProvider is not MsTestGeneratorProvider && + generationContext.UnitTestGeneratorProvider is not NUnit3TestGeneratorProvider) + return; // only MsTest & NUnit is implemented in this prototype // Step 6: "Release" the TestRunner, so that other threads can pick it up (moved from TestClassCleanupMethod) // TestRunnerManager.ReleaseTestRunner(testRunner); diff --git a/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs index 237aab2f9..62d993917 100644 --- a/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs @@ -51,11 +51,17 @@ public virtual void SetTestMethodIgnore(TestClassGenerationContext generationCon public virtual void SetTestClassInitializeMethod(TestClassGenerationContext generationContext) { CodeDomHelper.AddAttribute(generationContext.TestClassInitializeMethod, TESTFIXTURESETUP_ATTR_NUNIT3); + + // Step 2: Remove TestClassInitializeMethod (not needed) + generationContext.TestClass.Members.Remove(generationContext.TestClassInitializeMethod); } public virtual void SetTestClassCleanupMethod(TestClassGenerationContext generationContext) { CodeDomHelper.AddAttribute(generationContext.TestClassCleanupMethod, TESTFIXTURETEARDOWN_ATTR_NUNIT3); + + // Step 3: Remove TestClassCleanupMethod (not needed) + generationContext.TestClass.Members.Remove(generationContext.TestClassCleanupMethod); } public virtual void SetTestClassNonParallelizable(TestClassGenerationContext generationContext) diff --git a/Tests/Reqnroll.GeneratorTests/UnitTestProvider/NUnit3GeneratorProviderTests.cs b/Tests/Reqnroll.GeneratorTests/UnitTestProvider/NUnit3GeneratorProviderTests.cs index dce0efc83..1ce0c3a2d 100644 --- a/Tests/Reqnroll.GeneratorTests/UnitTestProvider/NUnit3GeneratorProviderTests.cs +++ b/Tests/Reqnroll.GeneratorTests/UnitTestProvider/NUnit3GeneratorProviderTests.cs @@ -211,60 +211,6 @@ Then something should happen testMethod.CustomAttributes().Should().ContainSingle(a => a.Name == NUnit3TestCaseAttributeName && (string)a.ArgumentValues().First() == "and another"); } - [Fact] - public void NUnit3TestGeneratorProvider_ShouldNotGenerateObsoleteTestFixtureSetUpAttribute() - { - var code = GenerateCodeNamespaceFromFeature(SampleFeatureFile); - - var featureSetupMethod = code.Class().Members().Single(m => m.Name == "FeatureSetupAsync"); - - featureSetupMethod.CustomAttributes() - .FirstOrDefault(a => a.Name == "NUnit.Framework.TestFixtureSetUpAttribute") - .Should() - .BeNull(); - } - - [Fact] - public void NUnit3TestGeneratorProvider_ShouldGenerateNewOneTimeSetUpAttribute() - { - var code = GenerateCodeNamespaceFromFeature(SampleFeatureFile); - - var featureSetupMethod = code.Class().Members().Single(m => m.Name == "FeatureSetupAsync"); - - // Assert that we do use the NUnit3 attribute - featureSetupMethod.CustomAttributes() - .FirstOrDefault(a => a.Name == "NUnit.Framework.OneTimeSetUpAttribute") - .Should() - .NotBeNull(); - } - - [Fact] - public void NUnit3TestGeneratorProvider_ShouldNotGenerateObsoleteTestFixtureTearDownAttribute() - { - var code = GenerateCodeNamespaceFromFeature(SampleFeatureFile); - - var featureSetupMethod = code.Class().Members().Single(m => m.Name == "FeatureTearDownAsync"); - - featureSetupMethod.CustomAttributes() - .FirstOrDefault(a => a.Name == "NUnit.Framework.TestFixtureTearDownAttribute") - .Should() - .BeNull(); - } - - [Fact] - public void NUnit3TestGeneratorProvider_ShouldGenerateNewOneTimeTearDownAttribute() - { - var code = GenerateCodeNamespaceFromFeature(SampleFeatureFile); - - var featureSetupMethod = code.Class().Members().Single(m => m.Name == "FeatureTearDownAsync"); - - // Assert that we do use the NUnit3 attribute - featureSetupMethod.CustomAttributes() - .FirstOrDefault(a => a.Name == "NUnit.Framework.OneTimeTearDownAttribute") - .Should() - .NotBeNull(); - } - [Fact] public void NUnit3TestGeneratorProvider_ShouldHaveRowTestsTrait() { diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs index b37dbcbb6..55afcfe95 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs @@ -367,7 +367,7 @@ private void AddUnitTestProviderSpecificConfig() _project.AddFile(new ProjectFile("XUnitConfiguration.cs", "Compile", "using Xunit; [assembly: CollectionBehavior(CollectionBehavior.CollectionPerClass, MaxParallelThreads = 4)]")); break; case UnitTestProvider.NUnit3 when _parallelTestExecution: - _project.AddFile(new ProjectFile("NUnitConfiguration.cs", "Compile", "[assembly: NUnit.Framework.Parallelizable(NUnit.Framework.ParallelScope.Fixtures)]")); + _project.AddFile(new ProjectFile("NUnitConfiguration.cs", "Compile", "[assembly: NUnit.Framework.Parallelizable(NUnit.Framework.ParallelScope.All)]")); break; case UnitTestProvider.MSTest when _parallelTestExecution: _project.AddFile( From c4880d70c5d27eb3f04c465d04a0e8a64e23bdac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 7 Oct 2024 11:05:38 +0200 Subject: [PATCH 09/26] Support new execution structure for xUnit --- .../Generation/UnitTestFeatureGenerator.cs | 15 ++++----------- .../XUnit2TestGeneratorProvider.cs | 12 +++++++++++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs index 2bfb43c75..7095f2ef7 100644 --- a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs +++ b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs @@ -174,6 +174,7 @@ private void SetupTestClassInitializeMethod(TestClassGenerationContext generatio _testGeneratorProvider.SetTestClassInitializeMethod(generationContext); + /* //testRunner = TestRunnerManager.GetTestRunnerForAssembly(null, [test_worker_id]); var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression(); @@ -208,7 +209,7 @@ private void SetupTestClassInitializeMethod(TestClassGenerationContext generatio _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureStartExpression); - testClassInitializeMethod.Statements.Add(onFeatureStartExpression); + testClassInitializeMethod.Statements.Add(onFeatureStartExpression);*/ } private void SetupTestClassCleanupMethod(TestClassGenerationContext generationContext) @@ -221,7 +222,7 @@ private void SetupTestClassCleanupMethod(TestClassGenerationContext generationCo _codeDomHelper.MarkCodeMemberMethodAsAsync(testClassCleanupMethod); _testGeneratorProvider.SetTestClassCleanupMethod(generationContext); - + /* var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression(); // await testRunner.OnFeatureEndAsync(); @@ -244,7 +245,7 @@ private void SetupTestClassCleanupMethod(TestClassGenerationContext generationCo testClassCleanupMethod.Statements.Add( new CodeAssignStatement( testRunnerField, - new CodePrimitiveExpression(null))); + new CodePrimitiveExpression(null)));*/ } private void SetupTestInitializeMethod(TestClassGenerationContext generationContext) @@ -258,10 +259,6 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont _testGeneratorProvider.SetTestInitializeMethod(generationContext); - if (generationContext.UnitTestGeneratorProvider is not MsTestGeneratorProvider && - generationContext.UnitTestGeneratorProvider is not NUnit3TestGeneratorProvider) - return; // only MsTest & NUnit is implemented in this prototype - // Step 4: Obtain the test runner for executing a single test // testRunner = global::Reqnroll.TestRunnerManager.GetTestRunnerForAssembly(); @@ -373,10 +370,6 @@ private void SetupTestCleanupMethod(TestClassGenerationContext generationContext testCleanupMethod.Statements.Add(expression); - if (generationContext.UnitTestGeneratorProvider is not MsTestGeneratorProvider && - generationContext.UnitTestGeneratorProvider is not NUnit3TestGeneratorProvider) - return; // only MsTest & NUnit is implemented in this prototype - // Step 6: "Release" the TestRunner, so that other threads can pick it up (moved from TestClassCleanupMethod) // TestRunnerManager.ReleaseTestRunner(testRunner); testCleanupMethod.Statements.Add( diff --git a/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs index 7b15c51c8..6749f61d5 100644 --- a/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs @@ -216,7 +216,8 @@ public void SetTestClassInitializeMethod(TestClassGenerationContext generationCo { // xUnit uses IUseFixture on the class generationContext.TestClassInitializeMethod.Attributes |= MemberAttributes.Static; - generationContext.TestRunnerField.Attributes |= MemberAttributes.Static; + // Step 1: make 'testRunner' instance field + // generationContext.TestRunnerField.Attributes |= MemberAttributes.Static; _currentFixtureDataTypeDeclaration = CodeDomHelper.CreateGeneratedTypeDeclaration("FixtureData"); generationContext.TestClass.Members.Add(_currentFixtureDataTypeDeclaration); @@ -245,6 +246,11 @@ public void SetTestClassInitializeMethod(TestClassGenerationContext generationCo CodeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression); initializeMethod.Statements.Add(expression); + + + // Step 2: Remove TestClassInitializeMethod (not needed) + generationContext.TestClassInitializeMethod.Statements.Clear(); + //generationContext.TestClass.Members.Remove(generationContext.TestClassInitializeMethod); } public void SetTestClassCleanupMethod(TestClassGenerationContext generationContext) @@ -345,6 +351,10 @@ public void SetTestCleanupMethod(TestClassGenerationContext generationContext) CodeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression); disposeMethod.Statements.Add(expression); + + // Step 3: Remove TestClassCleanupMethod (not needed) + generationContext.TestClassCleanupMethod.Statements.Clear(); + //generationContext.TestClass.Members.Remove(generationContext.TestClassCleanupMethod); } public void SetTestClassIgnore(TestClassGenerationContext generationContext) From 37bbfd5427c7f2982202a6eabeab002c97b0a856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 7 Oct 2024 11:41:17 +0200 Subject: [PATCH 10/26] code cleanup --- .../Generation/UnitTestFeatureGenerator.cs | 76 ++----------------- .../MsTestGeneratorProvider.cs | 55 +------------- .../NUnit3TestGeneratorProvider.cs | 21 +++-- .../XUnit2TestGeneratorProvider.cs | 35 +++------ .../Generation/GenerationTestBase.cs | 2 - 5 files changed, 29 insertions(+), 160 deletions(-) diff --git a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs index 7095f2ef7..26ba5101f 100644 --- a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs +++ b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs @@ -1,5 +1,6 @@ using System; using System.CodeDom; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Reflection; @@ -12,6 +13,7 @@ namespace Reqnroll.Generator.Generation { + [SuppressMessage("ReSharper", "BitwiseOperatorOnEnumWithoutFlags")] public class UnitTestFeatureGenerator : IFeatureGenerator { private readonly CodeDomHelper _codeDomHelper; @@ -173,43 +175,6 @@ private void SetupTestClassInitializeMethod(TestClassGenerationContext generatio _codeDomHelper.MarkCodeMemberMethodAsAsync(testClassInitializeMethod); _testGeneratorProvider.SetTestClassInitializeMethod(generationContext); - - /* - //testRunner = TestRunnerManager.GetTestRunnerForAssembly(null, [test_worker_id]); - var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression(); - - var getTestRunnerExpression = new CodeMethodInvokeExpression( - new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))), - nameof(TestRunnerManager.GetTestRunnerForAssembly)); - - testClassInitializeMethod.Statements.Add( - new CodeAssignStatement( - testRunnerField, - getTestRunnerExpression)); - - //FeatureInfo featureInfo = new FeatureInfo("xxxx"); - testClassInitializeMethod.Statements.Add( - new CodeVariableDeclarationStatement(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), "featureInfo", - new CodeObjectCreateExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), - new CodeObjectCreateExpression(typeof(CultureInfo), - new CodePrimitiveExpression(generationContext.Feature.Language)), - new CodePrimitiveExpression(generationContext.Document.DocumentLocation?.FeatureFolderPath), - new CodePrimitiveExpression(generationContext.Feature.Name), - new CodePrimitiveExpression(generationContext.Feature.Description), - new CodeFieldReferenceExpression( - new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(Reqnroll.ProgrammingLanguage))), - _codeDomHelper.TargetLanguage.ToString()), - new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME)))); - - //await testRunner.OnFeatureStartAsync(featureInfo); - var onFeatureStartExpression = new CodeMethodInvokeExpression( - testRunnerField, - nameof(ITestRunner.OnFeatureStartAsync), - new CodeVariableReferenceExpression("featureInfo")); - - _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureStartExpression); - - testClassInitializeMethod.Statements.Add(onFeatureStartExpression);*/ } private void SetupTestClassCleanupMethod(TestClassGenerationContext generationContext) @@ -222,30 +187,6 @@ private void SetupTestClassCleanupMethod(TestClassGenerationContext generationCo _codeDomHelper.MarkCodeMemberMethodAsAsync(testClassCleanupMethod); _testGeneratorProvider.SetTestClassCleanupMethod(generationContext); - /* - var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression(); - - // await testRunner.OnFeatureEndAsync(); - var expression = new CodeMethodInvokeExpression( - testRunnerField, - nameof(ITestRunner.OnFeatureEndAsync)); - - _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression); - - testClassCleanupMethod.Statements.Add(expression); - - // TestRunnerManager.ReleaseTestRunner(testRunner); - testClassCleanupMethod.Statements.Add( - new CodeMethodInvokeExpression( - new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))), - nameof(TestRunnerManager.ReleaseTestRunner), - testRunnerField)); - - // testRunner = null; - testClassCleanupMethod.Statements.Add( - new CodeAssignStatement( - testRunnerField, - new CodePrimitiveExpression(null)));*/ } private void SetupTestInitializeMethod(TestClassGenerationContext generationContext) @@ -259,8 +200,7 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont _testGeneratorProvider.SetTestInitializeMethod(generationContext); - // Step 4: Obtain the test runner for executing a single test - + // Obtain the test runner for executing a single test // testRunner = global::Reqnroll.TestRunnerManager.GetTestRunnerForAssembly(); var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression(); @@ -275,9 +215,7 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont getTestRunnerExpression)); - // Step 5 (part 1): "Finish" current feature if needed & "Start" feature if needed - // The similar code in custom test runner codes is not needed - // The feature initialization steps are copied from TestClassInitializeMethod + // "Finish" current feature if needed //if (testRunner.FeatureContext != null && testRunner.FeatureContext.FeatureInfo.Title != "") // await testRunner.OnFeatureEndAsync(); // finish if different @@ -313,6 +251,8 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont onFeatureEndAsyncExpression))); + // "Start" the feature if needed + //if (testRunner.FeatureContext == null) { // FeatureInfo featureInfo = new FeatureInfo("xxxx"); // await testRunner.OnFeatureStartAsync(featureInfo); @@ -327,7 +267,7 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont new CodePrimitiveExpression(generationContext.Feature.Name), new CodePrimitiveExpression(generationContext.Feature.Description), new CodeFieldReferenceExpression( - new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(Reqnroll.ProgrammingLanguage))), + new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(ProgrammingLanguage))), _codeDomHelper.TargetLanguage.ToString()), new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME))); @@ -370,7 +310,7 @@ private void SetupTestCleanupMethod(TestClassGenerationContext generationContext testCleanupMethod.Statements.Add(expression); - // Step 6: "Release" the TestRunner, so that other threads can pick it up (moved from TestClassCleanupMethod) + // "Release" the TestRunner, so that other threads can pick it up // TestRunnerManager.ReleaseTestRunner(testRunner); testCleanupMethod.Statements.Add( new CodeMethodInvokeExpression( diff --git a/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs index df8e4a57d..3fa6db578 100644 --- a/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs @@ -3,9 +3,11 @@ using System.Collections.Generic; using Reqnroll.Generator.CodeDom; using Reqnroll.BoDi; +using System.Diagnostics.CodeAnalysis; namespace Reqnroll.Generator.UnitTestProvider { + [SuppressMessage("ReSharper", "BitwiseOperatorOnEnumWithoutFlags")] public class MsTestGeneratorProvider : IUnitTestGeneratorProvider { protected internal const string TESTFIXTURE_ATTR = "Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute"; @@ -103,16 +105,11 @@ public virtual void SetTestClassNonParallelizable(TestClassGenerationContext gen public virtual void SetTestClassInitializeMethod(TestClassGenerationContext generationContext) { generationContext.TestClassInitializeMethod.Attributes |= MemberAttributes.Static; - // Step 1: make 'testRunner' instance field - //generationContext.TestRunnerField.Attributes |= MemberAttributes.Static; generationContext.TestClassInitializeMethod.Parameters.Add(new CodeParameterDeclarationExpression( TESTCONTEXT_TYPE, "testContext")); CodeDomHelper.AddAttribute(generationContext.TestClassInitializeMethod, TESTFIXTURESETUP_ATTR); - - // Step 2: Remove TestClassInitializeMethod (not needed) - generationContext.TestClass.Members.Remove(generationContext.TestClassInitializeMethod); } public void SetTestClassCleanupMethod(TestClassGenerationContext generationContext) @@ -121,59 +118,12 @@ public void SetTestClassCleanupMethod(TestClassGenerationContext generationConte // [Microsoft.VisualStudio.TestTools.UnitTesting.ClassCleanupAttribute(Microsoft.VisualStudio.TestTools.UnitTesting.ClassCleanupBehavior.EndOfClass)] var attribute = CodeDomHelper.AddAttribute(generationContext.TestClassCleanupMethod, TESTFIXTURETEARDOWN_ATTR); attribute.Arguments.Add(new CodeAttributeArgument(new CodeFieldReferenceExpression(new CodeTypeReferenceExpression(CLASSCLEANUPBEHAVIOR_ENUM), CLASSCLEANUPBEHAVIOR_ENDOFCLASS))); - - // Step 3: Remove TestClassCleanupMethod (not needed) - generationContext.TestClass.Members.Remove(generationContext.TestClassCleanupMethod); } public virtual void SetTestInitializeMethod(TestClassGenerationContext generationContext) { CodeDomHelper.AddAttribute(generationContext.TestInitializeMethod, TESTSETUP_ATTR); - - // Step 5 (part 2): remove feature handling from here as it is moved to UnitTestGenerator.SetupTestInitializeMethod - //FixTestRunOrderingIssue(generationContext); - } - - protected virtual void FixTestRunOrderingIssue(TestClassGenerationContext generationContext) - { - //see https://github.com/SpecFlowOSS/SpecFlow/issues/96 - - //if (testRunner.FeatureContext != null && testRunner.FeatureContext.FeatureInfo.Title != "") - // .(null); - - var featureContextExpression = new CodePropertyReferenceExpression( - new CodeFieldReferenceExpression(null, generationContext.TestRunnerField.Name), - "FeatureContext"); - - var callTestClassInitializeMethodExpression = new CodeMethodInvokeExpression( - new CodeTypeReferenceExpression( - new CodeTypeReference( - generationContext.Namespace.Name + "." + generationContext.TestClass.Name, - CodeTypeReferenceOptions.GlobalReference)), - generationContext.TestClassInitializeMethod.Name, - new CodePrimitiveExpression(null)); - - CodeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(callTestClassInitializeMethodExpression); - - generationContext.TestInitializeMethod.Statements.Add( - new CodeConditionStatement( - new CodeBinaryOperatorExpression( - new CodeBinaryOperatorExpression( - featureContextExpression, - CodeBinaryOperatorType.IdentityInequality, - new CodePrimitiveExpression(null)), - CodeBinaryOperatorType.BooleanAnd, - new CodeBinaryOperatorExpression( - new CodePropertyReferenceExpression( - new CodePropertyReferenceExpression( - featureContextExpression, - "FeatureInfo"), - "Title"), - CodeBinaryOperatorType.IdentityInequality, - new CodePrimitiveExpression(generationContext.Feature.Name))), - new CodeExpressionStatement( - callTestClassInitializeMethodExpression))); } public void SetTestCleanupMethod(TestClassGenerationContext generationContext) @@ -181,7 +131,6 @@ public void SetTestCleanupMethod(TestClassGenerationContext generationContext) CodeDomHelper.AddAttribute(generationContext.TestCleanupMethod, TESTTEARDOWN_ATTR); } - public virtual void SetTestMethod(TestClassGenerationContext generationContext, CodeMemberMethod testMethod, string friendlyTestName) { CodeDomHelper.AddAttribute(testMethod, TEST_ATTR); diff --git a/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs index 62d993917..34cfd81e0 100644 --- a/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs @@ -1,12 +1,13 @@ using System.CodeDom; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; -using System.Threading.Tasks; using Reqnroll.BoDi; using Reqnroll.Generator.CodeDom; namespace Reqnroll.Generator.UnitTestProvider { + [SuppressMessage("ReSharper", "BitwiseOperatorOnEnumWithoutFlags")] public class NUnit3TestGeneratorProvider : IUnitTestGeneratorProvider { protected internal const string TESTFIXTURESETUP_ATTR_NUNIT3 = "NUnit.Framework.OneTimeSetUpAttribute"; @@ -50,18 +51,14 @@ public virtual void SetTestMethodIgnore(TestClassGenerationContext generationCon public virtual void SetTestClassInitializeMethod(TestClassGenerationContext generationContext) { + generationContext.TestClassInitializeMethod.Attributes |= MemberAttributes.Static; CodeDomHelper.AddAttribute(generationContext.TestClassInitializeMethod, TESTFIXTURESETUP_ATTR_NUNIT3); - - // Step 2: Remove TestClassInitializeMethod (not needed) - generationContext.TestClass.Members.Remove(generationContext.TestClassInitializeMethod); } public virtual void SetTestClassCleanupMethod(TestClassGenerationContext generationContext) { + generationContext.TestClassInitializeMethod.Attributes |= MemberAttributes.Static; CodeDomHelper.AddAttribute(generationContext.TestClassCleanupMethod, TESTFIXTURETEARDOWN_ATTR_NUNIT3); - - // Step 3: Remove TestClassCleanupMethod (not needed) - generationContext.TestClass.Members.Remove(generationContext.TestClassCleanupMethod); } public virtual void SetTestClassNonParallelizable(TestClassGenerationContext generationContext) @@ -129,9 +126,11 @@ public void SetRow(TestClassGenerationContext generationContext, CodeMemberMetho var args = arguments.Select( arg => new CodeAttributeArgument(new CodePrimitiveExpression(arg))).ToList(); - // addressing ReSharper bug: TestCase attribute with empty string[] param causes inconclusive result - https://github.com/reqnroll/Reqnroll/issues/116 - bool hasExampleTags = tags.Any(); - var exampleTagExpressionList = tags.Select(t => new CodePrimitiveExpression(t)); + var tagsArray = tags.ToArray(); + + // addressing ReSharper bug: TestCase attribute with empty string[] param causes inconclusive result - https://github.com/SpecFlowOSS/SpecFlow/issues/116 + bool hasExampleTags = tagsArray.Any(); + var exampleTagExpressionList = tagsArray.Select(t => (CodeExpression)new CodePrimitiveExpression(t)); var exampleTagsExpression = hasExampleTags ? new CodeArrayCreateExpression(typeof(string[]), exampleTagExpressionList.ToArray()) : (CodeExpression) new CodePrimitiveExpression(null); @@ -141,7 +140,7 @@ public void SetRow(TestClassGenerationContext generationContext, CodeMemberMetho // adds 'Category' named parameter so that NUnit also understands that this test case belongs to the given categories if (hasExampleTags) { - CodeExpression exampleTagsStringExpr = new CodePrimitiveExpression(string.Join(",", tags.ToArray())); + CodeExpression exampleTagsStringExpr = new CodePrimitiveExpression(string.Join(",", tagsArray)); args.Add(new CodeAttributeArgument("Category", exampleTagsStringExpr)); } diff --git a/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs index 6749f61d5..698d8e717 100644 --- a/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs @@ -5,13 +5,15 @@ using Reqnroll.Generator.CodeDom; using Reqnroll.BoDi; using System.Text.RegularExpressions; +using System.Diagnostics.CodeAnalysis; namespace Reqnroll.Generator.UnitTestProvider { + [SuppressMessage("ReSharper", "BitwiseOperatorOnEnumWithoutFlags")] public class XUnit2TestGeneratorProvider : IUnitTestGeneratorProvider { private CodeTypeDeclaration _currentFixtureDataTypeDeclaration = null; - private readonly CodeTypeReference _objectCodeTypeReference = new CodeTypeReference(typeof(object)); + private readonly CodeTypeReference _objectCodeTypeReference = new(typeof(object)); protected internal const string THEORY_ATTRIBUTE = "Xunit.SkippableTheoryAttribute"; protected internal const string INLINEDATA_ATTRIBUTE = "Xunit.InlineDataAttribute"; protected internal const string ICLASSFIXTURE_INTERFACE = "Xunit.IClassFixture"; @@ -59,12 +61,8 @@ protected virtual CodeTypeReference CreateFixtureInterface(TestClassGenerationCo return new CodeTypeReference(ICLASSFIXTURE_INTERFACE, fixtureDataType); } - public virtual bool ImplementInterfaceExplicit => false; - protected CodeDomHelper CodeDomHelper { get; set; } - public bool GenerateParallelCodeForFeature { get; set; } - public virtual void SetRowTest(TestClassGenerationContext generationContext, CodeMemberMethod testMethod, string scenarioTitle) { CodeDomHelper.AddAttribute(testMethod, THEORY_ATTRIBUTE, new CodeAttributeArgument("DisplayName", new CodePrimitiveExpression(scenarioTitle))); @@ -85,7 +83,7 @@ public virtual void SetRow(TestClassGenerationContext generationContext, CodeMem args.Add( new CodeAttributeArgument( - new CodeArrayCreateExpression(typeof(string[]), tags.Select(t => new CodePrimitiveExpression(t)).ToArray()))); + new CodeArrayCreateExpression(typeof(string[]), tags.Select(t => (CodeExpression)new CodePrimitiveExpression(t)).ToArray()))); CodeDomHelper.AddAttribute(testMethod, INLINEDATA_ATTRIBUTE, args.ToArray()); } @@ -133,15 +131,16 @@ public virtual void SetTestMethodIgnore(TestClassGenerationContext generationCon public virtual void SetTestClassCategories(TestClassGenerationContext generationContext, IEnumerable featureCategories) { - IEnumerable collection = featureCategories.Where(f => f.StartsWith(COLLECTION_TAG, StringComparison.InvariantCultureIgnoreCase)).ToList(); - if (collection.Any()) + var featureCategoriesArray = featureCategories.ToArray(); + var collectionCategories = featureCategoriesArray.Where(f => f.StartsWith(COLLECTION_TAG, StringComparison.InvariantCultureIgnoreCase)).ToList(); + if (collectionCategories.Any()) { //Only one 'Xunit.Collection' can exist per class. - SetTestClassCollection(generationContext, collection.FirstOrDefault()); + SetTestClassCollection(generationContext, collectionCategories.FirstOrDefault()); } // Set Category trait which can be used with the /trait or /-trait xunit flags to include/exclude tests - foreach (string str in featureCategories) + foreach (string str in featureCategoriesArray) { SetProperty(generationContext.TestClass, CATEGORY_PROPERTY_NAME, str); } @@ -216,8 +215,6 @@ public void SetTestClassInitializeMethod(TestClassGenerationContext generationCo { // xUnit uses IUseFixture on the class generationContext.TestClassInitializeMethod.Attributes |= MemberAttributes.Static; - // Step 1: make 'testRunner' instance field - // generationContext.TestRunnerField.Attributes |= MemberAttributes.Static; _currentFixtureDataTypeDeclaration = CodeDomHelper.CreateGeneratedTypeDeclaration("FixtureData"); generationContext.TestClass.Members.Add(_currentFixtureDataTypeDeclaration); @@ -246,11 +243,6 @@ public void SetTestClassInitializeMethod(TestClassGenerationContext generationCo CodeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression); initializeMethod.Statements.Add(expression); - - - // Step 2: Remove TestClassInitializeMethod (not needed) - generationContext.TestClassInitializeMethod.Statements.Clear(); - //generationContext.TestClass.Members.Remove(generationContext.TestClassInitializeMethod); } public void SetTestClassCleanupMethod(TestClassGenerationContext generationContext) @@ -351,10 +343,6 @@ public void SetTestCleanupMethod(TestClassGenerationContext generationContext) CodeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression); disposeMethod.Statements.Add(expression); - - // Step 3: Remove TestClassCleanupMethod (not needed) - generationContext.TestClassCleanupMethod.Statements.Clear(); - //generationContext.TestClass.Members.Remove(generationContext.TestClassCleanupMethod); } public void SetTestClassIgnore(TestClassGenerationContext generationContext) @@ -409,10 +397,5 @@ public void MarkCodeMethodInvokeExpressionAsAwait(CodeMethodInvokeExpression exp { CodeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression); } - - private string GlobalNamespaceIfCSharp(string typeName) - { - return CodeDomHelper.TargetLanguage == CodeDomProviderLanguage.CSharp ? "global::" + typeName : typeName; - } } } \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs index b6573fdf5..bc2b1091d 100644 --- a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs +++ b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs @@ -2,8 +2,6 @@ using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; using FluentAssertions; -using Reqnroll.TestProjectGenerator; -using System.ComponentModel.DataAnnotations; using System.Text.Json; using System.Collections.Generic; From 3a4dc5340095e92a42866ef2800d6fc47f7dd6a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 7 Oct 2024 12:31:09 +0200 Subject: [PATCH 11/26] generate featureInfo as field --- .../Generation/GeneratorConstants.cs | 1 + .../Generation/UnitTestFeatureGenerator.cs | 61 +++++++++++-------- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/Reqnroll.Generator/Generation/GeneratorConstants.cs b/Reqnroll.Generator/Generation/GeneratorConstants.cs index 3a9a6b081..859c3a111 100644 --- a/Reqnroll.Generator/Generation/GeneratorConstants.cs +++ b/Reqnroll.Generator/Generation/GeneratorConstants.cs @@ -13,6 +13,7 @@ public class GeneratorConstants public const string TESTCLASS_CLEANUP_NAME = "FeatureTearDownAsync"; public const string BACKGROUND_NAME = "FeatureBackgroundAsync"; public const string TESTRUNNER_FIELD = "testRunner"; + public const string FEATUREINFO_FIELD = "featureInfo"; public const string REQNROLL_NAMESPACE = "Reqnroll"; public const string SCENARIO_OUTLINE_EXAMPLE_TAGS_PARAMETER = "exampleTags"; public const string SCENARIO_TAGS_VARIABLE_NAME = "tagsOfScenario"; diff --git a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs index 26ba5101f..012fb5d9d 100644 --- a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs +++ b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs @@ -151,11 +151,8 @@ private void SetupTestClass(TestClassGenerationContext generationContext) _testGeneratorProvider.SetTestClassCategories(generationContext, featureCategories); } - var featureTagsField = new CodeMemberField(typeof(string[]), GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME); - featureTagsField.Attributes |= MemberAttributes.Static; - featureTagsField.InitExpression = _scenarioPartHelper.GetStringArrayExpression(generationContext.Feature.Tags); - - generationContext.TestClass.Members.Add(featureTagsField); + DeclareFeatureTagsField(generationContext); + DeclareFeatureInfoMember(generationContext); } private CodeMemberField DeclareTestRunnerMember(CodeTypeDeclaration type) @@ -165,6 +162,33 @@ private CodeMemberField DeclareTestRunnerMember(CodeTypeDeclaration type) return testRunnerField; } + private void DeclareFeatureTagsField(TestClassGenerationContext generationContext) + { + var featureTagsField = new CodeMemberField(typeof(string[]), GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME); + featureTagsField.Attributes |= MemberAttributes.Static; + featureTagsField.InitExpression = _scenarioPartHelper.GetStringArrayExpression(generationContext.Feature.Tags); + generationContext.TestClass.Members.Add(featureTagsField); + } + + private void DeclareFeatureInfoMember(TestClassGenerationContext generationContext) + { + var featureInfoField = new CodeMemberField( + _codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), GeneratorConstants.FEATUREINFO_FIELD); + featureInfoField.Attributes |= MemberAttributes.Static; + featureInfoField.InitExpression = new CodeObjectCreateExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), + new CodeObjectCreateExpression(typeof(CultureInfo), + new CodePrimitiveExpression(generationContext.Feature.Language)), + new CodePrimitiveExpression(generationContext.Document.DocumentLocation?.FeatureFolderPath), + new CodePrimitiveExpression(generationContext.Feature.Name), + new CodePrimitiveExpression(generationContext.Feature.Description), + new CodeFieldReferenceExpression( + new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(ProgrammingLanguage))), + _codeDomHelper.TargetLanguage.ToString()), + new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME)); + + generationContext.TestClass.Members.Add(featureInfoField); + } + private void SetupTestClassInitializeMethod(TestClassGenerationContext generationContext) { var testClassInitializeMethod = generationContext.TestClassInitializeMethod; @@ -229,7 +253,7 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont nameof(ITestRunner.OnFeatureEndAsync)); _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureEndAsyncExpression); - //if (testRunner.FeatureContext != null && testRunner.FeatureContext.FeatureInfo.Title != "") + //if (testRunner.FeatureContext != null && testRunner.FeatureContext.FeatureInfo != featureInfo) // await testRunner.OnFeatureEndAsync(); // finish if different testInitializeMethod.Statements.Add( new CodeConditionStatement( @@ -241,12 +265,10 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont CodeBinaryOperatorType.BooleanAnd, new CodeBinaryOperatorExpression( new CodePropertyReferenceExpression( - new CodePropertyReferenceExpression( - featureContextExpression, - "FeatureInfo"), - "Title"), + featureContextExpression, + "FeatureInfo"), CodeBinaryOperatorType.IdentityInequality, - new CodePrimitiveExpression(generationContext.Feature.Name))), + new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD))), new CodeExpressionStatement( onFeatureEndAsyncExpression))); @@ -254,27 +276,13 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont // "Start" the feature if needed //if (testRunner.FeatureContext == null) { - // FeatureInfo featureInfo = new FeatureInfo("xxxx"); // await testRunner.OnFeatureStartAsync(featureInfo); //} - var featureInfoInitializeStatement = - new CodeVariableDeclarationStatement(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), "featureInfo", - new CodeObjectCreateExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), - new CodeObjectCreateExpression(typeof(CultureInfo), - new CodePrimitiveExpression(generationContext.Feature.Language)), - new CodePrimitiveExpression(generationContext.Document.DocumentLocation?.FeatureFolderPath), - new CodePrimitiveExpression(generationContext.Feature.Name), - new CodePrimitiveExpression(generationContext.Feature.Description), - new CodeFieldReferenceExpression( - new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(ProgrammingLanguage))), - _codeDomHelper.TargetLanguage.ToString()), - new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME))); - var onFeatureStartExpression = new CodeMethodInvokeExpression( testRunnerField, nameof(ITestRunner.OnFeatureStartAsync), - new CodeVariableReferenceExpression("featureInfo")); + new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD)); _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureStartExpression); testInitializeMethod.Statements.Add( @@ -283,7 +291,6 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont featureContextExpression, CodeBinaryOperatorType.IdentityEquality, new CodePrimitiveExpression(null)), - featureInfoInitializeStatement, new CodeExpressionStatement( onFeatureStartExpression))); } From f7b0d77be93d7d41c60b03017180331dc57f4242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 7 Oct 2024 12:36:30 +0200 Subject: [PATCH 12/26] slow down parallel test to avoid false errors (tests are so fast on Linux CI that they don't run in parallel) --- Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs index bc2b1091d..f049016a5 100644 --- a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs +++ b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs @@ -508,6 +508,8 @@ public void WhenExecuting(string scenarioName, string featureName) throw new System.Exception($"Invalid scenario context: {_featureContext.FeatureInfo.Title} should be {featureName}"); if (!_featureContext.TryGetValue("before_feature", out var value) || !value) throw new System.Exception($"BeforeFeature hook was not executed!"); + + System.Threading.Thread.Sleep(10); var afterStartIndex = startIndex; if (afterStartIndex != currentStartIndex) From 8fe31a5d9cb74edd2eee08f691b2e9b497a9b076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 7 Oct 2024 12:56:05 +0200 Subject: [PATCH 13/26] fix VB compatibility --- .../Generation/UnitTestFeatureGenerator.cs | 15 +++++++++------ .../Generation/GenerationTestBase.cs | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs index 012fb5d9d..2e471d654 100644 --- a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs +++ b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs @@ -253,7 +253,7 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont nameof(ITestRunner.OnFeatureEndAsync)); _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureEndAsyncExpression); - //if (testRunner.FeatureContext != null && testRunner.FeatureContext.FeatureInfo != featureInfo) + //if (testRunner.FeatureContext != null && !testRunner.FeatureContext.FeatureInfo.Equals(featureInfo)) // await testRunner.OnFeatureEndAsync(); // finish if different testInitializeMethod.Statements.Add( new CodeConditionStatement( @@ -264,11 +264,14 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont new CodePrimitiveExpression(null)), CodeBinaryOperatorType.BooleanAnd, new CodeBinaryOperatorExpression( - new CodePropertyReferenceExpression( - featureContextExpression, - "FeatureInfo"), - CodeBinaryOperatorType.IdentityInequality, - new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD))), + new CodeMethodInvokeExpression( + new CodePropertyReferenceExpression( + featureContextExpression, + "FeatureInfo"), + nameof(object.Equals), + new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD)), + CodeBinaryOperatorType.ValueEquality, + new CodePrimitiveExpression(false))), new CodeExpressionStatement( onFeatureEndAsyncExpression))); diff --git a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs index dadc3dd4f..b0f85c614 100644 --- a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs +++ b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs @@ -4,6 +4,7 @@ using FluentAssertions; using System.Text.Json; using System.Collections.Generic; +using Reqnroll.TestProjectGenerator; namespace Reqnroll.SystemTests.Generation; From 8a529ae9cb7200339bb7719fa295710d699d0060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 7 Oct 2024 12:58:54 +0200 Subject: [PATCH 14/26] update CHANGELOG --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecf0b48e1..f26ab2eb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,18 @@ # [vNext] +## Improvements: + +* Support method-level parallel execution (#570) + ## Bug fixes: -* Fix: Rule Backgounds cause External Data Plugin to fail (#271) +* Fix: Rule Backgounds cause External Data Plugin to fail (#119, #271) * Fix: VersionInfo class might provide the version of the runner instead of the version of Reqnroll (#248) * Fix: Reqnroll.CustomPlugin NuGet package has a version mismatch for the System.CodeDom dependency (#244) * Fix: Reqnroll.Verify fails to run parallel tests determinately (#254). See our [verify documentation](docs/integrations/verify.md) on how to set up your test code to enable parallel testing. * Fix: Reqnroll generates invalid code for rule backgrounds in Visual Basic (#283) -*Contributors of this release (in alphabetical order):* @ajeckmans, @clrudolphi, @gasparnagy, @UL-ChrisGlew +*Contributors of this release (in alphabetical order):* @ajeckmans, @clrudolphi, @gasparnagy, @obligaron, @UL-ChrisGlew # v2.1.0 - 2024-08-30 From f7a0126c86ada2adea309e7b21e88a506594d566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 7 Oct 2024 13:00:02 +0200 Subject: [PATCH 15/26] update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f26ab2eb1..30311b532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Improvements: -* Support method-level parallel execution (#570) +* Support scenario-level (method-level) parallel execution (#570) ## Bug fixes: From c93614b86c1c13f07ad2b17313591095e3448780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 7 Oct 2024 13:03:15 +0200 Subject: [PATCH 16/26] remove obsolete specs feature --- .../MsTest/MethodLevelParallelisation.feature | 170 ------------------ 1 file changed, 170 deletions(-) delete mode 100644 Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature diff --git a/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature b/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature deleted file mode 100644 index 86341a4f3..000000000 --- a/Tests/Reqnroll.Specs/Features/UnitTestProviderSpecific/MsTest/MethodLevelParallelisation.feature +++ /dev/null @@ -1,170 +0,0 @@ -@xUnit @NUnit3 @MSTest -Feature: MethodLevel Parallelisation - -Background: - Given there is a Reqnroll project - And parallel execution is enabled - And the following binding class - """ -using Reqnroll; -using Reqnroll.Tracing; -using System.Collections.Concurrent; -using System.Diagnostics; -using FluentAssertions; - -[Binding] -public class TraceSteps -{ - sealed class FeatureData - { - public Stopwatch Duration { get; } = Stopwatch.StartNew(); - public volatile int StepCount; - public ConcurrentDictionary TestRunners { get; } = new ConcurrentDictionary(); - public ConcurrentDictionary FeatureContexts { get; } = new ConcurrentDictionary(); - public ConcurrentDictionary ScenarioContexts { get; } = new ConcurrentDictionary(); - public ConcurrentDictionary BindingInstances { get; } = new ConcurrentDictionary(); - } - - private readonly ITraceListener _traceListener; - private readonly ITestRunner _testRunner; - private volatile int _ScenarioLocalCounter; - - public TraceSteps(ITraceListener traceListener, ITestRunner testRunner) - { - _traceListener = traceListener; - _testRunner = testRunner; - - Interlocked.Increment(ref _ScenarioLocalCounter); - } - - [BeforeFeature] - static void BeforeFeature(FeatureContext featureContext, ITestRunner testRunner) - { - testRunner.ScenarioContext.Should().BeNull(); - - var featureData = new FeatureData(); - featureData.TestRunners.TryAdd(testRunner, 1); - featureData.FeatureContexts.TryAdd(featureContext, 1); - featureContext.Set(featureData); - } - - static FeatureData GetFeatureData(FeatureContext featureContext) => featureContext.Get(); - const int WaitTimeInMS = 1_000; - - [AfterFeature] - static void AfterFeature(FeatureContext featureContext, ITestRunner testRunner) - { - testRunner.ScenarioContext.Should().BeNull(); - - var featureData = GetFeatureData(featureContext); - featureData.TestRunners.TryAdd(testRunner, 1).Should().BeFalse(); - featureData.Duration.Stop(); - //featureData.TestRunners.Count.Should().Be(11, because: "One TestRunner for before/after hooks and one for each test is created"); - //featureData.FeatureContexts.Count.Should().Be(1, because: "Only one FeatureContext should be created"); - //featureData.ScenarioContexts.Count.Should().Be(10, because: "One ScenarioContext for each test is created"); - //featureData.BindingInstances.Count.Should().Be(10, because: "One binding instance for each test is created"); - featureData.Duration.ElapsedMilliseconds.Should().BeLessThan(9 * WaitTimeInMS, because: "Test should be processed (parallel) in time"); - } - - public static int startIndex = 0; - - [When(@"I do something in Scenario '(.*)'")] - void WhenIDoSomething(string scenario) - { - _testRunner.ScenarioContext.Should().NotBeNull(); - _testRunner.ScenarioContext.ScenarioInfo.Title.Should().Be(scenario); - - Interlocked.Increment(ref _ScenarioLocalCounter); - _ScenarioLocalCounter.Should().Be(2); - - var featureData = GetFeatureData(_testRunner.FeatureContext); - featureData.TestRunners.TryAdd(_testRunner, 1); - featureData.FeatureContexts.TryAdd(_testRunner.FeatureContext, 1); - featureData.ScenarioContexts.TryAdd(_testRunner.ScenarioContext, 1); - featureData.BindingInstances.TryAdd(this, 1); - var currentStartIndex = Interlocked.Increment(ref featureData.StepCount); - _traceListener.WriteTestOutput($"Start index: {currentStartIndex}, Worker: {_testRunner.TestWorkerId}"); - - var currentStartIndex2 = System.Threading.Interlocked.Increment(ref startIndex); - _traceListener.WriteTestOutput($"Start index2: {currentStartIndex2}, Worker: {_testRunner.TestWorkerId}"); - - Thread.Sleep(WaitTimeInMS); - var afterStartIndex = featureData.StepCount; - if (afterStartIndex == currentStartIndex) - { - _traceListener.WriteTestOutput("Was not parallel"); - } - else - { - _traceListener.WriteTestOutput("Was parallel"); - } - - var afterStartIndex2 = startIndex; - if (afterStartIndex2 == currentStartIndex2) - { - _traceListener.WriteTestOutput("XWas not parallel"); - } - else - { - _traceListener.WriteTestOutput("XWas parallel"); - } - } -} - """ - - And there is a feature file in the project as - """ -Feature: Feature 1 -Scenario Outline: Simple Scenario Outline 1 - When I do something in Scenario 'Simple Scenario Outline 1' - -Examples: - | Count | - | 1 | - | 2 | - | 3 | - -Scenario Outline: Simple Scenario Outline 2 - When I do something in Scenario 'Simple Scenario Outline 2' - -Examples: - | Count | - | 1 | - | 2 | - | 3 | - -Scenario Outline: Simple Scenario Outline 3 - When I do something in Scenario 'Simple Scenario Outline 3' - -Scenario Outline: Simple Scenario Outline 4 - When I do something in Scenario 'Simple Scenario Outline 4' - -Scenario Outline: Simple Scenario Outline 5 - When I do something in Scenario 'Simple Scenario Outline 5' - -Scenario Outline: Simple Scenario Outline 6 - When I do something in Scenario 'Simple Scenario Outline 6' - """ - -Scenario: Precondition: Tests run parallel - When I execute the tests - Then the execution log should contain text 'Was parallel' - -Scenario: Tests should be processed parallel without failure - When I execute the tests - Then the execution log should contain text 'Was parallel' - And the execution summary should contain - | Total | Succeeded | - | 10 | 10 | - -Scenario Outline: Before/After TestRun hook should only be executed once - Given a hook 'HookFor' for '' - When I execute the tests - Then the execution log should contain text 'Was parallel' - And the hook 'HookFor' is executed once - -Examples: - | event | - | BeforeTestRun | - | AfterTestRun | - From 89760f70c72c280218c6d971c088cb6895b0f345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 7 Oct 2024 16:28:39 +0200 Subject: [PATCH 17/26] Update docs --- docs/execution/parallel-execution.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/execution/parallel-execution.md b/docs/execution/parallel-execution.md index 00bbb3dfa..8afed74c5 100644 --- a/docs/execution/parallel-execution.md +++ b/docs/execution/parallel-execution.md @@ -19,7 +19,7 @@ When using Reqnroll we can consider the parallel scheduling on the level of scen | Scheduling unit | Description | Runner support | | ---------------- | -------------------- | -------------------- | -| Scenario | Scenarios can run in parallel with each other (also from different features) | N/A | +| Scenario | Scenarios can run in parallel with each other (also from different features) | NUnit, MsTest | | Feature | Features can run in parallel with each other. Scenarios from the same feature are running on the same test thread. | NUnit, MsTest, xUnit | | Test assembly | Different test assemblies can run in parallel with each other | e.g. VSTest | @@ -36,7 +36,7 @@ When using Reqnroll we can consider the parallel scheduling on the level of scen ### Requirements -* You have to use a test runner that supports in-process parallel execution (currently NUnit v3, xUnit v2 and MSTest) +* You have to use a test runner that supports in-process parallel execution (NUnit and MsTest supports scenario-level, xUnit supports feature-level) * You have to ensure that your code does not conflict on static state. * You must not use the static context properties of Reqnroll `ScenarioContext.Current`, `FeatureContext.Current` or `ScenarioStepContext.Current` (see further information below). * You have to configure the test runner to execute the Reqnroll features in parallel with each other (see configuration details below). @@ -44,7 +44,7 @@ When using Reqnroll we can consider the parallel scheduling on the level of scen ### Execution Behavior * `[BeforeTestRun]` and `[AfterTestRun]` hooks (events) are executed only once on the first thread that initializes the framework. Executing tests in the other threads is blocked until the hooks have been fully executed on the first thread. -* All scenarios in a feature must be executed on the **same thread**. See the configuration of the test runners below. This ensures that the `[BeforeFeature]` and `[AfterFeature]` hooks are executed only once for each feature and that the thread has a separate (and isolated) `FeatureContext`. +* As a general guideline, **we do not suggest using `[BeforeFeature]` and `[AfterFeature]` hooks and the `FeatureContext` when running the tests parallel**, because in that case it is not guaranteed that these hooks will be executed only once and there will be only one instance of `FeatureContext` per feature. The lifetime of the `FeatureContext` (that starts and finishes by invoking the `[BeforeFeature]` and `[AfterFeature]` hooks) is the consecutive execution of scenarios of a feature on the same parallel execution worker thread. In case of running the scenarios parallel, the scenarios of a feature might be distributed to multiple workers and therefore might have their onw non-unique `FeatureContext`. Because of this behavior the `FeatureContext` is never shared between parallel threads so it does not have to be handled in a thread-safe way. If you wish to have a singleton `FeatureContext` and `[BeforeFeature]` and `[AfterFeature]` hook execution, scenarios in a feature must be executed on the **same thread**. * Scenarios and their related hooks (Before/After scenario, scenario block, step) are isolated in the different threads during execution and do not block each other. Each thread has a separate (and isolated) `ScenarioContext`. * The test trace listener (that outputs the scenario execution trace to the console by default) is invoked asynchronously from the multiple threads and the trace messages are queued and passed to the listener in serialized form. If the test trace listener implements `Reqnroll.Tracing.IThreadSafeTraceListener`, the messages are sent directly from the threads. @@ -54,14 +54,17 @@ By default, [NUnit does not run the tests in parallel](https://docs.nunit.org/ar Parallelization must be configured by setting an assembly-level attribute in the Reqnroll project. ```{code-block} csharp -:caption: C# File +:caption: C# file for configuring feature-level parallelization using NUnit.Framework; [assembly: Parallelizable(ParallelScope.Fixtures)] ``` -```{note} -Reqnroll does not support scenario level parallelization with NUnit (when scenarios from the same feature execute in parallel). If you configure a higher level NUnit parallelization than "Fixtures" your tests will fail with runtime errors. +```{code-block} csharp +:caption: C# file for configuring scenario-level parallelization + +using NUnit.Framework; +[assembly: Parallelizable(ParallelScope.Children)] ``` ### MSTest Configuration @@ -70,14 +73,17 @@ By default, [MsTest does not run the tests in parallel](https://devblogs.microso Parallelisation must be configured by setting an assembly-level attribute in the Reqnroll project. ```{code-block} csharp -:caption: C# File +:caption: C# file for configuring feature-level parallelization using Microsoft.VisualStudio.TestTools.UnitTesting; [assembly: Parallelize(Scope = ExecutionScope.ClassLevel)] ``` -```{note} -Reqnroll does not support scenario level parallelization with MsTest (when scenarios from the same feature execute in parallel). If you configure a higher level MsTest parallelization than "ClassLevel" your tests will fail with runtime errors. +```{code-block} csharp +:caption: C# file for configuring scenario-level parallelization + +using Microsoft.VisualStudio.TestTools.UnitTesting; +[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)] ``` ### xUnit Configuration From de0bc794d0fea26f4a9ba26183f71bb61fb89ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 14 Oct 2024 10:37:42 +0200 Subject: [PATCH 18/26] fix CHANGELOG --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1445e1bf7..b1640244d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Improvements: -* Support scenario-level (method-level) parallel execution (#570) +* Support scenario-level (method-level) parallel execution (#119, #277) ## Bug fixes: @@ -12,13 +12,13 @@ ## Bug fixes: -* Fix: Rule Backgounds cause External Data Plugin to fail (#119, #271) +* Fix: Rule Backgounds cause External Data Plugin to fail (#271) * Fix: VersionInfo class might provide the version of the runner instead of the version of Reqnroll (#248) * Fix: Reqnroll.CustomPlugin NuGet package has a version mismatch for the System.CodeDom dependency (#244) * Fix: Reqnroll.Verify fails to run parallel tests determinately (#254). See our [verify documentation](docs/integrations/verify.md) on how to set up your test code to enable parallel testing. * Fix: Reqnroll generates invalid code for rule backgrounds in Visual Basic (#283) -*Contributors of this release (in alphabetical order):* @ajeckmans, @clrudolphi, @gasparnagy, @obligaron, @UL-ChrisGlew +*Contributors of this release (in alphabetical order):* @ajeckmans, @clrudolphi, @gasparnagy, @UL-ChrisGlew # v2.1.0 - 2024-08-30 From dea281cb8a81ca31ed36dccd51907d7796ae0ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 14 Oct 2024 11:30:00 +0200 Subject: [PATCH 19/26] refactor Log class generation for system tests --- .../Driver/ProjectsDriver.cs | 10 - .../BaseBindingsGenerator.cs | 15 - .../CSharp10BindingsGenerator.cs | 63 +--- .../CSharpBindingsGenerator.cs | 288 +++++++----------- .../FSharpBindingsGenerator.cs | 6 - .../BindingsGenerator/VbBindingsGenerator.cs | 174 ++++------- .../ProjectBuilder.cs | 9 - 7 files changed, 170 insertions(+), 395 deletions(-) diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs index 9eee89bd3..88fb70632 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs @@ -70,16 +70,6 @@ private void AddHookBinding(ProjectBuilder project, string eventType, string nam project.AddHookBinding(eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); } - public void AddAsyncHookBindingIncludingLocking(string eventType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null) - { - AddAsyncHookBindingIncludingLocking(_solutionDriver.DefaultProject, eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); - } - - private void AddAsyncHookBindingIncludingLocking(ProjectBuilder project, string eventType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null) - { - project.AddAsyncHookBindingIncludingLocking(eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); - } - public void AddFeatureFile(string featureFileContent) { LastFeatureFile = _solutionDriver.DefaultProject.AddFeatureFile(featureFileContent); diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs index d799d3426..9f27b968d 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs @@ -28,12 +28,6 @@ public ProjectFile GenerateHookBinding(string eventType, string name, string cod return GenerateBindingClassFile(hookClass); } - public ProjectFile GenerateAsyncHookBindingIncludingLocking(string eventType, string name, string code = null, int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null) - { - string hookClass = GetAsyncHookIncludingLockingBindingClass(eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); - return GenerateBindingClassFile(hookClass); - } - public abstract ProjectFile GenerateLoggerClass(string pathToLogFile); protected abstract string GetBindingCode(string methodName, string methodImplementation, string attributeName, string regex, ParameterType parameterType, string argumentName); @@ -49,15 +43,6 @@ protected abstract string GetHookBindingClass( IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null); - protected abstract string GetAsyncHookIncludingLockingBindingClass( - string hookType, - string name, - string code = "", - int? order = null, - IList hookTypeAttributeTags = null, - IList methodScopeAttributeTags = null, - IList classScopeAttributeTags = null); - protected bool IsStaticEvent(string eventType) { return eventType == "BeforeFeature" || eventType == "AfterFeature" || eventType == "BeforeTestRun" || eventType == "AfterTestRun"; diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs index 2f7809437..f06ffe006 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs @@ -1,66 +1,13 @@ -using Reqnroll.TestProjectGenerator.Data; +using System; namespace Reqnroll.TestProjectGenerator.Factories.BindingsGenerator; public class CSharp10BindingsGenerator : CSharpBindingsGenerator { - public override ProjectFile GenerateLoggerClass(string pathToLogFile) + protected override string GetLogFileContent(string pathToLogFile) { - string fileContent = $$""" - using System; - using System.IO; - using System.Runtime.CompilerServices; - using System.Threading; - - internal static class Log - { - private const string LogFileLocation = @"{{pathToLogFile}}"; - - private static void Retry(int number, Action action) - { - try - { - action(); - } - catch (Exception) - { - var i = number - 1; - - if (i == 0) - throw; - - Thread.Sleep(500); - Retry(i, action); - } - } - - internal static void LogStep([CallerMemberName] string stepName = null!) - { - Retry(5, () => WriteToFile($@"-> step: {stepName}{Environment.NewLine}")); - } - - internal static void LogHook([CallerMemberName] string stepName = null!) - { - Retry(5, () => WriteToFile($@"-> hook: {stepName}{Environment.NewLine}")); - } - - internal static void LogCustom(string category, string value, [CallerMemberName] string memberName = null) - { - Retry(5, () => WriteToFile($@"-> {category}: {value}:{memberName}{Environment.NewLine}")); - } - - static void WriteToFile(string line) - { - using (FileStream fs = File.Open(LogFileLocation, FileMode.Append, FileAccess.Write, FileShare.None)) - { - byte[] bytes = System.Text.Encoding.UTF8.GetBytes(line); - fs.Write(bytes, 0, bytes.Length); - fs.Close(); - } - } - } - """; - return new ProjectFile("Log.cs", "Compile", fileContent); + string logFileContent = base.GetLogFileContent(pathToLogFile); + logFileContent = "#nullable disable" + Environment.NewLine + logFileContent; + return logFileContent; } - } diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs index 40c0229a6..3fad7f40d 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs @@ -10,26 +10,28 @@ namespace Reqnroll.TestProjectGenerator.Factories.BindingsGenerator { public class CSharpBindingsGenerator : BaseBindingsGenerator { - private const string BindingsClassTemplate = @" -using System; -using System.IO; -using System.Xml; -using System.Linq; -using System.Threading.Tasks; -using Reqnroll; - -[Binding] -public class {0} -{{ - private readonly ScenarioContext _scenarioContext; - - public {0}(ScenarioContext scenarioContext) - {{ - _scenarioContext = scenarioContext; - }} - - {1} -}}"; + private const string BindingsClassTemplate = """ + + using System; + using System.IO; + using System.Xml; + using System.Linq; + using System.Threading.Tasks; + using Reqnroll; + + [Binding] + public class {0} + {{ + private readonly ScenarioContext _scenarioContext; + + public {0}(ScenarioContext scenarioContext) + {{ + _scenarioContext = scenarioContext; + }} + + {1} + }} + """; public override ProjectFile GenerateBindingClassFile(string content) { @@ -63,95 +65,71 @@ public override ProjectFile GenerateStepDefinition(string method) public override ProjectFile GenerateLoggerClass(string pathToLogFile) { - string fileContent = $$""" - using System; - using System.IO; - using System.Runtime.CompilerServices; - using System.Threading; - using System.Threading.Tasks; - - internal static class Log - { - private const string LogFileLocation = @"{{pathToLogFile}}"; - - private static void Retry(int number, Action action) - { - try - { - action(); - } - catch (Exception) - { - var i = number - 1; - - if (i == 0) - throw; - - Thread.Sleep(500); - Retry(i, action); - } - } - - internal static void LogStep([CallerMemberName] string stepName = null) - { - Retry(5, () => WriteToFile($@"-> step: {stepName}{Environment.NewLine}")); - } - - internal static void LogHook([CallerMemberName] string stepName = null) - { - Retry(5, () => WriteToFile($@"-> hook: {stepName}{Environment.NewLine}")); - } - - internal static async Task LogHookIncludingLockingAsync([CallerMemberName] string stepName = null) - { - WriteToFile($@"-> waiting for hook lock: {stepName}{Environment.NewLine}"); - await WaitForLockAsync(); - WriteToFile($@"-> hook: {stepName}{Environment.NewLine}"); - } - - internal static void LogCustom(string category, string value, [CallerMemberName] string memberName = null) - { - Retry(5, () => WriteToFile($@"-> {category}: {value}:{memberName}{Environment.NewLine}")); - } - - static void WriteToFile(string line) - { - using (FileStream fs = File.Open(LogFileLocation, FileMode.Append, FileAccess.Write, FileShare.None)) - { - byte[] bytes = System.Text.Encoding.UTF8.GetBytes(line); - fs.Write(bytes, 0, bytes.Length); - fs.Close(); - } - } - - static async Task WaitForLockAsync() - { - var lockFile = LogFileLocation + ".lock"; - - while (true) - { - try - { - using (File.Open(lockFile, FileMode.CreateNew)) - { - } - - break; - } - catch (IOException) - { - //wait and retry - await Task.Delay(1000); - } - } - - File.Delete(lockFile); - } - } - """; + string fileContent = GetLogFileContent(pathToLogFile); return new ProjectFile("Log.cs", "Compile", fileContent); } + protected virtual string GetLogFileContent(string pathToLogFile) + { + string fileContent = $$""" + using System; + using System.IO; + using System.Runtime.CompilerServices; + using System.Threading; + using System.Threading.Tasks; + + internal static class Log + { + private const int RetryCount = 5; + private const string LogFileLocation = @"{{pathToLogFile}}"; + + private static void Retry(int number, Action action) + { + try + { + action(); + } + catch (Exception) + { + var i = number - 1; + + if (i == 0) + throw; + + Thread.Sleep(500); + Retry(i, action); + } + } + + internal static void LogStep([CallerMemberName] string stepName = null) + { + Retry(RetryCount, () => WriteToFile($@"-> step: {stepName}{Environment.NewLine}")); + } + + internal static void LogHook([CallerMemberName] string stepName = null) + { + Retry(RetryCount, () => WriteToFile($@"-> hook: {stepName}{Environment.NewLine}")); + } + + internal static void LogCustom(string category, string value, [CallerMemberName] string memberName = null) + { + Retry(RetryCount, () => WriteToFile($@"-> {category}: {value}:{memberName}{Environment.NewLine}")); + } + + static void WriteToFile(string line) + { + using (FileStream fs = File.Open(LogFileLocation, FileMode.Append, FileAccess.Write, FileShare.None)) + { + byte[] bytes = System.Text.Encoding.UTF8.GetBytes(line); + fs.Write(bytes, 0, bytes.Length); + fs.Close(); + } + } + } + """; + return fileContent; + } + protected override string GetBindingCode(string methodName, string methodImplementation, string attributeName, string regex, ParameterType parameterType, string argumentName) { string parameter = ""; @@ -239,80 +217,30 @@ protected override string GetHookBindingClass( string staticKeyword = isStatic ? "static" : string.Empty; - return $@" -using System; -using System.Collections; -using System.IO; -using System.Linq; -using System.Xml; -using System.Xml.Linq; -using Reqnroll; - -[Binding] -{scopeClassAttributes} -public class {$"HooksClass_{Guid.NewGuid():N}"} -{{ - [{hookType}({hookTypeAttributeTagsString})] - {scopeMethodAttributes} - public {staticKeyword} void {name}() - {{ - {code} - global::Log.LogHook(); - }} -}} -"; - } - - protected override string GetAsyncHookIncludingLockingBindingClass( - string hookType, - string name, - string code = "", - int? order = null, - IList hookTypeAttributeTags = null, - IList methodScopeAttributeTags = null, - IList classScopeAttributeTags = null) - { - string ToScopeTags(IList scopeTags) => scopeTags is null || !scopeTags.Any() ? null : $"[{string.Join(", ", scopeTags.Select(t => $@"Scope(Tag=""{t}"")"))}]"; - - bool isStatic = IsStaticEvent(hookType); - - string hookTags = hookTypeAttributeTags?.Select(t => $@"""{t}""").JoinToString(", "); - - var hookAttributeConstructorProperties = new[] - { - hookTypeAttributeTags is null || !hookTypeAttributeTags.Any() ? null : $"tags: new string[] {{{hookTags}}}", - order is null ? null : $"Order = {order}" - }.Where(p => p.IsNotNullOrWhiteSpace()); - - string hookTypeAttributeTagsString = string.Join(", ", hookAttributeConstructorProperties); - - string scopeClassAttributes = ToScopeTags(classScopeAttributeTags); - string scopeMethodAttributes = ToScopeTags(methodScopeAttributeTags); - string staticKeyword = isStatic ? "static" : string.Empty; - - - return $@" -using System; -using System.Collections; -using System.IO; -using System.Linq; -using System.Xml; -using System.Xml.Linq; -using Reqnroll; - -[Binding] -{scopeClassAttributes} -public class {$"HooksClass_{Guid.NewGuid():N}"} -{{ - [{hookType}({hookTypeAttributeTagsString})] - {scopeMethodAttributes} - public {staticKeyword} async Task {name}() - {{ - {code} - await global::Log.LogHookIncludingLockingAsync(); - }} -}} -"; + return $$""" + + using System; + using System.Collections; + using System.IO; + using System.Linq; + using System.Xml; + using System.Xml.Linq; + using Reqnroll; + + [Binding] + {{scopeClassAttributes}} + public class {{$"HooksClass_{Guid.NewGuid():N}"}} + { + [{{hookType}}({{hookTypeAttributeTagsString}})] + {{scopeMethodAttributes}} + public {{staticKeyword}} void {{name}}() + { + {{code}} + global::Log.LogHook(); + } + } + + """; } } } diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/FSharpBindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/FSharpBindingsGenerator.cs index 8e2739cb5..7381c8a05 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/FSharpBindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/FSharpBindingsGenerator.cs @@ -118,11 +118,5 @@ protected override string GetHookBindingClass( { throw new NotImplementedException(); } - - protected override string GetAsyncHookIncludingLockingBindingClass(string hookType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, - IList classScopeAttributeTags = null) - { - throw new NotImplementedException(); - } } } diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs index d21e1e102..b43c552de 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs @@ -10,13 +10,15 @@ namespace Reqnroll.TestProjectGenerator.Factories.BindingsGenerator { public class VbBindingsGenerator : BaseBindingsGenerator { - private const string BindingsClassTemplate = @" -Imports Reqnroll + private const string BindingsClassTemplate = + """ + Imports Reqnroll - _ -Public Class {0} - {1} -End Class"; + _ + Public Class {0} + {1} + End Class + """; public override ProjectFile GenerateLoggerClass(string pathToLogFile) { @@ -28,56 +30,37 @@ Imports System.Runtime.CompilerServices Imports System.Threading.Tasks Friend Module Log - Private Const LogFileLocation As String = "{{pathToLogFile}}" + Private Const RetryCount As Integer = 5 + Private Const LogFileLocation As String = "{{pathToLogFile}}" - Private Sub Retry(number As Integer, action As Action) - Try - action - Catch ex As Exception - Dim i = number - 1 - If (i = 0) - Throw - End If - System.Threading.Thread.Sleep(500) - Retry(i, action) - End Try - End Sub - - Friend Sub LogStep( Optional stepName As String = Nothing) - Retry(5, sub() File.AppendAllText(LogFileLocation, $"-> step: {stepName}{Environment.NewLine}")) - End Sub - - Friend Sub LogHook( Optional stepName As String = Nothing) - Retry(5, sub() File.AppendAllText(LogFileLocation, $"-> hook: {stepName}{Environment.NewLine}")) - End Sub + Private Sub Retry(number As Integer, action As Action) + Try + action + Catch ex As Exception + Dim i = number - 1 + If (i = 0) + Throw + End If + System.Threading.Thread.Sleep(500) + Retry(i, action) + End Try + End Sub + + Friend Sub LogStep( Optional stepName As String = Nothing) + Retry(RetryCount, sub() WriteToFile($"-> step: {stepName}{Environment.NewLine}")) + End Sub + + Friend Sub LogHook( Optional stepName As String = Nothing) + Retry(RetryCount, sub() WriteToFile($"-> hook: {stepName}{Environment.NewLine}")) + End Sub + + Friend Sub LogCustom(category As String, value As String, Optional memberName As String = Nothing) + Retry(RetryCount, sub() WriteToFile($"-> {category}: {value}:{memberName}{Environment.NewLine}")) + End Sub - Friend Async Function LogHookIncludingLockingAsync( - ByVal Optional stepName As String = Nothing) As Task - File.AppendAllText(LogFileLocation, $"->waiting for hook lock: {stepName}{Environment.NewLine}") - Await WaitForLockAsync() - File.AppendAllText(LogFileLocation, $"-> hook: {stepName}{Environment.NewLine}") - End Function - - Private Async Function WaitForLockAsync() As Task - Dim lockFile = LogFileLocation & ".lock" - - While True - - Dim succeeded = True - Try - Using File.Open(lockFile, FileMode.CreateNew) - End Using - Exit While - Catch __unusedIOException1__ As IOException - succeeded = False - End Try - If Not succeeded Then - Await Task.Delay(1000) - End If - End While - - File.Delete(lockFile) - End Function + Private Sub WriteToFile(line As String) + File.AppendAllText(LogFileLocation, line) + End Sub End Module """; return new ProjectFile("Log.vb", "Compile", fileContent); @@ -182,70 +165,27 @@ protected override string GetHookBindingClass( string methodScopeAttributes = ToScopeTags(methodScopeAttributeTags); string staticKeyword = isStatic ? "Static" : string.Empty; - return $@" -Imports System -Imports System.Collections -Imports System.IO -Imports System.Linq -Imports System.Xml -Imports System.Xml.Linq -Imports Reqnroll - -<[Binding]> _ -{classScopeAttributes} -Public Class {Guid.NewGuid()} - <[{hookType}({hookTypeAttributeTagsString})]>_ - {methodScopeAttributes} - Public {staticKeyword} Sub {name}() - {code} - Console.WriteLine(""-> hook: {name}"") - End Sub -End Class -"; + return $""" + Imports System + Imports System.Collections + Imports System.IO + Imports System.Linq + Imports System.Xml + Imports System.Xml.Linq + Imports Reqnroll + + <[Binding]> _ + {classScopeAttributes} + Public Class {Guid.NewGuid()} + <[{hookType}({hookTypeAttributeTagsString})]>_ + {methodScopeAttributes} + Public {staticKeyword} Sub {name}() + {code} + Console.WriteLine("-> hook: {name}") + End Sub + End Class + """; } - protected override string GetAsyncHookIncludingLockingBindingClass(string hookType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, - IList classScopeAttributeTags = null) - { - string ToScopeTags(IList scopeTags) => scopeTags.Any() ? $"{scopeTags.Select(t => $@"<[Scope](Tag=""{t}"")>").JoinToString("")}_" : null; - - bool isStatic = IsStaticEvent(hookType); - - string hookTypeTags = hookTypeAttributeTags?.Select(t => $@"""{t}""").JoinToString(", "); - - var hookAttributeConstructorProperties = new[] - { - hookTypeAttributeTags is null || !hookTypeAttributeTags.Any() ? null : $"tags:= New String() {{{hookTypeTags}}}", - order is null ? null : $"Order:= {order}" - }.Where(p => p.IsNotNullOrWhiteSpace()); - - string hookTypeAttributeTagsString = string.Join(", ", hookAttributeConstructorProperties); - string classScopeAttributes = ToScopeTags(classScopeAttributeTags); - string methodScopeAttributes = ToScopeTags(methodScopeAttributeTags); - - string staticKeyword = isStatic ? "Static" : string.Empty; - return $@" -Imports System -Imports System.Collections -Imports System.IO -Imports System.Linq -Imports System.Xml -Imports System.Xml.Linq -Imports Reqnroll -Imports System.Threading -Imports System.Threading.Tasks - -<[Binding]> _ -{classScopeAttributes} -Public Class {Guid.NewGuid()} - <[{hookType}({hookTypeAttributeTagsString})]>_ - {methodScopeAttributes} - Public {staticKeyword} Async Function {name}() as Task - {code} - Await Log.LogHookIncludingLockingAsync() - End Function -End Class -"; - } } } diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs index 55afcfe95..410b7cbb1 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs @@ -120,15 +120,6 @@ public void AddHookBinding(string eventType, string name, string code = "", int? _project.AddFile(bindingsGenerator.GenerateHookBinding(eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags)); } - public void AddAsyncHookBindingIncludingLocking(string eventType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, - IList classScopeAttributeTags = null) - { - EnsureProjectExists(); - - var bindingsGenerator = _bindingsGeneratorFactory.FromLanguage(_project.ProgrammingLanguage); - _project.AddFile(bindingsGenerator.GenerateAsyncHookBindingIncludingLocking(eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags)); - } - public void AddStepBinding(string bindingCode) { EnsureProjectExists(); From 1e4dacb1a4c355970e724ffc56cf5b54fed1c11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 14 Oct 2024 11:44:31 +0200 Subject: [PATCH 20/26] Improve System Tests parallel handling --- .../BindingsGenerator/CSharpBindingsGenerator.cs | 12 +++++------- .../BindingsGenerator/VbBindingsGenerator.cs | 8 ++++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs index 3fad7f40d..895131e89 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs @@ -82,6 +82,8 @@ internal static class Log { private const int RetryCount = 5; private const string LogFileLocation = @"{{pathToLogFile}}"; + private static readonly Random Rnd = new Random(); + private static readonly object LockObj = new object(); private static void Retry(int number, Action action) { @@ -96,7 +98,7 @@ private static void Retry(int number, Action action) if (i == 0) throw; - Thread.Sleep(500); + Thread.Sleep(50 + Rnd.Next(50)); Retry(i, action); } } @@ -118,12 +120,8 @@ internal static void LogCustom(string category, string value, [CallerMemberName] static void WriteToFile(string line) { - using (FileStream fs = File.Open(LogFileLocation, FileMode.Append, FileAccess.Write, FileShare.None)) - { - byte[] bytes = System.Text.Encoding.UTF8.GetBytes(line); - fs.Write(bytes, 0, bytes.Length); - fs.Close(); - } + lock(LockObj) + File.AppendAllText(LogFileLocation, line); } } """; diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs index b43c552de..5596f6927 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs @@ -32,6 +32,8 @@ Imports System.Threading.Tasks Friend Module Log Private Const RetryCount As Integer = 5 Private Const LogFileLocation As String = "{{pathToLogFile}}" + Private ReadOnly Rnd As Random = New Random() + Private ReadOnly LockObj As Object = New Object() Private Sub Retry(number As Integer, action As Action) Try @@ -41,7 +43,7 @@ Catch ex As Exception If (i = 0) Throw End If - System.Threading.Thread.Sleep(500) + System.Threading.Thread.Sleep(50 + Rnd.Next(50)) Retry(i, action) End Try End Sub @@ -59,7 +61,9 @@ Friend Sub LogCustom(category As String, value As String, O End Sub Private Sub WriteToFile(line As String) - File.AppendAllText(LogFileLocation, line) + SyncLock LockObj + File.AppendAllText(LogFileLocation, line) + End SyncLock End Sub End Module """; From 5456a0f8e49c1c7f01bee6bef88e5105d3f80088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 14 Oct 2024 13:00:31 +0200 Subject: [PATCH 21/26] apply hinting to choose the test runners in more optimal way --- Reqnroll.sln.DotSettings | 1 + Reqnroll/TestRunnerManager.cs | 44 +++++++++++++++---- .../TestRunnerManagerTests.cs | 32 ++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/Reqnroll.sln.DotSettings b/Reqnroll.sln.DotSettings index 00beadc2b..ff9f19376 100644 --- a/Reqnroll.sln.DotSettings +++ b/Reqnroll.sln.DotSettings @@ -16,6 +16,7 @@ VS MS + <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb"><ExtraRule Prefix="_" Suffix="" Style="aaBb" /></Policy> C:\Users\awi\AppData\Local\JetBrains\Transient\ReSharperPlatformVs15\v09_95175bdf\SolutionCaches Required Required diff --git a/Reqnroll/TestRunnerManager.cs b/Reqnroll/TestRunnerManager.cs index 9472305f3..4adef8283 100644 --- a/Reqnroll/TestRunnerManager.cs +++ b/Reqnroll/TestRunnerManager.cs @@ -24,7 +24,28 @@ public class TestRunnerManager : ITestRunnerManager protected readonly IRuntimeBindingRegistryBuilder _bindingRegistryBuilder; protected readonly ITestTracer _testTracer; - private readonly ConcurrentDictionary _availableTestWorkerContainers = new(); + /// + /// Contains hints to for the test runner manager to choose the test runners from multiple available + /// ones in a more optimal way. Our goal is to keep the test runner "sticky" to the test framework workers + /// so that no unnecessary feature context switches occur. + /// Currently, we remember the managed thread ID, and we try to give back the same test runner for the + /// same managed thread (if possible). + /// + class TestWorkerContainerHint + { + private int ReleasedOnManagedThreadId { get; } = Thread.CurrentThread.ManagedThreadId; + + /// + /// Returns information about how optimal the provided hint for the current situation. Smaller number means more optimal. + /// + public static int GetDistance(TestWorkerContainerHint hint) + { + return hint?.ReleasedOnManagedThreadId == null || hint.ReleasedOnManagedThreadId != Thread.CurrentThread.ManagedThreadId ? + 1 : 0; + } + } + + private readonly ConcurrentDictionary _availableTestWorkerContainers = new(); private readonly ConcurrentDictionary _usedTestWorkerContainers = new(); private int _nextTestWorkerContainerId; @@ -79,7 +100,7 @@ public virtual void ReleaseTestThreadContext(ITestThreadContext testThreadContex var testThreadContainer = testThreadContext.TestThreadContainer; if (!_usedTestWorkerContainers.TryRemove(testThreadContainer, out _)) throw new InvalidOperationException($"TestThreadContext with id {TestThreadContainerInfo.GetId(testThreadContainer)} was already released"); - if (!_availableTestWorkerContainers.TryAdd(testThreadContainer, null)) + if (!_availableTestWorkerContainers.TryAdd(testThreadContainer, new TestWorkerContainerHint())) throw new InvalidOperationException($"TestThreadContext with id {TestThreadContainerInfo.GetId(testThreadContainer)} was released twice"); } @@ -143,16 +164,21 @@ public async Task FireTestRunStartAsync() protected virtual ITestRunner CreateTestRunnerInstance() { IObjectContainer testThreadContainer = null; - for (int i = 0; i < 5 && testThreadContainer == null; i++) // Try to get a available Container max 5 times + for (int i = 0; i < 5 && testThreadContainer == null; i++) // Try to get an available Container max 5 times { var items = _availableTestWorkerContainers.ToArray(); if (items.Length == 0) break; // No Containers are available - foreach (var item in items) + + var prioritizedContainers = items + .OrderBy(it => TestWorkerContainerHint.GetDistance(it.Value)) + .Select(it => it.Key); + + foreach (var container in prioritizedContainers) { - if (!_availableTestWorkerContainers.TryRemove(item.Key, out _)) + if (!_availableTestWorkerContainers.TryRemove(container, out _)) continue; // Container was already taken by another thread - testThreadContainer = item.Key; + testThreadContainer = container; break; } } @@ -205,13 +231,13 @@ private ITestRunner GetTestRunnerWithoutExceptionHandling() private async Task FireRemainingAfterFeatureHooks() { - var testWorkerContainers = _availableTestWorkerContainers.Concat(_usedTestWorkerContainers).ToArray(); + var testWorkerContainers = _availableTestWorkerContainers.Keys.Concat(_usedTestWorkerContainers.Keys).ToArray(); foreach (var testWorkerContainer in testWorkerContainers) { - var contextManager = testWorkerContainer.Key.Resolve(); + var contextManager = testWorkerContainer.Resolve(); if (contextManager.FeatureContext != null) { - var testRunner = testWorkerContainer.Key.Resolve(); + var testRunner = testWorkerContainer.Resolve(); await testRunner.OnFeatureEndAsync(); } } diff --git a/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs b/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs index c10be0aa0..21e55c99d 100644 --- a/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs +++ b/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs @@ -1,5 +1,6 @@ using System; using System.Globalization; +using System.Linq; using System.Reflection; using System.Threading.Tasks; using FluentAssertions; @@ -71,6 +72,37 @@ public void Should_return_different_thread_ids_for_different_instances() TestRunnerManager.ReleaseTestRunner(testRunner3); } + [Fact] + public void Should_use_test_runner_that_was_active_last_time_on_the_same_thread_if_possible() + { + // Use an explicit new ITestRunnerManager to make sure that the Ids are created in a new way. + var container = new RuntimeTestsContainerBuilder().CreateGlobalContainer(_anAssembly); + var testRunnerManager = container.Resolve(); + testRunnerManager.Initialize(_anAssembly); + + var otherRunners1 = Enumerable.Range(0, 5).Select(_ => testRunnerManager.GetTestRunner()).ToList(); + var otherRunners2 = Enumerable.Range(0, 5).Select(_ => testRunnerManager.GetTestRunner()).ToList(); + var ourRunner = testRunnerManager.GetTestRunner(); + + // release otherRunners1 on a different thread + Task.Run(() => otherRunners1.ForEach(TestRunnerManager.ReleaseTestRunner)).Wait(); + // release ourRunner on our thread + TestRunnerManager.ReleaseTestRunner(ourRunner); + // release otherRunners2 on a different thread + Task.Run(() => otherRunners2.ForEach(TestRunnerManager.ReleaseTestRunner)).Wait(); + + // from the same thread, we should get our test runner + var ourRunnerAgain = testRunnerManager.GetTestRunner(); + ourRunnerAgain.TestWorkerId.Should().Be(ourRunner.TestWorkerId); + + // if there is no from the same thread, just provide one of the others + var otherRunnerWithoutHint = testRunnerManager.GetTestRunner(); + otherRunnerWithoutHint.TestWorkerId.Should().NotBe(ourRunner.TestWorkerId); + + TestRunnerManager.ReleaseTestRunner(ourRunnerAgain); + TestRunnerManager.ReleaseTestRunner(otherRunnerWithoutHint); + } + [Fact] public async Task Should_fire_remaining_AfterFeature_hooks_of_test_threads() { From 18cc5b4c600856236b1e7c80c4306d2aa810e1bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 14 Oct 2024 13:59:38 +0200 Subject: [PATCH 22/26] fix unit test warning --- Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs b/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs index 21e55c99d..f751bdce9 100644 --- a/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs +++ b/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs @@ -84,12 +84,17 @@ public void Should_use_test_runner_that_was_active_last_time_on_the_same_thread_ var otherRunners2 = Enumerable.Range(0, 5).Select(_ => testRunnerManager.GetTestRunner()).ToList(); var ourRunner = testRunnerManager.GetTestRunner(); + void RunOnOtherThreadAndWait(Action action) + { + Task.Run(action).Wait(500); + } + // release otherRunners1 on a different thread - Task.Run(() => otherRunners1.ForEach(TestRunnerManager.ReleaseTestRunner)).Wait(); + RunOnOtherThreadAndWait(() => otherRunners1.ForEach(TestRunnerManager.ReleaseTestRunner)); // release ourRunner on our thread TestRunnerManager.ReleaseTestRunner(ourRunner); // release otherRunners2 on a different thread - Task.Run(() => otherRunners2.ForEach(TestRunnerManager.ReleaseTestRunner)).Wait(); + RunOnOtherThreadAndWait(() => otherRunners2.ForEach(TestRunnerManager.ReleaseTestRunner)); // from the same thread, we should get our test runner var ourRunnerAgain = testRunnerManager.GetTestRunner(); From b9895315e31e878eabc3fedca475bcbf0aef2a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Mon, 14 Oct 2024 13:59:52 +0200 Subject: [PATCH 23/26] Add extra check to ensure scenario-level parallelization --- .../Generation/GenerationTestBase.cs | 20 +++++++++++++++++-- .../Generation/XUnitGenerationTest.cs | 5 +++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs index b0f85c614..1401a38e5 100644 --- a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs +++ b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs @@ -494,6 +494,8 @@ namespace ParallelExecution.StepDefinitions public class ParallelExecutionSteps { public static int startIndex = 0; + public static int featureAStartIndex = 0; + public static int featureBStartIndex = 0; private readonly FeatureContext _featureContext; private readonly ScenarioContext _scenarioContext; @@ -514,6 +516,9 @@ public static void BeforeFeature(FeatureContext featureContext) public void WhenExecuting(string scenarioName, string featureName) { var currentStartIndex = System.Threading.Interlocked.Increment(ref startIndex); + var currentFeatureStartIndex = featureName == "A" + ? System.Threading.Interlocked.Increment(ref featureAStartIndex) + : System.Threading.Interlocked.Increment(ref featureBStartIndex); global::Log.LogStep(); if (_scenarioContext.ScenarioInfo.Title != scenarioName) throw new System.Exception($"Invalid scenario context: {_scenarioContext.ScenarioInfo.Title} should be {scenarioName}"); @@ -527,6 +532,9 @@ public void WhenExecuting(string scenarioName, string featureName) var afterStartIndex = startIndex; if (afterStartIndex != currentStartIndex) Log.LogCustom("parallel", "true"); + var afterFeatureStartIndex = featureName == "A" ? featureAStartIndex : featureBStartIndex; + if (afterFeatureStartIndex != currentFeatureStartIndex) + Log.LogCustom("scenario-parallel", "true"); } } } @@ -535,8 +543,16 @@ public void WhenExecuting(string scenarioName, string featureName) ExecuteTests(); ShouldAllScenariosPass(); - var arguments = _bindingDriver.GetActualLogLines("parallel").ToList(); - arguments.Should().NotBeEmpty("the scenarios should have run parallel"); + var parallelLogs = _bindingDriver.GetActualLogLines("parallel").ToList(); + parallelLogs.Should().NotBeEmpty("the scenarios should have run parallel"); + + AssertScenarioLevelParallelExecution(); + } + + protected virtual void AssertScenarioLevelParallelExecution() + { + var scenarioParallelLogs = _bindingDriver.GetActualLogLines("scenario-parallel").ToList(); + scenarioParallelLogs.Should().NotBeEmpty("the scenarios should have run parallel using scenario-level parallelization"); } #endregion diff --git a/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs b/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs index 086f2ab37..f9978e5a8 100644 --- a/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs +++ b/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs @@ -19,4 +19,9 @@ protected override void TestInitialize() protected override string GetExpectedPendingOutcome() => "Failed"; protected override string GetExpectedUndefinedOutcome() => "Failed"; + + protected override void AssertScenarioLevelParallelExecution() + { + //nop - xUnit currently does not support method-level parallelization + } } \ No newline at end of file From c569e421b9445573f37c47ff7570d60ea909d39b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Tue, 15 Oct 2024 11:44:15 +0200 Subject: [PATCH 24/26] Use feature info to optimize test runner reuse --- Reqnroll.Generator/CodeDom/CodeDomHelper.cs | 12 ++++ .../Generation/UnitTestFeatureGenerator.cs | 6 +- Reqnroll/FeatureInfo.cs | 2 + Reqnroll/ITestRunnerManager.cs | 2 +- Reqnroll/TestRunnerManager.cs | 55 ++++++++++++------- .../TestRunnerManagerTests.cs | 48 +++++++++++----- 6 files changed, 88 insertions(+), 37 deletions(-) diff --git a/Reqnroll.Generator/CodeDom/CodeDomHelper.cs b/Reqnroll.Generator/CodeDom/CodeDomHelper.cs index 407d49280..4a9e8e5f7 100644 --- a/Reqnroll.Generator/CodeDom/CodeDomHelper.cs +++ b/Reqnroll.Generator/CodeDom/CodeDomHelper.cs @@ -298,6 +298,18 @@ public string GetGlobalizedTypeName(Type type) // Global namespaces not yet supported in VB return type.FullName!; } + + public CodeExpression CreateOptionalArgumentExpression(string parameterName, CodeVariableReferenceExpression valueExpression) + { + switch (TargetLanguage) + { + case CodeDomProviderLanguage.CSharp: + return new CodeSnippetExpression($"{parameterName}: {valueExpression.VariableName}"); + case CodeDomProviderLanguage.VB: + return new CodeSnippetExpression($"{parameterName} := {valueExpression.VariableName}"); + } + return valueExpression; + } } } diff --git a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs index 2e471d654..41b2e47ce 100644 --- a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs +++ b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs @@ -225,13 +225,15 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont _testGeneratorProvider.SetTestInitializeMethod(generationContext); // Obtain the test runner for executing a single test - // testRunner = global::Reqnroll.TestRunnerManager.GetTestRunnerForAssembly(); + // testRunner = global::Reqnroll.TestRunnerManager.GetTestRunnerForAssembly(featureHint: featureInfo); var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression(); var getTestRunnerExpression = new CodeMethodInvokeExpression( new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))), - nameof(TestRunnerManager.GetTestRunnerForAssembly)); + nameof(TestRunnerManager.GetTestRunnerForAssembly), + _codeDomHelper.CreateOptionalArgumentExpression("featureHint", + new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD))); testInitializeMethod.Statements.Add( new CodeAssignStatement( diff --git a/Reqnroll/FeatureInfo.cs b/Reqnroll/FeatureInfo.cs index cdb62f7b7..d6c7cfaed 100644 --- a/Reqnroll/FeatureInfo.cs +++ b/Reqnroll/FeatureInfo.cs @@ -1,9 +1,11 @@ using System; +using System.Diagnostics; using System.Globalization; using Reqnroll.Tracing; namespace Reqnroll { + [DebuggerDisplay("{Title}")] public class FeatureInfo { public string[] Tags { get; private set; } diff --git a/Reqnroll/ITestRunnerManager.cs b/Reqnroll/ITestRunnerManager.cs index 815926d33..b81627426 100644 --- a/Reqnroll/ITestRunnerManager.cs +++ b/Reqnroll/ITestRunnerManager.cs @@ -10,7 +10,7 @@ public interface ITestRunnerManager Assembly TestAssembly { get; } Assembly[] BindingAssemblies { get; } bool IsMultiThreaded { get; } - ITestRunner GetTestRunner(); + ITestRunner GetTestRunner(FeatureInfo featureHint = null); void ReleaseTestThreadContext(ITestThreadContext testThreadContext); void Initialize(Assembly testAssembly); Task FireTestRunEndAsync(); diff --git a/Reqnroll/TestRunnerManager.cs b/Reqnroll/TestRunnerManager.cs index 4adef8283..62dd12656 100644 --- a/Reqnroll/TestRunnerManager.cs +++ b/Reqnroll/TestRunnerManager.cs @@ -28,25 +28,30 @@ public class TestRunnerManager : ITestRunnerManager /// Contains hints to for the test runner manager to choose the test runners from multiple available /// ones in a more optimal way. Our goal is to keep the test runner "sticky" to the test framework workers /// so that no unnecessary feature context switches occur. - /// Currently, we remember the managed thread ID, and we try to give back the same test runner for the - /// same managed thread (if possible). + /// Currently, we remember the feature info and the managed thread ID to help this. /// - class TestWorkerContainerHint + class TestWorkerContainerHint(FeatureInfo lastUsedFeatureInfo, int? releasedOnManagedThreadId) { - private int ReleasedOnManagedThreadId { get; } = Thread.CurrentThread.ManagedThreadId; + public FeatureInfo LastUsedFeatureInfo { get; } = lastUsedFeatureInfo; + + private int? ReleasedOnManagedThreadId { get; } = releasedOnManagedThreadId; /// /// Returns information about how optimal the provided hint for the current situation. Smaller number means more optimal. /// - public static int GetDistance(TestWorkerContainerHint hint) + public static int GetDistance(TestWorkerContainerHint hint, FeatureInfo featureHint) { - return hint?.ReleasedOnManagedThreadId == null || hint.ReleasedOnManagedThreadId != Thread.CurrentThread.ManagedThreadId ? + int distance = 0; + distance += hint?.LastUsedFeatureInfo == null || featureHint == null || !ReferenceEquals(hint.LastUsedFeatureInfo, featureHint) ? + 2 : 0; + distance += hint?.ReleasedOnManagedThreadId == null || hint.ReleasedOnManagedThreadId != Thread.CurrentThread.ManagedThreadId ? 1 : 0; + return distance; } } private readonly ConcurrentDictionary _availableTestWorkerContainers = new(); - private readonly ConcurrentDictionary _usedTestWorkerContainers = new(); + private readonly ConcurrentDictionary _usedTestWorkerContainers = new(); private int _nextTestWorkerContainerId; public bool IsTestRunInitialized { get; private set; } @@ -76,9 +81,9 @@ private int GetWorkerTestRunnerCount() return _usedTestWorkerContainers.Count - (hasTestRunStartWorker ? 1 : 0); } - public virtual ITestRunner CreateTestRunner() + public virtual ITestRunner GetOrCreateTestRunner(FeatureInfo featureHint = null) { - var testRunner = CreateTestRunnerInstance(); + var testRunner = GetOrCreateTestRunnerInstance(featureHint); if (!IsTestRunInitialized) { @@ -98,9 +103,10 @@ public virtual ITestRunner CreateTestRunner() public virtual void ReleaseTestThreadContext(ITestThreadContext testThreadContext) { var testThreadContainer = testThreadContext.TestThreadContainer; - if (!_usedTestWorkerContainers.TryRemove(testThreadContainer, out _)) + if (!_usedTestWorkerContainers.TryRemove(testThreadContainer, out var usedContainerHint)) throw new InvalidOperationException($"TestThreadContext with id {TestThreadContainerInfo.GetId(testThreadContainer)} was already released"); - if (!_availableTestWorkerContainers.TryAdd(testThreadContainer, new TestWorkerContainerHint())) + var containerHint = new TestWorkerContainerHint(usedContainerHint?.LastUsedFeatureInfo, Thread.CurrentThread.ManagedThreadId); + if (!_availableTestWorkerContainers.TryAdd(testThreadContainer, containerHint)) throw new InvalidOperationException($"TestThreadContext with id {TestThreadContainerInfo.GetId(testThreadContainer)} was released twice"); } @@ -161,7 +167,7 @@ public async Task FireTestRunStartAsync() await onTestRunnerStartExecutionHost.OnTestRunStartAsync(); } - protected virtual ITestRunner CreateTestRunnerInstance() + protected virtual ITestRunner GetOrCreateTestRunnerInstance(FeatureInfo featureHint = null) { IObjectContainer testThreadContainer = null; for (int i = 0; i < 5 && testThreadContainer == null; i++) // Try to get an available Container max 5 times @@ -171,7 +177,7 @@ protected virtual ITestRunner CreateTestRunnerInstance() break; // No Containers are available var prioritizedContainers = items - .OrderBy(it => TestWorkerContainerHint.GetDistance(it.Value)) + .OrderBy(it => TestWorkerContainerHint.GetDistance(it.Value, featureHint)) .Select(it => it.Key); foreach (var container in prioritizedContainers) @@ -191,7 +197,7 @@ protected virtual ITestRunner CreateTestRunnerInstance() testThreadContainer.RegisterInstanceAs(testThreadContainerInfo); } - if (!_usedTestWorkerContainers.TryAdd(testThreadContainer, null)) + if (!_usedTestWorkerContainers.TryAdd(testThreadContainer, new TestWorkerContainerHint(featureHint, null))) throw new InvalidOperationException($"TestThreadContext with id {TestThreadContainerInfo.GetId(testThreadContainer)} is already in usage"); return testThreadContainer.Resolve(); @@ -202,11 +208,11 @@ public void Initialize(Assembly assignedTestAssembly) TestAssembly = assignedTestAssembly; } - public virtual ITestRunner GetTestRunner() + public virtual ITestRunner GetTestRunner(FeatureInfo featureHint = null) { try { - return GetTestRunnerWithoutExceptionHandling(); + return GetTestRunnerWithoutExceptionHandling(featureHint); } catch (Exception ex) { @@ -215,9 +221,9 @@ public virtual ITestRunner GetTestRunner() } } - private ITestRunner GetTestRunnerWithoutExceptionHandling() + private ITestRunner GetTestRunnerWithoutExceptionHandling(FeatureInfo featureHint) { - var testRunner = CreateTestRunner(); + var testRunner = GetOrCreateTestRunner(featureHint); if (IsMultiThreaded && Interlocked.CompareExchange(ref _wasSingletonInstanceDisabled, 1, 0) == 0) { @@ -359,11 +365,20 @@ public static async Task OnTestRunStartAsync(Assembly testAssembly = null, ICont await testRunnerManager.FireTestRunStartAsync(); } - public static ITestRunner GetTestRunnerForAssembly(Assembly testAssembly = null, IContainerBuilder containerBuilder = null) + /// + /// Provides a test runner for the specified or current assembly with optionally a custom container builder and a feature hint. + /// When a feature hint is provided the test runner manager will attempt to return the same test runner that was used for that + /// feature before. + /// + /// The test assembly. If omitted or invoked with null, the calling assembly is used. + /// The container builder to be used to set up Reqnroll dependencies. If omitted or invoked with null, the default container builder is used. + /// If specified, it is used as a hint for the test runner manager to choose the test runner that has been used for the feature before, if possible. + /// A test runner that can be used to interact with Reqnroll. + public static ITestRunner GetTestRunnerForAssembly(Assembly testAssembly = null, IContainerBuilder containerBuilder = null, FeatureInfo featureHint = null) { testAssembly ??= GetCallingAssembly(); var testRunnerManager = GetTestRunnerManager(testAssembly, containerBuilder); - return testRunnerManager.GetTestRunner(); + return testRunnerManager.GetTestRunner(featureHint); } public static void ReleaseTestRunner(ITestRunner testRunner) diff --git a/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs b/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs index f751bdce9..0ad07f57f 100644 --- a/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs +++ b/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs @@ -28,7 +28,7 @@ public async Task DisposeAsync() [Fact] public void CreateTestRunner_should_be_able_to_create_a_TestRunner() { - var testRunner = _testRunnerManager.CreateTestRunner(); + var testRunner = _testRunnerManager.GetOrCreateTestRunner(); testRunner.Should().NotBeNull(); testRunner.Should().BeOfType(); @@ -75,14 +75,14 @@ public void Should_return_different_thread_ids_for_different_instances() [Fact] public void Should_use_test_runner_that_was_active_last_time_on_the_same_thread_if_possible() { - // Use an explicit new ITestRunnerManager to make sure that the Ids are created in a new way. - var container = new RuntimeTestsContainerBuilder().CreateGlobalContainer(_anAssembly); - var testRunnerManager = container.Resolve(); - testRunnerManager.Initialize(_anAssembly); + var ourFeatureInfo = new FeatureInfo(new CultureInfo("en-US"), null, "F1", null); + var otherFeatureInfo = new FeatureInfo(new CultureInfo("en-US"), null, "F2", null); - var otherRunners1 = Enumerable.Range(0, 5).Select(_ => testRunnerManager.GetTestRunner()).ToList(); - var otherRunners2 = Enumerable.Range(0, 5).Select(_ => testRunnerManager.GetTestRunner()).ToList(); - var ourRunner = testRunnerManager.GetTestRunner(); + var otherRunners1 = Enumerable.Range(0, 5).Select(_ => TestRunnerManager.GetTestRunnerForAssembly(featureHint: otherFeatureInfo)).ToList(); + var otherRunners2 = Enumerable.Range(0, 5).Select(_ => TestRunnerManager.GetTestRunnerForAssembly(featureHint: otherFeatureInfo)).ToList(); + var runnerOnDifferentFeatureSameThread = TestRunnerManager.GetTestRunnerForAssembly(featureHint: otherFeatureInfo); + var runnerOnSameFeatureDifferentThread = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); + var ourRunner = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); void RunOnOtherThreadAndWait(Action action) { @@ -91,20 +91,40 @@ void RunOnOtherThreadAndWait(Action action) // release otherRunners1 on a different thread RunOnOtherThreadAndWait(() => otherRunners1.ForEach(TestRunnerManager.ReleaseTestRunner)); - // release ourRunner on our thread + + // release the selected runners TestRunnerManager.ReleaseTestRunner(ourRunner); + TestRunnerManager.ReleaseTestRunner(runnerOnDifferentFeatureSameThread); + RunOnOtherThreadAndWait(() => TestRunnerManager.ReleaseTestRunner(runnerOnSameFeatureDifferentThread)); + // release otherRunners2 on a different thread RunOnOtherThreadAndWait(() => otherRunners2.ForEach(TestRunnerManager.ReleaseTestRunner)); - // from the same thread, we should get our test runner - var ourRunnerAgain = testRunnerManager.GetTestRunner(); + // Priority: + // 1. Same feature, same thread + // 2. Same feature, different thread + // 3. Other feature, same thread + // 4. Other feature, other thread + + // Priority 1: from the same feature & same thread, we should get our test runner first + var ourRunnerAgain = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); ourRunnerAgain.TestWorkerId.Should().Be(ourRunner.TestWorkerId); - // if there is no from the same thread, just provide one of the others - var otherRunnerWithoutHint = testRunnerManager.GetTestRunner(); + // Priority 2: from the same feature & different thread + var runnerOnSameFeatureDifferentThreadAgain = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); + runnerOnSameFeatureDifferentThreadAgain.TestWorkerId.Should().Be(runnerOnSameFeatureDifferentThread.TestWorkerId); + + // Priority 3: from the different feature & same thread + var runnerOnDifferentFeatureSameThreadAgain = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); + runnerOnDifferentFeatureSameThreadAgain.TestWorkerId.Should().Be(runnerOnDifferentFeatureSameThread.TestWorkerId); + + // Priority 4: if there is no from the same thread, just provide one of the others + var otherRunnerWithoutHint = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); otherRunnerWithoutHint.TestWorkerId.Should().NotBe(ourRunner.TestWorkerId); TestRunnerManager.ReleaseTestRunner(ourRunnerAgain); + TestRunnerManager.ReleaseTestRunner(runnerOnSameFeatureDifferentThreadAgain); + TestRunnerManager.ReleaseTestRunner(runnerOnDifferentFeatureSameThreadAgain); TestRunnerManager.ReleaseTestRunner(otherRunnerWithoutHint); } @@ -176,7 +196,7 @@ public void First_call_to_CreateTestRunner_should_initialize_binding_registry() { _testRunnerManager.IsTestRunInitialized.Should().BeFalse("binding registry should not be initialized initially"); - _testRunnerManager.CreateTestRunner(); + _testRunnerManager.GetOrCreateTestRunner(); _testRunnerManager.IsTestRunInitialized.Should().BeTrue("binding registry be initialized"); } From 971e6bf80b0db30a1d764eee022cc5978cd7831c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Tue, 15 Oct 2024 12:13:35 +0200 Subject: [PATCH 25/26] fix a comment --- Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs index 41b2e47ce..151b3c3cc 100644 --- a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs +++ b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs @@ -243,9 +243,6 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont // "Finish" current feature if needed - //if (testRunner.FeatureContext != null && testRunner.FeatureContext.FeatureInfo.Title != "") - // await testRunner.OnFeatureEndAsync(); // finish if different - var featureContextExpression = new CodePropertyReferenceExpression( testRunnerField, "FeatureContext"); From c7012e8be8c660359385bb0483934d0231f3b78b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Tue, 5 Nov 2024 08:59:53 +0100 Subject: [PATCH 26/26] fix typo, make docs clearer --- docs/execution/parallel-execution.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/execution/parallel-execution.md b/docs/execution/parallel-execution.md index 8afed74c5..871ec880d 100644 --- a/docs/execution/parallel-execution.md +++ b/docs/execution/parallel-execution.md @@ -44,7 +44,7 @@ When using Reqnroll we can consider the parallel scheduling on the level of scen ### Execution Behavior * `[BeforeTestRun]` and `[AfterTestRun]` hooks (events) are executed only once on the first thread that initializes the framework. Executing tests in the other threads is blocked until the hooks have been fully executed on the first thread. -* As a general guideline, **we do not suggest using `[BeforeFeature]` and `[AfterFeature]` hooks and the `FeatureContext` when running the tests parallel**, because in that case it is not guaranteed that these hooks will be executed only once and there will be only one instance of `FeatureContext` per feature. The lifetime of the `FeatureContext` (that starts and finishes by invoking the `[BeforeFeature]` and `[AfterFeature]` hooks) is the consecutive execution of scenarios of a feature on the same parallel execution worker thread. In case of running the scenarios parallel, the scenarios of a feature might be distributed to multiple workers and therefore might have their onw non-unique `FeatureContext`. Because of this behavior the `FeatureContext` is never shared between parallel threads so it does not have to be handled in a thread-safe way. If you wish to have a singleton `FeatureContext` and `[BeforeFeature]` and `[AfterFeature]` hook execution, scenarios in a feature must be executed on the **same thread**. +* As a general guideline, **we do not suggest using `[BeforeFeature]` and `[AfterFeature]` hooks and the `FeatureContext` when running the tests parallel**, because in that case it is not guaranteed that these hooks will be executed only once and there will be only one instance of `FeatureContext` per feature. The lifetime of the `FeatureContext` (that starts and finishes by invoking the `[BeforeFeature]` and `[AfterFeature]` hooks) is the consecutive execution of scenarios of a feature on the same parallel execution worker thread. In case of running the scenarios parallel, the scenarios of a feature might be distributed to multiple workers and therefore might have their own dedicated `FeatureContext`. Because of this behavior the `FeatureContext` is never shared between parallel threads so it does not have to be handled in a thread-safe way. If you wish to have a singleton `FeatureContext` and `[BeforeFeature]` and `[AfterFeature]` hook execution, scenarios in a feature must be executed on the **same thread**. * Scenarios and their related hooks (Before/After scenario, scenario block, step) are isolated in the different threads during execution and do not block each other. Each thread has a separate (and isolated) `ScenarioContext`. * The test trace listener (that outputs the scenario execution trace to the console by default) is invoked asynchronously from the multiple threads and the trace messages are queued and passed to the listener in serialized form. If the test trace listener implements `Reqnroll.Tracing.IThreadSafeTraceListener`, the messages are sent directly from the threads.