diff --git a/CHANGELOG.md b/CHANGELOG.md index da2a268e0..c839fa14a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # [vNext] ## Improvements: + * Upgrade to Gherkin v30 from v29 (see [Gherkin changelog](https://github.com/cucumber/gherkin/blob/main/CHANGELOG.md)) (#305) +* Support scenario-level (method-level) parallel execution (#119, #277) ## Bug fixes: diff --git a/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.cs b/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.cs index e87a467fe..b04eee1f2 100644 --- a/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.cs +++ b/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.cs @@ -6,13 +6,15 @@ using global::System.Runtime.CompilerServices; using System.Threading.Tasks; +[assembly: NUnit.Framework.FixtureLifeCycle(NUnit.Framework.LifeCycle.InstancePerTestCase)] + [GeneratedCode("Reqnroll", "REQNROLL_VERSION")] [global::NUnit.Framework.SetUpFixture] -public class PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks +public static class PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks { [global::NUnit.Framework.OneTimeSetUp] [MethodImpl(MethodImplOptions.NoInlining)] - public async Task AssemblyInitializeAsync() + public static async Task AssemblyInitializeAsync() { var currentAssembly = typeof(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly; await global::Reqnroll.TestRunnerManager.OnTestRunStartAsync(currentAssembly); @@ -20,7 +22,7 @@ public async Task AssemblyInitializeAsync() [global::NUnit.Framework.OneTimeTearDown] [MethodImpl(MethodImplOptions.NoInlining)] - public async ValueTask AssemblyCleanupAsync() + public static async ValueTask AssemblyCleanupAsync() { var currentAssembly = typeof(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly; await global::Reqnroll.TestRunnerManager.OnTestRunEndAsync(currentAssembly); diff --git a/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.vb b/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.vb index 07a98e545..de1925645 100644 --- a/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.vb +++ b/Plugins/Reqnroll.NUnit.Generator.ReqnrollPlugin/build/NUnit.AssemblyHooks.template.vb @@ -7,19 +7,21 @@ Imports System.CodeDom.Compiler Imports System.Reflection Imports System.Runtime.CompilerServices + + Public NotInheritable Class PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks - Public Async Function AssemblyInitializeAsync() As Task + Public Shared Async Function AssemblyInitializeAsync() As Task Dim currentAssembly As Assembly = GetType(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly Await Global.Reqnroll.TestRunnerManager.OnTestRunStartAsync(currentAssembly) End Function - Public Async Function AssemblyCleanupAsync() As Task + Public Shared Async Function AssemblyCleanupAsync() As Task Dim currentAssembly As Assembly = GetType(PROJECT_ROOT_NAMESPACE_NUnitAssemblyHooks).Assembly Await Global.Reqnroll.TestRunnerManager.OnTestRunEndAsync(currentAssembly) End Function diff --git a/Reqnroll.Generator/CodeDom/CodeDomHelper.cs b/Reqnroll.Generator/CodeDom/CodeDomHelper.cs index 407d49280..4a9e8e5f7 100644 --- a/Reqnroll.Generator/CodeDom/CodeDomHelper.cs +++ b/Reqnroll.Generator/CodeDom/CodeDomHelper.cs @@ -298,6 +298,18 @@ public string GetGlobalizedTypeName(Type type) // Global namespaces not yet supported in VB return type.FullName!; } + + public CodeExpression CreateOptionalArgumentExpression(string parameterName, CodeVariableReferenceExpression valueExpression) + { + switch (TargetLanguage) + { + case CodeDomProviderLanguage.CSharp: + return new CodeSnippetExpression($"{parameterName}: {valueExpression.VariableName}"); + case CodeDomProviderLanguage.VB: + return new CodeSnippetExpression($"{parameterName} := {valueExpression.VariableName}"); + } + return valueExpression; + } } } diff --git a/Reqnroll.Generator/Generation/GeneratorConstants.cs b/Reqnroll.Generator/Generation/GeneratorConstants.cs index 3a9a6b081..859c3a111 100644 --- a/Reqnroll.Generator/Generation/GeneratorConstants.cs +++ b/Reqnroll.Generator/Generation/GeneratorConstants.cs @@ -13,6 +13,7 @@ public class GeneratorConstants public const string TESTCLASS_CLEANUP_NAME = "FeatureTearDownAsync"; public const string BACKGROUND_NAME = "FeatureBackgroundAsync"; public const string TESTRUNNER_FIELD = "testRunner"; + public const string FEATUREINFO_FIELD = "featureInfo"; public const string REQNROLL_NAMESPACE = "Reqnroll"; public const string SCENARIO_OUTLINE_EXAMPLE_TAGS_PARAMETER = "exampleTags"; public const string SCENARIO_TAGS_VARIABLE_NAME = "tagsOfScenario"; diff --git a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs index 404d22d75..151b3c3cc 100644 --- a/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs +++ b/Reqnroll.Generator/Generation/UnitTestFeatureGenerator.cs @@ -1,5 +1,6 @@ using System; using System.CodeDom; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Reflection; @@ -12,6 +13,7 @@ namespace Reqnroll.Generator.Generation { + [SuppressMessage("ReSharper", "BitwiseOperatorOnEnumWithoutFlags")] public class UnitTestFeatureGenerator : IFeatureGenerator { private readonly CodeDomHelper _codeDomHelper; @@ -149,11 +151,8 @@ private void SetupTestClass(TestClassGenerationContext generationContext) _testGeneratorProvider.SetTestClassCategories(generationContext, featureCategories); } - var featureTagsField = new CodeMemberField(typeof(string[]), GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME); - featureTagsField.Attributes |= MemberAttributes.Static; - featureTagsField.InitExpression = _scenarioPartHelper.GetStringArrayExpression(generationContext.Feature.Tags); - - generationContext.TestClass.Members.Add(featureTagsField); + DeclareFeatureTagsField(generationContext); + DeclareFeatureInfoMember(generationContext); } private CodeMemberField DeclareTestRunnerMember(CodeTypeDeclaration type) @@ -163,6 +162,33 @@ private CodeMemberField DeclareTestRunnerMember(CodeTypeDeclaration type) return testRunnerField; } + private void DeclareFeatureTagsField(TestClassGenerationContext generationContext) + { + var featureTagsField = new CodeMemberField(typeof(string[]), GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME); + featureTagsField.Attributes |= MemberAttributes.Static; + featureTagsField.InitExpression = _scenarioPartHelper.GetStringArrayExpression(generationContext.Feature.Tags); + generationContext.TestClass.Members.Add(featureTagsField); + } + + private void DeclareFeatureInfoMember(TestClassGenerationContext generationContext) + { + var featureInfoField = new CodeMemberField( + _codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), GeneratorConstants.FEATUREINFO_FIELD); + featureInfoField.Attributes |= MemberAttributes.Static; + featureInfoField.InitExpression = new CodeObjectCreateExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), + new CodeObjectCreateExpression(typeof(CultureInfo), + new CodePrimitiveExpression(generationContext.Feature.Language)), + new CodePrimitiveExpression(generationContext.Document.DocumentLocation?.FeatureFolderPath), + new CodePrimitiveExpression(generationContext.Feature.Name), + new CodePrimitiveExpression(generationContext.Feature.Description), + new CodeFieldReferenceExpression( + new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(ProgrammingLanguage))), + _codeDomHelper.TargetLanguage.ToString()), + new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME)); + + generationContext.TestClass.Members.Add(featureInfoField); + } + private void SetupTestClassInitializeMethod(TestClassGenerationContext generationContext) { var testClassInitializeMethod = generationContext.TestClassInitializeMethod; @@ -173,42 +199,6 @@ private void SetupTestClassInitializeMethod(TestClassGenerationContext generatio _codeDomHelper.MarkCodeMemberMethodAsAsync(testClassInitializeMethod); _testGeneratorProvider.SetTestClassInitializeMethod(generationContext); - - //testRunner = TestRunnerManager.GetTestRunnerForAssembly(null, [test_worker_id]); - var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression(); - - var getTestRunnerExpression = new CodeMethodInvokeExpression( - new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))), - nameof(TestRunnerManager.GetTestRunnerForAssembly)); - - testClassInitializeMethod.Statements.Add( - new CodeAssignStatement( - testRunnerField, - getTestRunnerExpression)); - - //FeatureInfo featureInfo = new FeatureInfo("xxxx"); - testClassInitializeMethod.Statements.Add( - new CodeVariableDeclarationStatement(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), "featureInfo", - new CodeObjectCreateExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(FeatureInfo)), - new CodeObjectCreateExpression(typeof(CultureInfo), - new CodePrimitiveExpression(generationContext.Feature.Language)), - new CodePrimitiveExpression(generationContext.Document.DocumentLocation?.FeatureFolderPath), - new CodePrimitiveExpression(generationContext.Feature.Name), - new CodePrimitiveExpression(generationContext.Feature.Description), - new CodeFieldReferenceExpression( - new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(Reqnroll.ProgrammingLanguage))), - _codeDomHelper.TargetLanguage.ToString()), - new CodeFieldReferenceExpression(null, GeneratorConstants.FEATURE_TAGS_VARIABLE_NAME)))); - - //await testRunner.OnFeatureStartAsync(featureInfo); - var onFeatureStartExpression = new CodeMethodInvokeExpression( - testRunnerField, - nameof(ITestRunner.OnFeatureStartAsync), - new CodeVariableReferenceExpression("featureInfo")); - - _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureStartExpression); - - testClassInitializeMethod.Statements.Add(onFeatureStartExpression); } private void SetupTestClassCleanupMethod(TestClassGenerationContext generationContext) @@ -221,30 +211,6 @@ private void SetupTestClassCleanupMethod(TestClassGenerationContext generationCo _codeDomHelper.MarkCodeMemberMethodAsAsync(testClassCleanupMethod); _testGeneratorProvider.SetTestClassCleanupMethod(generationContext); - - var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression(); - - // await testRunner.OnFeatureEndAsync(); - var expression = new CodeMethodInvokeExpression( - testRunnerField, - nameof(ITestRunner.OnFeatureEndAsync)); - - _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression); - - testClassCleanupMethod.Statements.Add(expression); - - // - testClassCleanupMethod.Statements.Add( - new CodeMethodInvokeExpression( - new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))), - nameof(TestRunnerManager.ReleaseTestRunner), - testRunnerField)); - - // testRunner = null; - testClassCleanupMethod.Statements.Add( - new CodeAssignStatement( - testRunnerField, - new CodePrimitiveExpression(null))); } private void SetupTestInitializeMethod(TestClassGenerationContext generationContext) @@ -257,6 +223,78 @@ private void SetupTestInitializeMethod(TestClassGenerationContext generationCont _codeDomHelper.MarkCodeMemberMethodAsAsync(testInitializeMethod); _testGeneratorProvider.SetTestInitializeMethod(generationContext); + + // Obtain the test runner for executing a single test + // testRunner = global::Reqnroll.TestRunnerManager.GetTestRunnerForAssembly(featureHint: featureInfo); + + var testRunnerField = _scenarioPartHelper.GetTestRunnerExpression(); + + var getTestRunnerExpression = new CodeMethodInvokeExpression( + new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))), + nameof(TestRunnerManager.GetTestRunnerForAssembly), + _codeDomHelper.CreateOptionalArgumentExpression("featureHint", + new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD))); + + testInitializeMethod.Statements.Add( + new CodeAssignStatement( + testRunnerField, + getTestRunnerExpression)); + + + // "Finish" current feature if needed + + var featureContextExpression = new CodePropertyReferenceExpression( + testRunnerField, + "FeatureContext"); + + var onFeatureEndAsyncExpression = new CodeMethodInvokeExpression( + testRunnerField, + nameof(ITestRunner.OnFeatureEndAsync)); + _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureEndAsyncExpression); + + //if (testRunner.FeatureContext != null && !testRunner.FeatureContext.FeatureInfo.Equals(featureInfo)) + // await testRunner.OnFeatureEndAsync(); // finish if different + testInitializeMethod.Statements.Add( + new CodeConditionStatement( + new CodeBinaryOperatorExpression( + new CodeBinaryOperatorExpression( + featureContextExpression, + CodeBinaryOperatorType.IdentityInequality, + new CodePrimitiveExpression(null)), + CodeBinaryOperatorType.BooleanAnd, + new CodeBinaryOperatorExpression( + new CodeMethodInvokeExpression( + new CodePropertyReferenceExpression( + featureContextExpression, + "FeatureInfo"), + nameof(object.Equals), + new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD)), + CodeBinaryOperatorType.ValueEquality, + new CodePrimitiveExpression(false))), + new CodeExpressionStatement( + onFeatureEndAsyncExpression))); + + + // "Start" the feature if needed + + //if (testRunner.FeatureContext == null) { + // await testRunner.OnFeatureStartAsync(featureInfo); + //} + + var onFeatureStartExpression = new CodeMethodInvokeExpression( + testRunnerField, + nameof(ITestRunner.OnFeatureStartAsync), + new CodeVariableReferenceExpression(GeneratorConstants.FEATUREINFO_FIELD)); + _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(onFeatureStartExpression); + + testInitializeMethod.Statements.Add( + new CodeConditionStatement( + new CodeBinaryOperatorExpression( + featureContextExpression, + CodeBinaryOperatorType.IdentityEquality, + new CodePrimitiveExpression(null)), + new CodeExpressionStatement( + onFeatureStartExpression))); } private void SetupTestCleanupMethod(TestClassGenerationContext generationContext) @@ -280,6 +318,14 @@ private void SetupTestCleanupMethod(TestClassGenerationContext generationContext _codeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression); testCleanupMethod.Statements.Add(expression); + + // "Release" the TestRunner, so that other threads can pick it up + // TestRunnerManager.ReleaseTestRunner(testRunner); + testCleanupMethod.Statements.Add( + new CodeMethodInvokeExpression( + new CodeTypeReferenceExpression(_codeDomHelper.GetGlobalizedTypeName(typeof(TestRunnerManager))), + nameof(TestRunnerManager.ReleaseTestRunner), + testRunnerField)); } private void SetupScenarioInitializeMethod(TestClassGenerationContext generationContext) diff --git a/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs index cb2b92a4d..3fa6db578 100644 --- a/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/MsTestGeneratorProvider.cs @@ -3,9 +3,11 @@ using System.Collections.Generic; using Reqnroll.Generator.CodeDom; using Reqnroll.BoDi; +using System.Diagnostics.CodeAnalysis; namespace Reqnroll.Generator.UnitTestProvider { + [SuppressMessage("ReSharper", "BitwiseOperatorOnEnumWithoutFlags")] public class MsTestGeneratorProvider : IUnitTestGeneratorProvider { protected internal const string TESTFIXTURE_ATTR = "Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute"; @@ -103,7 +105,6 @@ public virtual void SetTestClassNonParallelizable(TestClassGenerationContext gen public virtual void SetTestClassInitializeMethod(TestClassGenerationContext generationContext) { generationContext.TestClassInitializeMethod.Attributes |= MemberAttributes.Static; - generationContext.TestRunnerField.Attributes |= MemberAttributes.Static; generationContext.TestClassInitializeMethod.Parameters.Add(new CodeParameterDeclarationExpression( TESTCONTEXT_TYPE, "testContext")); @@ -123,48 +124,6 @@ public void SetTestClassCleanupMethod(TestClassGenerationContext generationConte public virtual void SetTestInitializeMethod(TestClassGenerationContext generationContext) { CodeDomHelper.AddAttribute(generationContext.TestInitializeMethod, TESTSETUP_ATTR); - FixTestRunOrderingIssue(generationContext); - } - - protected virtual void FixTestRunOrderingIssue(TestClassGenerationContext generationContext) - { - //see https://github.com/reqnroll/Reqnroll/issues/96 - - //if (testRunner.FeatureContext != null && testRunner.FeatureContext.FeatureInfo.Title != "") - // .(null); - - var featureContextExpression = new CodePropertyReferenceExpression( - new CodeFieldReferenceExpression(null, generationContext.TestRunnerField.Name), - "FeatureContext"); - - var callTestClassInitializeMethodExpression = new CodeMethodInvokeExpression( - new CodeTypeReferenceExpression( - new CodeTypeReference( - generationContext.Namespace.Name + "." + generationContext.TestClass.Name, - CodeTypeReferenceOptions.GlobalReference)), - generationContext.TestClassInitializeMethod.Name, - new CodePrimitiveExpression(null)); - - CodeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(callTestClassInitializeMethodExpression); - - generationContext.TestInitializeMethod.Statements.Add( - new CodeConditionStatement( - new CodeBinaryOperatorExpression( - new CodeBinaryOperatorExpression( - featureContextExpression, - CodeBinaryOperatorType.IdentityInequality, - new CodePrimitiveExpression(null)), - CodeBinaryOperatorType.BooleanAnd, - new CodeBinaryOperatorExpression( - new CodePropertyReferenceExpression( - new CodePropertyReferenceExpression( - featureContextExpression, - "FeatureInfo"), - "Title"), - CodeBinaryOperatorType.IdentityInequality, - new CodePrimitiveExpression(generationContext.Feature.Name))), - new CodeExpressionStatement( - callTestClassInitializeMethodExpression))); } public void SetTestCleanupMethod(TestClassGenerationContext generationContext) @@ -172,7 +131,6 @@ public void SetTestCleanupMethod(TestClassGenerationContext generationContext) CodeDomHelper.AddAttribute(generationContext.TestCleanupMethod, TESTTEARDOWN_ATTR); } - public virtual void SetTestMethod(TestClassGenerationContext generationContext, CodeMemberMethod testMethod, string friendlyTestName) { CodeDomHelper.AddAttribute(testMethod, TEST_ATTR); diff --git a/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs index 237aab2f9..34cfd81e0 100644 --- a/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs @@ -1,12 +1,13 @@ using System.CodeDom; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; -using System.Threading.Tasks; using Reqnroll.BoDi; using Reqnroll.Generator.CodeDom; namespace Reqnroll.Generator.UnitTestProvider { + [SuppressMessage("ReSharper", "BitwiseOperatorOnEnumWithoutFlags")] public class NUnit3TestGeneratorProvider : IUnitTestGeneratorProvider { protected internal const string TESTFIXTURESETUP_ATTR_NUNIT3 = "NUnit.Framework.OneTimeSetUpAttribute"; @@ -50,11 +51,13 @@ public virtual void SetTestMethodIgnore(TestClassGenerationContext generationCon public virtual void SetTestClassInitializeMethod(TestClassGenerationContext generationContext) { + generationContext.TestClassInitializeMethod.Attributes |= MemberAttributes.Static; CodeDomHelper.AddAttribute(generationContext.TestClassInitializeMethod, TESTFIXTURESETUP_ATTR_NUNIT3); } public virtual void SetTestClassCleanupMethod(TestClassGenerationContext generationContext) { + generationContext.TestClassInitializeMethod.Attributes |= MemberAttributes.Static; CodeDomHelper.AddAttribute(generationContext.TestClassCleanupMethod, TESTFIXTURETEARDOWN_ATTR_NUNIT3); } @@ -123,9 +126,11 @@ public void SetRow(TestClassGenerationContext generationContext, CodeMemberMetho var args = arguments.Select( arg => new CodeAttributeArgument(new CodePrimitiveExpression(arg))).ToList(); - // addressing ReSharper bug: TestCase attribute with empty string[] param causes inconclusive result - https://github.com/reqnroll/Reqnroll/issues/116 - bool hasExampleTags = tags.Any(); - var exampleTagExpressionList = tags.Select(t => new CodePrimitiveExpression(t)); + var tagsArray = tags.ToArray(); + + // addressing ReSharper bug: TestCase attribute with empty string[] param causes inconclusive result - https://github.com/SpecFlowOSS/SpecFlow/issues/116 + bool hasExampleTags = tagsArray.Any(); + var exampleTagExpressionList = tagsArray.Select(t => (CodeExpression)new CodePrimitiveExpression(t)); var exampleTagsExpression = hasExampleTags ? new CodeArrayCreateExpression(typeof(string[]), exampleTagExpressionList.ToArray()) : (CodeExpression) new CodePrimitiveExpression(null); @@ -135,7 +140,7 @@ public void SetRow(TestClassGenerationContext generationContext, CodeMemberMetho // adds 'Category' named parameter so that NUnit also understands that this test case belongs to the given categories if (hasExampleTags) { - CodeExpression exampleTagsStringExpr = new CodePrimitiveExpression(string.Join(",", tags.ToArray())); + CodeExpression exampleTagsStringExpr = new CodePrimitiveExpression(string.Join(",", tagsArray)); args.Add(new CodeAttributeArgument("Category", exampleTagsStringExpr)); } diff --git a/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs index 7b15c51c8..698d8e717 100644 --- a/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/XUnit2TestGeneratorProvider.cs @@ -5,13 +5,15 @@ using Reqnroll.Generator.CodeDom; using Reqnroll.BoDi; using System.Text.RegularExpressions; +using System.Diagnostics.CodeAnalysis; namespace Reqnroll.Generator.UnitTestProvider { + [SuppressMessage("ReSharper", "BitwiseOperatorOnEnumWithoutFlags")] public class XUnit2TestGeneratorProvider : IUnitTestGeneratorProvider { private CodeTypeDeclaration _currentFixtureDataTypeDeclaration = null; - private readonly CodeTypeReference _objectCodeTypeReference = new CodeTypeReference(typeof(object)); + private readonly CodeTypeReference _objectCodeTypeReference = new(typeof(object)); protected internal const string THEORY_ATTRIBUTE = "Xunit.SkippableTheoryAttribute"; protected internal const string INLINEDATA_ATTRIBUTE = "Xunit.InlineDataAttribute"; protected internal const string ICLASSFIXTURE_INTERFACE = "Xunit.IClassFixture"; @@ -59,12 +61,8 @@ protected virtual CodeTypeReference CreateFixtureInterface(TestClassGenerationCo return new CodeTypeReference(ICLASSFIXTURE_INTERFACE, fixtureDataType); } - public virtual bool ImplementInterfaceExplicit => false; - protected CodeDomHelper CodeDomHelper { get; set; } - public bool GenerateParallelCodeForFeature { get; set; } - public virtual void SetRowTest(TestClassGenerationContext generationContext, CodeMemberMethod testMethod, string scenarioTitle) { CodeDomHelper.AddAttribute(testMethod, THEORY_ATTRIBUTE, new CodeAttributeArgument("DisplayName", new CodePrimitiveExpression(scenarioTitle))); @@ -85,7 +83,7 @@ public virtual void SetRow(TestClassGenerationContext generationContext, CodeMem args.Add( new CodeAttributeArgument( - new CodeArrayCreateExpression(typeof(string[]), tags.Select(t => new CodePrimitiveExpression(t)).ToArray()))); + new CodeArrayCreateExpression(typeof(string[]), tags.Select(t => (CodeExpression)new CodePrimitiveExpression(t)).ToArray()))); CodeDomHelper.AddAttribute(testMethod, INLINEDATA_ATTRIBUTE, args.ToArray()); } @@ -133,15 +131,16 @@ public virtual void SetTestMethodIgnore(TestClassGenerationContext generationCon public virtual void SetTestClassCategories(TestClassGenerationContext generationContext, IEnumerable featureCategories) { - IEnumerable collection = featureCategories.Where(f => f.StartsWith(COLLECTION_TAG, StringComparison.InvariantCultureIgnoreCase)).ToList(); - if (collection.Any()) + var featureCategoriesArray = featureCategories.ToArray(); + var collectionCategories = featureCategoriesArray.Where(f => f.StartsWith(COLLECTION_TAG, StringComparison.InvariantCultureIgnoreCase)).ToList(); + if (collectionCategories.Any()) { //Only one 'Xunit.Collection' can exist per class. - SetTestClassCollection(generationContext, collection.FirstOrDefault()); + SetTestClassCollection(generationContext, collectionCategories.FirstOrDefault()); } // Set Category trait which can be used with the /trait or /-trait xunit flags to include/exclude tests - foreach (string str in featureCategories) + foreach (string str in featureCategoriesArray) { SetProperty(generationContext.TestClass, CATEGORY_PROPERTY_NAME, str); } @@ -216,7 +215,6 @@ public void SetTestClassInitializeMethod(TestClassGenerationContext generationCo { // xUnit uses IUseFixture on the class generationContext.TestClassInitializeMethod.Attributes |= MemberAttributes.Static; - generationContext.TestRunnerField.Attributes |= MemberAttributes.Static; _currentFixtureDataTypeDeclaration = CodeDomHelper.CreateGeneratedTypeDeclaration("FixtureData"); generationContext.TestClass.Members.Add(_currentFixtureDataTypeDeclaration); @@ -399,10 +397,5 @@ public void MarkCodeMethodInvokeExpressionAsAwait(CodeMethodInvokeExpression exp { CodeDomHelper.MarkCodeMethodInvokeExpressionAsAwait(expression); } - - private string GlobalNamespaceIfCSharp(string typeName) - { - return CodeDomHelper.TargetLanguage == CodeDomProviderLanguage.CSharp ? "global::" + typeName : typeName; - } } } \ No newline at end of file diff --git a/Reqnroll.sln.DotSettings b/Reqnroll.sln.DotSettings index 00beadc2b..ff9f19376 100644 --- a/Reqnroll.sln.DotSettings +++ b/Reqnroll.sln.DotSettings @@ -16,6 +16,7 @@ VS MS + <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb"><ExtraRule Prefix="_" Suffix="" Style="aaBb" /></Policy> C:\Users\awi\AppData\Local\JetBrains\Transient\ReSharperPlatformVs15\v09_95175bdf\SolutionCaches Required Required diff --git a/Reqnroll/FeatureInfo.cs b/Reqnroll/FeatureInfo.cs index cdb62f7b7..d6c7cfaed 100644 --- a/Reqnroll/FeatureInfo.cs +++ b/Reqnroll/FeatureInfo.cs @@ -1,9 +1,11 @@ using System; +using System.Diagnostics; using System.Globalization; using Reqnroll.Tracing; namespace Reqnroll { + [DebuggerDisplay("{Title}")] public class FeatureInfo { public string[] Tags { get; private set; } diff --git a/Reqnroll/ITestRunnerManager.cs b/Reqnroll/ITestRunnerManager.cs index 815926d33..b81627426 100644 --- a/Reqnroll/ITestRunnerManager.cs +++ b/Reqnroll/ITestRunnerManager.cs @@ -10,7 +10,7 @@ public interface ITestRunnerManager Assembly TestAssembly { get; } Assembly[] BindingAssemblies { get; } bool IsMultiThreaded { get; } - ITestRunner GetTestRunner(); + ITestRunner GetTestRunner(FeatureInfo featureHint = null); void ReleaseTestThreadContext(ITestThreadContext testThreadContext); void Initialize(Assembly testAssembly); Task FireTestRunEndAsync(); diff --git a/Reqnroll/TestRunnerManager.cs b/Reqnroll/TestRunnerManager.cs index 5d196fe72..62dd12656 100644 --- a/Reqnroll/TestRunnerManager.cs +++ b/Reqnroll/TestRunnerManager.cs @@ -24,8 +24,34 @@ public class TestRunnerManager : ITestRunnerManager protected readonly IRuntimeBindingRegistryBuilder _bindingRegistryBuilder; protected readonly ITestTracer _testTracer; - private readonly ConcurrentDictionary _availableTestWorkerContainers = new(); - private readonly ConcurrentDictionary _usedTestWorkerContainers = new(); + /// + /// Contains hints to for the test runner manager to choose the test runners from multiple available + /// ones in a more optimal way. Our goal is to keep the test runner "sticky" to the test framework workers + /// so that no unnecessary feature context switches occur. + /// Currently, we remember the feature info and the managed thread ID to help this. + /// + class TestWorkerContainerHint(FeatureInfo lastUsedFeatureInfo, int? releasedOnManagedThreadId) + { + public FeatureInfo LastUsedFeatureInfo { get; } = lastUsedFeatureInfo; + + private int? ReleasedOnManagedThreadId { get; } = releasedOnManagedThreadId; + + /// + /// Returns information about how optimal the provided hint for the current situation. Smaller number means more optimal. + /// + public static int GetDistance(TestWorkerContainerHint hint, FeatureInfo featureHint) + { + int distance = 0; + distance += hint?.LastUsedFeatureInfo == null || featureHint == null || !ReferenceEquals(hint.LastUsedFeatureInfo, featureHint) ? + 2 : 0; + distance += hint?.ReleasedOnManagedThreadId == null || hint.ReleasedOnManagedThreadId != Thread.CurrentThread.ManagedThreadId ? + 1 : 0; + return distance; + } + } + + private readonly ConcurrentDictionary _availableTestWorkerContainers = new(); + private readonly ConcurrentDictionary _usedTestWorkerContainers = new(); private int _nextTestWorkerContainerId; public bool IsTestRunInitialized { get; private set; } @@ -55,9 +81,9 @@ private int GetWorkerTestRunnerCount() return _usedTestWorkerContainers.Count - (hasTestRunStartWorker ? 1 : 0); } - public virtual ITestRunner CreateTestRunner() + public virtual ITestRunner GetOrCreateTestRunner(FeatureInfo featureHint = null) { - var testRunner = CreateTestRunnerInstance(); + var testRunner = GetOrCreateTestRunnerInstance(featureHint); if (!IsTestRunInitialized) { @@ -77,9 +103,10 @@ public virtual ITestRunner CreateTestRunner() public virtual void ReleaseTestThreadContext(ITestThreadContext testThreadContext) { var testThreadContainer = testThreadContext.TestThreadContainer; - if (!_usedTestWorkerContainers.TryRemove(testThreadContainer, out _)) + if (!_usedTestWorkerContainers.TryRemove(testThreadContainer, out var usedContainerHint)) throw new InvalidOperationException($"TestThreadContext with id {TestThreadContainerInfo.GetId(testThreadContainer)} was already released"); - if (!_availableTestWorkerContainers.TryAdd(testThreadContainer, null)) + var containerHint = new TestWorkerContainerHint(usedContainerHint?.LastUsedFeatureInfo, Thread.CurrentThread.ManagedThreadId); + if (!_availableTestWorkerContainers.TryAdd(testThreadContainer, containerHint)) throw new InvalidOperationException($"TestThreadContext with id {TestThreadContainerInfo.GetId(testThreadContainer)} was released twice"); } @@ -140,19 +167,24 @@ public async Task FireTestRunStartAsync() await onTestRunnerStartExecutionHost.OnTestRunStartAsync(); } - protected virtual ITestRunner CreateTestRunnerInstance() + protected virtual ITestRunner GetOrCreateTestRunnerInstance(FeatureInfo featureHint = null) { IObjectContainer testThreadContainer = null; - for (int i = 0; i < 5 && testThreadContainer == null; i++) // Try to get a available Container max 5 times + for (int i = 0; i < 5 && testThreadContainer == null; i++) // Try to get an available Container max 5 times { var items = _availableTestWorkerContainers.ToArray(); if (items.Length == 0) break; // No Containers are available - foreach (var item in items) + + var prioritizedContainers = items + .OrderBy(it => TestWorkerContainerHint.GetDistance(it.Value, featureHint)) + .Select(it => it.Key); + + foreach (var container in prioritizedContainers) { - if (!_availableTestWorkerContainers.TryRemove(item.Key, out _)) + if (!_availableTestWorkerContainers.TryRemove(container, out _)) continue; // Container was already taken by another thread - testThreadContainer = item.Key; + testThreadContainer = container; break; } } @@ -165,7 +197,7 @@ protected virtual ITestRunner CreateTestRunnerInstance() testThreadContainer.RegisterInstanceAs(testThreadContainerInfo); } - if (!_usedTestWorkerContainers.TryAdd(testThreadContainer, null)) + if (!_usedTestWorkerContainers.TryAdd(testThreadContainer, new TestWorkerContainerHint(featureHint, null))) throw new InvalidOperationException($"TestThreadContext with id {TestThreadContainerInfo.GetId(testThreadContainer)} is already in usage"); return testThreadContainer.Resolve(); @@ -176,11 +208,11 @@ public void Initialize(Assembly assignedTestAssembly) TestAssembly = assignedTestAssembly; } - public virtual ITestRunner GetTestRunner() + public virtual ITestRunner GetTestRunner(FeatureInfo featureHint = null) { try { - return GetTestRunnerWithoutExceptionHandling(); + return GetTestRunnerWithoutExceptionHandling(featureHint); } catch (Exception ex) { @@ -189,9 +221,9 @@ public virtual ITestRunner GetTestRunner() } } - private ITestRunner GetTestRunnerWithoutExceptionHandling() + private ITestRunner GetTestRunnerWithoutExceptionHandling(FeatureInfo featureHint) { - var testRunner = CreateTestRunner(); + var testRunner = GetOrCreateTestRunner(featureHint); if (IsMultiThreaded && Interlocked.CompareExchange(ref _wasSingletonInstanceDisabled, 1, 0) == 0) { @@ -203,10 +235,26 @@ private ITestRunner GetTestRunnerWithoutExceptionHandling() return testRunner; } + private async Task FireRemainingAfterFeatureHooks() + { + var testWorkerContainers = _availableTestWorkerContainers.Keys.Concat(_usedTestWorkerContainers.Keys).ToArray(); + foreach (var testWorkerContainer in testWorkerContainers) + { + var contextManager = testWorkerContainer.Resolve(); + if (contextManager.FeatureContext != null) + { + var testRunner = testWorkerContainer.Resolve(); + await testRunner.OnFeatureEndAsync(); + } + } + } + public virtual async Task DisposeAsync() { if (Interlocked.CompareExchange(ref _wasDisposed, 1, 0) == 0) { + await FireRemainingAfterFeatureHooks(); + await FireTestRunEndAsync(); if (_globalTestRunner != null) @@ -214,21 +262,21 @@ public virtual async Task DisposeAsync() ReleaseTestThreadContext(_globalTestRunner.TestThreadContext); } - var items = _availableTestWorkerContainers.ToArray(); - while (items.Length > 0) + var testWorkerContainers = _availableTestWorkerContainers.ToArray(); + while (testWorkerContainers.Length > 0) { - foreach (var item in items) + foreach (var item in testWorkerContainers) { item.Key.Dispose(); _availableTestWorkerContainers.TryRemove(item.Key, out _); } - items = _availableTestWorkerContainers.ToArray(); + testWorkerContainers = _availableTestWorkerContainers.ToArray(); } - var notReleasedRunner = _usedTestWorkerContainers.ToArray(); - if (notReleasedRunner.Length > 0) + var notReleasedTestWorkerContainers = _usedTestWorkerContainers.ToArray(); + if (notReleasedTestWorkerContainers.Length > 0) { - var errorText = $"Found {notReleasedRunner.Length} not released TestRunners (ids: {string.Join(",", notReleasedRunner.Select(x => TestThreadContainerInfo.GetId(x.Key)))})"; + var errorText = $"Found {notReleasedTestWorkerContainers.Length} not released TestRunners (ids: {string.Join(",", notReleasedTestWorkerContainers.Select(x => TestThreadContainerInfo.GetId(x.Key)))})"; _globalContainer.Resolve().TraceWarning(errorText); } @@ -317,11 +365,20 @@ public static async Task OnTestRunStartAsync(Assembly testAssembly = null, ICont await testRunnerManager.FireTestRunStartAsync(); } - public static ITestRunner GetTestRunnerForAssembly(Assembly testAssembly = null, IContainerBuilder containerBuilder = null) + /// + /// Provides a test runner for the specified or current assembly with optionally a custom container builder and a feature hint. + /// When a feature hint is provided the test runner manager will attempt to return the same test runner that was used for that + /// feature before. + /// + /// The test assembly. If omitted or invoked with null, the calling assembly is used. + /// The container builder to be used to set up Reqnroll dependencies. If omitted or invoked with null, the default container builder is used. + /// If specified, it is used as a hint for the test runner manager to choose the test runner that has been used for the feature before, if possible. + /// A test runner that can be used to interact with Reqnroll. + public static ITestRunner GetTestRunnerForAssembly(Assembly testAssembly = null, IContainerBuilder containerBuilder = null, FeatureInfo featureHint = null) { testAssembly ??= GetCallingAssembly(); var testRunnerManager = GetTestRunnerManager(testAssembly, containerBuilder); - return testRunnerManager.GetTestRunner(); + return testRunnerManager.GetTestRunner(featureHint); } public static void ReleaseTestRunner(ITestRunner testRunner) diff --git a/Tests/Reqnroll.GeneratorTests/UnitTestProvider/MsTestGeneratorProviderTests.cs b/Tests/Reqnroll.GeneratorTests/UnitTestProvider/MsTestGeneratorProviderTests.cs index 55ee1b379..694a5b901 100644 --- a/Tests/Reqnroll.GeneratorTests/UnitTestProvider/MsTestGeneratorProviderTests.cs +++ b/Tests/Reqnroll.GeneratorTests/UnitTestProvider/MsTestGeneratorProviderTests.cs @@ -1,4 +1,3 @@ -using System.CodeDom; using System.Globalization; using System.IO; using System.Linq; @@ -187,36 +186,6 @@ public void MsTestGeneratorProvider_ExampleSetIdentifiers_ShouldSetDescriptionCo descriptionAttributeForFourthScenarioOutline.ArgumentValues().First().Should().Be("Simple Scenario Outline: and another"); } - [Fact] - public void MsTestGeneratorProvider_ShouldInvokeFeatureSetupMethodWithGlobalNamespaceAlias() - { - // ARRANGE - var document = ParseDocumentFromString(SampleFeatureFileWithMultipleExampleSets); - var sampleTestGeneratorProvider = new MsTestGeneratorProvider(new CodeDomHelper(CodeDomProviderLanguage.CSharp)); - var converter = sampleTestGeneratorProvider.CreateUnitTestConverter(); - - // ACT - var code = converter.GenerateUnitTestFixture(document, "TestClassName", "Target.Namespace"); - - // ASSERT - var featureSetupCall = code - .Class() - .Members() - .Single(m => m.Name == "TestInitializeAsync") - .Statements - .OfType() - .First() - .TrueStatements - .OfType() - .First() - .Expression - .As(); - - featureSetupCall.Should().NotBeNull(); - featureSetupCall.Method.MethodName.Should().Be("FeatureSetupAsync"); - featureSetupCall.Method.TargetObject.As().Type.BaseType.Should().Contain("global::"); - } - [Fact] public void MsTestGeneratorProvider_ShouldNotHaveParallelExecutionTrait() { @@ -240,7 +209,7 @@ public void MsTestGeneratorProvider_WithFeatureWithMatchingTag_ShouldNotAddDoNot Given there is something"); var provider = new MsTestGeneratorProvider(new CodeDomHelper(CodeDomProviderLanguage.CSharp)); - var featureGenerator = provider.CreateFeatureGenerator(addNonParallelizableMarkerForTags: new string[] { "nonparallelizable" }); + var featureGenerator = provider.CreateFeatureGenerator(addNonParallelizableMarkerForTags: new[] { "nonparallelizable" }); // ACT var code = featureGenerator.GenerateUnitTestFixture(document, "TestClassName", "Target.Namespace"); @@ -261,7 +230,7 @@ public void MsTestGeneratorProvider_WithFeatureWithNoMatchingTag_ShouldNotAddDoN Given there is something"); var provider = new MsTestGeneratorProvider(new CodeDomHelper(CodeDomProviderLanguage.CSharp)); - var featureGenerator = provider.CreateFeatureGenerator(addNonParallelizableMarkerForTags: new string[] { "nonparallelizable" }); + var featureGenerator = provider.CreateFeatureGenerator(addNonParallelizableMarkerForTags: ["nonparallelizable"]); // ACT var code = featureGenerator.GenerateUnitTestFixture(document, "TestClassName", "Target.Namespace"); diff --git a/Tests/Reqnroll.GeneratorTests/UnitTestProvider/NUnit3GeneratorProviderTests.cs b/Tests/Reqnroll.GeneratorTests/UnitTestProvider/NUnit3GeneratorProviderTests.cs index dce0efc83..1ce0c3a2d 100644 --- a/Tests/Reqnroll.GeneratorTests/UnitTestProvider/NUnit3GeneratorProviderTests.cs +++ b/Tests/Reqnroll.GeneratorTests/UnitTestProvider/NUnit3GeneratorProviderTests.cs @@ -211,60 +211,6 @@ Then something should happen testMethod.CustomAttributes().Should().ContainSingle(a => a.Name == NUnit3TestCaseAttributeName && (string)a.ArgumentValues().First() == "and another"); } - [Fact] - public void NUnit3TestGeneratorProvider_ShouldNotGenerateObsoleteTestFixtureSetUpAttribute() - { - var code = GenerateCodeNamespaceFromFeature(SampleFeatureFile); - - var featureSetupMethod = code.Class().Members().Single(m => m.Name == "FeatureSetupAsync"); - - featureSetupMethod.CustomAttributes() - .FirstOrDefault(a => a.Name == "NUnit.Framework.TestFixtureSetUpAttribute") - .Should() - .BeNull(); - } - - [Fact] - public void NUnit3TestGeneratorProvider_ShouldGenerateNewOneTimeSetUpAttribute() - { - var code = GenerateCodeNamespaceFromFeature(SampleFeatureFile); - - var featureSetupMethod = code.Class().Members().Single(m => m.Name == "FeatureSetupAsync"); - - // Assert that we do use the NUnit3 attribute - featureSetupMethod.CustomAttributes() - .FirstOrDefault(a => a.Name == "NUnit.Framework.OneTimeSetUpAttribute") - .Should() - .NotBeNull(); - } - - [Fact] - public void NUnit3TestGeneratorProvider_ShouldNotGenerateObsoleteTestFixtureTearDownAttribute() - { - var code = GenerateCodeNamespaceFromFeature(SampleFeatureFile); - - var featureSetupMethod = code.Class().Members().Single(m => m.Name == "FeatureTearDownAsync"); - - featureSetupMethod.CustomAttributes() - .FirstOrDefault(a => a.Name == "NUnit.Framework.TestFixtureTearDownAttribute") - .Should() - .BeNull(); - } - - [Fact] - public void NUnit3TestGeneratorProvider_ShouldGenerateNewOneTimeTearDownAttribute() - { - var code = GenerateCodeNamespaceFromFeature(SampleFeatureFile); - - var featureSetupMethod = code.Class().Members().Single(m => m.Name == "FeatureTearDownAsync"); - - // Assert that we do use the NUnit3 attribute - featureSetupMethod.CustomAttributes() - .FirstOrDefault(a => a.Name == "NUnit.Framework.OneTimeTearDownAttribute") - .Should() - .NotBeNull(); - } - [Fact] public void NUnit3TestGeneratorProvider_ShouldHaveRowTestsTrait() { diff --git a/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs b/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs index 355bbe724..0ad07f57f 100644 --- a/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs +++ b/Tests/Reqnroll.RuntimeTests/TestRunnerManagerTests.cs @@ -1,5 +1,6 @@ using System; using System.Globalization; +using System.Linq; using System.Reflection; using System.Threading.Tasks; using FluentAssertions; @@ -27,7 +28,7 @@ public async Task DisposeAsync() [Fact] public void CreateTestRunner_should_be_able_to_create_a_TestRunner() { - var testRunner = _testRunnerManager.CreateTestRunner(); + var testRunner = _testRunnerManager.GetOrCreateTestRunner(); testRunner.Should().NotBeNull(); testRunner.Should().BeOfType(); @@ -71,6 +72,95 @@ public void Should_return_different_thread_ids_for_different_instances() TestRunnerManager.ReleaseTestRunner(testRunner3); } + [Fact] + public void Should_use_test_runner_that_was_active_last_time_on_the_same_thread_if_possible() + { + var ourFeatureInfo = new FeatureInfo(new CultureInfo("en-US"), null, "F1", null); + var otherFeatureInfo = new FeatureInfo(new CultureInfo("en-US"), null, "F2", null); + + var otherRunners1 = Enumerable.Range(0, 5).Select(_ => TestRunnerManager.GetTestRunnerForAssembly(featureHint: otherFeatureInfo)).ToList(); + var otherRunners2 = Enumerable.Range(0, 5).Select(_ => TestRunnerManager.GetTestRunnerForAssembly(featureHint: otherFeatureInfo)).ToList(); + var runnerOnDifferentFeatureSameThread = TestRunnerManager.GetTestRunnerForAssembly(featureHint: otherFeatureInfo); + var runnerOnSameFeatureDifferentThread = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); + var ourRunner = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); + + void RunOnOtherThreadAndWait(Action action) + { + Task.Run(action).Wait(500); + } + + // release otherRunners1 on a different thread + RunOnOtherThreadAndWait(() => otherRunners1.ForEach(TestRunnerManager.ReleaseTestRunner)); + + // release the selected runners + TestRunnerManager.ReleaseTestRunner(ourRunner); + TestRunnerManager.ReleaseTestRunner(runnerOnDifferentFeatureSameThread); + RunOnOtherThreadAndWait(() => TestRunnerManager.ReleaseTestRunner(runnerOnSameFeatureDifferentThread)); + + // release otherRunners2 on a different thread + RunOnOtherThreadAndWait(() => otherRunners2.ForEach(TestRunnerManager.ReleaseTestRunner)); + + // Priority: + // 1. Same feature, same thread + // 2. Same feature, different thread + // 3. Other feature, same thread + // 4. Other feature, other thread + + // Priority 1: from the same feature & same thread, we should get our test runner first + var ourRunnerAgain = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); + ourRunnerAgain.TestWorkerId.Should().Be(ourRunner.TestWorkerId); + + // Priority 2: from the same feature & different thread + var runnerOnSameFeatureDifferentThreadAgain = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); + runnerOnSameFeatureDifferentThreadAgain.TestWorkerId.Should().Be(runnerOnSameFeatureDifferentThread.TestWorkerId); + + // Priority 3: from the different feature & same thread + var runnerOnDifferentFeatureSameThreadAgain = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); + runnerOnDifferentFeatureSameThreadAgain.TestWorkerId.Should().Be(runnerOnDifferentFeatureSameThread.TestWorkerId); + + // Priority 4: if there is no from the same thread, just provide one of the others + var otherRunnerWithoutHint = TestRunnerManager.GetTestRunnerForAssembly(featureHint: ourFeatureInfo); + otherRunnerWithoutHint.TestWorkerId.Should().NotBe(ourRunner.TestWorkerId); + + TestRunnerManager.ReleaseTestRunner(ourRunnerAgain); + TestRunnerManager.ReleaseTestRunner(runnerOnSameFeatureDifferentThreadAgain); + TestRunnerManager.ReleaseTestRunner(runnerOnDifferentFeatureSameThreadAgain); + TestRunnerManager.ReleaseTestRunner(otherRunnerWithoutHint); + } + + [Fact] + public async Task Should_fire_remaining_AfterFeature_hooks_of_test_threads() + { + _testRunnerManager.Initialize(_anAssembly); + + var testRunnerWithFeatureContext = _testRunnerManager.GetTestRunner(); + var testRunnerWithoutFeatureContext = _testRunnerManager.GetTestRunner(); + + FeatureHookTracker.AfterFeatureCalled = false; + await testRunnerWithFeatureContext.OnFeatureStartAsync(new FeatureInfo(new CultureInfo("en-US", false), string.Empty, "F", null)); + var disposableClass1 = new DisposableClass(); + testRunnerWithFeatureContext.FeatureContext.FeatureContainer.RegisterInstanceAs(disposableClass1, dispose: true); + + TestRunnerManager.ReleaseTestRunner(testRunnerWithFeatureContext); + TestRunnerManager.ReleaseTestRunner(testRunnerWithoutFeatureContext); + + await TestRunnerManager.OnTestRunEndAsync(_anAssembly); + FeatureHookTracker.AfterFeatureCalled.Should().BeTrue(); + disposableClass1.IsDisposed.Should().BeTrue(); + } + + [Binding] + static class FeatureHookTracker + { + public static bool AfterFeatureCalled = false; + + [AfterFeature] + public static void AfterFeature() + { + AfterFeatureCalled = true; + } + } + class DisposableClass : IDisposable { public bool IsDisposed { get; private set; } @@ -106,7 +196,7 @@ public void First_call_to_CreateTestRunner_should_initialize_binding_registry() { _testRunnerManager.IsTestRunInitialized.Should().BeFalse("binding registry should not be initialized initially"); - _testRunnerManager.CreateTestRunner(); + _testRunnerManager.GetOrCreateTestRunner(); _testRunnerManager.IsTestRunInitialized.Should().BeTrue("binding registry be initialized"); } diff --git a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs index 04cd07cdf..1401a38e5 100644 --- a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs +++ b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs @@ -1,10 +1,10 @@ -using System.Linq; +using System; +using System.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; using FluentAssertions; -using Reqnroll.TestProjectGenerator; -using System.ComponentModel.DataAnnotations; using System.Text.Json; using System.Collections.Generic; +using Reqnroll.TestProjectGenerator; namespace Reqnroll.SystemTests.Generation; @@ -443,5 +443,117 @@ public async Task WhenThisTableIsProcessed(DataTable table) #endregion - //TODO: test parallel execution (details TBD) - maybe this should be in a separate test class + #region It is able to run tests parallel + + // We need to verify + // Tests are run in parallel + // Each scenario execution has a feature context and scenario context that is matching to the feature/scenario + // For each scenario the before feature has been executed for that feature context + + [TestMethod] + public void Tests_can_be_executed_parallel() + { + _projectsDriver.EnableTestParallelExecution(); + + var scenarioTemplate = + """ + Scenario: {0} + When executing '{0}' in '{1}' + """; + + var scenarioOutlineTemplate = + """ + Scenario Outline: {0} + When executing '{0}' in '{1}' + Examples: + | Count | + | 1 | + | 2 | + | 3 | + """; + + const int scenariosPerFile = 3; + const int scenarioOutlinesPerFile = 3; + string[] features = ["A", "B"]; + int scenarioIdCounter = 0; + + foreach (string feature in features) + { + AddFeatureFile($"Feature: {feature}" + Environment.NewLine); + for (int i = 0; i < scenariosPerFile; i++) + AddScenario(string.Format(scenarioTemplate, $"S{++scenarioIdCounter}", feature)); + for (int i = 0; i < scenarioOutlinesPerFile; i++) + AddScenario(string.Format(scenarioOutlineTemplate, $"S{++scenarioIdCounter}", feature)); + } + + AddBindingClass( + """ + namespace ParallelExecution.StepDefinitions + { + [Binding] + public class ParallelExecutionSteps + { + public static int startIndex = 0; + public static int featureAStartIndex = 0; + public static int featureBStartIndex = 0; + + private readonly FeatureContext _featureContext; + private readonly ScenarioContext _scenarioContext; + + public ParallelExecutionSteps(FeatureContext featureContext, ScenarioContext scenarioContext) + { + _featureContext = featureContext; + _scenarioContext = scenarioContext; + } + + [BeforeFeature] + public static void BeforeFeature(FeatureContext featureContext) + { + featureContext.Set(true, "before_feature"); + } + + [When("executing {string} in {string}")] + public void WhenExecuting(string scenarioName, string featureName) + { + var currentStartIndex = System.Threading.Interlocked.Increment(ref startIndex); + var currentFeatureStartIndex = featureName == "A" + ? System.Threading.Interlocked.Increment(ref featureAStartIndex) + : System.Threading.Interlocked.Increment(ref featureBStartIndex); + global::Log.LogStep(); + if (_scenarioContext.ScenarioInfo.Title != scenarioName) + throw new System.Exception($"Invalid scenario context: {_scenarioContext.ScenarioInfo.Title} should be {scenarioName}"); + if (_featureContext.FeatureInfo.Title != featureName) + throw new System.Exception($"Invalid scenario context: {_featureContext.FeatureInfo.Title} should be {featureName}"); + if (!_featureContext.TryGetValue("before_feature", out var value) || !value) + throw new System.Exception($"BeforeFeature hook was not executed!"); + + System.Threading.Thread.Sleep(10); + + var afterStartIndex = startIndex; + if (afterStartIndex != currentStartIndex) + Log.LogCustom("parallel", "true"); + var afterFeatureStartIndex = featureName == "A" ? featureAStartIndex : featureBStartIndex; + if (afterFeatureStartIndex != currentFeatureStartIndex) + Log.LogCustom("scenario-parallel", "true"); + } + } + } + """); + + ExecuteTests(); + ShouldAllScenariosPass(); + + var parallelLogs = _bindingDriver.GetActualLogLines("parallel").ToList(); + parallelLogs.Should().NotBeEmpty("the scenarios should have run parallel"); + + AssertScenarioLevelParallelExecution(); + } + + protected virtual void AssertScenarioLevelParallelExecution() + { + var scenarioParallelLogs = _bindingDriver.GetActualLogLines("scenario-parallel").ToList(); + scenarioParallelLogs.Should().NotBeEmpty("the scenarios should have run parallel using scenario-level parallelization"); + } + + #endregion } diff --git a/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs b/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs index 086f2ab37..f9978e5a8 100644 --- a/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs +++ b/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs @@ -19,4 +19,9 @@ protected override void TestInitialize() protected override string GetExpectedPendingOutcome() => "Failed"; protected override string GetExpectedUndefinedOutcome() => "Failed"; + + protected override void AssertScenarioLevelParallelExecution() + { + //nop - xUnit currently does not support method-level parallelization + } } \ No newline at end of file diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs index 9eee89bd3..88fb70632 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs @@ -70,16 +70,6 @@ private void AddHookBinding(ProjectBuilder project, string eventType, string nam project.AddHookBinding(eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); } - public void AddAsyncHookBindingIncludingLocking(string eventType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null) - { - AddAsyncHookBindingIncludingLocking(_solutionDriver.DefaultProject, eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); - } - - private void AddAsyncHookBindingIncludingLocking(ProjectBuilder project, string eventType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null) - { - project.AddAsyncHookBindingIncludingLocking(eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); - } - public void AddFeatureFile(string featureFileContent) { LastFeatureFile = _solutionDriver.DefaultProject.AddFeatureFile(featureFileContent); diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs index d799d3426..9f27b968d 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs @@ -28,12 +28,6 @@ public ProjectFile GenerateHookBinding(string eventType, string name, string cod return GenerateBindingClassFile(hookClass); } - public ProjectFile GenerateAsyncHookBindingIncludingLocking(string eventType, string name, string code = null, int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null) - { - string hookClass = GetAsyncHookIncludingLockingBindingClass(eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); - return GenerateBindingClassFile(hookClass); - } - public abstract ProjectFile GenerateLoggerClass(string pathToLogFile); protected abstract string GetBindingCode(string methodName, string methodImplementation, string attributeName, string regex, ParameterType parameterType, string argumentName); @@ -49,15 +43,6 @@ protected abstract string GetHookBindingClass( IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null); - protected abstract string GetAsyncHookIncludingLockingBindingClass( - string hookType, - string name, - string code = "", - int? order = null, - IList hookTypeAttributeTags = null, - IList methodScopeAttributeTags = null, - IList classScopeAttributeTags = null); - protected bool IsStaticEvent(string eventType) { return eventType == "BeforeFeature" || eventType == "AfterFeature" || eventType == "BeforeTestRun" || eventType == "AfterTestRun"; diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs index 2f7809437..f06ffe006 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs @@ -1,66 +1,13 @@ -using Reqnroll.TestProjectGenerator.Data; +using System; namespace Reqnroll.TestProjectGenerator.Factories.BindingsGenerator; public class CSharp10BindingsGenerator : CSharpBindingsGenerator { - public override ProjectFile GenerateLoggerClass(string pathToLogFile) + protected override string GetLogFileContent(string pathToLogFile) { - string fileContent = $$""" - using System; - using System.IO; - using System.Runtime.CompilerServices; - using System.Threading; - - internal static class Log - { - private const string LogFileLocation = @"{{pathToLogFile}}"; - - private static void Retry(int number, Action action) - { - try - { - action(); - } - catch (Exception) - { - var i = number - 1; - - if (i == 0) - throw; - - Thread.Sleep(500); - Retry(i, action); - } - } - - internal static void LogStep([CallerMemberName] string stepName = null!) - { - Retry(5, () => WriteToFile($@"-> step: {stepName}{Environment.NewLine}")); - } - - internal static void LogHook([CallerMemberName] string stepName = null!) - { - Retry(5, () => WriteToFile($@"-> hook: {stepName}{Environment.NewLine}")); - } - - internal static void LogCustom(string category, string value, [CallerMemberName] string memberName = null) - { - Retry(5, () => WriteToFile($@"-> {category}: {value}:{memberName}{Environment.NewLine}")); - } - - static void WriteToFile(string line) - { - using (FileStream fs = File.Open(LogFileLocation, FileMode.Append, FileAccess.Write, FileShare.None)) - { - byte[] bytes = System.Text.Encoding.UTF8.GetBytes(line); - fs.Write(bytes, 0, bytes.Length); - fs.Close(); - } - } - } - """; - return new ProjectFile("Log.cs", "Compile", fileContent); + string logFileContent = base.GetLogFileContent(pathToLogFile); + logFileContent = "#nullable disable" + Environment.NewLine + logFileContent; + return logFileContent; } - } diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs index 40c0229a6..895131e89 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs @@ -10,26 +10,28 @@ namespace Reqnroll.TestProjectGenerator.Factories.BindingsGenerator { public class CSharpBindingsGenerator : BaseBindingsGenerator { - private const string BindingsClassTemplate = @" -using System; -using System.IO; -using System.Xml; -using System.Linq; -using System.Threading.Tasks; -using Reqnroll; - -[Binding] -public class {0} -{{ - private readonly ScenarioContext _scenarioContext; - - public {0}(ScenarioContext scenarioContext) - {{ - _scenarioContext = scenarioContext; - }} - - {1} -}}"; + private const string BindingsClassTemplate = """ + + using System; + using System.IO; + using System.Xml; + using System.Linq; + using System.Threading.Tasks; + using Reqnroll; + + [Binding] + public class {0} + {{ + private readonly ScenarioContext _scenarioContext; + + public {0}(ScenarioContext scenarioContext) + {{ + _scenarioContext = scenarioContext; + }} + + {1} + }} + """; public override ProjectFile GenerateBindingClassFile(string content) { @@ -63,95 +65,69 @@ public override ProjectFile GenerateStepDefinition(string method) public override ProjectFile GenerateLoggerClass(string pathToLogFile) { - string fileContent = $$""" - using System; - using System.IO; - using System.Runtime.CompilerServices; - using System.Threading; - using System.Threading.Tasks; - - internal static class Log - { - private const string LogFileLocation = @"{{pathToLogFile}}"; - - private static void Retry(int number, Action action) - { - try - { - action(); - } - catch (Exception) - { - var i = number - 1; - - if (i == 0) - throw; - - Thread.Sleep(500); - Retry(i, action); - } - } - - internal static void LogStep([CallerMemberName] string stepName = null) - { - Retry(5, () => WriteToFile($@"-> step: {stepName}{Environment.NewLine}")); - } - - internal static void LogHook([CallerMemberName] string stepName = null) - { - Retry(5, () => WriteToFile($@"-> hook: {stepName}{Environment.NewLine}")); - } - - internal static async Task LogHookIncludingLockingAsync([CallerMemberName] string stepName = null) - { - WriteToFile($@"-> waiting for hook lock: {stepName}{Environment.NewLine}"); - await WaitForLockAsync(); - WriteToFile($@"-> hook: {stepName}{Environment.NewLine}"); - } - - internal static void LogCustom(string category, string value, [CallerMemberName] string memberName = null) - { - Retry(5, () => WriteToFile($@"-> {category}: {value}:{memberName}{Environment.NewLine}")); - } - - static void WriteToFile(string line) - { - using (FileStream fs = File.Open(LogFileLocation, FileMode.Append, FileAccess.Write, FileShare.None)) - { - byte[] bytes = System.Text.Encoding.UTF8.GetBytes(line); - fs.Write(bytes, 0, bytes.Length); - fs.Close(); - } - } - - static async Task WaitForLockAsync() - { - var lockFile = LogFileLocation + ".lock"; - - while (true) - { - try - { - using (File.Open(lockFile, FileMode.CreateNew)) - { - } - - break; - } - catch (IOException) - { - //wait and retry - await Task.Delay(1000); - } - } - - File.Delete(lockFile); - } - } - """; + string fileContent = GetLogFileContent(pathToLogFile); return new ProjectFile("Log.cs", "Compile", fileContent); } + protected virtual string GetLogFileContent(string pathToLogFile) + { + string fileContent = $$""" + using System; + using System.IO; + using System.Runtime.CompilerServices; + using System.Threading; + using System.Threading.Tasks; + + internal static class Log + { + private const int RetryCount = 5; + private const string LogFileLocation = @"{{pathToLogFile}}"; + private static readonly Random Rnd = new Random(); + private static readonly object LockObj = new object(); + + private static void Retry(int number, Action action) + { + try + { + action(); + } + catch (Exception) + { + var i = number - 1; + + if (i == 0) + throw; + + Thread.Sleep(50 + Rnd.Next(50)); + Retry(i, action); + } + } + + internal static void LogStep([CallerMemberName] string stepName = null) + { + Retry(RetryCount, () => WriteToFile($@"-> step: {stepName}{Environment.NewLine}")); + } + + internal static void LogHook([CallerMemberName] string stepName = null) + { + Retry(RetryCount, () => WriteToFile($@"-> hook: {stepName}{Environment.NewLine}")); + } + + internal static void LogCustom(string category, string value, [CallerMemberName] string memberName = null) + { + Retry(RetryCount, () => WriteToFile($@"-> {category}: {value}:{memberName}{Environment.NewLine}")); + } + + static void WriteToFile(string line) + { + lock(LockObj) + File.AppendAllText(LogFileLocation, line); + } + } + """; + return fileContent; + } + protected override string GetBindingCode(string methodName, string methodImplementation, string attributeName, string regex, ParameterType parameterType, string argumentName) { string parameter = ""; @@ -239,80 +215,30 @@ protected override string GetHookBindingClass( string staticKeyword = isStatic ? "static" : string.Empty; - return $@" -using System; -using System.Collections; -using System.IO; -using System.Linq; -using System.Xml; -using System.Xml.Linq; -using Reqnroll; - -[Binding] -{scopeClassAttributes} -public class {$"HooksClass_{Guid.NewGuid():N}"} -{{ - [{hookType}({hookTypeAttributeTagsString})] - {scopeMethodAttributes} - public {staticKeyword} void {name}() - {{ - {code} - global::Log.LogHook(); - }} -}} -"; - } - - protected override string GetAsyncHookIncludingLockingBindingClass( - string hookType, - string name, - string code = "", - int? order = null, - IList hookTypeAttributeTags = null, - IList methodScopeAttributeTags = null, - IList classScopeAttributeTags = null) - { - string ToScopeTags(IList scopeTags) => scopeTags is null || !scopeTags.Any() ? null : $"[{string.Join(", ", scopeTags.Select(t => $@"Scope(Tag=""{t}"")"))}]"; - - bool isStatic = IsStaticEvent(hookType); - - string hookTags = hookTypeAttributeTags?.Select(t => $@"""{t}""").JoinToString(", "); - - var hookAttributeConstructorProperties = new[] - { - hookTypeAttributeTags is null || !hookTypeAttributeTags.Any() ? null : $"tags: new string[] {{{hookTags}}}", - order is null ? null : $"Order = {order}" - }.Where(p => p.IsNotNullOrWhiteSpace()); - - string hookTypeAttributeTagsString = string.Join(", ", hookAttributeConstructorProperties); - - string scopeClassAttributes = ToScopeTags(classScopeAttributeTags); - string scopeMethodAttributes = ToScopeTags(methodScopeAttributeTags); - string staticKeyword = isStatic ? "static" : string.Empty; - - - return $@" -using System; -using System.Collections; -using System.IO; -using System.Linq; -using System.Xml; -using System.Xml.Linq; -using Reqnroll; - -[Binding] -{scopeClassAttributes} -public class {$"HooksClass_{Guid.NewGuid():N}"} -{{ - [{hookType}({hookTypeAttributeTagsString})] - {scopeMethodAttributes} - public {staticKeyword} async Task {name}() - {{ - {code} - await global::Log.LogHookIncludingLockingAsync(); - }} -}} -"; + return $$""" + + using System; + using System.Collections; + using System.IO; + using System.Linq; + using System.Xml; + using System.Xml.Linq; + using Reqnroll; + + [Binding] + {{scopeClassAttributes}} + public class {{$"HooksClass_{Guid.NewGuid():N}"}} + { + [{{hookType}}({{hookTypeAttributeTagsString}})] + {{scopeMethodAttributes}} + public {{staticKeyword}} void {{name}}() + { + {{code}} + global::Log.LogHook(); + } + } + + """; } } } diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/FSharpBindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/FSharpBindingsGenerator.cs index 8e2739cb5..7381c8a05 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/FSharpBindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/FSharpBindingsGenerator.cs @@ -118,11 +118,5 @@ protected override string GetHookBindingClass( { throw new NotImplementedException(); } - - protected override string GetAsyncHookIncludingLockingBindingClass(string hookType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, - IList classScopeAttributeTags = null) - { - throw new NotImplementedException(); - } } } diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs index d21e1e102..5596f6927 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/VbBindingsGenerator.cs @@ -10,13 +10,15 @@ namespace Reqnroll.TestProjectGenerator.Factories.BindingsGenerator { public class VbBindingsGenerator : BaseBindingsGenerator { - private const string BindingsClassTemplate = @" -Imports Reqnroll + private const string BindingsClassTemplate = + """ + Imports Reqnroll - _ -Public Class {0} - {1} -End Class"; + _ + Public Class {0} + {1} + End Class + """; public override ProjectFile GenerateLoggerClass(string pathToLogFile) { @@ -28,56 +30,41 @@ Imports System.Runtime.CompilerServices Imports System.Threading.Tasks Friend Module Log - Private Const LogFileLocation As String = "{{pathToLogFile}}" + Private Const RetryCount As Integer = 5 + Private Const LogFileLocation As String = "{{pathToLogFile}}" + Private ReadOnly Rnd As Random = New Random() + Private ReadOnly LockObj As Object = New Object() - Private Sub Retry(number As Integer, action As Action) - Try - action - Catch ex As Exception - Dim i = number - 1 - If (i = 0) - Throw - End If - System.Threading.Thread.Sleep(500) - Retry(i, action) - End Try - End Sub - - Friend Sub LogStep( Optional stepName As String = Nothing) - Retry(5, sub() File.AppendAllText(LogFileLocation, $"-> step: {stepName}{Environment.NewLine}")) - End Sub - - Friend Sub LogHook( Optional stepName As String = Nothing) - Retry(5, sub() File.AppendAllText(LogFileLocation, $"-> hook: {stepName}{Environment.NewLine}")) - End Sub + Private Sub Retry(number As Integer, action As Action) + Try + action + Catch ex As Exception + Dim i = number - 1 + If (i = 0) + Throw + End If + System.Threading.Thread.Sleep(50 + Rnd.Next(50)) + Retry(i, action) + End Try + End Sub + + Friend Sub LogStep( Optional stepName As String = Nothing) + Retry(RetryCount, sub() WriteToFile($"-> step: {stepName}{Environment.NewLine}")) + End Sub + + Friend Sub LogHook( Optional stepName As String = Nothing) + Retry(RetryCount, sub() WriteToFile($"-> hook: {stepName}{Environment.NewLine}")) + End Sub + + Friend Sub LogCustom(category As String, value As String, Optional memberName As String = Nothing) + Retry(RetryCount, sub() WriteToFile($"-> {category}: {value}:{memberName}{Environment.NewLine}")) + End Sub - Friend Async Function LogHookIncludingLockingAsync( - ByVal Optional stepName As String = Nothing) As Task - File.AppendAllText(LogFileLocation, $"->waiting for hook lock: {stepName}{Environment.NewLine}") - Await WaitForLockAsync() - File.AppendAllText(LogFileLocation, $"-> hook: {stepName}{Environment.NewLine}") - End Function - - Private Async Function WaitForLockAsync() As Task - Dim lockFile = LogFileLocation & ".lock" - - While True - - Dim succeeded = True - Try - Using File.Open(lockFile, FileMode.CreateNew) - End Using - Exit While - Catch __unusedIOException1__ As IOException - succeeded = False - End Try - If Not succeeded Then - Await Task.Delay(1000) - End If - End While - - File.Delete(lockFile) - End Function + Private Sub WriteToFile(line As String) + SyncLock LockObj + File.AppendAllText(LogFileLocation, line) + End SyncLock + End Sub End Module """; return new ProjectFile("Log.vb", "Compile", fileContent); @@ -182,70 +169,27 @@ protected override string GetHookBindingClass( string methodScopeAttributes = ToScopeTags(methodScopeAttributeTags); string staticKeyword = isStatic ? "Static" : string.Empty; - return $@" -Imports System -Imports System.Collections -Imports System.IO -Imports System.Linq -Imports System.Xml -Imports System.Xml.Linq -Imports Reqnroll - -<[Binding]> _ -{classScopeAttributes} -Public Class {Guid.NewGuid()} - <[{hookType}({hookTypeAttributeTagsString})]>_ - {methodScopeAttributes} - Public {staticKeyword} Sub {name}() - {code} - Console.WriteLine(""-> hook: {name}"") - End Sub -End Class -"; + return $""" + Imports System + Imports System.Collections + Imports System.IO + Imports System.Linq + Imports System.Xml + Imports System.Xml.Linq + Imports Reqnroll + + <[Binding]> _ + {classScopeAttributes} + Public Class {Guid.NewGuid()} + <[{hookType}({hookTypeAttributeTagsString})]>_ + {methodScopeAttributes} + Public {staticKeyword} Sub {name}() + {code} + Console.WriteLine("-> hook: {name}") + End Sub + End Class + """; } - protected override string GetAsyncHookIncludingLockingBindingClass(string hookType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, - IList classScopeAttributeTags = null) - { - string ToScopeTags(IList scopeTags) => scopeTags.Any() ? $"{scopeTags.Select(t => $@"<[Scope](Tag=""{t}"")>").JoinToString("")}_" : null; - - bool isStatic = IsStaticEvent(hookType); - - string hookTypeTags = hookTypeAttributeTags?.Select(t => $@"""{t}""").JoinToString(", "); - - var hookAttributeConstructorProperties = new[] - { - hookTypeAttributeTags is null || !hookTypeAttributeTags.Any() ? null : $"tags:= New String() {{{hookTypeTags}}}", - order is null ? null : $"Order:= {order}" - }.Where(p => p.IsNotNullOrWhiteSpace()); - - string hookTypeAttributeTagsString = string.Join(", ", hookAttributeConstructorProperties); - string classScopeAttributes = ToScopeTags(classScopeAttributeTags); - string methodScopeAttributes = ToScopeTags(methodScopeAttributeTags); - - string staticKeyword = isStatic ? "Static" : string.Empty; - return $@" -Imports System -Imports System.Collections -Imports System.IO -Imports System.Linq -Imports System.Xml -Imports System.Xml.Linq -Imports Reqnroll -Imports System.Threading -Imports System.Threading.Tasks - -<[Binding]> _ -{classScopeAttributes} -Public Class {Guid.NewGuid()} - <[{hookType}({hookTypeAttributeTagsString})]>_ - {methodScopeAttributes} - Public {staticKeyword} Async Function {name}() as Task - {code} - Await Log.LogHookIncludingLockingAsync() - End Function -End Class -"; - } } } diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs index 984f99e52..c2d4659b5 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs @@ -7,7 +7,7 @@ public class FeatureFileGenerator { public ProjectFile Generate(string featureFileContent, string featureFileName = null) { - featureFileName = featureFileName ?? $"FeatureFile{Guid.NewGuid():N}.feature"; + featureFileName ??= $"FeatureFile{Guid.NewGuid():N}.feature"; string fileContent = featureFileContent.Replace("'''", "\"\"\""); return new ProjectFile(featureFileName, "None", fileContent); diff --git a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs index f5194d638..1eb012049 100644 --- a/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs +++ b/Tests/TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs @@ -120,15 +120,6 @@ public void AddHookBinding(string eventType, string name, string code = "", int? _project.AddFile(bindingsGenerator.GenerateHookBinding(eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags)); } - public void AddAsyncHookBindingIncludingLocking(string eventType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, - IList classScopeAttributeTags = null) - { - EnsureProjectExists(); - - var bindingsGenerator = _bindingsGeneratorFactory.FromLanguage(_project.ProgrammingLanguage); - _project.AddFile(bindingsGenerator.GenerateAsyncHookBindingIncludingLocking(eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags)); - } - public void AddStepBinding(string bindingCode) { EnsureProjectExists(); @@ -357,11 +348,11 @@ private void AddUnitTestProviderSpecificConfig() _project.AddFile(new ProjectFile("XUnitConfiguration.cs", "Compile", "using Xunit; [assembly: CollectionBehavior(CollectionBehavior.CollectionPerClass, MaxParallelThreads = 4)]")); break; case UnitTestProvider.NUnit3 when _parallelTestExecution: - _project.AddFile(new ProjectFile("NUnitConfiguration.cs", "Compile", "[assembly: NUnit.Framework.Parallelizable(NUnit.Framework.ParallelScope.Fixtures)]")); + _project.AddFile(new ProjectFile("NUnitConfiguration.cs", "Compile", "[assembly: NUnit.Framework.Parallelizable(NUnit.Framework.ParallelScope.All)]")); break; case UnitTestProvider.MSTest when _parallelTestExecution: _project.AddFile( - new ProjectFile("MsTestConfiguration.cs", "Compile", "using Microsoft.VisualStudio.TestTools.UnitTesting; [assembly: Parallelize(Workers = 4, Scope = ExecutionScope.ClassLevel)]")); + new ProjectFile("MsTestConfiguration.cs", "Compile", "using Microsoft.VisualStudio.TestTools.UnitTesting; [assembly: Parallelize(Workers = 4, Scope = ExecutionScope.MethodLevel)]")); break; case UnitTestProvider.MSTest when !_parallelTestExecution: _project.AddFile(new ProjectFile("MsTestConfiguration.cs", "Compile", "using Microsoft.VisualStudio.TestTools.UnitTesting; [assembly: DoNotParallelize]")); diff --git a/docs/execution/parallel-execution.md b/docs/execution/parallel-execution.md index 00bbb3dfa..871ec880d 100644 --- a/docs/execution/parallel-execution.md +++ b/docs/execution/parallel-execution.md @@ -19,7 +19,7 @@ When using Reqnroll we can consider the parallel scheduling on the level of scen | Scheduling unit | Description | Runner support | | ---------------- | -------------------- | -------------------- | -| Scenario | Scenarios can run in parallel with each other (also from different features) | N/A | +| Scenario | Scenarios can run in parallel with each other (also from different features) | NUnit, MsTest | | Feature | Features can run in parallel with each other. Scenarios from the same feature are running on the same test thread. | NUnit, MsTest, xUnit | | Test assembly | Different test assemblies can run in parallel with each other | e.g. VSTest | @@ -36,7 +36,7 @@ When using Reqnroll we can consider the parallel scheduling on the level of scen ### Requirements -* You have to use a test runner that supports in-process parallel execution (currently NUnit v3, xUnit v2 and MSTest) +* You have to use a test runner that supports in-process parallel execution (NUnit and MsTest supports scenario-level, xUnit supports feature-level) * You have to ensure that your code does not conflict on static state. * You must not use the static context properties of Reqnroll `ScenarioContext.Current`, `FeatureContext.Current` or `ScenarioStepContext.Current` (see further information below). * You have to configure the test runner to execute the Reqnroll features in parallel with each other (see configuration details below). @@ -44,7 +44,7 @@ When using Reqnroll we can consider the parallel scheduling on the level of scen ### Execution Behavior * `[BeforeTestRun]` and `[AfterTestRun]` hooks (events) are executed only once on the first thread that initializes the framework. Executing tests in the other threads is blocked until the hooks have been fully executed on the first thread. -* All scenarios in a feature must be executed on the **same thread**. See the configuration of the test runners below. This ensures that the `[BeforeFeature]` and `[AfterFeature]` hooks are executed only once for each feature and that the thread has a separate (and isolated) `FeatureContext`. +* As a general guideline, **we do not suggest using `[BeforeFeature]` and `[AfterFeature]` hooks and the `FeatureContext` when running the tests parallel**, because in that case it is not guaranteed that these hooks will be executed only once and there will be only one instance of `FeatureContext` per feature. The lifetime of the `FeatureContext` (that starts and finishes by invoking the `[BeforeFeature]` and `[AfterFeature]` hooks) is the consecutive execution of scenarios of a feature on the same parallel execution worker thread. In case of running the scenarios parallel, the scenarios of a feature might be distributed to multiple workers and therefore might have their own dedicated `FeatureContext`. Because of this behavior the `FeatureContext` is never shared between parallel threads so it does not have to be handled in a thread-safe way. If you wish to have a singleton `FeatureContext` and `[BeforeFeature]` and `[AfterFeature]` hook execution, scenarios in a feature must be executed on the **same thread**. * Scenarios and their related hooks (Before/After scenario, scenario block, step) are isolated in the different threads during execution and do not block each other. Each thread has a separate (and isolated) `ScenarioContext`. * The test trace listener (that outputs the scenario execution trace to the console by default) is invoked asynchronously from the multiple threads and the trace messages are queued and passed to the listener in serialized form. If the test trace listener implements `Reqnroll.Tracing.IThreadSafeTraceListener`, the messages are sent directly from the threads. @@ -54,14 +54,17 @@ By default, [NUnit does not run the tests in parallel](https://docs.nunit.org/ar Parallelization must be configured by setting an assembly-level attribute in the Reqnroll project. ```{code-block} csharp -:caption: C# File +:caption: C# file for configuring feature-level parallelization using NUnit.Framework; [assembly: Parallelizable(ParallelScope.Fixtures)] ``` -```{note} -Reqnroll does not support scenario level parallelization with NUnit (when scenarios from the same feature execute in parallel). If you configure a higher level NUnit parallelization than "Fixtures" your tests will fail with runtime errors. +```{code-block} csharp +:caption: C# file for configuring scenario-level parallelization + +using NUnit.Framework; +[assembly: Parallelizable(ParallelScope.Children)] ``` ### MSTest Configuration @@ -70,14 +73,17 @@ By default, [MsTest does not run the tests in parallel](https://devblogs.microso Parallelisation must be configured by setting an assembly-level attribute in the Reqnroll project. ```{code-block} csharp -:caption: C# File +:caption: C# file for configuring feature-level parallelization using Microsoft.VisualStudio.TestTools.UnitTesting; [assembly: Parallelize(Scope = ExecutionScope.ClassLevel)] ``` -```{note} -Reqnroll does not support scenario level parallelization with MsTest (when scenarios from the same feature execute in parallel). If you configure a higher level MsTest parallelization than "ClassLevel" your tests will fail with runtime errors. +```{code-block} csharp +:caption: C# file for configuring scenario-level parallelization + +using Microsoft.VisualStudio.TestTools.UnitTesting; +[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)] ``` ### xUnit Configuration