From e066e07166e4e275c36b5a144d37813483fbbc41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Gn=C3=BCchtel?= Date: Fri, 4 Jan 2019 19:05:00 +0100 Subject: [PATCH] Fix tests that should be skipped in Gherkin are failing (#1174) Fix tests that should be skipped in Gherkin are failing Fix #1173 --- ...ResultShowsFeatureAndScenarioNames.feature | 22 ++- Examples/Gherkin/PesterResult.Steps.ps1 | 11 +- Functions/Gherkin.Tests.ps1 | 152 ++++++++++++++++-- Functions/Gherkin.ps1 | 4 +- Functions/Output.ps1 | 6 +- Functions/Set-ItResult.ps1 | 39 ++++- 6 files changed, 201 insertions(+), 33 deletions(-) diff --git a/Examples/Gherkin/Gherkin-PesterResultShowsFeatureAndScenarioNames.feature b/Examples/Gherkin/Gherkin-PesterResultShowsFeatureAndScenarioNames.feature index 8699e97a2..1b405d07e 100644 --- a/Examples/Gherkin/Gherkin-PesterResultShowsFeatureAndScenarioNames.feature +++ b/Examples/Gherkin/Gherkin-PesterResultShowsFeatureAndScenarioNames.feature @@ -17,9 +17,21 @@ Feature: PesterResult shows executed feature names Then the scenario name is displayed in the '' array of the PesterResults object Examples: A Passing Scenario - | Outcome | Status | - | Passed | PassedScenarios | + | Outcome | Status | + | Passed | PassedScenarios | - Examples: A Failing Scenario - | Outcome | Status | - | Failed | FailedScenarios | + Examples: Failing Scenario (later) + | Outcome | Status | + # The following example fails in the Then code block + | FailedLater | FailedLaterScenarios | + + Examples: Failing Scenario (early) + | Outcome | Status | + # The following example fails in the Given code block + | FailedEarly | FailedEarlyScenarios | + + Examples: Failing Scenario (inconclusive) + | Outcome | Status | + # In PesterResult.Steps.ps1 we do not implement anything for the following example + # to produce an inconclusive test result + | Does not matter | Does not matter | diff --git a/Examples/Gherkin/PesterResult.Steps.ps1 b/Examples/Gherkin/PesterResult.Steps.ps1 index cd94a4348..7b3e948ea 100644 --- a/Examples/Gherkin/PesterResult.Steps.ps1 +++ b/Examples/Gherkin/PesterResult.Steps.ps1 @@ -2,14 +2,19 @@ Given "this feature and scenario" { } When "(it|the scenario) is executed" { } Then "the feature name is displayed in the test report" { } -Given "this is a '(?(Passed|Failed))' scenario" { +Given "this is a '(?(Passed|Failed(?:Early|Later)))' scenario" { param($Outcome) + if ($Outcome -eq 'FailedEarly') { + throw "We fail for test by intention in the Given code block" + } } -Then "the scenario name is displayed in the '(?(Passed|Failed)Scenarios)' array of the PesterResults object" { +Then "the scenario name is displayed in the '(?(Passed|Failed(?:Early|Later))Scenarios)' array of the PesterResults object" { param($Status) # Forcing a failure on FailedScenarios to ensure that the FailedScenarios array # of the PesterResult object has a value. - $Status | Should -Be "PassedScenarios" + if ($Status -match "Failed") { + throw "We fail for test by intention in the Then code block" + } } diff --git a/Functions/Gherkin.Tests.ps1 b/Functions/Gherkin.Tests.ps1 index f0dd3ca95..1664769a2 100644 --- a/Functions/Gherkin.Tests.ps1 +++ b/Functions/Gherkin.Tests.ps1 @@ -258,13 +258,106 @@ Describe "When displaying PesterResults in the console" -Tag Gherkin { $gherkin.Results.Features | Should -Be "PesterResult shows executed feature names" } - It 'Should show the names of the passed secnarios' { - $gherkin.Results.PassedScenarios | Should -Be @('The PesterResult object shows the executed feature names', 'The Pester test report shows scenario names with examples: Examples: A Passing Scenario') + It 'Should show the names of the passed scenarios' { + $gherkin.Results.PassedScenarios | Should -Be @( + 'The PesterResult object shows the executed feature names', + 'The Pester test report shows scenario names with examples: Examples: A Passing Scenario' + ) } It 'Should show the names of the failed scenarios' { - $gherkin.Results.FailedScenarios | Should -Be "The Pester test report shows scenario names with examples: Examples: A Failing Scenario" + $gherkin.Results.FailedScenarios | Should -Be @( + 'The Pester test report shows scenario names with examples: Examples: Failing Scenario (later)' + 'The Pester test report shows scenario names with examples: Examples: Failing Scenario (early)' + 'The Pester test report shows scenario names with examples: Examples: Failing Scenario (inconclusive)' + ) } + +} + +Describe "Check test results of steps" -Tag Gherkin { + # Calling this in a job so we don't monkey with the active pester state that's already running + $job = Start-Job -ArgumentList $scriptRoot -ScriptBlock { + param ($scriptRoot) + Get-Module Pester | Remove-Module -Force + Import-Module $scriptRoot\Pester.psd1 -Force + + New-Object psobject -Property @{ + Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\Gherkin-PesterResultShowsFeatureAndScenarioNames.feature) -PassThru -Show None + } + } + + $gherkin = $job | Wait-Job | Receive-Job + Remove-Job $job + + $testResults = $gherkin.Results | + Select-Object -ExpandProperty TestResult | + Select-Object -ExpandProperty Result + + It "Should have the expected number of test results" { + $testResults.Count | Should -Be 15 + } + + It "Test result 1 is correct" { + $testResults[0] | Should -Be 'Passed' + } + + It "Test result 2 is correct" { + $testResults[1] | Should -Be 'Passed' + } + + It "Test result 3 is correct" { + $testResults[2] | Should -Be 'Passed' + } + + It "Test result 4 is correct" { + $testResults[3] | Should -Be 'Passed' + } + + It "Test result 5 is correct" { + $testResults[4] | Should -Be 'Passed' + } + + It "Test result 6 is correct" { + $testResults[5] | Should -Be 'Passed' + } + + It "Test result 7 is correct" { + $testResults[6] | Should -Be 'Passed' + } + + It "Test result 8 is correct" { + $testResults[7] | Should -Be 'Passed' + } + + It "Test result 9 is correct" { + $testResults[8] | Should -Be 'Failed' + } + + It "Test result 10 is correct" { + $testResults[9] | Should -Be "Failed" + } + + It "Test result 11 is correct" { + $testResults[10] | Should -Be 'Passed' + } + + It "Test result 12 is correct" { + $testResults[11] | Should -Be 'Failed' + } + + It "Test result 13 is correct" { + $testResults[12] | Should -Be 'Inconclusive' + } + + It "Test result 14 is correct" { + $testResults[13] | Should -Be 'Passed' + } + + It "Test result 15 is correct" { + $testResults[14] | Should -Be 'Inconclusive' + } + } @@ -299,6 +392,11 @@ Describe "A generated NUnit report" -Tag Gherkin { return (Select-Xml -Xml $nUnitReportXml -XPath $xPath | Select-Object -ExpandProperty Node) } + # Helper function to get the inner text of a XML node from a XPath expression + function Get-XmlInnerText($xPath) { + return (Get-XmlNode $xPath).InnerText + } + # Helper function to get the value of a XML node from a XPath expression function Get-XmlValue($xPath) { return (Get-XmlNode $xPath).Value @@ -309,10 +407,16 @@ Describe "A generated NUnit report" -Tag Gherkin { return (Get-XmlNode $xPath).Count } + $expectedFeatureFileName1 = (Join-Path $scriptRoot Examples\Gherkin\JustForReporting1.feature) + $expectedFeatureFileName2 = (Join-Path $scriptRoot Examples\Gherkin\JustForReporting2.feature) + $expectedImplementationFileName = (Join-Path $scriptRoot Examples\Gherkin\JustForReporting.Steps.ps1) + $featuresXPath = "/test-results/test-suite/results/test-suite" $feature1ScenariosXPath = "$featuresXPath[1]/results/test-suite" $feature2ScenariosXPath = "$featuresXPath[2]/results/test-suite" + $expectFeatureFileNameInStackTrace = $PSVersionTable.PSVersion.Major -gt 2 + It 'should be an existing and well formed XML file' { $reportFile | Should -Exist $nUnitReportXml | Should -Not -BeNullOrEmpty @@ -384,6 +488,12 @@ Describe "A generated NUnit report" -Tag Gherkin { Get-XmlValue "($scenario2Examples1StepsXPath/@result)[4]" | Should -Be "Success" Get-XmlValue "($scenario2Examples1StepsXPath/@result)[5]" | Should -Be "Failure" Get-XmlValue "($scenario2Examples1StepsXPath/@result)[6]" | Should -Be "Success" + + Get-XmlInnerText "$scenario2Examples1StepsXPath[5]/failure/message" | Should -Be "An example error" + if ($expectFeatureFileNameInStackTrace) { + Get-XmlInnerText "($scenario2Examples1StepsXPath)[5]/failure/stack-trace" | Should -BeLike "*From $($expectedFeatureFileName1): line 15*" + } + Get-XmlInnerText "($scenario2Examples1StepsXPath)[5]/failure/stack-trace" | Should -BeLike "*at , $($expectedImplementationFileName): line 23*" } It 'should contain all steps of scenario 2 (examples 2) with correct names and test results' { @@ -422,6 +532,12 @@ Describe "A generated NUnit report" -Tag Gherkin { Get-XmlValue "($scenario3StepsXPath/@result)[3]" | Should -Be "Success" Get-XmlValue "($scenario3StepsXPath/@result)[4]" | Should -Be "Success" Get-XmlValue "($scenario3StepsXPath/@result)[5]" | Should -Be "Failure" + + Get-XmlInnerText "$scenario3StepsXPath[5]/failure/message" | Should -Be "Another example error" + if ($expectFeatureFileNameInStackTrace) { + Get-XmlInnerText "($scenario3StepsXPath)[5]/failure/stack-trace" | Should -BeLike "*From $($expectedFeatureFileName1): line 32*" + } + Get-XmlInnerText "($scenario3StepsXPath)[5]/failure/stack-trace" | Should -BeLike "*at , $($expectedImplementationFileName): line 57*" } It 'should contain all steps of scenario 4 with correct names and test results' { @@ -433,9 +549,13 @@ Describe "A generated NUnit report" -Tag Gherkin { Get-XmlValue "($scenario4StepsXPath/@name)[2]" | Should -Be "Scenario 4.When step_402" Get-XmlValue "($scenario4StepsXPath/@name)[3]" | Should -Be "Scenario 4.Then step_403" - Get-XmlValue "($scenario4StepsXPath/@result)[1]" | Should -Be "Failure" - Get-XmlValue "($scenario4StepsXPath/@result)[2]" | Should -Be "Failure" - Get-XmlValue "($scenario4StepsXPath/@result)[3]" | Should -Be "Failure" + Get-XmlValue "($scenario4StepsXPath/@result)[1]" | Should -Be "Inconclusive" + Get-XmlValue "($scenario4StepsXPath/@result)[2]" | Should -Be "Inconclusive" + Get-XmlValue "($scenario4StepsXPath/@result)[3]" | Should -Be "Inconclusive" + + Get-XmlInnerText "($scenario4StepsXPath)[1]/reason/message" | Should -Be "Could not find implementation for step!" + Get-XmlInnerText "($scenario4StepsXPath)[2]/reason/message" | Should -Be "Could not find implementation for step!" + Get-XmlInnerText "($scenario4StepsXPath)[3]/reason/message" | Should -Be "Could not find implementation for step!" } It 'should contain all steps of scenario 5 (examples 1) with correct names and test results' { @@ -447,9 +567,13 @@ Describe "A generated NUnit report" -Tag Gherkin { Get-XmlValue "($scenario5Examples1StepsXPath/@name)[2]" | Should -Match "(?s)Scenario 5.+Examples 1\.When step_502" Get-XmlValue "($scenario5Examples1StepsXPath/@name)[3]" | Should -Match "(?s)Scenario 5.+Examples 1\.Then step_503" - Get-XmlValue "($scenario5Examples1StepsXPath/@result)[1]" | Should -Be "Failure" - Get-XmlValue "($scenario5Examples1StepsXPath/@result)[2]" | Should -Be "Failure" - Get-XmlValue "($scenario5Examples1StepsXPath/@result)[3]" | Should -Be "Failure" + Get-XmlValue "($scenario5Examples1StepsXPath/@result)[1]" | Should -Be "Inconclusive" + Get-XmlValue "($scenario5Examples1StepsXPath/@result)[2]" | Should -Be "Inconclusive" + Get-XmlValue "($scenario5Examples1StepsXPath/@result)[3]" | Should -Be "Inconclusive" + + Get-XmlInnerText "($scenario5Examples1StepsXPath)[1]/reason/message" | Should -Be "Could not find implementation for step!" + Get-XmlInnerText "($scenario5Examples1StepsXPath)[2]/reason/message" | Should -Be "Could not find implementation for step!" + Get-XmlInnerText "($scenario5Examples1StepsXPath)[3]/reason/message" | Should -Be "Could not find implementation for step!" } It 'should contain all steps of scenario 5 (examples 2) with correct names and test results' { @@ -461,9 +585,13 @@ Describe "A generated NUnit report" -Tag Gherkin { Get-XmlValue "($scenario5Examples2StepsXPath/@name)[2]" | Should -Match "(?s)Scenario 5.+Examples 2\.When step_602" Get-XmlValue "($scenario5Examples2StepsXPath/@name)[3]" | Should -Match "(?s)Scenario 5.+Examples 2\.Then step_603" - Get-XmlValue "($scenario5Examples2StepsXPath/@result)[1]" | Should -Be "Failure" - Get-XmlValue "($scenario5Examples2StepsXPath/@result)[2]" | Should -Be "Failure" - Get-XmlValue "($scenario5Examples2StepsXPath/@result)[3]" | Should -Be "Failure" + Get-XmlValue "($scenario5Examples2StepsXPath/@result)[1]" | Should -Be "Inconclusive" + Get-XmlValue "($scenario5Examples2StepsXPath/@result)[2]" | Should -Be "Inconclusive" + Get-XmlValue "($scenario5Examples2StepsXPath/@result)[3]" | Should -Be "Inconclusive" + + Get-XmlInnerText "($scenario5Examples2StepsXPath)[1]/reason/message" | Should -Be "Could not find implementation for step!" + Get-XmlInnerText "($scenario5Examples2StepsXPath)[2]/reason/message" | Should -Be "Could not find implementation for step!" + Get-XmlInnerText "($scenario5Examples2StepsXPath)[3]/reason/message" | Should -Be "Could not find implementation for step!" } } diff --git a/Functions/Gherkin.ps1 b/Functions/Gherkin.ps1 index 883c47dc6..a1fdf9cf0 100644 --- a/Functions/Gherkin.ps1 +++ b/Functions/Gherkin.ps1 @@ -697,7 +697,7 @@ function Invoke-GherkinStep { ) | & $SafeCommands["Sort-Object"] MatchCount | & $SafeCommands["Select-Object"] -First 1 if (!$StepCommand) { - $PesterErrorRecord = New-InconclusiveErrorRecord -Message "Could not find implementation for step!" -File $Step.Location.Path -Line $Step.Location.Line -LineText $DisplayText + $PesterErrorRecord = New-PesterErrorRecord -Result Inconclusive -Message "Could not find implementation for step!" -File $Step.Location.Path -Line $Step.Location.Line -LineText $DisplayText } else { $NamedArguments, $Parameters = Get-StepParameters $Step $StepCommand @@ -747,7 +747,7 @@ function Invoke-GherkinStep { # Unless we really are a StackTrace... ${Pester Result}.StackTrace += "`nFrom " + $Step.Location.Path + ': line ' + $Step.Location.Line } - $Pester.AddTestResult($DisplayText, ${Pester Result}.Result, $Elapsed, $PesterErrorRecord.Exception.Message, ${Pester Result}.StackTrace, $null, $NamedArguments, $PesterErrorRecord ) + $Pester.AddTestResult($DisplayText, ${Pester Result}.Result, $Elapsed, ${Pester Result}.FailureMessage, ${Pester Result}.StackTrace, $null, $NamedArguments, $PesterErrorRecord) $Pester.TestResult[-1] | Write-PesterResult } } diff --git a/Functions/Output.ps1 b/Functions/Output.ps1 index 6d5c55ae4..3d6127e90 100644 --- a/Functions/Output.ps1 +++ b/Functions/Output.ps1 @@ -267,7 +267,7 @@ function Write-PesterResult { } Skipped { - $because = if ($testresult.FailureMessage) { ", because $($testresult.FailureMessage)"} else { $null } + $because = if ($testresult.ErrorRecord.TargetObject.Data.Because) { ", because $($testresult.ErrorRecord.TargetObject.Data.Because)"} else { $null } & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Skipped "$margin[!] $output" -NoNewLine & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Skipped ", is skipped$because" -NoNewLine & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.SkippedTime " $humanTime" @@ -275,7 +275,7 @@ function Write-PesterResult { } Pending { - $because = if ($testresult.FailureMessage) { ", because $($testresult.FailureMessage)"} else { $null } + $because = if ($testresult.ErrorRecord.TargetObject.Data.Because) { ", because $($testresult.ErrorRecord.TargetObject.Data.Because)"} else { $null } & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Pending "$margin[?] $output" -NoNewLine & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Pending ", is pending$because" -NoNewLine & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.PendingTime " $humanTime" @@ -283,7 +283,7 @@ function Write-PesterResult { } Inconclusive { - $because = if ($testresult.FailureMessage) { ", because $($testresult.FailureMessage)"} else { $null } + $because = if ($testresult.ErrorRecord.TargetObject.Data.Because) { ", because $($testresult.ErrorRecord.TargetObject.Data.Because)"} else { $null } & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Inconclusive "$margin[?] $output" -NoNewLine & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Inconclusive ", is inconclusive$because" -NoNewLine & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.InconclusiveTime " $humanTime" diff --git a/Functions/Set-ItResult.ps1 b/Functions/Set-ItResult.ps1 index 995c2f5d0..c2cd39cbf 100644 --- a/Functions/Set-ItResult.ps1 +++ b/Functions/Set-ItResult.ps1 @@ -60,18 +60,41 @@ function Set-ItResult { Assert-DescribeInProgress -CommandName Set-ItResult - $state = $PSCmdlet.ParameterSetName - $exception = New-Object Exception "It result set to $state" - $errorID = "PesterTest$state" + $result = $PSCmdlet.ParameterSetName + $message = "It result set to $result$(if ($Because) { ", $Because" })" + $data = @{ + Result = $result + Because = $Because + } + $errorRecord = New-PesterErrorRecord -Result $result -Message $message -Data $data + throw $errorRecord +} + +function New-PesterErrorRecord { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [string]$Result, + [Parameter(Mandatory=$true)] + [string]$Message, + [string]$File, + [string]$Line, + [string]$LineText, + [hashtable]$Data + ) + + $exception = New-Object Exception $Message + $errorID = "PesterTest$Result" $errorCategory = [Management.Automation.ErrorCategory]::InvalidResult # we use ErrorRecord.TargetObject to pass structured information about the error to a reporting system. $targetObject = @{ - Message = $Because; - File = $MyInvocation.ScriptName; - Line = $MyInvocation.ScriptLineNumber; - LineText = $MyInvocation.Line.TrimEnd($([System.Environment]::NewLine)) + Message = $Message + Data = $Data + File = $(if ($File -ne $null) { $File } else { $MyInvocation.ScriptName}) + Line = $(if ($Line -ne $null) { $Line } else { $MyInvocation.ScriptLineNumber}) + LineText = $(if ($LineText -ne $null) { $LineText } else { $MyInvocation.Line }).TrimEnd($([System.Environment]::NewLine)) } - throw New-Object Management.Automation.ErrorRecord $exception, $errorID, $errorCategory, $targetObject + New-Object Management.Automation.ErrorRecord $exception, $errorID, $errorCategory, $targetObject }