Fix tests that should be skipped in Gherkin are failing (#1174)

Fix tests that should be skipped in Gherkin are failing
Fix #1173

Examples/Gherkin/Gherkin-PesterResultShowsFeatureAndScenarioNames.feature

@@ -17,9 +17,21 @@ Feature: PesterResult shows executed feature names
      Then the scenario name is displayed in the '<Status>' array of the PesterResults object
 
       Examples: A Passing Scenario
-        | Outcome | Status          |
-        | Passed  | PassedScenarios |
+        | Outcome         | Status               |
+        | Passed          | PassedScenarios      |
 
-      Examples: A Failing Scenario
-        | Outcome | Status          |
-        | Failed  | FailedScenarios |
+      Examples: Failing Scenario (later)
+        | Outcome         | Status               |
+        # The following example fails in the Then code block
+        | FailedLater     | FailedLaterScenarios |
+
+      Examples: Failing Scenario (early)
+        | Outcome         | Status               |
+        # The following example fails in the Given code block
+        | FailedEarly     | FailedEarlyScenarios |
+
+      Examples: Failing Scenario (inconclusive)
+        | Outcome         | Status               |
+        # In PesterResult.Steps.ps1 we do not implement anything for the following example
+        # to produce an inconclusive test result
+        | Does not matter | Does not matter      |

Examples/Gherkin/PesterResult.Steps.ps1

@@ -2,14 +2,19 @@ Given "this feature and scenario" { }
 When "(it|the scenario) is executed" { }
 Then "the feature name is displayed in the test report" { }
 
-Given "this is a '(?<Outcome>(Passed|Failed))' scenario" {
+Given "this is a '(?<Outcome>(Passed|Failed(?:Early|Later)))' scenario" {
     param($Outcome)
+    if ($Outcome -eq 'FailedEarly') {
+        throw "We fail for test by intention in the Given code block"
+    }
 }
 
-Then "the scenario name is displayed in the '(?<Status>(Passed|Failed)Scenarios)' array of the PesterResults object" {
+Then "the scenario name is displayed in the '(?<Status>(Passed|Failed(?:Early|Later))Scenarios)' array of the PesterResults object" {
     param($Status)
 
     # Forcing a failure on FailedScenarios to ensure that the FailedScenarios array
     # of the PesterResult object has a value.
-    $Status | Should -Be "PassedScenarios"
+    if ($Status -match "Failed") {
+        throw "We fail for test by intention in the Then code block"
+    }
 }

Functions/Gherkin.Tests.ps1

@@ -258,13 +258,106 @@ Describe "When displaying PesterResults in the console" -Tag Gherkin {
         $gherkin.Results.Features | Should -Be "PesterResult shows executed feature names"
     }
 
-    It 'Should show the names of the passed secnarios' {
-        $gherkin.Results.PassedScenarios | Should -Be @('The PesterResult object shows the executed feature names', 'The Pester test report shows scenario names with examples: Examples: A Passing Scenario')
+    It 'Should show the names of the passed scenarios' {
+        $gherkin.Results.PassedScenarios | Should -Be @(
+            'The PesterResult object shows the executed feature names',
+            'The Pester test report shows scenario names with examples: Examples: A Passing Scenario'
+        )
     }
 
     It 'Should show the names of the failed scenarios' {
-        $gherkin.Results.FailedScenarios | Should -Be "The Pester test report shows scenario names with examples: Examples: A Failing Scenario"
+        $gherkin.Results.FailedScenarios | Should -Be @(
+            'The Pester test report shows scenario names with examples: Examples: Failing Scenario (later)'
+            'The Pester test report shows scenario names with examples: Examples: Failing Scenario (early)'
+            'The Pester test report shows scenario names with examples: Examples: Failing Scenario (inconclusive)'
+        )
     }
+
+}
+
+Describe "Check test results of steps" -Tag Gherkin {
+    # Calling this in a job so we don't monkey with the active pester state that's already running
+    $job = Start-Job -ArgumentList $scriptRoot -ScriptBlock {
+        param ($scriptRoot)
+        Get-Module Pester | Remove-Module -Force
+        Import-Module $scriptRoot\Pester.psd1 -Force
+
+        New-Object psobject -Property @{
+            Results = Invoke-Gherkin (Join-Path $scriptRoot Examples\Gherkin\Gherkin-PesterResultShowsFeatureAndScenarioNames.feature) -PassThru -Show None
+        }
+    }
+
+    $gherkin = $job | Wait-Job | Receive-Job
+    Remove-Job $job
+
+    $testResults = $gherkin.Results |
+        Select-Object -ExpandProperty TestResult |
+        Select-Object -ExpandProperty Result
+
+    It "Should have the expected number of test results" {
+        $testResults.Count | Should -Be 15
+    }
+
+    It "Test result 1 is correct" {
+        $testResults[0] | Should -Be 'Passed'
+    }
+
+    It "Test result 2 is correct" {
+        $testResults[1] | Should -Be 'Passed'
+    }
+
+    It "Test result 3 is correct" {
+        $testResults[2] | Should -Be 'Passed'
+    }
+
+    It "Test result 4 is correct" {
+        $testResults[3] | Should -Be 'Passed'
+    }
+
+    It "Test result 5 is correct" {
+        $testResults[4] | Should -Be 'Passed'
+    }
+
+    It "Test result 6 is correct" {
+        $testResults[5] | Should -Be 'Passed'
+    }
+
+    It "Test result 7 is correct" {
+        $testResults[6] | Should -Be 'Passed'
+    }
+
+    It "Test result 8 is correct" {
+        $testResults[7] | Should -Be 'Passed'
+    }
+
+    It "Test result 9 is correct" {
+        $testResults[8] | Should -Be 'Failed'
+    }
+
+    It "Test result 10 is correct" {
+        $testResults[9] | Should -Be "Failed"
+    }
+
+    It "Test result 11 is correct" {
+        $testResults[10] | Should -Be 'Passed'
+    }
+
+    It "Test result 12 is correct" {
+        $testResults[11] | Should -Be 'Failed'
+    }
+
+    It "Test result 13 is correct" {
+        $testResults[12] | Should -Be 'Inconclusive'
+    }
+
+    It "Test result 14 is correct" {
+        $testResults[13] | Should -Be 'Passed'
+    }
+
+    It "Test result 15 is correct" {
+        $testResults[14] | Should -Be 'Inconclusive'
+    }
+
 }
 
 
@@ -299,6 +392,11 @@ Describe "A generated NUnit report" -Tag Gherkin {
         return (Select-Xml -Xml $nUnitReportXml -XPath $xPath | Select-Object -ExpandProperty Node)
     }
 
+    # Helper function to get the inner text of a XML node from a XPath expression
+    function Get-XmlInnerText($xPath) {
+        return (Get-XmlNode $xPath).InnerText
+    }
+
     # Helper function to get the value of a XML node from a XPath expression
     function Get-XmlValue($xPath) {
         return (Get-XmlNode $xPath).Value
@@ -309,10 +407,16 @@ Describe "A generated NUnit report" -Tag Gherkin {
         return (Get-XmlNode $xPath).Count
     }
 
+    $expectedFeatureFileName1 = (Join-Path $scriptRoot Examples\Gherkin\JustForReporting1.feature)
+    $expectedFeatureFileName2 = (Join-Path $scriptRoot Examples\Gherkin\JustForReporting2.feature)
+    $expectedImplementationFileName = (Join-Path $scriptRoot Examples\Gherkin\JustForReporting.Steps.ps1)
+
     $featuresXPath = "/test-results/test-suite/results/test-suite"
     $feature1ScenariosXPath = "$featuresXPath[1]/results/test-suite"
     $feature2ScenariosXPath = "$featuresXPath[2]/results/test-suite"
 
+    $expectFeatureFileNameInStackTrace = $PSVersionTable.PSVersion.Major -gt 2
+
     It 'should be an existing and well formed XML file' {
         $reportFile | Should -Exist
         $nUnitReportXml | Should -Not -BeNullOrEmpty
@@ -384,6 +488,12 @@ Describe "A generated NUnit report" -Tag Gherkin {
         Get-XmlValue "($scenario2Examples1StepsXPath/@result)[4]" | Should -Be "Success"
         Get-XmlValue "($scenario2Examples1StepsXPath/@result)[5]" | Should -Be "Failure"
         Get-XmlValue "($scenario2Examples1StepsXPath/@result)[6]" | Should -Be "Success"
+
+        Get-XmlInnerText "$scenario2Examples1StepsXPath[5]/failure/message" | Should -Be "An example error"
+        if ($expectFeatureFileNameInStackTrace) {
+            Get-XmlInnerText "($scenario2Examples1StepsXPath)[5]/failure/stack-trace" | Should -BeLike "*From $($expectedFeatureFileName1): line 15*"
+        }
+        Get-XmlInnerText "($scenario2Examples1StepsXPath)[5]/failure/stack-trace" | Should -BeLike "*at <ScriptBlock>, $($expectedImplementationFileName): line 23*"
     }
 
     It 'should contain all steps of scenario 2 (examples 2) with correct names and test results' {
@@ -422,6 +532,12 @@ Describe "A generated NUnit report" -Tag Gherkin {
         Get-XmlValue "($scenario3StepsXPath/@result)[3]" | Should -Be "Success"
         Get-XmlValue "($scenario3StepsXPath/@result)[4]" | Should -Be "Success"
         Get-XmlValue "($scenario3StepsXPath/@result)[5]" | Should -Be "Failure"
+
+        Get-XmlInnerText "$scenario3StepsXPath[5]/failure/message" | Should -Be "Another example error"
+        if ($expectFeatureFileNameInStackTrace) {
+            Get-XmlInnerText "($scenario3StepsXPath)[5]/failure/stack-trace" | Should -BeLike "*From $($expectedFeatureFileName1): line 32*"
+        }
+        Get-XmlInnerText "($scenario3StepsXPath)[5]/failure/stack-trace" | Should -BeLike "*at <ScriptBlock>, $($expectedImplementationFileName): line 57*"
     }
 
     It 'should contain all steps of scenario 4 with correct names and test results' {
@@ -433,9 +549,13 @@ Describe "A generated NUnit report" -Tag Gherkin {
         Get-XmlValue "($scenario4StepsXPath/@name)[2]" | Should -Be "Scenario 4.When step_402"
         Get-XmlValue "($scenario4StepsXPath/@name)[3]" | Should -Be "Scenario 4.Then step_403"
 
-        Get-XmlValue "($scenario4StepsXPath/@result)[1]" | Should -Be "Failure"
-        Get-XmlValue "($scenario4StepsXPath/@result)[2]" | Should -Be "Failure"
-        Get-XmlValue "($scenario4StepsXPath/@result)[3]" | Should -Be "Failure"
+        Get-XmlValue "($scenario4StepsXPath/@result)[1]" | Should -Be "Inconclusive"
+        Get-XmlValue "($scenario4StepsXPath/@result)[2]" | Should -Be "Inconclusive"
+        Get-XmlValue "($scenario4StepsXPath/@result)[3]" | Should -Be "Inconclusive"
+
+        Get-XmlInnerText "($scenario4StepsXPath)[1]/reason/message" | Should -Be "Could not find implementation for step!"
+        Get-XmlInnerText "($scenario4StepsXPath)[2]/reason/message" | Should -Be "Could not find implementation for step!"
+        Get-XmlInnerText "($scenario4StepsXPath)[3]/reason/message" | Should -Be "Could not find implementation for step!"
     }
 
     It 'should contain all steps of scenario 5 (examples 1) with correct names and test results' {
@@ -447,9 +567,13 @@ Describe "A generated NUnit report" -Tag Gherkin {
         Get-XmlValue "($scenario5Examples1StepsXPath/@name)[2]" | Should -Match "(?s)Scenario 5.+Examples 1\.When step_502"
         Get-XmlValue "($scenario5Examples1StepsXPath/@name)[3]" | Should -Match "(?s)Scenario 5.+Examples 1\.Then step_503"
 
-        Get-XmlValue "($scenario5Examples1StepsXPath/@result)[1]" | Should -Be "Failure"
-        Get-XmlValue "($scenario5Examples1StepsXPath/@result)[2]" | Should -Be "Failure"
-        Get-XmlValue "($scenario5Examples1StepsXPath/@result)[3]" | Should -Be "Failure"
+        Get-XmlValue "($scenario5Examples1StepsXPath/@result)[1]" | Should -Be "Inconclusive"
+        Get-XmlValue "($scenario5Examples1StepsXPath/@result)[2]" | Should -Be "Inconclusive"
+        Get-XmlValue "($scenario5Examples1StepsXPath/@result)[3]" | Should -Be "Inconclusive"
+
+        Get-XmlInnerText "($scenario5Examples1StepsXPath)[1]/reason/message" | Should -Be "Could not find implementation for step!"
+        Get-XmlInnerText "($scenario5Examples1StepsXPath)[2]/reason/message" | Should -Be "Could not find implementation for step!"
+        Get-XmlInnerText "($scenario5Examples1StepsXPath)[3]/reason/message" | Should -Be "Could not find implementation for step!"
     }
 
     It 'should contain all steps of scenario 5 (examples 2) with correct names and test results' {
@@ -461,9 +585,13 @@ Describe "A generated NUnit report" -Tag Gherkin {
         Get-XmlValue "($scenario5Examples2StepsXPath/@name)[2]" | Should -Match "(?s)Scenario 5.+Examples 2\.When step_602"
         Get-XmlValue "($scenario5Examples2StepsXPath/@name)[3]" | Should -Match "(?s)Scenario 5.+Examples 2\.Then step_603"
 
-        Get-XmlValue "($scenario5Examples2StepsXPath/@result)[1]" | Should -Be "Failure"
-        Get-XmlValue "($scenario5Examples2StepsXPath/@result)[2]" | Should -Be "Failure"
-        Get-XmlValue "($scenario5Examples2StepsXPath/@result)[3]" | Should -Be "Failure"
+        Get-XmlValue "($scenario5Examples2StepsXPath/@result)[1]" | Should -Be "Inconclusive"
+        Get-XmlValue "($scenario5Examples2StepsXPath/@result)[2]" | Should -Be "Inconclusive"
+        Get-XmlValue "($scenario5Examples2StepsXPath/@result)[3]" | Should -Be "Inconclusive"
+
+        Get-XmlInnerText "($scenario5Examples2StepsXPath)[1]/reason/message" | Should -Be "Could not find implementation for step!"
+        Get-XmlInnerText "($scenario5Examples2StepsXPath)[2]/reason/message" | Should -Be "Could not find implementation for step!"
+        Get-XmlInnerText "($scenario5Examples2StepsXPath)[3]/reason/message" | Should -Be "Could not find implementation for step!"
     }
 
 }

Functions/Gherkin.ps1

@@ -697,7 +697,7 @@ function Invoke-GherkinStep {
         ) | & $SafeCommands["Sort-Object"] MatchCount | & $SafeCommands["Select-Object"] -First 1
 
         if (!$StepCommand) {
-            $PesterErrorRecord = New-InconclusiveErrorRecord -Message "Could not find implementation for step!" -File $Step.Location.Path -Line $Step.Location.Line -LineText $DisplayText
+            $PesterErrorRecord = New-PesterErrorRecord -Result Inconclusive -Message "Could not find implementation for step!" -File $Step.Location.Path -Line $Step.Location.Line -LineText $DisplayText
         } else {
 
             $NamedArguments, $Parameters = Get-StepParameters $Step $StepCommand
@@ -747,7 +747,7 @@ function Invoke-GherkinStep {
             # Unless we really are a StackTrace...
             ${Pester Result}.StackTrace += "`nFrom " + $Step.Location.Path + ': line ' + $Step.Location.Line
         }
-        $Pester.AddTestResult($DisplayText, ${Pester Result}.Result, $Elapsed, $PesterErrorRecord.Exception.Message, ${Pester Result}.StackTrace, $null, $NamedArguments, $PesterErrorRecord )
+        $Pester.AddTestResult($DisplayText, ${Pester Result}.Result, $Elapsed, ${Pester Result}.FailureMessage, ${Pester Result}.StackTrace, $null, $NamedArguments, $PesterErrorRecord)
         $Pester.TestResult[-1] | Write-PesterResult
     }
 }

Functions/Output.ps1

@@ -267,23 +267,23 @@ function Write-PesterResult {
                 }
 
                 Skipped {
-                    $because = if ($testresult.FailureMessage) { ", because $($testresult.FailureMessage)"} else { $null }
+                    $because = if ($testresult.ErrorRecord.TargetObject.Data.Because) { ", because $($testresult.ErrorRecord.TargetObject.Data.Because)"} else { $null }
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Skipped "$margin[!] $output" -NoNewLine
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Skipped ", is skipped$because" -NoNewLine
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.SkippedTime " $humanTime"
                     break
                 }
 
                 Pending {
-                    $because = if ($testresult.FailureMessage) { ", because $($testresult.FailureMessage)"} else { $null }
+                    $because = if ($testresult.ErrorRecord.TargetObject.Data.Because) { ", because $($testresult.ErrorRecord.TargetObject.Data.Because)"} else { $null }
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Pending "$margin[?] $output" -NoNewLine
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Pending ", is pending$because" -NoNewLine
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.PendingTime " $humanTime"
                     break
                 }
 
                 Inconclusive {
-                    $because = if ($testresult.FailureMessage) { ", because $($testresult.FailureMessage)"} else { $null }
+                    $because = if ($testresult.ErrorRecord.TargetObject.Data.Because) { ", because $($testresult.ErrorRecord.TargetObject.Data.Because)"} else { $null }
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Inconclusive "$margin[?] $output" -NoNewLine
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Inconclusive ", is inconclusive$because" -NoNewLine
                     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.InconclusiveTime " $humanTime"

Functions/Set-ItResult.ps1

@@ -60,18 +60,41 @@ function Set-ItResult {
 
     Assert-DescribeInProgress -CommandName Set-ItResult
 
-    $state = $PSCmdlet.ParameterSetName
-    $exception = New-Object Exception "It result set to $state"
-    $errorID = "PesterTest$state"
+    $result = $PSCmdlet.ParameterSetName
+    $message = "It result set to $result$(if ($Because) { ", $Because" })"
+    $data = @{
+        Result = $result
+        Because = $Because
+    }
+    $errorRecord = New-PesterErrorRecord -Result $result  -Message $message -Data $data
+    throw $errorRecord
+}
+
+function New-PesterErrorRecord {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory=$true)]
+        [string]$Result,
+        [Parameter(Mandatory=$true)]
+        [string]$Message,
+        [string]$File,
+        [string]$Line,
+        [string]$LineText,
+        [hashtable]$Data
+    )
+
+    $exception = New-Object Exception $Message
+    $errorID = "PesterTest$Result"
     $errorCategory = [Management.Automation.ErrorCategory]::InvalidResult
 
     # we use ErrorRecord.TargetObject to pass structured information about the error to a reporting system.
     $targetObject = @{
-        Message = $Because;
-        File = $MyInvocation.ScriptName;
-        Line = $MyInvocation.ScriptLineNumber;
-        LineText = $MyInvocation.Line.TrimEnd($([System.Environment]::NewLine))
+        Message = $Message
+        Data = $Data
+        File = $(if ($File -ne $null) { $File } else { $MyInvocation.ScriptName})
+        Line = $(if ($Line -ne $null) { $Line } else { $MyInvocation.ScriptLineNumber})
+        LineText = $(if ($LineText -ne $null) { $LineText } else { $MyInvocation.Line }).TrimEnd($([System.Environment]::NewLine))
     }
 
-    throw New-Object Management.Automation.ErrorRecord $exception, $errorID, $errorCategory, $targetObject
+    New-Object Management.Automation.ErrorRecord $exception, $errorID, $errorCategory, $targetObject
 }