fix: make sure hook failures print outer status + attempts when the e…

…rror is the hook itself.
cypress-io · Sep 25, 2023 · 63b6be7 · 63b6be7
1 parent 3ec51b3
commit 63b6be7
Show file tree

Hide file tree

Showing 7 changed files with 824 additions and 28 deletions.
diff --git a/packages/app/cypress/e2e/runner/retries.experimentalRetries.mochaEvents.snapshots.ts b/packages/app/cypress/e2e/runner/retries.experimentalRetries.mochaEvents.snapshots.ts
@@ -40285,6 +40285,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            strategy: 'detect-flake-and-pass-on-threshold',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before all" hook for "test 1"',
           hookName: 'before all',
@@ -40612,6 +40618,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            strategy: 'detect-flake-but-always-fail',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before all" hook for "test 1"',
           hookName: 'before all',
@@ -40940,6 +40952,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            strategy: 'detect-flake-but-always-fail',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before all" hook for "test 1"',
           hookName: 'before all',

diff --git a/packages/app/cypress/e2e/runner/retries.mochaEvents.snapshots.ts b/packages/app/cypress/e2e/runner/retries.mochaEvents.snapshots.ts
@@ -6327,6 +6327,11 @@ export const snapshots = {
       'mocha',
       'fail',
       {
+        _cypressTestStatusInfo: {
+          attempts: 1,
+          outerStatus: 'failed',
+          shouldAttemptsContinue: false,
+        },
         id: 'r3',
         title: '"before all" hook for "test 1"',
         hookName: 'before all',

diff --git a/packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.snapshots.ts b/packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.snapshots.ts
@@ -89,6 +89,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            strategy: 'detect-flake-and-pass-on-threshold',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before all" hook for "test 1"',
           hookName: 'before all',
@@ -358,6 +364,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            strategy: 'detect-flake-but-always-fail',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before all" hook for "test 1"',
           hookName: 'before all',
@@ -627,6 +639,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            strategy: 'detect-flake-but-always-fail',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before all" hook for "test 1"',
           hookName: 'before all',
@@ -2167,6 +2185,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 6,
+            strategy: 'detect-flake-and-pass-on-threshold',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before each" hook for "test 1"',
           hookName: 'before each',
@@ -4742,6 +4766,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 10,
+            strategy: 'detect-flake-but-always-fail',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before each" hook for "test 1"',
           hookName: 'before each',
@@ -7317,6 +7347,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 10,
+            strategy: 'detect-flake-but-always-fail',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before each" hook for "test 1"',
           hookName: 'before each',
@@ -7863,6 +7899,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            strategy: 'detect-flake-and-pass-on-threshold',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r4',
           title: '"after all" hook for "test 2"',
           hookName: 'after all',
@@ -8404,6 +8446,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            strategy: 'detect-flake-but-always-fail',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r4',
           title: '"after all" hook for "test 2"',
           hookName: 'after all',
@@ -8945,6 +8993,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            strategy: 'detect-flake-but-always-fail',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r4',
           title: '"after all" hook for "test 2"',
           hookName: 'after all',
@@ -10780,6 +10834,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 6,
+            strategy: 'detect-flake-and-pass-on-threshold',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"after each" hook for "test 1"',
           hookName: 'after each',
@@ -13807,6 +13867,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 10,
+            strategy: 'detect-flake-but-always-fail',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"after each" hook for "test 1"',
           hookName: 'after each',
@@ -16530,6 +16596,12 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 10,
+            strategy: 'detect-flake-but-always-fail',
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"after each" hook for "test 1"',
           hookName: 'after each',

diff --git a/packages/app/cypress/e2e/runner/runner.mochaEvents.snapshots.ts b/packages/app/cypress/e2e/runner/runner.mochaEvents.snapshots.ts
@@ -1,13 +1,7 @@
 export const snapshots = {
   'src/cypress/runner tests finish with correct state hook failures fail in [before] #1':
     [
-      [
-        'mocha',
-        'start',
-        {
-          start: 'match.date',
-        },
-      ],
+      ['mocha', 'start', { start: 'match.date' }],
       [
         'mocha',
         'suite',
@@ -95,6 +89,11 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before all" hook for "test 1"',
           hookName: 'before all',
@@ -272,13 +271,7 @@ export const snapshots = {
           _slow: 10000,
         },
       ],
-      [
-        'mocha',
-        'end',
-        {
-          end: 'match.date',
-        },
-      ],
+      ['mocha', 'end', { end: 'match.date' }],
     ],
 
   'src/cypress/runner tests finish with correct state hook failures fail in [beforeEach] #1':
@@ -388,6 +381,11 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"before each" hook for "test 1"',
           hookName: 'before each',
@@ -609,13 +607,7 @@ export const snapshots = {
 
   'src/cypress/runner tests finish with correct state hook failures fail in [after] #1':
     [
-      [
-        'mocha',
-        'start',
-        {
-          start: 'match.date',
-        },
-      ],
+      ['mocha', 'start', { start: 'match.date' }],
       [
         'mocha',
         'suite',
@@ -940,6 +932,11 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r4',
           title: '"after all" hook for "test 2"',
           hookName: 'after all',
@@ -1144,13 +1141,7 @@ export const snapshots = {
           _slow: 10000,
         },
       ],
-      [
-        'mocha',
-        'end',
-        {
-          end: 'match.date',
-        },
-      ],
+      ['mocha', 'end', { end: 'match.date' }],
     ],
 
   'src/cypress/runner tests finish with correct state hook failures fail in [afterEach] #1':
@@ -1260,6 +1251,11 @@ export const snapshots = {
         'mocha',
         'fail',
         {
+          _cypressTestStatusInfo: {
+            attempts: 1,
+            outerStatus: 'failed',
+            shouldAttemptsContinue: false,
+          },
           id: 'r3',
           title: '"after each" hook for "test 1"',
           hookName: 'after each',

diff --git a/packages/driver/src/cypress/runner.ts b/packages/driver/src/cypress/runner.ts
@@ -1157,6 +1157,20 @@ const _runnerListeners = (_runner, Cypress, _emissions, getTestById, getTest, se
       } else {
         err = $errUtils.appendErrMsg(err, errMessage)
       }
+
+      // If the test never failed and only the hooks did,
+      // we need to attach the metadata of the test to the hook to report the failure correctly to the server reporter.
+      // We calculate it fresh here since it may not be available on the test, which is the case with a beforeEach hook.
+      // as well as maybe incorrect (test passed on first attempt, but after hooks failed)
+      const testStatus = test.calculateTestStatus()
+
+      runnable._cypressTestStatusInfo = {
+        attempts: testStatus.attempts,
+        strategy: testStatus.strategy,
+        // regardless of the test state, we should ultimately fail the test here.
+        outerStatus: runnable.state,
+        shouldAttemptsContinue: false,
+      }
     }
 
     // always set runnable err so we can tap into