Add failureDetails property to circus test results (#9496)

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ### Features
 
+- `[jest-circus, jest-jasmine2]` Include `failureDetails` property in test results ([#9496](https://github.com/facebook/jest/pull/9496))
+
 ### Fixes
 
 ### Chore & Maintenance

e2e/__tests__/failureDetailsProperty.test.ts

@@ -0,0 +1,217 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {isJestCircusRun} from '@jest/test-utils';
+import runJest from '../runJest';
+
+const removeStackTraces = (stdout: string) =>
+  stdout.replace(
+    /at (new Promise \(<anonymous>\)|.+:\d+:\d+\)?)/g,
+    'at <stacktrace>',
+  );
+
+test('that the failureDetails property is set', () => {
+  const {stdout, stderr} = runJest('failureDetails-property', [
+    'tests.test.js',
+  ]);
+
+  // safety check: if the reporter errors it'll show up here
+  expect(stderr).toStrictEqual('');
+
+  const output = JSON.parse(removeStackTraces(stdout));
+
+  if (isJestCircusRun()) {
+    expect(output).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          Object {
+            "matcherResult": Object {
+              "actual": true,
+              "expected": false,
+              "name": "toBe",
+              "pass": false,
+            },
+          },
+        ],
+        Array [
+          Object {
+            "matcherResult": Object {
+              "actual": true,
+              "expected": false,
+              "name": "toBe",
+              "pass": false,
+            },
+          },
+        ],
+        Array [
+          Object {
+            "matcherResult": Object {
+              "actual": "Object {
+        \\"p1\\": \\"hello\\",
+        \\"p2\\": \\"world\\",
+      }",
+              "expected": "Object {
+        \\"p1\\": \\"hello\\",
+        \\"p2\\": \\"sunshine\\",
+      }",
+              "name": "toMatchInlineSnapshot",
+              "pass": false,
+            },
+          },
+        ],
+        Array [
+          Object {},
+        ],
+        Array [
+          Object {
+            "message": "expect(received).rejects.toThrowError()
+
+      Received promise resolved instead of rejected
+      Resolved to value: 1",
+          },
+        ],
+      ]
+    `);
+  } else {
+    expect(output).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          Object {
+            "actual": "",
+            "error": Object {
+              "matcherResult": Object {
+                "actual": true,
+                "expected": false,
+                "name": "toBe",
+                "pass": false,
+              },
+            },
+            "expected": "",
+            "matcherName": "",
+            "message": "Error: expect(received).toBe(expected) // Object.is equality
+
+      Expected: false
+      Received: true",
+            "passed": false,
+            "stack": "Error: expect(received).toBe(expected) // Object.is equality
+
+      Expected: false
+      Received: true
+          at <stacktrace>",
+          },
+        ],
+        Array [
+          Object {
+            "actual": "",
+            "error": Object {
+              "matcherResult": Object {
+                "actual": true,
+                "expected": false,
+                "name": "toBe",
+                "pass": false,
+              },
+            },
+            "expected": "",
+            "matcherName": "",
+            "message": "Error: expect(received).toBe(expected) // Object.is equality
+
+      Expected: false
+      Received: true",
+            "passed": false,
+            "stack": "Error: expect(received).toBe(expected) // Object.is equality
+
+      Expected: false
+      Received: true
+          at <stacktrace>",
+          },
+        ],
+        Array [
+          Object {
+            "actual": "",
+            "error": Object {
+              "matcherResult": Object {
+                "actual": "Object {
+        \\"p1\\": \\"hello\\",
+        \\"p2\\": \\"world\\",
+      }",
+                "expected": "Object {
+        \\"p1\\": \\"hello\\",
+        \\"p2\\": \\"sunshine\\",
+      }",
+                "name": "toMatchInlineSnapshot",
+                "pass": false,
+              },
+            },
+            "expected": "",
+            "matcherName": "",
+            "message": "expect(received).toMatchInlineSnapshot(snapshot)
+
+      Snapshot name: \`my test a snapshot failure 1\`
+
+      - Snapshot  - 1
+      + Received  + 1
+
+        Object {
+          \\"p1\\": \\"hello\\",
+      -   \\"p2\\": \\"sunshine\\",
+      +   \\"p2\\": \\"world\\",
+        }",
+            "passed": false,
+            "stack": "Error: expect(received).toMatchInlineSnapshot(snapshot)
+
+      Snapshot name: \`my test a snapshot failure 1\`
+
+      - Snapshot  - 1
+      + Received  + 1
+
+        Object {
+          \\"p1\\": \\"hello\\",
+      -   \\"p2\\": \\"sunshine\\",
+      +   \\"p2\\": \\"world\\",
+        }
+          at <stacktrace>",
+          },
+        ],
+        Array [
+          Object {
+            "actual": "",
+            "error": Object {},
+            "expected": "",
+            "matcherName": "",
+            "message": "Error",
+            "passed": false,
+            "stack": "Error: 
+          at <stacktrace>",
+          },
+        ],
+        Array [
+          Object {
+            "actual": "",
+            "error": Object {
+              "message": "expect(received).rejects.toThrowError()
+
+      Received promise resolved instead of rejected
+      Resolved to value: 1",
+            },
+            "expected": "",
+            "matcherName": "",
+            "message": "Error: expect(received).rejects.toThrowError()
+
+      Received promise resolved instead of rejected
+      Resolved to value: 1",
+            "passed": false,
+            "stack": "Error: expect(received).rejects.toThrowError()
+
+      Received promise resolved instead of rejected
+      Resolved to value: 1
+          at <stacktrace>",
+          },
+        ],
+      ]
+    `);
+  }
+});

e2e/failureDetails-property/__tests__/tests.test.js

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+describe('my test', () => {
+  test('it passes', () => {
+    expect(true).toBe(false);
+  });
+
+  it('fails :(', () => {
+    expect(true).toBe(false);
+  });
+
+  test('a snapshot failure', () => {
+    expect({
+      p1: 'hello',
+      p2: 'world',
+    }).toMatchInlineSnapshot(`
+      Object {
+        "p1": "hello",
+        "p2": "sunshine",
+      }
+    `);
+  });
+});
+
+it('throws!', () => {
+  throw new Error();
+});
+
+test('promise rejection', async () => {
+  await expect(Promise.resolve(1)).rejects.toThrowError();
+});

e2e/failureDetails-property/myreporter.js

@@ -0,0 +1,22 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+class MyCustomReporter {
+  onRunComplete(contexts, results) {
+    console.log(
+      JSON.stringify(
+        results.testResults[0].testResults.map(f => f.failureDetails),
+        null,
+        2,
+      ),
+    );
+  }
+}
+
+module.exports = MyCustomReporter;

e2e/failureDetails-property/package.json

@@ -0,0 +1,8 @@
+{
+  "jest": {
+    "testEnvironment": "node",
+    "reporters": [
+      "<rootDir>/myreporter.js"
+    ]
+  }
+}

packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts

@@ -193,6 +193,7 @@ export const runAndTransformResultsToJestFormat = async ({
       return {
         ancestorTitles,
         duration: testResult.duration,
+        failureDetails: testResult.errorsDetailed,
         failureMessages: testResult.errors,
         fullName: title
           ? ancestorTitles.concat(title).join(' ')

packages/jest-circus/src/utils.ts

@@ -280,7 +280,7 @@ export const makeRunResult = (
   unhandledErrors: Array<Error>,
 ): Circus.RunResult => ({
   testResults: makeTestResults(describeBlock),
-  unhandledErrors: unhandledErrors.map(_formatError),
+  unhandledErrors: unhandledErrors.map(_getError).map(getErrorStack),
 });
 
 export const makeSingleTestResult = (
@@ -313,9 +313,12 @@ export const makeSingleTestResult = (
     }
   }
 
+  const errorsDetailed = test.errors.map(_getError);
+
   return {
     duration: test.duration,
-    errors: test.errors.map(_formatError),
+    errors: errorsDetailed.map(getErrorStack),
+    errorsDetailed,
     invocations: test.invocations,
     location,
     status,
@@ -357,9 +360,9 @@ export const getTestID = (test: Circus.TestEntry): string => {
   return titles.join(' ');
 };
 
-const _formatError = (
+const _getError = (
   errors?: Circus.Exception | [Circus.Exception | undefined, Circus.Exception],
-): string => {
+): Error => {
   let error;
   let asyncError;
 
@@ -371,20 +374,17 @@ const _formatError = (
     asyncError = new Error();
   }
 
-  if (error) {
-    if (error.stack) {
-      return error.stack;
-    }
-    if (error.message) {
-      return error.message;
-    }
+  if (error && (error.stack || error.message)) {
+    return error;
   }
 
   asyncError.message = `thrown: ${prettyFormat(error, {maxDepth: 3})}`;
 
-  return asyncError.stack;
+  return asyncError;
 };
 
+const getErrorStack = (error: Error): string => error.stack || error.message;
+
 export const addErrorToEachTestUnderDescribe = (
   describeBlock: Circus.DescribeBlock,
   error: Circus.Exception,
@@ -433,6 +433,7 @@ export const parseSingleTestResult = (
   return {
     ancestorTitles,
     duration: testResult.duration,
+    failureDetails: testResult.errorsDetailed,
     failureMessages: Array.from(testResult.errors),
     fullName: title
       ? ancestorTitles.concat(title).join(' ')

packages/jest-jasmine2/src/reporter.ts

@@ -141,6 +141,7 @@ export default class Jasmine2Reporter implements Reporter {
     const results: AssertionResult = {
       ancestorTitles,
       duration,
+      failureDetails: [],
       failureMessages: [],
       fullName: specResult.fullName,
       location,
@@ -155,6 +156,7 @@ export default class Jasmine2Reporter implements Reporter {
           ? this._addMissingMessageToStack(failed.stack, failed.message)
           : failed.message || '';
       results.failureMessages.push(message);
+      results.failureDetails.push(failed);
     });
 
     return results;