Add nightly check for failures

.github/workflows/nightly.yml

@@ -3,15 +3,15 @@ name: Nightly checks
 # runs every day at midnight
 on:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: '0 0 * * *'
   workflow_dispatch:
   # To test fixes on push rather than wait for the scheduling
   push:
     branches:
       - fix/nightly
 
 jobs:
-  test_storybook_prerelease:
+  assert_test_runner:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -37,7 +37,6 @@ jobs:
       - name: Fix local @storybook/csf version
         run: |
           yarn add @storybook/[email protected]
-
       - name: Run test runner
         uses: mathiasvr/command-output@v1
         id: tests
@@ -57,7 +56,7 @@ jobs:
         id: slack
         uses: slackapi/[email protected]
         with:
-          channel-id: "${{ secrets.SLACK_CHANNEL_ID }}"
+          channel-id: '${{ secrets.SLACK_CHANNEL_ID }}'
           payload: |
             {
               "blocks": [
@@ -97,3 +96,101 @@ jobs:
             }
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
+  assert_test_runner_failures:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+
+      - name: Install dependencies
+        uses: bahmutov/npm-install@v1
+
+      - name: Get current version of Storybook
+        run: |
+          echo "prev_sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+          echo "prev_sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+
+      - name: Upgrade to storybook@future
+        run: |
+          npx storybook@future upgrade --prerelease --yes
+
+      # TODO: This should not be necessary once @storybook/csf is properly updated
+      - name: Fix local @storybook/csf version
+        run: |
+          yarn add @storybook/[email protected]
+
+      - name: Run test runner and expect failure
+        uses: mathiasvr/command-output@v1
+        with:
+          run: |
+            yarn build
+            yarn test-storybook:ci-failures
+
+      - name: Process test results
+        if: ${{ always() }}
+        id: tests
+        uses: sergeysova/jq-action@v2
+        with:
+          cmd: 'jq .numPassedTests test-results.json -r'
+
+      - name: Set failure check to env
+        if: ${{ always() }}
+        run: |
+          echo "FAILED=${{ steps.tests.outputs.value > 0 }}" >> $GITHUB_ENV
+
+      - name: Get prerelease version of Storybook
+        if: ${{ always() && env.FAILED == 'true' }}
+        run: |
+          echo "sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+          echo "sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+
+      - name: Report if any test passes
+        if: ${{ always() && env.FAILED == 'true' }}
+        id: slack
+        uses: slackapi/[email protected]
+        with:
+          channel-id: '${{ secrets.SLACK_CHANNEL_ID }}'
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": ":storybook: :runner: [Test Runner] The Nightly check for **failures** has passed :thinking_face:",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*@storybook/react version:*\n${{ env.prev_sb_version }} >> ${{ env.sb_version }}"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*@storybook/csf version:*\n${{ env.prev_sb_csf_version }} >> ${{ env.sb_csf_version }}"
+                    }
+                  ],
+                  "accessory": {
+                    "type": "button",
+                    "text": {
+                      "type": "plain_text",
+                      "text": "View failure",
+                      "emoji": true
+                    },
+                    "value": "view_failure",
+                    "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                    "action_id": "button-action"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.gitignore

@@ -8,4 +8,5 @@ build-storybook.log
 stories/atoms/StressTest.stories.js
 yarn-error.log
 .nyc_output
-coverage
+coverage
+test-results.json

.storybook/main.js

@@ -1,4 +1,4 @@
-const stories = [
+let stories = [
   '../stories/docs/**/*.stories.mdx',
   // default title prefix
   {
@@ -18,6 +18,10 @@ if (process.env.STRESS_TEST) {
   stories.push('../stories/stress-test/*.stories.@(js|jsx|ts|tsx)');
 }
 
+if(process.env.TEST_FAILURES) {
+  stories = ['../stories/expected-failures/*.stories.@(js|jsx|ts|tsx)'];
+}
+
 const addons = [
   process.env.WITHOUT_DOCS
     ? {

.storybook/test-runner.ts

@@ -4,6 +4,7 @@ import type { TestRunnerConfig } from '../dist/ts';
 
 const snapshotsDir = process.env.SNAPSHOTS_DIR || '__snapshots__';
 const customSnapshotsDir = `${process.cwd()}/${snapshotsDir}`;
+const skipSnapshots = process.env.SKIP_SNAPSHOTS === 'true';
 
 const config: TestRunnerConfig = {
   setup() {
@@ -17,6 +18,10 @@ const config: TestRunnerConfig = {
       return;
     }
 
+    if (skipSnapshots) {
+      return;
+    }
+
     // Visual snapshot tests
     const image = await page.screenshot({ fullPage: true });
     expect(image).toMatchImageSnapshot({

package.json

@@ -42,9 +42,11 @@
     "build-storybook": "build-storybook",
     "release": "yarn build && auto shipit",
     "test-storybook": "node bin/test-storybook.js",
+    "test-storybook:failures": "SKIP_SNAPSHOTS=true TEST_FAILURES=1 yarn test-storybook --json --outputFile test-results.json",
     "test-storybook:no-cache": "yarn test-storybook --no-cache",
     "test-storybook:json": "yarn test-storybook --stories-json",
     "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook\"",
+    "test-storybook:ci-failures": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"TEST_FAILURES=1 yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:failures\"",
     "test-storybook:ci-coverage": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook --coverage\"",
     "test-storybook:ci-json": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:json\"",
     "generate-dynamic-stories": "node scripts/generate-dynamic-stories.js"

src/util/getParsedCliOptions.ts

@@ -44,6 +44,14 @@ export const getParsedCliOptions = () => {
       '-u, --updateSnapshot',
       'Use this flag to re-record every snapshot that fails during this test run'
     )
+    .option(
+      '--json',
+      'Prints the test results in JSON. This mode will send all other test output and user messages to stderr.'
+    )
+    .option(
+      '--outputFile',
+      'Write test results to a file when the --json option is also specified.'
+    )
     .option(
       '--coverage',
       'Indicates that test coverage information should be collected and reported in the output'

stories/expected-failures/Failure.stories.jsx

@@ -0,0 +1,28 @@
+import React from 'react';
+import { within, userEvent } from '@storybook/testing-library';
+
+import { Page } from '../pages/Page';
+
+export default {
+  title: 'Stories with failures',
+  component: Page,
+};
+
+const Template = (args) => <Page {...args} />;
+
+export const ComponentThrowsErrors = () => {
+  // throw new Error('Component has a failure');
+  return <div>Oi</div>
+}
+
+export const PlayFnThrowsErrors = Template.bind({});
+PlayFnThrowsErrors.play = () => {
+  throw new Error('Play function has a failure');
+};
+
+export const PlayFnAssertionFails = Template.bind({});
+PlayFnAssertionFails.play = async ({ canvasElement }) => {
+  const canvas = within(canvasElement);
+  const unexistentButton = await canvas.getByRole('button', { name: /I do not exist/i });
+  await userEvent.click(unexistentButton);
+};