add failure test to nightly check [skip-release]

.github/workflows/nightly.yml

@@ -3,15 +3,15 @@ name: Nightly checks
 # runs every day at midnight
 on:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: '0 0 * * *'
   workflow_dispatch:
   # To test fixes on push rather than wait for the scheduling
   push:
     branches:
       - fix/nightly
 
 jobs:
-  test_storybook_prerelease:
+  assert_test_runner:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -37,7 +37,6 @@ jobs:
       - name: Fix local @storybook/csf version
         run: |
           yarn add @storybook/[email protected]
-
       - name: Run test runner
         uses: mathiasvr/command-output@v1
         id: tests
@@ -57,7 +56,7 @@ jobs:
         id: slack
         uses: slackapi/[email protected]
         with:
-          channel-id: "${{ secrets.SLACK_CHANNEL_ID }}"
+          channel-id: '${{ secrets.SLACK_CHANNEL_ID }}'
           payload: |
             {
               "blocks": [
@@ -97,3 +96,101 @@ jobs:
             }
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
+  assert_test_runner_failures:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+
+      - name: Install dependencies
+        uses: bahmutov/npm-install@v1
+
+      - name: Get current version of Storybook
+        run: |
+          echo "prev_sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+          echo "prev_sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+
+      - name: Upgrade to storybook@future
+        run: |
+          npx storybook@future upgrade --prerelease --yes
+
+      # TODO: This should not be necessary once @storybook/csf is properly updated
+      - name: Fix local @storybook/csf version
+        run: |
+          yarn add @storybook/[email protected]
+      
+      - name: Run test runner and expect failure
+        uses: mathiasvr/command-output@v1
+        with:
+          run: |
+            yarn build
+            yarn test-storybook:ci-failures
+      
+      - name: Process test results
+        if: ${{ always() }}
+        id: tests
+        uses: sergeysova/jq-action@v2
+        with:
+          cmd: 'jq .numPassedTests test-results.json -r'
+
+      - name: Set failure check to env
+        if: ${{ always() }}
+        run: |
+          echo "FAILED=${{ steps.tests.outputs.value > 0 }}" >> $GITHUB_ENV
+
+      - name: Get prerelease version of Storybook
+        if: ${{ always() && env.FAILED == 'true' }}
+        run: |
+          echo "sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+          echo "sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+
+      - name: Report if any test passes
+        if: ${{ always() && env.FAILED == 'true' }}
+        id: slack
+        uses: slackapi/[email protected]
+        with:
+          channel-id: '${{ secrets.SLACK_CHANNEL_ID }}'
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": ":storybook: :runner: [Test Runner] The Nightly check for **failures** has passed :thinking_face:",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*@storybook/react version:*\n${{ env.prev_sb_version }} >> ${{ env.sb_version }}"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*@storybook/csf version:*\n${{ env.prev_sb_csf_version }} >> ${{ env.sb_csf_version }}"
+                    }
+                  ],
+                  "accessory": {
+                    "type": "button",
+                    "text": {
+                      "type": "plain_text",
+                      "text": "View failure",
+                      "emoji": true
+                    },
+                    "value": "view_failure",
+                    "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                    "action_id": "button-action"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.gitignore

@@ -8,4 +8,5 @@ build-storybook.log
 stories/atoms/StressTest.stories.js
 yarn-error.log
 .nyc_output
-coverage
+coverage
+test-results.json

package.json

@@ -42,10 +42,11 @@
     "build-storybook": "build-storybook",
     "release": "yarn build && auto shipit",
     "test-storybook": "node bin/test-storybook.js",
-    "test-storybook:failures": "TEST_FAILURES=1 yarn test-storybook",
+    "test-storybook:failures": "SKIP_SNAPSHOTS=true TEST_FAILURES=1 yarn test-storybook --json --outputFile test-results.json",
     "test-storybook:no-cache": "yarn test-storybook --no-cache",
     "test-storybook:json": "yarn test-storybook --stories-json",
     "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook\"",
+    "test-storybook:ci-failures": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"TEST_FAILURES=1 yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:failures\"",
     "test-storybook:ci-coverage": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook --coverage\"",
     "test-storybook:ci-json": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:json\"",
     "generate-dynamic-stories": "node scripts/generate-dynamic-stories.js"

src/util/getParsedCliOptions.ts

@@ -44,6 +44,14 @@ export const getParsedCliOptions = () => {
       '-u, --updateSnapshot',
       'Use this flag to re-record every snapshot that fails during this test run'
     )
+    .option(
+      '--json',
+      'Prints the test results in JSON. This mode will send all other test output and user messages to stderr.'
+    )
+    .option(
+      '--outputFile',
+      'Write test results to a file when the --json option is also specified.'
+    )
     .option(
       '--coverage',
       'Indicates that test coverage information should be collected and reported in the output'