diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d347e0cb..fa06bb32 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,7 +3,7 @@ name: Nightly checks # runs every day at midnight on: schedule: - - cron: "0 0 * * *" + - cron: '0 0 * * *' workflow_dispatch: # To test fixes on push rather than wait for the scheduling push: @@ -11,7 +11,7 @@ on: - fix/nightly jobs: - test_storybook_prerelease: + assert_test_runner: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -37,7 +37,6 @@ jobs: - name: Fix local @storybook/csf version run: | yarn add @storybook/csf@0.0.2--canary.4566f4d.1 - - name: Run test runner uses: mathiasvr/command-output@v1 id: tests @@ -57,7 +56,7 @@ jobs: id: slack uses: slackapi/slack-github-action@v1.19.0 with: - channel-id: "${{ secrets.SLACK_CHANNEL_ID }}" + channel-id: '${{ secrets.SLACK_CHANNEL_ID }}' payload: | { "blocks": [ @@ -97,3 +96,101 @@ jobs: } env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + + assert_test_runner_failures: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Use Node.js 14.x + uses: actions/setup-node@v1 + with: + node-version: 14.x + + - name: Install dependencies + uses: bahmutov/npm-install@v1 + + - name: Get current version of Storybook + run: | + echo "prev_sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV + echo "prev_sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV + + - name: Upgrade to storybook@future + run: | + npx storybook@future upgrade --prerelease --yes + + # TODO: This should not be necessary once @storybook/csf is properly updated + - name: Fix local @storybook/csf version + run: | + yarn add @storybook/csf@0.0.2--canary.4566f4d.1 + + - name: Run test runner and expect failure + uses: mathiasvr/command-output@v1 + with: + run: | + yarn build + yarn test-storybook:ci-failures + + - name: Process test results + if: ${{ always() }} + id: tests + uses: sergeysova/jq-action@v2 + with: + cmd: 'jq .numPassedTests test-results.json -r' + + - name: Set failure check to env + if: ${{ always() }} + run: | + echo "FAILED=${{ steps.tests.outputs.value > 0 }}" >> $GITHUB_ENV + + - name: Get prerelease version of Storybook + if: ${{ always() && env.FAILED == 'true' }} + run: | + echo "sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV + echo "sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV + + - name: Report if any test passes + if: ${{ always() && env.FAILED == 'true' }} + id: slack + uses: slackapi/slack-github-action@v1.19.0 + with: + channel-id: '${{ secrets.SLACK_CHANNEL_ID }}' + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": ":storybook: :runner: [Test Runner] The Nightly check for **failures** has passed :thinking_face:", + "emoji": true + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*@storybook/react version:*\n${{ env.prev_sb_version }} >> ${{ env.sb_version }}" + }, + { + "type": "mrkdwn", + "text": "*@storybook/csf version:*\n${{ env.prev_sb_csf_version }} >> ${{ env.sb_csf_version }}" + } + ], + "accessory": { + "type": "button", + "text": { + "type": "plain_text", + "text": "View failure", + "emoji": true + }, + "value": "view_failure", + "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", + "action_id": "button-action" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.gitignore b/.gitignore index 46710295..86ff7083 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ build-storybook.log stories/atoms/StressTest.stories.js yarn-error.log .nyc_output -coverage \ No newline at end of file +coverage +test-results.json \ No newline at end of file diff --git a/package.json b/package.json index 9449ddb7..2a9c5279 100644 --- a/package.json +++ b/package.json @@ -42,10 +42,11 @@ "build-storybook": "build-storybook", "release": "yarn build && auto shipit", "test-storybook": "node bin/test-storybook.js", - "test-storybook:failures": "TEST_FAILURES=1 yarn test-storybook", + "test-storybook:failures": "SKIP_SNAPSHOTS=true TEST_FAILURES=1 yarn test-storybook --json --outputFile test-results.json", "test-storybook:no-cache": "yarn test-storybook --no-cache", "test-storybook:json": "yarn test-storybook --stories-json", "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook\"", + "test-storybook:ci-failures": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"TEST_FAILURES=1 yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:failures\"", "test-storybook:ci-coverage": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook --coverage\"", "test-storybook:ci-json": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:json\"", "generate-dynamic-stories": "node scripts/generate-dynamic-stories.js" diff --git a/src/util/getParsedCliOptions.ts b/src/util/getParsedCliOptions.ts index 04013486..d0862bf5 100644 --- a/src/util/getParsedCliOptions.ts +++ b/src/util/getParsedCliOptions.ts @@ -44,6 +44,14 @@ export const getParsedCliOptions = () => { '-u, --updateSnapshot', 'Use this flag to re-record every snapshot that fails during this test run' ) + .option( + '--json', + 'Prints the test results in JSON. This mode will send all other test output and user messages to stderr.' + ) + .option( + '--outputFile', + 'Write test results to a file when the --json option is also specified.' + ) .option( '--coverage', 'Indicates that test coverage information should be collected and reported in the output'