From 472e0647a079c022f00058eec8b1fe4430b5bbdb Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Thu, 6 Jan 2022 14:44:58 +0100 Subject: [PATCH 1/4] docs: update README with instructions --- README.md | 220 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 138 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 03fd4f22..c2be39fa 100644 --- a/README.md +++ b/README.md @@ -2,138 +2,194 @@ Storybook test runner turns all of your stories into executable tests. -It's currently a prototype that configures Jest to run smoke tests on your stories in either: +## Table of Contents -- **JSDOM** - render stories to JSDOM using TestingLibrary -- **Playwright** - render stories in a browser using Playwright +- [1. Features](#features) +- [2. Getting Started](#getting-started) +- [3. Configuration](#configuration) +- [3. Running against a deployed Storybook](#running-against-a-deployed-storybook) +- [4. Running in CI](#running-in-ci) +- [5. Troubleshooting](#troubleshooting) +- [6. Future work](#future-work) -The goal of this prototype is to help evaluate both approaches: primarily for performance, but also to understand the general strengths and weaknesses. +## Features -## Install - -The package has not yet been published so you'll have to link it for now. - -In the `test-runner` package: - -```sh -yarn && yarn build -yarn link -``` - -In your project directory: - -```sh -yarn link @storybook/test-runner -yarn add jest babel-jest @testing-library/react @storybook/testing-react --dev -``` - -For JSDOM (React-only for now): +- ⚡️ Zero config setup +- 💨 Smoke test all stories +- ▶️ Test stories with play functions +- 🏃 Test your stories in parallel in a headless browser +- 👷 Get feedback from error with a link directly to the story +- 🐛 Debug them visually and interactively in a live browser with [addon-interactions](https://storybook.js.org/docs/react/essentials/interactions) +- 🎭 Powered by [Jest](https://jestjs.io/) and [Playwright](https://playwright.dev/) +- 👀 Watch mode, filters, and the conveniences you'd expect -```sh -yarn add jest-environment-jsdom --dev -``` +## Getting started -For Playwright: +1. Install the test runner and the interactions addon in Storybook: -```sh -yarn add jest-playwright-preset playwright --dev +```jsx +yarn add @storybook/test-runner -D ``` -This simply installs the package in `node_modules`. Using the package is fully manual at this point. +
+ 1.1 Optional instructions to install the Interactions addon for visual debugging of play functions -## Configure + ```jsx + yarn add @storybook/addon-interactions @storybook/jest @storybook/testing-library -D + ``` -To use the addon, you need to manually configure it: + Then add it to your `.storybook/main.js` config and enable debugging: -- Create a script in `package.json` -- Create a jest config for your use case + ```jsx + module.exports = { + stories: ['@storybook/addon-interactions'], + features: { + interactionsDebugger: true, + } + }; + ``` +
-### JSDOM - -Add a script to `package.json`: +2. Add a `test-storybook` script to your package.json ```json { "scripts": { - "test-storybook:jsdom": "jest --config ./jsdom-jest.config.js" + "test-storybook": "test-storybook" } } ``` -Then create a config file `jsdom-jest.config.js` that sets up your environment. For example: +3. Run Storybook (the test runner runs against a running Storybook instance): -```js -module.exports = { - cacheDirectory: '.cache/jest', - testMatch: ['**/*.stories.[jt]s(x)?'], - moduleNameMapper: { - // non-js files - '\\.(jpg|jpeg|png|apng|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$': - '@storybook/test-runner/mocks/fileMock.js', - '\\.(css|scss|stylesheet)$': '@storybook/test-runner/mocks/styleMock.js', - }, - transform: { - '^.+\\.stories\\.[jt]sx?$': '@storyboook/test-runner/jsdom/transform', - '^.+\\.[jt]sx?$': 'babel-jest', - }, - testEnvironment: 'jest-environment-jsdom', -}; +```jsx +yarn storybook ``` -### Playwright - -Add a script to `package.json`: +4. Run the test runner: -```json -{ - "scripts": { - "test-storybook:playwright": "jest --config ./test-runner-jest.config.js" - } -} +```jsx +yarn test-storybook --watch ``` -Then create a config file `test-runner-jest.config.js` that sets up your environment. For example: +> **NOTE:** The runner assumes that your Storybook is running on `process.env.STORYBOOK_PORT` which defaults to `6006`. If you're running Storybook in another port, set the STORYBOOK_PORT before running your command like: +>```jsx +>STORYBOOK_PORT=9009 yarn test-storybook --watch +>``` + +## Configuration + +The test runner works out of the box, but you can override its configuration by adding a `test-runner-jest.config.js` that sets up your environment in the root folder of your project. ```js +// test-runner-jest.config.js module.exports = { - cacheDirectory: '.cache/jest', + cacheDirectory: 'node_modules/.cache/storybook/test-runner', testMatch: ['**/*.stories.[jt]s(x)?'], - moduleNameMapper: { - // non-js files - '\\.(jpg|jpeg|png|apng|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$': - '@storybook/test-runner/mocks/fileMock.js', - '\\.(css|scss|stylesheet)$': '@storybook/test-runner/mocks/styleMock.js', - }, transform: { '^.+\\.stories\\.[jt]sx?$': '@storybook/test-runner/playwright/transform', '^.+\\.[jt]sx?$': 'babel-jest', }, preset: 'jest-playwright-preset', testEnvironment: '@storybook/test-runner/playwright/custom-environment.js', + testEnvironmentOptions: { + 'jest-playwright': { + browsers: ['chromium', 'firefox', 'webkit'], // run tests against all browsers + }, + }, }; ``` -> **NOTE:** The code currently assumes that your Storybook is ALREADY running on `process.env.STORYBOOK_PORT` which defaults to `6006`. +The runner uses [jest-playwright](https://github.com/playwright-community/jest-playwright) and you can pass [testEnvironmentOptions](https://github.com/playwright-community/jest-playwright#configuration) to further configure it, such as how it's done above to run tests against all browsers instead of just chromium. -#### Running against a deployed Storybook +## Running against a deployed Storybook -By default, the playwright assumes that you're running it against a locally served Storybook. +By default, the test runner assumes that you're running it against a locally served Storybook. If you want to define a target url so it runs against deployed Storybooks, you can do so by passing the `TARGET_URL` environment variable: +```bash +TARGET_URL=https://the-storybook-url-here.com yarn test-storybook +``` + +## Running in CI + +### Running against deployed Storybooks on Github Actions deployment + +On Github actions, once services like Vercel, Netlify and others do deployment run, they follow a pattern of emitting a `deployment_status` event containing the newly generated URL under `deployment_status.target_url`. Here's an example of an action to run tests based on that: + +```yml +name: Storybook Tests +on: deployment_status +jobs: + test: + timeout-minutes: 60 + runs-on: ubuntu-latest + if: github.event.deployment_status.state == 'success' + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '14.x' + - name: Install dependencies + run: yarn + - name: Run Storybook tests + run: yarn test-storybook + env: + TARGET_URL: '${{ github.event.deployment_status.target_url }}' +``` + +### Running againts locally built Storybooks in CI + +In order to build and run tests against your Storybook in CI, you might need to use a combination of commands involving the [concurrently](https://www.npmjs.com/package/concurrently), [http-server](https://www.npmjs.com/package/http-server) and [wait-on](https://www.npmjs.com/package/wait-on) libraries. Here's a recipe that does the following: Storybook is built and served locally, and once it is ready, the test runner will run against it. + ```json { - "scripts": { - "test-storybook:playwright": "TARGET_URL=the-storybook-url-here jest --config ./test-runner-jest.config.js" - } + "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook\"" } ``` +And then you can essentially run `test-storybook:ci` in your CI: + +```yml +name: Storybook Tests +on: push +jobs: + test: + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '14.x' + - name: Install dependencies + run: yarn + - name: Run Storybook tests + run: yarn test-storybook:ci +``` + + +## Troubleshooting + +#### The test runner seems flaky and keeps timing out + +If your tests are timing out with `Timeout - Async callback was not invoked within the 15000 ms timeout specified by jest.setTimeout`, it might be that playwright couldn't handle to test the amount of stories you have in your project. Maybe you have a large amount of stories or your CI has a really low RAM configuration. + +In either way, to fix it you should limit the amount of workers that run in parallel by passing the [--maxWorkers](https://jestjs.io/docs/cli#--maxworkersnumstring) option to your command: + +```json +{ + "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook --maxWorkers=2\"" +} +``` + +#### Adding the test runner to other CI environments + +As the test runner is based on playwright, depending on your CI setup you might need to use specific docker images or other configuration. In that case, you can refer to the [Playwright CI docs](https://playwright.dev/docs/ci) for more information. + ## Future work -In the future it will support the following features: +Future plans involve adding support for the following features: -- 💨 Smoke test all stories -- ▶️ Test CSF3 play functions - 🧪 Custom test functions - 📄 Run addon reports -- ⚡️ Zero config setup From 23dc24785651362b93a67ffac4368001120f8979 Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Thu, 6 Jan 2022 15:21:34 +0100 Subject: [PATCH 2/4] docs: improve README based on PR comments --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c2be39fa..c69dc82c 100644 --- a/README.md +++ b/README.md @@ -69,17 +69,19 @@ yarn storybook 4. Run the test runner: ```jsx -yarn test-storybook --watch +yarn test-storybook ``` -> **NOTE:** The runner assumes that your Storybook is running on `process.env.STORYBOOK_PORT` which defaults to `6006`. If you're running Storybook in another port, set the STORYBOOK_PORT before running your command like: +> **NOTE:** The runner assumes that your Storybook is running on port `6006`. If you're running Storybook in another port, set the TARGET_URL before running your command like: >```jsx ->STORYBOOK_PORT=9009 yarn test-storybook --watch +>TARGET_URL=http://localhost:9009 yarn test-storybook >``` ## Configuration -The test runner works out of the box, but you can override its configuration by adding a `test-runner-jest.config.js` that sets up your environment in the root folder of your project. +The test runner is based on [Jest](https://jestjs.io/) and will accept the [CLI options](https://jestjs.io/docs/cli) that Jest does, like `--watch`, `--marWorkers`, etc. + +The test runner works out of the box, but you can override its Jest configuration by adding a `test-runner-jest.config.js` that sets up your environment in the root folder of your project. ```js // test-runner-jest.config.js From 2106f55a9d5f0116a680a58726232e8c13d69b3e Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Thu, 6 Jan 2022 15:22:33 +0100 Subject: [PATCH 3/4] fix: remove STORYBOOK_PORT check --- bin/test-storybook.js | 3 +-- playwright/custom-environment.js | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/test-storybook.js b/bin/test-storybook.js index 4e414ed8..06bc18ac 100644 --- a/bin/test-storybook.js +++ b/bin/test-storybook.js @@ -34,8 +34,7 @@ function sanitizeURL(url) { return finalURL; } -const port = process.env.STORYBOOK_PORT || '6006'; -const targetURL = sanitizeURL(process.env.TARGET_URL || `http://localhost:${port}`); +const targetURL = sanitizeURL(process.env.TARGET_URL || `http://localhost:6006`); urlExists(targetURL, function (err, exists) { if (!exists) { diff --git a/playwright/custom-environment.js b/playwright/custom-environment.js index b0e3d062..40b08bf3 100644 --- a/playwright/custom-environment.js +++ b/playwright/custom-environment.js @@ -24,8 +24,7 @@ class CustomEnvironment extends PlaywrightEnvironment { await super.setup(); const page = this.global.page; const start = new Date(); - const port = process.env.STORYBOOK_PORT || '6006'; - const targetURL = sanitizeURL(process.env.TARGET_URL || `http://localhost:${port}`); + const targetURL = sanitizeURL(process.env.TARGET_URL || `http://localhost:6006`); const referenceURL = process.env.REFERENCE_URL && sanitizeURL(process.env.REFERENCE_URL); From b258153e3be3cb9f59ccfb852368326a184b9cd8 Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Thu, 6 Jan 2022 15:27:34 +0100 Subject: [PATCH 4/4] test: update snapshots --- src/jsdom/transformJsdom.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jsdom/transformJsdom.test.ts b/src/jsdom/transformJsdom.test.ts index 701d3593..49dac20f 100644 --- a/src/jsdom/transformJsdom.test.ts +++ b/src/jsdom/transformJsdom.test.ts @@ -23,7 +23,7 @@ describe('transformJsdom', () => { if (!require.main) { describe("foo/bar", () => { describe("A", () => { - it("smoke", async () => { + it("smoke-test", async () => { const Composed = await composeStory(A, exports.default); const { container