test: improve VRT developer experience (#653)

.ci/run.sh

@@ -131,4 +131,4 @@ echo " -- upload code coverage"
 ### visual testing
 ###
 echo " -- visual testing"
-yarn jest:integration --ci
+yarn test:integration --ci

integration/defaults.js

@@ -19,4 +19,5 @@
 module.exports = {
   PORT: '9009',
   HOST: 'host.docker.internal',
+  USE_LOCAL_STORYBOOK: false,
 };

integration/helpers.ts

@@ -20,16 +20,9 @@ import { join, resolve } from 'path';
 import { lstatSync, readdirSync } from 'fs';
 import { getStorybook, configure } from '@storybook/react';
 
-export interface StoryInfo {
-  title: string;
-  encodedTitle: string;
-}
+export type StoryInfo = [string, string];
 
-export interface StoryGroupInfo {
-  group: string;
-  encodedGroup: string;
-  stories: StoryInfo[];
-}
+export type StoryGroupInfo = [string, string, StoryInfo[]];
 
 function requireAllStories(basedir: string, directory: string) {
   function enumerateFiles(basedir: string, dir: string) {
@@ -74,6 +67,7 @@ function encodeString(string: string) {
 
 export function getStorybookInfo(): StoryGroupInfo[] {
   configure(requireAllStories(__dirname, '../stories'), module);
+
   return getStorybook()
     .filter(({ kind }) => kind)
     .map(({ kind: group, stories: storiesRaw }) => {
@@ -82,19 +76,11 @@ export function getStorybookInfo(): StoryGroupInfo[] {
         .map(({ name: title }) => {
           // cleans story name to match url params
           const encodedTitle = encodeString(title);
-
-          return {
-            title,
-            encodedTitle,
-          };
+          return [title, encodedTitle];
         });
 
       const encodedGroup = encodeString(group);
 
-      return {
-        group,
-        encodedGroup,
-        stories,
-      };
+      return [group, encodedGroup, stories];
     });
 }

integration/jest_puppeteer.config.js

@@ -22,6 +22,7 @@ const defaults = require('./defaults');
 
 const port = process.env.PORT || defaults.PORT;
 const host = process.env.HOST || defaults.HOST;
+const useLocalStorybook = process.env.USE_LOCAL_STORYBOOK || defaults.USE_LOCAL_STORYBOOK;
 
 /**
  * combined config object
@@ -37,13 +38,15 @@ const customConfig = Object.assign(
       browserUrl: `http://${host}:${port}/iframe.html`,
       args: ['--no-sandbox', '--disable-setuid-sandbox'],
     },
-    server: {
-      command: `RNG_SEED='elastic-charts' yarn start --port=${port} --quiet`,
-      port,
-      usedPortAction: 'error',
-      launchTimeout: 120000,
-      debug: false,
-    },
+    server: useLocalStorybook
+      ? null
+      : {
+          command: `RNG_SEED='elastic-charts' yarn start --port=${port} --quiet`,
+          port,
+          usedPortAction: 'error',
+          launchTimeout: 120000,
+          debug: false,
+        },
   },
   baseConfig,
 );

integration/tests/all.test.ts

@@ -27,15 +27,11 @@ jest.mock('../../.storybook/theme_service.ts', () => ({
 const storyGroups = getStorybookInfo();
 
 describe('Baseline Visual tests for all stories', () => {
-  storyGroups.forEach(({ group, encodedGroup, stories }) => {
-    describe(group, () => {
-      stories.forEach(({ title, encodedTitle }) => {
-        describe(title, () => {
-          it('visually looks correct', async () => {
-            const url = `http://localhost:9001?id=${encodedGroup}--${encodedTitle}`;
-            await common.expectChartAtUrlToMatchScreenshot(url);
-          });
-        });
+  describe.each(storyGroups)('%s', (_group, encodedGroup, stories) => {
+    describe.each(stories)('%s', (_title, encodedTitle) => {
+      it('visually looks correct', async () => {
+        const url = `http://localhost:9001?id=${encodedGroup}--${encodedTitle}`;
+        await common.expectChartAtUrlToMatchScreenshot(url);
       });
     });
   });

package.json

@@ -39,7 +39,8 @@
     "typecheck:all": "tsc -p ./tsconfig.json --noEmit",
     "playground": "cd .playground && webpack-dev-server",
     "playground:ie": "cd .playground && webpack-dev-server --host=0.0.0.0 --disable-host-check --useLocalIp",
-    "jest:integration": "rm -rf ./integration/tests/__image_snapshots__/__diff_output__ && TZ=UTC JEST_PUPPETEER_CONFIG=integration/jest_puppeteer.config.js jest --verbose --rootDir=integration -c=integration/jest.config.js --runInBand",
+    "test:integration": "rm -rf ./integration/tests/__image_snapshots__/__diff_output__ && TZ=UTC JEST_PUPPETEER_CONFIG=integration/jest_puppeteer.config.js jest --verbose --rootDir=integration -c=integration/jest.config.js --runInBand",
+    "test:integration:local": "LOCAL_STORYBOOK_VRT=true PORT=9001 yarn test:integration",
     "test:browsers": "cd .playground && jest -c=../browsers/jest.config.js ../browsers/browsers.test.ts",
     "ts:prune": "ts-prune",
     "backport": "backport"

wiki/testing.md

@@ -28,3 +28,52 @@ your_file_name.tz.test.jp.ts
 Each test with the specific timezone postfix will be executed only on that timezone.
 
 These tests are not included in the code coverage yet.
+
+
+## Storybook screenshot testing aka VRT
+
+Every story in storybook is tested through a Visual Regression Test suite.
+The tests are available in `integration/tests` folder. One main test `all.test.ts` goes through all stories, takes screenshots of each story
+and compare it to the existing baseline available at `integration/test/__image_snapshots__`.
+
+To run the suite
+```
+yarn test:integration
+```
+
+This will start a separate storybook server and a docker machine with chromium, controlled by pupetteer and jest, that takes and compare screeshots.
+
+If the screenshot differ from the baseline, a test error is rised and a diff image is stored in `integration/test/__image_snapshots__/__diff_output__`.
+
+If a new test is added, a new screenshot `.png` file is written as part of the baseline.
+
+To update all existing screenshot baselines to the new version run:
+```
+yarn test:integration -u
+```
+
+
+To run the VRT against your current storybook server (will reduce the running time of the test locally) run:
+```
+yarn test:integration:local
+```
+
+To run a specific test file run
+```
+yarn test:integration test_file_name
+# or
+yarn test:integration:local test_file_name
+```
+
+To run the test on a specific story name or story group name use `--testNamePattern=<regex>` or `-t`
+see [Jest](https://jestjs.io/docs/en/cli.html#--testnamepatternregex). This example will run the integration test
+on all.test.ts file for all matching test name in `describe` or `it` with `tree*` regex.
+
+```
+yarn test:integration all.test.ts --testNamePattern tree*
+
+# or
+
+yarn test:integration all.test.ts -t tree*
+
+```