Automated axe testing

package.json

@@ -10,7 +10,7 @@
   "sideEffects": false,
   "scripts": {
     "start": "cross-env BABEL_MODULES=false webpack-dev-server --port 8030 --inline --hot --config=src-docs/webpack.config.js",
-    "test-docker": "docker pull $npm_package_docker_image && docker run --rm -i -e GIT_COMMITTER_NAME=test -e GIT_COMMITTER_EMAIL=test --user=$(id -u):$(id -g) -e HOME=/tmp -v $(pwd):/app -w /app $npm_package_docker_image bash -c 'npm config set spin false && /opt/yarn*/bin/yarn && npm run test && npm run build'",
+    "test-docker": "node ./scripts/test-docker.js",
     "sync-docs": "node ./scripts/docs-sync.js",
     "build-docs": "cross-env BABEL_MODULES=false cross-env NODE_ENV=production NODE_OPTIONS=--max-old-space-size=4096 webpack --config=src-docs/webpack.config.js",
     "build": "yarn extract-i18n-strings && node ./scripts/compile-clean.js && node ./scripts/compile-eui.js && node ./scripts/compile-scss.js $npm_package_name",
@@ -24,13 +24,15 @@
     "lint-sass-fix": "sass-lint-auto-fix -c ./.sass-lint-fix.yml",
     "test": "yarn lint && yarn test-unit",
     "test-unit": "cross-env NODE_ENV=test jest --config ./scripts/jest/config.json",
+    "test-a11y": "node ./scripts/a11y-testing",
     "test-staged": "yarn lint && node scripts/test-staged.js",
-    "start-test-server": "webpack-dev-server --config src-docs/webpack.config.js --port 9999",
+    "start-test-server": "BABEL_MODULES=false NODE_ENV=puppeteer webpack-dev-server --config src-docs/webpack.config.js --port 9999",
     "test-visual": "wdio test/wdio.conf.js",
     "yo-component": "yo ./generator-eui/app/component.js",
     "test-visual-tests": "node ./scripts/run-visual-tests.js",
     "update-token-changelog": "node ./scripts/update-token-changelog.js",
     "start-test-server-and-visual-test": "start-server-and-test start-test-server http-get://localhost:9999 test-visual",
+    "start-test-server-and-a11y-test": "start-server-and-test start-test-server http-get://localhost:9999 test-a11y",
     "yo-doc": "yo ./generator-eui/app/documentation.js",
     "release": "node ./scripts/release.js",
     "postinstall": "node ./scripts/postinstall.js",
@@ -98,6 +100,8 @@
     "@typescript-eslint/eslint-plugin": "^1.13.0",
     "@typescript-eslint/parser": "^1.13.0",
     "autoprefixer": "^7.1.5",
+    "axe-core": "^3.3.2",
+    "axe-puppeteer": "^1.0.0",
     "babel-core": "7.0.0-bridge.0",
     "babel-eslint": "^10.0.1",
     "babel-jest": "^24.1.0",
@@ -113,7 +117,7 @@
     "chai-webdriverio": "^0.4.3",
     "chalk": "^2.4.1",
     "chokidar": "^1.7.0",
-    "chromedriver": "2.37.0",
+    "chromedriver": "^77.0.0",
     "circular-dependency-plugin": "^5.0.2",
     "core-js": "^2.5.1",
     "cross-env": "^5.2.0",
@@ -161,6 +165,7 @@
     "prettier": "^1.17.0",
     "prompt": "^1.0.0",
     "prop-types": "^15.6.0",
+    "puppeteer": "^2.0.0",
     "raw-loader": "^0.5.1",
     "react": "^16.12.0",
     "react-dom": "^16.12.0",

scripts/a11y-testing.js

@@ -0,0 +1,94 @@
+const chalk = require('chalk');
+const puppeteer = require('puppeteer');
+const { AxePuppeteer } = require('axe-puppeteer');
+
+const docsPages = async (root, page) => {
+  let links = [
+    root,
+    ...(await page.$$eval('nav a', anchors => anchors.map(a => a.href))),
+  ];
+
+  links = links.splice(0, 9);
+
+  return links;
+};
+
+const printResult = result =>
+  console.log(`[${result.id}]: ${result.description}
+  Help: ${chalk.blue(result.helpUrl)}
+  Elements:
+    - ${result.nodes.map(node => node.target).join('\n    - ')}`);
+
+(async () => {
+  let totalViolationsCount = 0;
+  let root = 'http://localhost:9999/';
+  let browser;
+  let page;
+
+  try {
+    browser = await puppeteer.launch({ args: ['--no-sandbox'] });
+    page = await browser.newPage();
+
+    await page.setBypassCSP(true);
+  } catch (e) {
+    console.log(chalk.red('Failed to setup puppeteer'));
+    console.log(e);
+    process.exit(1);
+  }
+
+  try {
+    await page.goto(root);
+  } catch (e) {
+    root = 'http://localhost:8030/';
+    try {
+      await page.goto(root);
+    } catch (e) {
+      console.log(
+        chalk.red(
+          'No local server found. Expecting localhost:9999 or localhost:8030 to resolve.'
+        )
+      );
+      process.exit(1);
+    }
+  }
+
+  const links = await docsPages(root, page);
+
+  for (const link of links) {
+    await page.goto(link);
+
+    const { violations } = await new AxePuppeteer(page)
+      .disableRules('color-contrast')
+      .exclude(['figure[role="figure"']) // excluding figure[role="figure"] the duplicatory role is there for ie11 support
+      .analyze();
+
+    if (violations.length > 0) {
+      totalViolationsCount += violations.length;
+
+      const pageName = link.length > 24 ? link.substr(2) : 'the home page';
+      console.log(chalk.red(`Errors on ${pageName}`));
+    }
+
+    violations.forEach(result => {
+      printResult(result);
+    });
+  }
+
+  await page.close();
+  await browser.close();
+
+  if (totalViolationsCount > 0) {
+    const errorsCount = chalk.red(
+      `${totalViolationsCount} accessibility errors`
+    );
+
+    console.log(`${errorsCount}
+
+Install axe for Chrome or Firefox to debug:
+Chrome: https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd
+Firefox: https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/`);
+    process.exit(1);
+  } else {
+    console.log(chalk.green('axe found no accessibility errors!'));
+  }
+})();

scripts/test-docker.js

@@ -0,0 +1,18 @@
+const { execSync } = require('child_process');
+
+execSync('docker pull zenato/puppeteer', {
+  stdio: 'inherit',
+});
+/* eslint-disable-next-line no-multi-str */
+execSync("docker run \
+  -i --rm --cap-add=SYS_ADMIN --volume=$(pwd):/app --workdir=/app \
+  -e GIT_COMMITTER_NAME=test -e GIT_COMMITTER_EMAIL=test -e HOME=/tmp \
+  --user=$(id -u):$(id -g) \
+  zenato/puppeteer \
+  bash -c 'npm config set spin false \
+    && /opt/yarn*/bin/yarn \
+    && npm run test \
+    && npm run start-test-server-and-a11y-test \
+    && npm run build'", {
+  stdio: 'inherit',
+});

src-docs/src/views/guidelines/button.js

@@ -49,7 +49,7 @@ export default () => (
 
       <EuiFlexItem>
         <EuiText className="guideSection__text">
-          <h4>Filled buttons are for the primary action</h4>
+          <h3>Filled buttons are for the primary action</h3>
           <p>
             This button has the heaviest visual weight to draw users&apos;
             attention.
@@ -67,7 +67,7 @@ export default () => (
 
       <EuiFlexItem>
         <EuiText className="guideSection__text">
-          <h4>Standard buttons are for secondary actions</h4>
+          <h3>Standard buttons are for secondary actions</h3>
           <p>
             Such actions include Add and Apply. This button type works well for
             multiple actions of equal weight.
@@ -85,7 +85,7 @@ export default () => (
 
       <EuiFlexItem>
         <EuiText className="guideSection__text">
-          <h4>Empty buttons are for complementary, UI-specific actions</h4>
+          <h3>Empty buttons are for complementary, UI-specific actions</h3>
           <p>
             Close, cancel, filter, refresh, and other actions that reconfigure
             the UI are appropriate for empty buttons.
@@ -111,7 +111,7 @@ export default () => (
 
       <EuiFlexItem>
         <EuiText className="guideSection__text">
-          <h4>Icon buttons are for saving space</h4>
+          <h3>Icon buttons are for saving space</h3>
           <p>
             The icon must be immediately understood, for example, a trash can
             for delete. Use these buttons sparingly, and never for the primary

src-docs/src/views/guidelines/modals.js

@@ -70,17 +70,17 @@ export default () => (
 
       <EuiFlexItem>
         <EuiText className="guideSection__text">
-          <h4>The header sets the context</h4>
+          <h3>The header sets the context</h3>
           <p>
             Short and sentence-case, the header should indicate what the modal
             is about.
           </p>
-          <h4>The body is for a single task</h4>
+          <h3>The body is for a single task</h3>
           <p>
             This task should not require a lot of explanation or user
             interaction.
           </p>
-          <h4>Buttons are right-aligned</h4>
+          <h3>Buttons are right-aligned</h3>
           <p>
             The primary action is a filled button, and the secondary action is a
             link button. Labels should use strong action verbs.

src-docs/src/views/guidelines/sass.js

@@ -388,7 +388,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
         <EuiFlexItem>
           <div>
             <EuiTitle size="s">
-              <h4>Sizing</h4>
+              <h3>Sizing</h3>
             </EuiTitle>
 
             <EuiSpacer />
@@ -400,7 +400,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
             <EuiSpacer />
 
             <EuiTitle size="s">
-              <h4>Z-index</h4>
+              <h3>Z-index</h3>
             </EuiTitle>
 
             <EuiSpacer />
@@ -412,7 +412,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
         </EuiFlexItem>
         <EuiFlexItem>
           <EuiTitle size="s">
-            <h4>Color</h4>
+            <h3>Color</h3>
           </EuiTitle>
 
           <EuiSpacer />
@@ -432,7 +432,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
       <EuiFlexGrid columns={2}>
         <EuiFlexItem>
           <EuiTitle size="s">
-            <h4>Theming patterns</h4>
+            <h3>Theming patterns</h3>
           </EuiTitle>
 
           <EuiSpacer />
@@ -554,7 +554,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
 
         <EuiFlexItem>
           <EuiTitle size="s">
-            <h4>Color contrast patterns</h4>
+            <h3>Color contrast patterns</h3>
           </EuiTitle>
 
           <EuiSpacer />
@@ -601,7 +601,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
           <EuiSpacer />
 
           <EuiTitle size="s">
-            <h4>More on color contrast</h4>
+            <h3>More on color contrast</h3>
           </EuiTitle>
 
           <EuiSpacer />
@@ -654,7 +654,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
       <EuiFlexGrid columns={2}>
         <EuiFlexItem>
           <EuiTitle size="s">
-            <h4>Text sizes</h4>
+            <h3>Text sizes</h3>
           </EuiTitle>
 
           <EuiSpacer />
@@ -665,7 +665,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
         <EuiFlexItem>
           <div>
             <EuiTitle size="s">
-              <h4>Text colors</h4>
+              <h3>Text colors</h3>
             </EuiTitle>
 
             <EuiSpacer />
@@ -677,7 +677,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
             <EuiSpacer />
 
             <EuiTitle>
-              <h4>Font families</h4>
+              <h3>Font families</h3>
             </EuiTitle>
 
             <EuiSpacer />
@@ -752,7 +752,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
       <EuiFlexGrid columns={2}>
         <EuiFlexItem>
           <EuiTitle size="s">
-            <h4>Use mixins for shadows</h4>
+            <h3>Use mixins for shadows</h3>
           </EuiTitle>
 
           <EuiText>
@@ -773,7 +773,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
           <EuiSpacer />
 
           <EuiTitle size="s">
-            <h4>Adding color to shadows</h4>
+            <h3>Adding color to shadows</h3>
           </EuiTitle>
 
           <EuiText>
@@ -793,7 +793,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
         </EuiFlexItem>
         <EuiFlexItem>
           <EuiTitle size="s">
-            <h4>Shadows to create graceful overflows</h4>
+            <h3>Shadows to create graceful overflows</h3>
           </EuiTitle>
 
           <EuiText>
@@ -806,9 +806,9 @@ export const SassGuidelines = ({ selectedTheme }) => {
           <EuiSpacer />
 
           <EuiTitle size="xs">
-            <h5>
+            <h4>
               Vertical scrolling with <EuiCode>euiYScrollWithShadows</EuiCode>
-            </h5>
+            </h4>
           </EuiTitle>
 
           <EuiSpacer size="s" />
@@ -859,9 +859,9 @@ export const SassGuidelines = ({ selectedTheme }) => {
           <EuiSpacer />
 
           <EuiTitle size="xs">
-            <h5>
+            <h4>
               Horizontal scrolling with <EuiCode>euiXScrollWithShadows</EuiCode>
-            </h5>
+            </h4>
           </EuiTitle>
 
           <EuiSpacer size="s" />
@@ -927,7 +927,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
         <EuiFlexItem>
           <div>
             <EuiTitle size="s">
-              <h4>Breakpoint sizing</h4>
+              <h3>Breakpoint sizing</h3>
             </EuiTitle>
 
             <EuiSpacer />
@@ -939,7 +939,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
         </EuiFlexItem>
         <EuiFlexItem>
           <EuiTitle size="s">
-            <h4>Mixin usage</h4>
+            <h3>Mixin usage</h3>
           </EuiTitle>
 
           <EuiSpacer />
@@ -1004,7 +1004,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
       <EuiFlexGrid columns={2}>
         <EuiFlexItem>
           <EuiTitle size="s">
-            <h4>Speed</h4>
+            <h3>Speed</h3>
           </EuiTitle>
 
           <EuiSpacer />
@@ -1015,7 +1015,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
         </EuiFlexItem>
         <EuiFlexItem>
           <EuiTitle size="s">
-            <h4>Timing</h4>
+            <h3>Timing</h3>
           </EuiTitle>
 
           <EuiSpacer />