Skip to content

Commit

Permalink
Automated axe testing (#2569)
Browse files Browse the repository at this point in the history
* PoC of automated axe testing

* fixing a11y bugs guidelines pages

* Adds a11y tests to test command

* adding a11y script

* adding docs

* fixing logging for yarn

* catching errors in puppeteer setup to fail builds

* experimenting with docker

* retest

* one step forward

* deps

* trial run

* trial run

* Revert "trial run"

This reverts commit f0c8f9f.

* user

* fix typo

* bypass webpack loader cache

* DEBUG

* env changes

* sandbox trial

* move to script file

Co-authored-by: Greg Thompson <[email protected]>
  • Loading branch information
thompsongl authored Mar 5, 2020
1 parent 1343725 commit 68299f4
Show file tree
Hide file tree
Showing 13 changed files with 367 additions and 76 deletions.
11 changes: 8 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"sideEffects": false,
"scripts": {
"start": "cross-env BABEL_MODULES=false webpack-dev-server --port 8030 --inline --hot --config=src-docs/webpack.config.js",
"test-docker": "docker pull $npm_package_docker_image && docker run --rm -i -e GIT_COMMITTER_NAME=test -e GIT_COMMITTER_EMAIL=test --user=$(id -u):$(id -g) -e HOME=/tmp -v $(pwd):/app -w /app $npm_package_docker_image bash -c 'npm config set spin false && /opt/yarn*/bin/yarn && npm run test && npm run build'",
"test-docker": "node ./scripts/test-docker.js",
"sync-docs": "node ./scripts/docs-sync.js",
"build-docs": "cross-env BABEL_MODULES=false cross-env NODE_ENV=production NODE_OPTIONS=--max-old-space-size=4096 webpack --config=src-docs/webpack.config.js",
"build": "yarn extract-i18n-strings && node ./scripts/compile-clean.js && node ./scripts/compile-eui.js && node ./scripts/compile-scss.js $npm_package_name",
Expand All @@ -24,13 +24,15 @@
"lint-sass-fix": "sass-lint-auto-fix -c ./.sass-lint-fix.yml",
"test": "yarn lint && yarn test-unit",
"test-unit": "cross-env NODE_ENV=test jest --config ./scripts/jest/config.json",
"test-a11y": "node ./scripts/a11y-testing",
"test-staged": "yarn lint && node scripts/test-staged.js",
"start-test-server": "webpack-dev-server --config src-docs/webpack.config.js --port 9999",
"start-test-server": "BABEL_MODULES=false NODE_ENV=puppeteer webpack-dev-server --config src-docs/webpack.config.js --port 9999",
"test-visual": "wdio test/wdio.conf.js",
"yo-component": "yo ./generator-eui/app/component.js",
"test-visual-tests": "node ./scripts/run-visual-tests.js",
"update-token-changelog": "node ./scripts/update-token-changelog.js",
"start-test-server-and-visual-test": "start-server-and-test start-test-server http-get://localhost:9999 test-visual",
"start-test-server-and-a11y-test": "start-server-and-test start-test-server http-get://localhost:9999 test-a11y",
"yo-doc": "yo ./generator-eui/app/documentation.js",
"release": "node ./scripts/release.js",
"postinstall": "node ./scripts/postinstall.js",
Expand Down Expand Up @@ -98,6 +100,8 @@
"@typescript-eslint/eslint-plugin": "^1.13.0",
"@typescript-eslint/parser": "^1.13.0",
"autoprefixer": "^7.1.5",
"axe-core": "^3.3.2",
"axe-puppeteer": "^1.0.0",
"babel-core": "7.0.0-bridge.0",
"babel-eslint": "^10.0.1",
"babel-jest": "^24.1.0",
Expand All @@ -113,7 +117,7 @@
"chai-webdriverio": "^0.4.3",
"chalk": "^2.4.1",
"chokidar": "^1.7.0",
"chromedriver": "2.37.0",
"chromedriver": "^77.0.0",
"circular-dependency-plugin": "^5.0.2",
"core-js": "^2.5.1",
"cross-env": "^5.2.0",
Expand Down Expand Up @@ -161,6 +165,7 @@
"prettier": "^1.17.0",
"prompt": "^1.0.0",
"prop-types": "^15.6.0",
"puppeteer": "^2.0.0",
"raw-loader": "^0.5.1",
"react": "^16.12.0",
"react-dom": "^16.12.0",
Expand Down
94 changes: 94 additions & 0 deletions scripts/a11y-testing.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
const chalk = require('chalk');
const puppeteer = require('puppeteer');
const { AxePuppeteer } = require('axe-puppeteer');

const docsPages = async (root, page) => {
let links = [
root,
...(await page.$$eval('nav a', anchors => anchors.map(a => a.href))),
];

links = links.splice(0, 9);

return links;
};

const printResult = result =>
console.log(`[${result.id}]: ${result.description}
Help: ${chalk.blue(result.helpUrl)}
Elements:
- ${result.nodes.map(node => node.target).join('\n - ')}`);

(async () => {
let totalViolationsCount = 0;
let root = 'http://localhost:9999/';
let browser;
let page;

try {
browser = await puppeteer.launch({ args: ['--no-sandbox'] });
page = await browser.newPage();

await page.setBypassCSP(true);
} catch (e) {
console.log(chalk.red('Failed to setup puppeteer'));
console.log(e);
process.exit(1);
}

try {
await page.goto(root);
} catch (e) {
root = 'http://localhost:8030/';
try {
await page.goto(root);
} catch (e) {
console.log(
chalk.red(
'No local server found. Expecting localhost:9999 or localhost:8030 to resolve.'
)
);
process.exit(1);
}
}

const links = await docsPages(root, page);

for (const link of links) {
await page.goto(link);

const { violations } = await new AxePuppeteer(page)
.disableRules('color-contrast')
.exclude(['figure[role="figure"']) // excluding figure[role="figure"] the duplicatory role is there for ie11 support
.analyze();

if (violations.length > 0) {
totalViolationsCount += violations.length;

const pageName = link.length > 24 ? link.substr(2) : 'the home page';
console.log(chalk.red(`Errors on ${pageName}`));
}

violations.forEach(result => {
printResult(result);
});
}

await page.close();
await browser.close();

if (totalViolationsCount > 0) {
const errorsCount = chalk.red(
`${totalViolationsCount} accessibility errors`
);

console.log(`${errorsCount}
Install axe for Chrome or Firefox to debug:
Chrome: https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd
Firefox: https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/`);
process.exit(1);
} else {
console.log(chalk.green('axe found no accessibility errors!'));
}
})();
18 changes: 18 additions & 0 deletions scripts/test-docker.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
const { execSync } = require('child_process');

execSync('docker pull zenato/puppeteer', {
stdio: 'inherit',
});
/* eslint-disable-next-line no-multi-str */
execSync("docker run \
-i --rm --cap-add=SYS_ADMIN --volume=$(pwd):/app --workdir=/app \
-e GIT_COMMITTER_NAME=test -e GIT_COMMITTER_EMAIL=test -e HOME=/tmp \
--user=$(id -u):$(id -g) \
zenato/puppeteer \
bash -c 'npm config set spin false \
&& /opt/yarn*/bin/yarn \
&& npm run test \
&& npm run start-test-server-and-a11y-test \
&& npm run build'", {
stdio: 'inherit',
});
8 changes: 4 additions & 4 deletions src-docs/src/views/guidelines/button.js
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ export default () => (

<EuiFlexItem>
<EuiText className="guideSection__text">
<h4>Filled buttons are for the primary action</h4>
<h3>Filled buttons are for the primary action</h3>
<p>
This button has the heaviest visual weight to draw users&apos;
attention.
Expand All @@ -67,7 +67,7 @@ export default () => (

<EuiFlexItem>
<EuiText className="guideSection__text">
<h4>Standard buttons are for secondary actions</h4>
<h3>Standard buttons are for secondary actions</h3>
<p>
Such actions include Add and Apply. This button type works well for
multiple actions of equal weight.
Expand All @@ -85,7 +85,7 @@ export default () => (

<EuiFlexItem>
<EuiText className="guideSection__text">
<h4>Empty buttons are for complementary, UI-specific actions</h4>
<h3>Empty buttons are for complementary, UI-specific actions</h3>
<p>
Close, cancel, filter, refresh, and other actions that reconfigure
the UI are appropriate for empty buttons.
Expand All @@ -111,7 +111,7 @@ export default () => (

<EuiFlexItem>
<EuiText className="guideSection__text">
<h4>Icon buttons are for saving space</h4>
<h3>Icon buttons are for saving space</h3>
<p>
The icon must be immediately understood, for example, a trash can
for delete. Use these buttons sparingly, and never for the primary
Expand Down
6 changes: 3 additions & 3 deletions src-docs/src/views/guidelines/modals.js
Original file line number Diff line number Diff line change
Expand Up @@ -70,17 +70,17 @@ export default () => (

<EuiFlexItem>
<EuiText className="guideSection__text">
<h4>The header sets the context</h4>
<h3>The header sets the context</h3>
<p>
Short and sentence-case, the header should indicate what the modal
is about.
</p>
<h4>The body is for a single task</h4>
<h3>The body is for a single task</h3>
<p>
This task should not require a lot of explanation or user
interaction.
</p>
<h4>Buttons are right-aligned</h4>
<h3>Buttons are right-aligned</h3>
<p>
The primary action is a filled button, and the secondary action is a
link button. Labels should use strong action verbs.
Expand Down
40 changes: 20 additions & 20 deletions src-docs/src/views/guidelines/sass.js
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiFlexItem>
<div>
<EuiTitle size="s">
<h4>Sizing</h4>
<h3>Sizing</h3>
</EuiTitle>

<EuiSpacer />
Expand All @@ -400,7 +400,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiSpacer />

<EuiTitle size="s">
<h4>Z-index</h4>
<h3>Z-index</h3>
</EuiTitle>

<EuiSpacer />
Expand All @@ -412,7 +412,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
</EuiFlexItem>
<EuiFlexItem>
<EuiTitle size="s">
<h4>Color</h4>
<h3>Color</h3>
</EuiTitle>

<EuiSpacer />
Expand All @@ -432,7 +432,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiFlexGrid columns={2}>
<EuiFlexItem>
<EuiTitle size="s">
<h4>Theming patterns</h4>
<h3>Theming patterns</h3>
</EuiTitle>

<EuiSpacer />
Expand Down Expand Up @@ -554,7 +554,7 @@ export const SassGuidelines = ({ selectedTheme }) => {

<EuiFlexItem>
<EuiTitle size="s">
<h4>Color contrast patterns</h4>
<h3>Color contrast patterns</h3>
</EuiTitle>

<EuiSpacer />
Expand Down Expand Up @@ -601,7 +601,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiSpacer />

<EuiTitle size="s">
<h4>More on color contrast</h4>
<h3>More on color contrast</h3>
</EuiTitle>

<EuiSpacer />
Expand Down Expand Up @@ -654,7 +654,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiFlexGrid columns={2}>
<EuiFlexItem>
<EuiTitle size="s">
<h4>Text sizes</h4>
<h3>Text sizes</h3>
</EuiTitle>

<EuiSpacer />
Expand All @@ -665,7 +665,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiFlexItem>
<div>
<EuiTitle size="s">
<h4>Text colors</h4>
<h3>Text colors</h3>
</EuiTitle>

<EuiSpacer />
Expand All @@ -677,7 +677,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiSpacer />

<EuiTitle>
<h4>Font families</h4>
<h3>Font families</h3>
</EuiTitle>

<EuiSpacer />
Expand Down Expand Up @@ -752,7 +752,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiFlexGrid columns={2}>
<EuiFlexItem>
<EuiTitle size="s">
<h4>Use mixins for shadows</h4>
<h3>Use mixins for shadows</h3>
</EuiTitle>

<EuiText>
Expand All @@ -773,7 +773,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiSpacer />

<EuiTitle size="s">
<h4>Adding color to shadows</h4>
<h3>Adding color to shadows</h3>
</EuiTitle>

<EuiText>
Expand All @@ -793,7 +793,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
</EuiFlexItem>
<EuiFlexItem>
<EuiTitle size="s">
<h4>Shadows to create graceful overflows</h4>
<h3>Shadows to create graceful overflows</h3>
</EuiTitle>

<EuiText>
Expand All @@ -806,9 +806,9 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiSpacer />

<EuiTitle size="xs">
<h5>
<h4>
Vertical scrolling with <EuiCode>euiYScrollWithShadows</EuiCode>
</h5>
</h4>
</EuiTitle>

<EuiSpacer size="s" />
Expand Down Expand Up @@ -859,9 +859,9 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiSpacer />

<EuiTitle size="xs">
<h5>
<h4>
Horizontal scrolling with <EuiCode>euiXScrollWithShadows</EuiCode>
</h5>
</h4>
</EuiTitle>

<EuiSpacer size="s" />
Expand Down Expand Up @@ -927,7 +927,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiFlexItem>
<div>
<EuiTitle size="s">
<h4>Breakpoint sizing</h4>
<h3>Breakpoint sizing</h3>
</EuiTitle>

<EuiSpacer />
Expand All @@ -939,7 +939,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
</EuiFlexItem>
<EuiFlexItem>
<EuiTitle size="s">
<h4>Mixin usage</h4>
<h3>Mixin usage</h3>
</EuiTitle>

<EuiSpacer />
Expand Down Expand Up @@ -1004,7 +1004,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
<EuiFlexGrid columns={2}>
<EuiFlexItem>
<EuiTitle size="s">
<h4>Speed</h4>
<h3>Speed</h3>
</EuiTitle>

<EuiSpacer />
Expand All @@ -1015,7 +1015,7 @@ export const SassGuidelines = ({ selectedTheme }) => {
</EuiFlexItem>
<EuiFlexItem>
<EuiTitle size="s">
<h4>Timing</h4>
<h3>Timing</h3>
</EuiTitle>

<EuiSpacer />
Expand Down
Loading

0 comments on commit 68299f4

Please sign in to comment.