diff --git a/addon-test-support/index.ts b/addon-test-support/index.ts index 83232ab5..3c7b6c70 100644 --- a/addon-test-support/index.ts +++ b/addon-test-support/index.ts @@ -7,5 +7,10 @@ export { teardownGlobalA11yHooks, } from './setup-global-a11y-hooks'; export { setCustomReporter } from './reporter'; +export { + TEST_SUITE_RESULTS as _TEST_SUITE_RESULTS, + middlewareReporter as _middlewareReporter, + setupMiddlewareReporter, +} from './setup-middleware-reporter'; export { InvocationStrategy, A11yAuditReporter } from './types'; diff --git a/addon-test-support/setup-middleware-reporter.ts b/addon-test-support/setup-middleware-reporter.ts index d382517d..88f9d8d8 100644 --- a/addon-test-support/setup-middleware-reporter.ts +++ b/addon-test-support/setup-middleware-reporter.ts @@ -1,23 +1,38 @@ import QUnit from 'qunit'; +import { getContext, getTestMetadata } from '@ember/test-helpers'; import { AxeResults } from 'axe-core'; import { setCustomReporter } from './reporter'; +import { DEBUG } from '@glimmer/env'; -const TEST_SUITE_RESULTS: { +export const TEST_SUITE_RESULTS: { moduleName: string; testName: string; + helperName: string; + stack: string; axeResults: AxeResults; }[] = []; -export function setupMiddlewareReporter() { - setCustomReporter(async (axeResults: AxeResults) => { - let { module, testName } = QUnit.config.current; +export function buildResult(axeResults: AxeResults) { + let { module, testName } = QUnit.config.current; + let testMetaData = getTestMetadata(getContext()); - TEST_SUITE_RESULTS.push({ - moduleName: module.name, - testName, - axeResults, - }); - }); + let stack = (!DEBUG && new Error().stack) || ''; + + return { + moduleName: module.name, + testName, + helperName: testMetaData.usedHelpers.pop() || '', + stack, + axeResults, + }; +} + +export async function middlewareReporter(axeResults: AxeResults) { + TEST_SUITE_RESULTS.push(buildResult(axeResults)); +} + +export function setupMiddlewareReporter() { + setCustomReporter(middlewareReporter); QUnit.done(async function () { let response = await fetch('/report-violations', { diff --git a/node-tests/setup-middleware-test.js b/node-tests/setup-middleware-test.js index 6ee76b5b..5de43384 100644 --- a/node-tests/setup-middleware-test.js +++ b/node-tests/setup-middleware-test.js @@ -11,6 +11,18 @@ function createTmpDir() { return fs.realpathSync(tmp.dirSync({ unsafeCleanup: true }).name); } +function buildResult(axeResults) { + let { module, testName } = QUnit.config.current; + + return { + moduleName: module.name, + testName, + helperName: 'visit', + stack: 'STACK', + axeResults, + }; +} + QUnit.module('setupMiddleware', function (hooks) { let tmpDir; let app; @@ -32,14 +44,16 @@ QUnit.module('setupMiddleware', function (hooks) { QUnit.test('can respond to requests to report violations', async function ( assert ) { + let data = buildResult(violationsFixture); + let json = await fetch('http://localhost:3000/report-violations', { method: 'POST', headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify(violationsFixture), + body: JSON.stringify(data), }).then((res) => res.json()); - assert.deepEqual(readJSONSync(json.outputPath), violationsFixture); + assert.deepEqual(readJSONSync(json.outputPath), data); }); }); diff --git a/tests/acceptance/setup-middleware-reporter-test.ts b/tests/acceptance/setup-middleware-reporter-test.ts new file mode 100644 index 00000000..151037ea --- /dev/null +++ b/tests/acceptance/setup-middleware-reporter-test.ts @@ -0,0 +1,39 @@ +import { module, test } from 'qunit'; +import { setupApplicationTest } from 'ember-qunit'; +import { visit } from '@ember/test-helpers'; +import { + setCustomReporter, + setEnableA11yAudit, + setupGlobalA11yHooks, + teardownGlobalA11yHooks, + _middlewareReporter, + _TEST_SUITE_RESULTS, +} from 'ember-a11y-testing/test-support'; + +module('setupMiddlewareReporter', function (hooks) { + setupApplicationTest(hooks); + + function invokeAll(): boolean { + return true; + } + + hooks.beforeEach(function () { + setCustomReporter(_middlewareReporter); + setupGlobalA11yHooks(invokeAll); + setEnableA11yAudit(true); + }); + + hooks.afterEach(function () { + setCustomReporter(); + teardownGlobalA11yHooks(); + setEnableA11yAudit(); + }); + + test('gathers results from failed a11yAudit calls', async function (assert) { + assert.expect(1); + + await visit('/'); + + assert.deepEqual(_TEST_SUITE_RESULTS[0].axeResults.violations.length, 3); + }); +}); diff --git a/tests/fixtures/violations.json b/tests/fixtures/violations.json new file mode 100644 index 00000000..c0e689c9 --- /dev/null +++ b/tests/fixtures/violations.json @@ -0,0 +1,3333 @@ +[ + { + "moduleName": "setupMiddlewareReporter", + "testName": "gathers results from failed a11yAudit calls", + "helperName": "visit", + "stack": "", + "axeResults": { + "testEngine": { + "name": "axe-core", + "version": "4.0.1" + }, + "testRunner": { + "name": "axe" + }, + "testEnvironment": { + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36", + "windowWidth": 2030, + "windowHeight": 933, + "orientationAngle": 0, + "orientationType": "landscape-primary" + }, + "timestamp": "2020-09-04T23:32:13.625Z", + "url": "http://localhost:4200/tests/index.html?testId=7541800d&enableA11yAudit=", + "toolOptions": { + "reporter": "v1" + }, + "violations": [ + { + "id": "button-name", + "impact": "critical", + "tags": [ + "cat.name-role-value", + "wcag2a", + "wcag412", + "section508", + "section508.22.a" + ], + "description": "Ensures buttons have discernible text", + "help": "Buttons must have discernible text", + "helpUrl": "https://dequeuniversity.com/rules/axe/4.0/button-name?application=axeAPI", + "nodes": [ + { + "any": [ + { + "id": "button-has-visible-text", + "data": null, + "relatedNodes": [], + "impact": "critical", + "message": "Element does not have inner text that is visible to screen readers" + }, + { + "id": "aria-label", + "data": null, + "relatedNodes": [], + "impact": "serious", + "message": "aria-label attribute does not exist or is empty" + }, + { + "id": "aria-labelledby", + "data": null, + "relatedNodes": [], + "impact": "serious", + "message": "aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty" + }, + { + "id": "role-presentation", + "data": null, + "relatedNodes": [], + "impact": "minor", + "message": "Element's default semantics were not overridden with role=\"presentation\"" + }, + { + "id": "role-none", + "data": null, + "relatedNodes": [], + "impact": "minor", + "message": "Element's default semantics were not overridden with role=\"none\"" + }, + { + "id": "non-empty-title", + "data": null, + "relatedNodes": [], + "impact": "serious", + "message": "Element has no title attribute or the title attribute is empty" + } + ], + "all": [], + "none": [], + "impact": "critical", + "html": "", + "target": [ + "#violations__empty-button" + ], + "failureSummary": "Fix any of the following:\n Element does not have inner text that is visible to screen readers\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n Element's default semantics were not overridden with role=\"presentation\"\n Element's default semantics were not overridden with role=\"none\"\n Element has no title attribute or the title attribute is empty" + } + ] + }, + { + "id": "image-alt", + "impact": "critical", + "tags": [ + "cat.text-alternatives", + "wcag2a", + "wcag111", + "section508", + "section508.22.a" + ], + "description": "Ensures elements have alternate text or a role of none or presentation", + "help": "Images must have alternate text", + "helpUrl": "https://dequeuniversity.com/rules/axe/4.0/image-alt?application=axeAPI", + "nodes": [ + { + "any": [ + { + "id": "has-alt", + "data": null, + "relatedNodes": [], + "impact": "critical", + "message": "Element does not have an alt attribute" + }, + { + "id": "aria-label", + "data": null, + "relatedNodes": [], + "impact": "serious", + "message": "aria-label attribute does not exist or is empty" + }, + { + "id": "aria-labelledby", + "data": null, + "relatedNodes": [], + "impact": "serious", + "message": "aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty" + }, + { + "id": "non-empty-title", + "data": null, + "relatedNodes": [], + "impact": "serious", + "message": "Element has no title attribute or the title attribute is empty" + }, + { + "id": "role-presentation", + "data": null, + "relatedNodes": [], + "impact": "minor", + "message": "Element's default semantics were not overridden with role=\"presentation\"" + }, + { + "id": "role-none", + "data": null, + "relatedNodes": [], + "impact": "minor", + "message": "Element's default semantics were not overridden with role=\"none\"" + } + ], + "all": [], + "none": [], + "impact": "critical", + "html": "", + "target": [ + "#violations__img-without-alt" + ], + "failureSummary": "Fix any of the following:\n Element does not have an alt attribute\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n Element has no title attribute or the title attribute is empty\n Element's default semantics were not overridden with role=\"presentation\"\n Element's default semantics were not overridden with role=\"none\"" + } + ] + }, + { + "id": "label", + "impact": "critical", + "tags": [ + "cat.forms", + "wcag2a", + "wcag412", + "wcag131", + "section508", + "section508.22.n" + ], + "description": "Ensures every form element has a label", + "help": "Form elements must have labels", + "helpUrl": "https://dequeuniversity.com/rules/axe/4.0/label?application=axeAPI", + "nodes": [ + { + "any": [ + { + "id": "aria-label", + "data": null, + "relatedNodes": [], + "impact": "serious", + "message": "aria-label attribute does not exist or is empty" + }, + { + "id": "aria-labelledby", + "data": null, + "relatedNodes": [], + "impact": "serious", + "message": "aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty" + }, + { + "id": "implicit-label", + "data": null, + "relatedNodes": [], + "impact": "critical", + "message": "Form element does not have an implicit (wrapped)