Skip to content

Commit

Permalink
Merge pull request #29661 from storybookjs/valentin/add-a11y-discrepa…
Browse files Browse the repository at this point in the history
…ncy-handling

Add discrepancy handling to A11yPanel
valentinpalkovic authored Nov 20, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
2 parents d3923d2 + 79ad98c commit 6dfea64
Showing 8 changed files with 374 additions and 99 deletions.
45 changes: 25 additions & 20 deletions code/addons/a11y/src/components/A11YPanel.tsx
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@ import { CheckIcon, SyncIcon } from '@storybook/icons';
import { useA11yContext } from './A11yContext';
import { Report } from './Report';
import { Tabs } from './Tabs';
import { TestDiscrepancyMessage } from './TestDiscrepancyMessage';

export enum RuleType {
VIOLATION,
@@ -43,7 +44,7 @@ const Centered = styled.span({
});

export const A11YPanel: React.FC = () => {
const { results, status, handleManual, error } = useA11yContext();
const { results, status, handleManual, error, discrepancy } = useA11yContext();

const manualActionItems = useMemo(
() => [{ title: 'Run test', onClick: handleManual }],
@@ -106,31 +107,35 @@ export const A11YPanel: React.FC = () => {

return (
<>
{status === 'initial' && <Centered>Initializing...</Centered>}
{status === 'manual' && (
<>
<Centered>Manually run the accessibility scan.</Centered>
<ActionBar key="actionbar" actionItems={manualActionItems} />
</>
)}
{status === 'running' && (
<Centered>
<RotatingIcon size={12} /> Please wait while the accessibility scan is running ...
</Centered>
)}
{(status === 'ready' || status === 'ran') && (
{discrepancy && <TestDiscrepancyMessage discrepancy={discrepancy} />}
{status === 'ready' || status === 'ran' ? (
<>
<ScrollArea vertical horizontal>
<Tabs key="tabs" tabs={tabs} />
</ScrollArea>
<ActionBar key="actionbar" actionItems={readyActionItems} />
</>
)}
{status === 'error' && (
<Centered>
The accessibility scan encountered an error.
<br />
{typeof error === 'string' ? error : JSON.stringify(error)}
) : (
<Centered style={{ marginTop: discrepancy ? '1em' : 0 }}>
{status === 'initial' && 'Initializing...'}
{status === 'manual' && (
<>
<>Manually run the accessibility scan.</>
<ActionBar key="actionbar" actionItems={manualActionItems} />
</>
)}
{status === 'running' && (
<>
<RotatingIcon size={12} /> Please wait while the accessibility scan is running ...
</>
)}
{status === 'error' && (
<>
The accessibility scan encountered an error.
<br />
{typeof error === 'string' ? error : JSON.stringify(error)}
</>
)}
</Centered>
)}
</>
77 changes: 68 additions & 9 deletions code/addons/a11y/src/components/A11yContext.test.tsx
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@ import * as api from 'storybook/internal/manager-api';

import type { AxeResults } from 'axe-core';

import { EVENTS } from '../constants';
import { EVENTS, TEST_PROVIDER_ID } from '../constants';
import { A11yContextProvider, useA11yContext } from './A11yContext';

vi.mock('storybook/internal/manager-api');
@@ -65,16 +65,22 @@ describe('A11yContext', () => {

const getCurrentStoryData = vi.fn();
const getParameters = vi.fn();

mockedApi.useAddonState.mockImplementation((_, defaultState) => React.useState(defaultState));
mockedApi.useChannel.mockReturnValue(vi.fn());
getCurrentStoryData.mockReturnValue({ id: storyId, type: 'story' });
getParameters.mockReturnValue({});
mockedApi.useStorybookApi.mockReturnValue({ getCurrentStoryData, getParameters } as any);
mockedApi.useParameter.mockReturnValue({ manual: false });
mockedApi.useStorybookState.mockReturnValue({ storyId } as any);
const getCurrentStoryStatus = vi.fn();

beforeEach(() => {
mockedApi.useAddonState.mockImplementation((_, defaultState) => React.useState(defaultState));
mockedApi.useChannel.mockReturnValue(vi.fn());
getCurrentStoryData.mockReturnValue({ id: storyId, type: 'story' });
getParameters.mockReturnValue({});
getCurrentStoryStatus.mockReturnValue({ [TEST_PROVIDER_ID]: { status: 'success' } });
mockedApi.useStorybookApi.mockReturnValue({
getCurrentStoryData,
getParameters,
getCurrentStoryStatus,
} as any);
mockedApi.useParameter.mockReturnValue({ manual: false });
mockedApi.useStorybookState.mockReturnValue({ storyId } as any);

mockedApi.useChannel.mockClear();
mockedApi.useStorybookApi.mockClear();
mockedApi.useAddonState.mockClear();
@@ -146,6 +152,59 @@ describe('A11yContext', () => {
);
});

it('should set discrepancy to cliFailedButModeManual when in manual mode', () => {
mockedApi.useParameter.mockReturnValue({ manual: true });
getCurrentStoryStatus.mockReturnValue({ [TEST_PROVIDER_ID]: { status: 'error' } });

const Component = () => {
const { discrepancy } = useA11yContext();
return <div data-testid="discrepancy">{discrepancy}</div>;
};

const { getByTestId } = render(
<A11yContextProvider>
<Component />
</A11yContextProvider>
);

expect(getByTestId('discrepancy').textContent).toBe('cliFailedButModeManual');
});

it('should set discrepancy to cliPassedBrowserFailed', () => {
mockedApi.useParameter.mockReturnValue({ manual: true });
getCurrentStoryStatus.mockReturnValue({ [TEST_PROVIDER_ID]: { status: 'success' } });

const Component = () => {
const { discrepancy } = useA11yContext();
return <div data-testid="discrepancy">{discrepancy}</div>;
};

const { getByTestId } = render(
<A11yContextProvider>
<Component />
</A11yContextProvider>
);

const storyFinishedPayload: StoryFinishedPayload = {
storyId,
status: 'error',
reporters: [
{
id: 'a11y',
result: axeResult as any,
status: 'failed',
version: 1,
},
],
};

const useChannelArgs = mockedApi.useChannel.mock.calls[0][0];

act(() => useChannelArgs[STORY_FINISHED](storyFinishedPayload));

expect(getByTestId('discrepancy').textContent).toBe('cliPassedBrowserFailed');
});

it('should handle STORY_RENDER_PHASE_CHANGED event correctly', () => {
const emit = vi.fn();
mockedApi.useChannel.mockReturnValue(emit);
34 changes: 32 additions & 2 deletions code/addons/a11y/src/components/A11yContext.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { FC, PropsWithChildren } from 'react';
import React, { createContext, useCallback, useContext, useEffect, useState } from 'react';
import React, { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react';

import {
STORY_FINISHED,
@@ -10,6 +10,7 @@ import {
useAddonState,
useChannel,
useParameter,
useStorybookApi,
useStorybookState,
} from 'storybook/internal/manager-api';
import type { Report } from 'storybook/internal/preview-api';
@@ -19,9 +20,10 @@ import { HIGHLIGHT } from '@storybook/addon-highlight';

import type { AxeResults, Result } from 'axe-core';

import { ADDON_ID, EVENTS } from '../constants';
import { ADDON_ID, EVENTS, TEST_PROVIDER_ID } from '../constants';
import type { A11yParameters } from '../params';
import type { A11YReport } from '../types';
import type { TestDiscrepancy } from './TestDiscrepancyMessage';

export interface Results {
passes: Result[];
@@ -40,6 +42,7 @@ export interface A11yContextStore {
setStatus: (status: Status) => void;
error: unknown;
handleManual: () => void;
discrepancy: TestDiscrepancy;
}

const colorsByType = [
@@ -63,6 +66,7 @@ export const A11yContext = createContext<A11yContextStore>({
status: 'initial',
error: undefined,
handleManual: () => {},
discrepancy: null,
});

const defaultResult = {
@@ -80,12 +84,15 @@ export const A11yContextProvider: FC<PropsWithChildren> = (props) => {

const getInitialStatus = useCallback((manual = false) => (manual ? 'manual' : 'initial'), []);

const api = useStorybookApi();
const [results, setResults] = useAddonState<Results>(ADDON_ID, defaultResult);
const [tab, setTab] = useState(0);
const [error, setError] = React.useState<unknown>(undefined);
const [status, setStatus] = useState<Status>(getInitialStatus(parameters.manual!));
const [highlighted, setHighlighted] = useState<string[]>([]);

const { storyId } = useStorybookState();
const storyStatus = api.getCurrentStoryStatus();

const handleToggleHighlight = useCallback((target: string[], highlight: boolean) => {
setHighlighted((prevHighlighted) =>
@@ -178,6 +185,28 @@ export const A11yContextProvider: FC<PropsWithChildren> = (props) => {
emit(HIGHLIGHT, { elements: highlighted, color: colorsByType[tab] });
}, [emit, highlighted, tab]);

const discrepancy: TestDiscrepancy = useMemo(() => {
const storyStatusA11y = storyStatus[TEST_PROVIDER_ID]?.status;

if (storyStatusA11y) {
if (storyStatusA11y === 'success' && results.violations.length > 0) {
return 'cliPassedBrowserFailed';
}

if (storyStatusA11y === 'error' && results.violations.length === 0) {
if (status === 'ready' || status === 'ran') {
return 'browserPassedCliFailed';
}

if (status === 'manual') {
return 'cliFailedButModeManual';
}
}
}

return null;
}, [results.violations.length, status, storyStatus]);

return (
<A11yContext.Provider
value={{
@@ -191,6 +220,7 @@ export const A11yContextProvider: FC<PropsWithChildren> = (props) => {
setStatus,
error,
handleManual,
discrepancy,
}}
{...props}
/>
73 changes: 73 additions & 0 deletions code/addons/a11y/src/components/TestDiscrepancyMessage.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import React, { useMemo } from 'react';

import { Link } from 'storybook/internal/components';
import { useStorybookApi } from 'storybook/internal/manager-api';
import { styled } from 'storybook/internal/theming';

import { DOCUMENTATION_DISCREPANCY_LINK } from '../constants';

const Wrapper = styled.div(({ theme: { color, typography, background } }) => ({
textAlign: 'start',
padding: '11px 15px',
fontSize: `${typography.size.s2}px`,
fontWeight: typography.weight.regular,
lineHeight: '1rem',
background: background.app,
borderBottom: `1px solid ${color.border}`,
color: color.defaultText,
backgroundClip: 'padding-box',
position: 'relative',
code: {
fontSize: `${typography.size.s1 - 1}px`,
color: 'inherit',
margin: '0 0.2em',
padding: '0 0.2em',
background: 'rgba(255, 255, 255, 0.8)',
borderRadius: '2px',
boxShadow: '0 0 0 1px rgba(0, 0, 0, 0.1)',
},
}));

export type TestDiscrepancy =
| 'browserPassedCliFailed'
| 'cliPassedBrowserFailed'
| 'cliFailedButModeManual'
| null;

interface TestDiscrepancyMessageProps {
discrepancy: TestDiscrepancy;
}
export const TestDiscrepancyMessage = ({ discrepancy }: TestDiscrepancyMessageProps) => {
const api = useStorybookApi();
const docsUrl = api.getDocsUrl({
subpath: DOCUMENTATION_DISCREPANCY_LINK,
versioned: true,
renderer: true,
});

const message = useMemo(() => {
switch (discrepancy) {
case 'browserPassedCliFailed':
return 'Accessibility checks passed in this browser but failed in the CLI.';
case 'cliPassedBrowserFailed':
return 'Accessibility checks passed in the CLI but failed in this browser.';
case 'cliFailedButModeManual':
return 'Accessibility checks failed in the CLI. Run the tests manually to see the results.';
default:
return null;
}
}, [discrepancy]);

if (!message) {
return null;
}

return (
<Wrapper>
{message}{' '}
<Link href={docsUrl} target="_blank" withArrow>
Learn what could cause this
</Link>
</Wrapper>
);
};
5 changes: 5 additions & 0 deletions code/addons/a11y/src/constants.ts
Original file line number Diff line number Diff line change
@@ -7,4 +7,9 @@ const RUNNING = `${ADDON_ID}/running`;
const ERROR = `${ADDON_ID}/error`;
const MANUAL = `${ADDON_ID}/manual`;

export const DOCUMENTATION_LINK = 'writing-tests/accessibility-testing';
export const DOCUMENTATION_DISCREPANCY_LINK = `${DOCUMENTATION_LINK}#what-happens-when-there-are-different-results-in-multiple-environments`;

export const TEST_PROVIDER_ID = 'storybook/addon-a11y/test-provider';

export const EVENTS = { RESULT, REQUEST, RUNNING, ERROR, MANUAL };
176 changes: 108 additions & 68 deletions code/addons/a11y/template/stories/A11YPanel.stories.tsx
Original file line number Diff line number Diff line change
@@ -1,22 +1,34 @@
import React from 'react';

import { ManagerContext } from 'storybook/internal/manager-api';
import { ThemeProvider, convert, themes } from 'storybook/internal/theming';

import type { Meta, StoryObj } from '@storybook/react';
import { fn } from '@storybook/test';

import type axe from 'axe-core';

import { A11YPanel } from '../../src/components/A11YPanel';
import { A11yContext } from '../../src/components/A11yContext';
import type { A11yContextStore } from '../../src/components/A11yContext';

const managerContext: any = {
state: {},
api: {
getDocsUrl: fn().mockName('api::getDocsUrl'),
},
};

const meta: Meta = {
title: 'A11YPanel',
component: A11YPanel,
decorators: [
(Story) => (
<ThemeProvider theme={convert(themes.light)}>
<Story />
</ThemeProvider>
<ManagerContext.Provider value={managerContext}>
<ThemeProvider theme={convert(themes.light)}>
<Story />
</ThemeProvider>
</ManagerContext.Provider>
),
],
} satisfies Meta<typeof A11YPanel>;
@@ -25,7 +37,62 @@ export default meta;

type Story = StoryObj<typeof meta>;

const Template = (args: Pick<A11yContextStore, 'results' | 'error' | 'status'>) => (
const violations: axe.Result[] = [
{
id: 'aria-command-name',
impact: 'serious',
tags: ['cat.aria', 'wcag2a', 'wcag412', 'TTv5', 'TT6.a', 'EN-301-549', 'EN-9.4.1.2', 'ACT'],
description: 'Ensures every ARIA button, link and menuitem has an accessible name',
help: 'ARIA commands must have an accessible name',
helpUrl: 'https://dequeuniversity.com/rules/axe/4.8/aria-command-name?application=axeAPI',
nodes: [
{
any: [
{
id: 'has-visible-text',
data: null,
relatedNodes: [],
impact: 'serious',
message: 'Element does not have text that is visible to screen readers',
},
{
id: 'aria-label',
data: null,
relatedNodes: [],
impact: 'serious',
message: 'aria-label attribute does not exist or is empty',
},
{
id: 'aria-labelledby',
data: null,
relatedNodes: [],
impact: 'serious',
message:
'aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty',
},
{
id: 'non-empty-title',
data: {
messageKey: 'noAttr',
},
relatedNodes: [],
impact: 'serious',
message: 'Element has no title attribute',
},
],
all: [],
none: [],
impact: 'serious',
html: '<div role="button" class="css-12jpz5t">',
target: ['.css-12jpz5t'],
failureSummary:
'Fix any of the following:\n Element does not have text that is visible to screen readers\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n Element has no title attribute',
},
],
},
];

const Template = (args: Pick<A11yContextStore, 'results' | 'error' | 'status' | 'discrepancy'>) => (
<A11yContext.Provider
value={{
handleManual: fn(),
@@ -49,6 +116,7 @@ export const Initializing: Story = {
results={{ passes: [], incomplete: [], violations: [] }}
status="initial"
error={null}
discrepancy={null}
/>
);
},
@@ -61,6 +129,20 @@ export const Manual: Story = {
results={{ passes: [], incomplete: [], violations: [] }}
status="manual"
error={null}
discrepancy={null}
/>
);
},
};

export const ManualWithDiscrepancy: Story = {
render: () => {
return (
<Template
results={{ passes: [], incomplete: [], violations: [] }}
status="manual"
error={null}
discrepancy={'cliFailedButModeManual'}
/>
);
},
@@ -73,6 +155,7 @@ export const Running: Story = {
results={{ passes: [], incomplete: [], violations: [] }}
status="running"
error={null}
discrepancy={null}
/>
);
},
@@ -85,73 +168,28 @@ export const ReadyWithResults: Story = {
results={{
passes: [],
incomplete: [],
violations: [
{
id: 'aria-command-name',
impact: 'serious',
tags: [
'cat.aria',
'wcag2a',
'wcag412',
'TTv5',
'TT6.a',
'EN-301-549',
'EN-9.4.1.2',
'ACT',
],
description: 'Ensures every ARIA button, link and menuitem has an accessible name',
help: 'ARIA commands must have an accessible name',
helpUrl:
'https://dequeuniversity.com/rules/axe/4.8/aria-command-name?application=axeAPI',
nodes: [
{
any: [
{
id: 'has-visible-text',
data: null,
relatedNodes: [],
impact: 'serious',
message: 'Element does not have text that is visible to screen readers',
},
{
id: 'aria-label',
data: null,
relatedNodes: [],
impact: 'serious',
message: 'aria-label attribute does not exist or is empty',
},
{
id: 'aria-labelledby',
data: null,
relatedNodes: [],
impact: 'serious',
message:
'aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty',
},
{
id: 'non-empty-title',
data: {
messageKey: 'noAttr',
},
relatedNodes: [],
impact: 'serious',
message: 'Element has no title attribute',
},
],
all: [],
none: [],
impact: 'serious',
html: '<div role="button" class="css-12jpz5t">',
target: ['.css-12jpz5t'],
failureSummary:
'Fix any of the following:\n Element does not have text that is visible to screen readers\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n Element has no title attribute',
},
],
},
],
violations,
}}
status="ready"
error={null}
discrepancy={null}
/>
);
},
};

export const ReadyWithResultsDiscrepancyCLIPassedBrowserFailed: Story = {
render: () => {
return (
<Template
results={{
passes: [],
incomplete: [],
violations,
}}
status="ready"
error={null}
discrepancy={'cliPassedBrowserFailed'}
/>
);
},
@@ -164,6 +202,7 @@ export const Error: Story = {
results={{ passes: [], incomplete: [], violations: [] }}
status="error"
error="Test error message"
discrepancy={null}
/>
);
},
@@ -176,6 +215,7 @@ export const ErrorStateWithObject: Story = {
results={{ passes: [], incomplete: [], violations: [] }}
status="error"
error={{ message: 'Test error object message' }}
discrepancy={null}
/>
);
},
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import React from 'react';

import { ManagerContext } from 'storybook/internal/manager-api';

import type { Meta, StoryObj } from '@storybook/react';
import { fn } from '@storybook/test';

import { TestDiscrepancyMessage } from '../../src/components/TestDiscrepancyMessage';

type Story = StoryObj<typeof TestDiscrepancyMessage>;

const managerContext: any = {
state: {},
api: {
getDocsUrl: fn().mockName('api::getDocsUrl'),
},
};

export default {
title: 'TestDiscrepancyMessage',
component: TestDiscrepancyMessage,
parameters: {
layout: 'fullscreen',
},
args: {
storyId: 'story-id',
},
decorators: [
(storyFn) => (
<ManagerContext.Provider value={managerContext}>{storyFn()}</ManagerContext.Provider>
),
],
} as Meta<typeof TestDiscrepancyMessage>;

export const BrowserPassedCliFailed: Story = {
args: {
discrepancy: 'browserPassedCliFailed',
},
};

export const CliPassedBrowserFailed: Story = {
args: {
discrepancy: 'cliPassedBrowserFailed',
},
};

export const CliFailedButModeManual: Story = {
args: {
discrepancy: 'cliFailedButModeManual',
},
};
12 changes: 12 additions & 0 deletions code/core/src/manager-api/modules/stories.ts
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ import type {
API_LeafEntry,
API_LoadedRefData,
API_PreparedStoryIndex,
API_StatusObject,
API_StatusState,
API_StatusUpdate,
API_StoryEntry,
@@ -268,6 +269,12 @@ export interface SubAPI {
* @returns {Promise<void>} A promise that resolves when the preview has been set as initialized.
*/
setPreviewInitialized: (ref?: ComposedRef) => Promise<void>;
/**
* Returns the current status of the stories.
*
* @returns {API_StatusState} The current status of the stories.
*/
getCurrentStoryStatus: () => Record<string, API_StatusObject>;
/**
* Updates the status of a collection of stories.
*
@@ -630,6 +637,11 @@ export const init: ModuleFn<SubAPI, SubState> = ({
}
},

getCurrentStoryStatus: () => {
const { status, storyId } = store.getState();
return status[storyId as StoryId];
},

/* EXPERIMENTAL APIs */
experimental_updateStatus: async (id, input) => {
const { status, internal_index: index } = store.getState();

0 comments on commit 6dfea64

Please sign in to comment.