From fbae626e2e3905b558a40dd1cb8d2285741b3eb2 Mon Sep 17 00:00:00 2001 From: Stalgia Grigg Date: Tue, 17 Sep 2024 08:42:53 -0700 Subject: [PATCH] feat: Enhanced conflict review (#1195) Addresses #975. Introduces a new dedicated "Conflicts" page that allows a user to compare the changes between all testers while going through the review process. --- .../TestPlanReportStatusDialog/index.jsx | 1 + .../Conflicts/AssertionConflictsTable.jsx | 62 ++ .../Conflicts/ConflictIssueDetails.jsx | 118 ++++ .../Conflicts/ConflictSummaryTable.jsx | 101 ++++ .../Conflicts/TestConflictsActions.jsx | 68 +++ .../UnexpectedBehaviorsConflictsTable.jsx | 76 +++ .../components/TestQueue/Conflicts/index.jsx | 324 +++++++++++ client/components/TestQueue/index.jsx | 1 + client/components/TestQueue/queries.js | 74 +++ .../common/ReportStatusSummary/index.jsx | 32 +- client/components/common/proptypes/index.js | 2 +- client/routes/index.js | 10 + client/tests/e2e/TestQueue.e2e.test.js | 4 +- .../e2e/snapshots/saved/_test-queue.html | 19 +- .../saved/_test-queue_2_conflicts.html | 545 ++++++++++++++++++ client/tests/e2e/snapshots/utils.js | 3 +- client/utils/generateConflictMarkdown.js | 89 +++ 17 files changed, 1513 insertions(+), 16 deletions(-) create mode 100644 client/components/TestQueue/Conflicts/AssertionConflictsTable.jsx create mode 100644 client/components/TestQueue/Conflicts/ConflictIssueDetails.jsx create mode 100644 client/components/TestQueue/Conflicts/ConflictSummaryTable.jsx create mode 100644 client/components/TestQueue/Conflicts/TestConflictsActions.jsx create mode 100644 client/components/TestQueue/Conflicts/UnexpectedBehaviorsConflictsTable.jsx create mode 100644 client/components/TestQueue/Conflicts/index.jsx create mode 100644 client/tests/e2e/snapshots/saved/_test-queue_2_conflicts.html create mode 100644 client/utils/generateConflictMarkdown.js diff --git a/client/components/TestPlanReportStatusDialog/index.jsx b/client/components/TestPlanReportStatusDialog/index.jsx index 7bd981b0b..21d96cca0 100644 --- a/client/components/TestPlanReportStatusDialog/index.jsx +++ b/client/components/TestPlanReportStatusDialog/index.jsx @@ -60,6 +60,7 @@ const TestPlanReportStatusDialog = ({ {isSignedIn && isAdmin && !testPlanReport ? ( { + const commandString = scenario => { + return `Assertions for 'After ${scenario.commands + .map(command => command.text) + .join(' then ')}'`; + }; + + const allAssertions = + conflictingResults[0].scenarioResult.assertionResults.map( + ar => ar.assertion.text + ); + + return ( + <> +

+ {commandString(conflictingResults[0].scenario)} +

+ + + + + Assertion + {testers.map(tester => ( + {tester.username} + ))} + + + + {allAssertions.map((assertion, index) => { + const results = conflictingResults.map( + cr => cr.scenarioResult.assertionResults[index].passed + ); + const hasConflict = results.some(r => r !== results[0]); + if (!hasConflict) { + return null; + } + return ( + + {assertion} + {results.map((result, i) => ( + {result ? 'Passed' : 'Failed'} + ))} + + ); + })} + + + + ); +}; + +AssertionConflictsTable.propTypes = { + conflictingResults: PropTypes.arrayOf(PropTypes.object).isRequired, + testers: PropTypes.arrayOf(UserPropType).isRequired +}; + +export default AssertionConflictsTable; diff --git a/client/components/TestQueue/Conflicts/ConflictIssueDetails.jsx b/client/components/TestQueue/Conflicts/ConflictIssueDetails.jsx new file mode 100644 index 000000000..91074e42f --- /dev/null +++ b/client/components/TestQueue/Conflicts/ConflictIssueDetails.jsx @@ -0,0 +1,118 @@ +import React from 'react'; +import styled from '@emotion/styled'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { + faExclamationCircle, + faExternalLinkAlt +} from '@fortawesome/free-solid-svg-icons'; +import PropTypes from 'prop-types'; +import { IssuePropType } from '../../common/proptypes'; +import { dates } from 'shared'; + +const IssuesContainer = styled.div` + margin-bottom: 2rem; +`; + +const IssueContainer = styled.div` + background-color: #f8f9fa; + border: 1px solid #e9ecef; + border-radius: 4px; + padding: 1rem; + margin: 1rem 0; +`; + +const IssueHeader = styled.div` + display: flex; + align-items: center; + margin-bottom: 0.5rem; +`; + +const IssueTitle = styled.h4` + margin: 0; + margin-left: 0.5rem; + flex-grow: 1; +`; + +const IssueGrid = styled.div` + display: grid; + grid-template-columns: auto 1fr; + gap: 0.5rem 1rem; +`; + +const IssueLabel = styled.span` + font-weight: bold; +`; + +const IssueValue = styled.span` + word-break: break-word; +`; + +const IssueLink = styled.a` + color: #0366d6; + text-decoration: none; + display: inline-flex; + align-items: center; + margin-top: 0.5rem; + &:hover { + text-decoration: underline; + } +`; + +const ConflictIssueDetails = ({ issues }) => { + if (!issues || issues.length === 0) return null; + + return ( + +

Related GitHub Issues

+ {issues.map((issue, index) => ( + + + + {issue.title} + + + Status: + {issue.isOpen ? 'Open' : 'Closed'} + Author: + {issue.author} + Type: + {issue.feedbackType} + Created: + + {dates.convertDateToString(issue.createdAt)} + + {issue.closedAt && ( + <> + Closed: + + {dates.convertDateToString(issue.closedAt)} + + + )} + + + View on GitHub  + + + + ))} + + ); +}; + +ConflictIssueDetails.propTypes = { + issues: PropTypes.arrayOf(IssuePropType).isRequired +}; + +export default ConflictIssueDetails; diff --git a/client/components/TestQueue/Conflicts/ConflictSummaryTable.jsx b/client/components/TestQueue/Conflicts/ConflictSummaryTable.jsx new file mode 100644 index 000000000..d282c5a3a --- /dev/null +++ b/client/components/TestQueue/Conflicts/ConflictSummaryTable.jsx @@ -0,0 +1,101 @@ +import React, { useMemo } from 'react'; +import PropTypes from 'prop-types'; +import styled from '@emotion/styled'; +import { ThemeTable } from '../../common/ThemeTable'; +import { IssuePropType } from '../../common/proptypes'; +import AssertionConflictsTable from './AssertionConflictsTable'; +import UnexpectedBehaviorsConflictsTable from './UnexpectedBehaviorsConflictsTable'; + +export const ConflictTable = styled(ThemeTable)` + th, + td { + text-align: left; + padding: 0.75rem; + } + margin-bottom: 2rem; +`; + +const ConflictSummaryTable = ({ conflictingResults }) => { + const commandString = scenario => { + return `Output for 'After ${scenario.commands + .map(command => command.text) + .join(' then ')}'`; + }; + + const testers = useMemo( + () => conflictingResults.map(result => result.testPlanRun.tester), + [conflictingResults] + ); + + const hasAssertionConflicts = useMemo( + () => + conflictingResults[0].scenarioResult.assertionResults.some((ar, index) => + conflictingResults.some( + cr => cr.scenarioResult.assertionResults[index].passed !== ar.passed + ) + ), + [conflictingResults] + ); + + const hasUnexpectedBehaviorConflicts = useMemo( + () => + conflictingResults.some( + result => result.scenarioResult.unexpectedBehaviors.length > 0 + ), + [conflictingResults] + ); + + return ( + <> +

+ {commandString(conflictingResults[0].scenario)} +

+ + + + + Tester + Output + + + + {testers.map(tester => ( + + {tester.username} + + { + conflictingResults.find( + cr => cr.testPlanRun.tester.id === tester.id + ).scenarioResult.output + } + + + ))} + + + + {hasAssertionConflicts && ( + + )} + {hasUnexpectedBehaviorConflicts && ( + + )} + + ); +}; + +ConflictSummaryTable.propTypes = { + conflictingResults: PropTypes.arrayOf(PropTypes.object).isRequired, + issues: PropTypes.arrayOf(IssuePropType), + issueLink: PropTypes.string.isRequired, + isAdmin: PropTypes.bool.isRequired, + testIndex: PropTypes.number.isRequired +}; + +export default ConflictSummaryTable; diff --git a/client/components/TestQueue/Conflicts/TestConflictsActions.jsx b/client/components/TestQueue/Conflicts/TestConflictsActions.jsx new file mode 100644 index 000000000..9f3f751be --- /dev/null +++ b/client/components/TestQueue/Conflicts/TestConflictsActions.jsx @@ -0,0 +1,68 @@ +import React from 'react'; +import styled from '@emotion/styled'; +import { Button, Dropdown } from 'react-bootstrap'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { + faExclamationCircle, + faFileImport +} from '@fortawesome/free-solid-svg-icons'; +import PropTypes from 'prop-types'; +import { TestPlanRunPropType } from '../../common/proptypes'; + +const ActionContainer = styled.div` + display: flex; + gap: 1rem; + margin-top: 2rem; + max-width: 500px; + & > * { + flex-grow: 1; + flex-basis: 0; + min-width: 0; + } +`; + +const ActionButton = styled(Button)` + flex-grow: 1; + flex-basis: 0; + min-width: 0; + width: 100%; + margin: 0; +`; + +const TestConflictsActions = ({ issueLink, isAdmin, testPlanRuns }) => { + return ( + + + {isAdmin && ( + + + + Open run as... + + + {testPlanRuns.map(testPlanRun => ( + + {testPlanRun.tester.username} + + ))} + + + )} + + ); +}; + +TestConflictsActions.propTypes = { + issueLink: PropTypes.string.isRequired, + isAdmin: PropTypes.bool.isRequired, + testPlanRuns: PropTypes.arrayOf(TestPlanRunPropType).isRequired, + testIndex: PropTypes.number.isRequired +}; + +export default TestConflictsActions; diff --git a/client/components/TestQueue/Conflicts/UnexpectedBehaviorsConflictsTable.jsx b/client/components/TestQueue/Conflicts/UnexpectedBehaviorsConflictsTable.jsx new file mode 100644 index 000000000..54c680d71 --- /dev/null +++ b/client/components/TestQueue/Conflicts/UnexpectedBehaviorsConflictsTable.jsx @@ -0,0 +1,76 @@ +import PropTypes from 'prop-types'; +import React, { useMemo } from 'react'; +import { ConflictTable } from './ConflictSummaryTable'; +import { UserPropType } from '../../common/proptypes'; + +const UnexpectedBehaviorsConflictsTable = ({ conflictingResults, testers }) => { + const commandString = scenario => { + return `Unexpected Behaviors for 'After ${scenario.commands + .map(command => command.text) + .join(' then ')}'`; + }; + + const allUnexpectedBehaviors = useMemo( + () => + Array.from( + new Set( + conflictingResults.flatMap(result => + result.scenarioResult.unexpectedBehaviors.map(ub => ub.text) + ) + ) + ), + [conflictingResults] + ); + + return ( + <> +

+ {commandString(conflictingResults[0].scenario)} +

+ + + + + Unexpected Behavior + {testers.map(tester => ( + {tester.username} + ))} + + + + {allUnexpectedBehaviors.map((behaviorText, index) => ( + + {behaviorText} + {conflictingResults.map((result, i) => { + const matchingBehavior = + result.scenarioResult.unexpectedBehaviors.find( + ub => ub.text === behaviorText + ); + return ( + + {matchingBehavior ? ( + <> + Impact: {matchingBehavior.impact} +
+ Details: {matchingBehavior.details} + + ) : ( + 'Not reported' + )} + + ); + })} + + ))} + + + + ); +}; + +UnexpectedBehaviorsConflictsTable.propTypes = { + conflictingResults: PropTypes.arrayOf(PropTypes.object).isRequired, + testers: PropTypes.arrayOf(UserPropType).isRequired +}; + +export default UnexpectedBehaviorsConflictsTable; diff --git a/client/components/TestQueue/Conflicts/index.jsx b/client/components/TestQueue/Conflicts/index.jsx new file mode 100644 index 000000000..c75c26376 --- /dev/null +++ b/client/components/TestQueue/Conflicts/index.jsx @@ -0,0 +1,324 @@ +import { useQuery } from '@apollo/client'; +import React, { useEffect, useMemo, useState } from 'react'; +import { Container } from 'react-bootstrap'; +import { Helmet } from 'react-helmet'; +import { useParams } from 'react-router-dom'; +import styled from '@emotion/styled'; +import { TEST_QUEUE_CONFLICTS_PAGE_QUERY } from '../queries'; +import PageStatus from '../../common/PageStatus'; +import DisclosureComponent from '../../common/DisclosureComponent'; +import ConflictSummaryTable from './ConflictSummaryTable'; +import createIssueLink from '../../../utils/createIssueLink'; +import { evaluateAuth } from '../../../utils/evaluateAuth'; +import ConflictIssueDetails from './ConflictIssueDetails'; +import TestConflictsActions from './TestConflictsActions'; +import generateConflictMarkdown from '../../../utils/generateConflictMarkdown'; + +const PageContainer = styled(Container)` + max-width: 1200px; + padding: 2rem; +`; + +const Section = styled.section` + margin-bottom: 3rem; +`; + +const SectionTitle = styled.h2` + margin-bottom: 1rem; +`; + +const MetadataList = styled.ul` + list-style-type: none; + padding-left: 0; + margin-bottom: 2rem; + background-color: #f8f9fa; + border-radius: 4px; + padding: 1rem; +`; + +const MetadataItem = styled.li` + margin-bottom: 0.5rem; + display: flex; + align-items: center; +`; + +const MetadataLabel = styled.span` + font-weight: bold; + margin-right: 0.5rem; + min-width: 200px; +`; + +const ConflictCount = styled.p` + font-size: 1.1rem; + margin-bottom: 1rem; + background-color: #e9ecef; + padding: 0.5rem 1rem; + border-radius: 4px; + display: inline-block; +`; + +const TestQueueConflicts = () => { + const [openDisclosures, setOpenDisclosures] = useState([]); + const [uniqueConflictsByAssertion, setUniqueConflictsByAssertion] = useState( + [] + ); + + const [conflictsByTest, setConflictsByTest] = useState({}); + + const { testPlanReportId } = useParams(); + const { data, error, loading } = useQuery(TEST_QUEUE_CONFLICTS_PAGE_QUERY, { + fetchPolicy: 'cache-and-network', + variables: { + testPlanReportId: testPlanReportId + } + }); + + useEffect(() => { + if (data) { + // Each conflict per test is counted so it could cause duplicate + // disclosures + // + // eg. tester1 and tester2 marking 2 assertions in test1 the opposite way + // would create 2 disclosures + const createUniqueConflictId = (testId = '', commands = []) => + `${testId}-${commands.map(({ text }) => text).join('')}`; + + const uniqueConflictsByAssertions = []; + data?.testPlanReport?.conflicts?.map(conflict => { + const conflictId = createUniqueConflictId( + conflict.conflictingResults[0].test.id, + conflict.conflictingResults[0].scenario.commands + ); + + if ( + !uniqueConflictsByAssertions.find( + conflict => + createUniqueConflictId( + conflict.conflictingResults[0].test.id, + conflict.conflictingResults[0].scenario.commands + ) === conflictId + ) + ) { + uniqueConflictsByAssertions.push(conflict); + } + }); + setUniqueConflictsByAssertion(uniqueConflictsByAssertions); + + const conflictsByTestObj = {}; + data?.testPlanReport?.conflicts?.forEach(conflict => { + const testId = conflict.conflictingResults[0].test.id; + const commandKey = conflict.conflictingResults[0].scenario.commands + .map(cmd => cmd.text) + .join(' then '); + + if (!conflictsByTestObj[testId]) { + conflictsByTestObj[testId] = { + test: conflict.conflictingResults[0].test, + conflicts: {} + }; + } + + if (!conflictsByTestObj[testId].conflicts[commandKey]) { + conflictsByTestObj[testId].conflicts[commandKey] = conflict; + } + }); + setConflictsByTest(conflictsByTestObj); + setOpenDisclosures(Object.keys(conflictsByTestObj).map(() => false)); + } + }, [data]); + + const getTestNumberFilteredByAt = test => { + const testIndex = data.testPlanReport.runnableTests.findIndex( + t => t.id === test.id + ); + return testIndex + 1; + }; + + const getIssueLink = test => { + if (!test) { + return; + } + + const { testPlanVersion } = data.testPlanReport; + const conflictMarkdown = generateConflictMarkdown( + data.testPlanReport, + test + ); + return createIssueLink({ + testPlanTitle: testPlanVersion.title, + testPlanDirectory: testPlanVersion.testPlan.directory, + versionString: testPlanVersion.versionString, + testTitle: test.title, + testRenderedUrl: test.renderedUrl, + testRowNumber: test.rowNumber, + testSequenceNumber: getTestNumberFilteredByAt(test), + atName: data.testPlanReport.at.name, + browserName: data.testPlanReport.browser.name, + atVersionName: data.testPlanReport.exactAtVersion?.name + ? data.testPlanReport.exactAtVersion?.name + : `${data.testPlanReport.minimumAtVersion?.name} and above`, + conflictMarkdown + }); + }; + + const { isAdmin } = useMemo(() => evaluateAuth(data?.me), [data?.me]); + + const disclosureLabels = useMemo(() => { + return Object.values(conflictsByTest).map(({ test }) => { + const testIndex = + data.testPlanReport.runnableTests.findIndex(t => t.id === test.id) + 1; + return `Test ${testIndex}: ${test.title}`; + }); + }, [conflictsByTest, data]); + + const disclosureContents = useMemo(() => { + const uniqueTestPlanRuns = Object.values(conflictsByTest) + .flatMap(({ conflicts }) => + Object.values(conflicts).map(conflict => + conflict.conflictingResults.map( + conflictingResult => conflictingResult.testPlanRun + ) + ) + ) + .flat() + .filter( + (testPlanRun, index, self) => + index === self.findIndex(t => t.id === testPlanRun.id) + ); + + return Object.values(conflictsByTest).map(({ test, conflicts }) => { + const issues = data?.testPlanReport?.issues?.filter( + issue => + issue.testNumberFilteredByAt === getTestNumberFilteredByAt(test) + ); + return ( +
+ {Object.entries(conflicts).map(([commandKey, conflict]) => ( + + ))} + {issues.length > 0 && } + +
+ ); + }); + }, [conflictsByTest, data, isAdmin]); + + const disclosureClickHandlers = useMemo(() => { + return Object.keys(conflictsByTest).map((_, index) => () => { + setOpenDisclosures(prevState => { + const newOpenDisclosures = [...prevState]; + newOpenDisclosures[index] = !newOpenDisclosures[index]; + return newOpenDisclosures; + }); + }); + }, [conflictsByTest]); + + if (error) { + return ( + + ); + } + + if (loading) { + return ( + + ); + } + + const { + title, + versionString, + id: testPlanVersionId + } = data?.testPlanReport.testPlanVersion ?? {}; + const { name: browserName } = data?.testPlanReport.browser ?? {}; + const { name: atName } = data?.testPlanReport.at ?? {}; + const { name: exactAtVersionName } = + data?.testPlanReport.exactAtVersion ?? {}; + const { name: minimumAtVersionName } = + data?.testPlanReport.minimumAtVersion ?? {}; + + const uniqueTestsLength = Object.keys(conflictsByTest).length; + + return ( + + + + Conflicts {title} {versionString} | ARIA-AT + + +

+ Conflicts for Test Plan Report {title} {versionString} +

+ +
+ Introduction +

+ This page displays conflicts identified in the current test plan + report. Conflicts occur when different testers report different + outcomes for the same test assertions or unexpected behaviors. +

+
+ +
+ Test Plan Report + + + Test Plan Version: + + {title} {versionString} + + + + Assistive Technology: + {atName} + {exactAtVersionName + ? ` (${exactAtVersionName})` + : ` (${minimumAtVersionName} and above)`} + + + Browser: + {browserName} + + +
+ +
+ Conflicts + + There are currently + {data?.testPlanReport?.conflicts?.length} conflicts + across + {uniqueTestsLength} tests + and + {uniqueConflictsByAssertion.length} assertions + for this test plan report. + + +
+ + ); +}; + +export default TestQueueConflicts; diff --git a/client/components/TestQueue/index.jsx b/client/components/TestQueue/index.jsx index 3cec38396..e7404db2b 100644 --- a/client/components/TestQueue/index.jsx +++ b/client/components/TestQueue/index.jsx @@ -332,6 +332,7 @@ const TestQueue = () => { {hasBotRun ? ( diff --git a/client/components/TestQueue/queries.js b/client/components/TestQueue/queries.js index 7dd6f7411..a4ceff963 100644 --- a/client/components/TestQueue/queries.js +++ b/client/components/TestQueue/queries.js @@ -10,6 +10,7 @@ import { TEST_PLAN_RUN_FIELDS, TEST_PLAN_VERSION_FIELDS, TEST_RESULT_FIELDS, + ISSUE_FIELDS, USER_FIELDS } from '@components/common/fragments'; @@ -82,6 +83,79 @@ export const TEST_QUEUE_PAGE_QUERY = gql` } `; +export const TEST_QUEUE_CONFLICTS_PAGE_QUERY = gql` + ${ME_FIELDS} + ${TEST_PLAN_REPORT_FIELDS} + ${ISSUE_FIELDS('all')} + ${TEST_PLAN_RUN_FIELDS} + query TestQueueConflictsPage($testPlanReportId: ID!) { + me { + ...MeFields + } + testPlanReport(id: $testPlanReportId) { + ...TestPlanReportFields + testPlanVersion { + title + versionString + id + testPlan { + directory + } + } + minimumAtVersion { + name + } + recommendedAtVersion { + name + } + browser { + name + } + at { + name + } + runnableTests { + id + } + issues { + ...IssueFieldsAll + } + conflicts { + conflictingResults { + testPlanRun { + ...TestPlanRunFields + } + test { + id + rowNumber + title + renderedUrl + } + scenario { + commands { + text + } + } + scenarioResult { + output + unexpectedBehaviors { + text + details + impact + } + assertionResults { + assertion { + text + } + passed + } + } + } + } + } + } +`; + export const ASSIGN_TESTER_MUTATION = gql` ${TEST_PLAN_RUN_FIELDS} mutation AssignTester( diff --git a/client/components/common/ReportStatusSummary/index.jsx b/client/components/common/ReportStatusSummary/index.jsx index aaae6acb8..48ab86946 100644 --- a/client/components/common/ReportStatusSummary/index.jsx +++ b/client/components/common/ReportStatusSummary/index.jsx @@ -3,7 +3,12 @@ import PropTypes from 'prop-types'; import styled from '@emotion/styled'; import { dates } from 'shared'; import { calculatePercentComplete } from '../../../utils/calculatePercentComplete'; -import { TestPlanVersionPropType, TestPlanRunPropType } from '../proptypes'; +import { + TestPlanVersionPropType, + TestPlanRunPropType, + UserPropType +} from '../proptypes'; +import { evaluateAuth } from '../../../utils/evaluateAuth'; const IncompleteStatusReport = styled.span` min-width: 5rem; @@ -13,8 +18,11 @@ const IncompleteStatusReport = styled.span` const ReportStatusSummary = ({ testPlanVersion, testPlanReport, + me, fromTestQueue = false }) => { + const { isSignedIn, isAdmin, isTester, isVendor } = evaluateAuth(me); + const renderCompleteReportStatus = testPlanReport => { const formattedDate = dates.convertDateToString( testPlanReport.markedFinalAt, @@ -27,6 +35,20 @@ const ReportStatusSummary = ({ ); }; + const getConflictsAnchor = conflictsCount => { + if (conflictsCount === 0) return null; + if (!isSignedIn) return null; + if (!isTester && !isVendor && !isAdmin) return null; + return ( + + with {conflictsCount} conflicts + + ); + }; + const renderPartialCompleteReportStatus = testPlanReport => { const { metrics, draftTestPlanRuns } = testPlanReport; @@ -48,15 +70,16 @@ const ReportStatusSummary = ({ > {draftTestPlanRuns[0].tester.username} -  with {conflictsCount} conflicts +   + {getConflictsAnchor(conflictsCount)} ); default: return ( {percentComplete}% complete by  - {draftTestPlanRuns.length} testers with {conflictsCount} -  conflicts + {draftTestPlanRuns.length} testers  + {getConflictsAnchor(conflictsCount)} ); } @@ -82,6 +105,7 @@ ReportStatusSummary.propTypes = { metrics: PropTypes.object, draftTestPlanRuns: PropTypes.arrayOf(TestPlanRunPropType).isRequired }), + me: UserPropType, fromTestQueue: PropTypes.bool }; diff --git a/client/components/common/proptypes/index.js b/client/components/common/proptypes/index.js index 7d98d1d78..27d0fad2c 100644 --- a/client/components/common/proptypes/index.js +++ b/client/components/common/proptypes/index.js @@ -285,7 +285,7 @@ export const TestResultPropType = PropTypes.shape({ export const UserPropType = PropTypes.shape({ __typename: PropTypes.string, - id: PropTypes.string.isRequired, + id: PropTypes.string, username: PropTypes.string.isRequired, isBot: PropTypes.bool }); diff --git a/client/routes/index.js b/client/routes/index.js index d8714aea6..a374c3718 100644 --- a/client/routes/index.js +++ b/client/routes/index.js @@ -14,6 +14,7 @@ import CandidateTestPlanRun from '@components/CandidateReview/CandidateTestPlanR import DataManagement from 'client/components/DataManagement'; import TestPlanVersionsPage from '../components/TestPlanVersionsPage'; import TestReview from '../components/TestReview'; +import TestQueueConflicts from '../components/TestQueue/Conflicts'; export default () => ( @@ -38,6 +39,15 @@ export default () => ( } /> } /> + + + + } + /> { const statusColumn = cells[3]; const statusColumnText = sanitizedText(statusColumn.innerText); const statusColumnCondition = statusColumnText.includes( - '100% complete by esmeralda-baggins with 0 conflicts' + '100% complete by esmeralda-baggins' ); // Actions Column @@ -350,7 +350,7 @@ describe('Test Queue tester traits when reports exist', () => { const statusColumn = cells[3]; const statusColumnText = sanitizedText(statusColumn.innerText); const statusColumnCondition = statusColumnText.includes( - '100% complete by esmeralda-baggins with 0 conflicts' + '100% complete by esmeralda-baggins' ); // Actions Column diff --git a/client/tests/e2e/snapshots/saved/_test-queue.html b/client/tests/e2e/snapshots/saved/_test-queue.html index a8cac1033..6adea8a7e 100644 --- a/client/tests/e2e/snapshots/saved/_test-queue.html +++ b/client/tests/e2e/snapshots/saved/_test-queue.html @@ -651,7 +651,7 @@

>100% complete by esmeralda-baggins with 0 conflicts  @@ -863,7 +863,7 @@

>100% complete by esmeralda-baggins with 0 conflicts  @@ -988,7 +988,7 @@

>100% complete by esmeralda-baggins with 0 conflicts  @@ -1115,7 +1115,7 @@

>100% complete by esmeralda-baggins with 0 conflicts  @@ -1242,7 +1242,7 @@

>100% complete by esmeralda-baggins with 0 conflicts  @@ -1465,8 +1465,11 @@

85% complete by 2 testers with - 3 conflicts85% complete by 2 testers with 3 conflicts @@ -1684,7 +1687,7 @@

>75% complete by esmeralda-baggins with 0 conflicts  diff --git a/client/tests/e2e/snapshots/saved/_test-queue_2_conflicts.html b/client/tests/e2e/snapshots/saved/_test-queue_2_conflicts.html new file mode 100644 index 000000000..85f1a231b --- /dev/null +++ b/client/tests/e2e/snapshots/saved/_test-queue_2_conflicts.html @@ -0,0 +1,545 @@ + + + + + + + + + Conflicts Select Only Combobox Example V22.03.17 | ARIA-AT + + +
+
+ +
+
+
+

+ Conflicts for Test Plan Report Select Only Combobox Example + V22.03.17 +

+
+

Introduction

+

+ This page displays conflicts identified in the current test plan + report. Conflicts occur when different testers report different + outcomes for the same test assertions or unexpected behaviors. +

+
+
+

Test Plan Report

+ +
+
+

Conflicts

+

+ There are currently 3 conflicts across + 3 tests and 3 assertions for this test plan report. +

+
+

+ +

+ +

+ +

+
+
+

+ Output for 'After Insert+Tab' +

+
+ + + + + + + + + + + + + + + + + +
TesterOutput
esmeralda-bagginsautomatically seeded sample output
tom-proudfeetautomatically seeded sample output
+
+

+ Unexpected Behaviors for 'After Insert+Tab' +

+
+ + + + + + + + + + + + + + + +
Unexpected Behavioresmeralda-bagginstom-proudfeet
OtherNot reported + Impact: MODERATE
Details: Seeded other + unexpected behavior +
+
+
+ Raise an Issue for Conflict +
+ +
+
+
+
+

+ +

+
+
+

+ Output for 'After Alt+Down' +

+
+ + + + + + + + + + + + + + + + + +
TesterOutput
esmeralda-bagginsautomatically seeded sample output
tom-proudfeetautomatically seeded sample output
+
+

+ Unexpected Behaviors for 'After Alt+Down' +

+
+ + + + + + + + + + + + + + + + + + + + +
Unexpected Behavioresmeralda-bagginstom-proudfeet
+ Output is excessively verbose, e.g., includes + redundant and/or irrelevant speech + Not reportedImpact: MODERATE
Details: N/A
OtherNot reported + Impact: SEVERE
Details: Seeded other unexpected + behavior +
+
+
+ Raise an Issue for Conflict +
+ +
+
+
+
+
+
+
+
+ +
+ + + diff --git a/client/tests/e2e/snapshots/utils.js b/client/tests/e2e/snapshots/utils.js index 259e3bf10..9984eefc0 100644 --- a/client/tests/e2e/snapshots/utils.js +++ b/client/tests/e2e/snapshots/utils.js @@ -13,7 +13,8 @@ const snapshotRoutes = [ '/test-review/8', '/run/2', '/data-management/meter', - '/candidate-test-plan/24/1' + '/candidate-test-plan/24/1', + '/test-queue/2/conflicts' ]; async function cleanAndNormalizeSnapshot(page) { diff --git a/client/utils/generateConflictMarkdown.js b/client/utils/generateConflictMarkdown.js new file mode 100644 index 000000000..552f45e83 --- /dev/null +++ b/client/utils/generateConflictMarkdown.js @@ -0,0 +1,89 @@ +const generateConflictMarkdown = (testPlanReport, test) => { + const conflicts = testPlanReport.conflicts.filter( + conflict => conflict.conflictingResults[0].test.id === test.id + ); + + if (conflicts.length === 0) return ''; + + const commandString = scenario => { + return scenario.commands.map(command => command.text).join(' then '); + }; + + const renderConflict = (conflict, index) => { + const hasUnexpectedBehaviors = conflict.conflictingResults.some( + result => result.scenarioResult.unexpectedBehaviors.length > 0 + ); + if (hasUnexpectedBehaviors) + return renderUnexpectedBehaviorConflict(conflict, index); + return renderAssertionResultConflict(conflict, index); + }; + + const renderAssertionResultConflict = ({ conflictingResults }, index) => { + const scenario = conflictingResults[0].scenario; + const command = commandString(scenario); + const assertion = + conflictingResults[0].scenarioResult.assertionResults.find( + ar => !ar.passed + ); + const assertionText = assertion + ? assertion.assertion.text + : 'Unknown assertion'; + + const results = conflictingResults + .map(result => { + const { testPlanRun, scenarioResult } = result; + const assertionResult = scenarioResult.assertionResults.find( + ar => ar.assertion.text === assertionText + ); + const passed = assertionResult ? assertionResult.passed : false; + return `* Tester ${testPlanRun.tester.username} recorded output "${ + scenarioResult.output + }" and marked assertion as ${passed ? 'passing' : 'failing'}.`; + }) + .join('\n'); + + return ` +${ + index + 1 +}. ### Assertion Results for "${command}" Command and "${assertionText}" Assertion + +${results}`; + }; + + const renderUnexpectedBehaviorConflict = ({ conflictingResults }, index) => { + const scenario = conflictingResults[0].scenario; + const command = commandString(scenario); + + const results = conflictingResults + .map(result => { + const { testPlanRun, scenarioResult } = result; + let resultFormatted; + if (scenarioResult.unexpectedBehaviors.length) { + resultFormatted = scenarioResult.unexpectedBehaviors + .map(({ text, impact, details }) => { + return `"${text}" (Details: ${details}, Impact: ${impact})`; + }) + .join(' and '); + } else { + resultFormatted = 'no unexpected behavior'; + } + return `* Tester ${testPlanRun.tester.username} recorded output "${scenarioResult.output}" and noted ${resultFormatted}.`; + }) + .join('\n'); + + return ` +${index + 1}. ### Unexpected Behaviors for "${command}" Command + +${results}`; + }; + + return ` +## Review Conflicts for "${test.title}" + +${conflicts.map(renderConflict).join('\n')} +`; + + // return content.trim(); +}; + +export default generateConflictMarkdown;