diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dd3df830..ff2705762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +## [1.6.0](https://github.com/w3c/aria-at-app/compare/v1.5.0...v1.6.0) (2024-07-29) + + +### Features + +* permit automated testing for firefox and chrome with voiceover ([#1170](https://github.com/w3c/aria-at-app/issues/1170)) ([9c73ab5](https://github.com/w3c/aria-at-app/commit/9c73ab516ac8f6bfbc0138191351ce84d680c732)) + + +### Bug Fixes + +* Update metrics calculations and related UI components ([#1172](https://github.com/w3c/aria-at-app/issues/1172)) ([e4242bc](https://github.com/w3c/aria-at-app/commit/e4242bcb38eec20eb50794f96c0bc8ae9a5aba97)) + + +### Reverts + +* revert accidental commit directly to develop ([d1c409c](https://github.com/w3c/aria-at-app/commit/d1c409c6f33005467f426771593c2c7eb00eef9b)) + ## [1.5.0](https://github.com/w3c/aria-at-app/compare/v1.4.0...v1.5.0) (2024-07-22) diff --git a/client/components/AddTestToQueueWithConfirmation/index.jsx b/client/components/AddTestToQueueWithConfirmation/index.jsx index 45c6931fb..54be4d16b 100644 --- a/client/components/AddTestToQueueWithConfirmation/index.jsx +++ b/client/components/AddTestToQueueWithConfirmation/index.jsx @@ -166,7 +166,8 @@ function AddTestToQueueWithConfirmation({ } catch (e) { console.error(e); } - } + }, + testId: 'add-run-later' }); if (!alreadyHasBotInTestPlanReport) { @@ -186,7 +187,8 @@ function AddTestToQueueWithConfirmation({ } catch (e) { console.error(e); } - } + }, + testId: 'add-run-bot' }); } } diff --git a/client/components/CandidateReview/TestPlans/index.jsx b/client/components/CandidateReview/TestPlans/index.jsx index ba5f8ac85..5ae550b90 100644 --- a/client/components/CandidateReview/TestPlans/index.jsx +++ b/client/components/CandidateReview/TestPlans/index.jsx @@ -20,6 +20,7 @@ import { import ClippedProgressBar from '@components/common/ClippedProgressBar'; import { convertDateToString } from '@client/utils/formatter'; import './TestPlans.css'; +import { calculations } from 'shared'; const FullHeightContainer = styled(Container)` min-height: calc(100vh - 64px); @@ -486,13 +487,12 @@ const TestPlans = ({ testPlanVersions }) => { obj.mustAssertionsFailedCount, 0 ), - totalSupportPercent: - Math.round( - allMetrics.reduce( - (acc, obj) => acc + obj.supportPercent, - 0 - ) / allMetrics.length - ) || 0 + totalSupportPercent: calculations.trimDecimals( + allMetrics.reduce( + (acc, obj) => acc + obj.supportPercent, + 0 + ) / allMetrics.length + ) }; // Make sure issues are unique @@ -550,10 +550,10 @@ const TestPlans = ({ testPlanVersions }) => { diff --git a/client/components/Reports/SummarizeTestPlanReports.jsx b/client/components/Reports/SummarizeTestPlanReports.jsx index c7f97f62d..26867cd5a 100644 --- a/client/components/Reports/SummarizeTestPlanReports.jsx +++ b/client/components/Reports/SummarizeTestPlanReports.jsx @@ -132,12 +132,10 @@ const SummarizeTestPlanReports = ({ testPlanVersions }) => { `/report/${testPlanVersion.id}` + `/targets/${testPlanReport.id}` } + aria-label={`${metrics.supportPercent}%`} > diff --git a/client/components/common/BasicModal/index.jsx b/client/components/common/BasicModal/index.jsx index 11d487264..fdb015742 100644 --- a/client/components/common/BasicModal/index.jsx +++ b/client/components/common/BasicModal/index.jsx @@ -61,6 +61,7 @@ const BasicModal = ({ variant={action.variant ?? 'primary'} onClick={action.onClick} className={action.className ?? ''} + data-testid={action.testId ?? ''} > {action.label ?? 'Continue'} @@ -138,6 +139,7 @@ BasicModal.propTypes = { onClick: PropTypes.func, variant: PropTypes.string, className: PropTypes.string, + testId: PropTypes.string, component: PropTypes.elementType, props: PropTypes.object }) diff --git a/client/components/common/ClippedProgressBar/index.jsx b/client/components/common/ClippedProgressBar/index.jsx index ebe7d8bcd..d47102a57 100644 --- a/client/components/common/ClippedProgressBar/index.jsx +++ b/client/components/common/ClippedProgressBar/index.jsx @@ -2,16 +2,11 @@ import React from 'react'; import PropTypes from 'prop-types'; import './ClippedProgressBar.css'; -const ProgressBar = ({ - progress = 0, - label = '', - clipped = true, - decorative -}) => { +const ProgressBar = ({ progress = 0, clipped = true, decorative }) => { return ( <> {clipped ? ( -
+
{decorative ? null : `${progress}%`}
) : ( -
+
{ test('calls mutation on button click with correct variables', async () => { fireEvent.click(getByTestId('add-button')); + fireEvent.click(await getByTestId('add-run-later')); await waitFor(() => { expect(mutationMock).toHaveBeenCalled(); diff --git a/client/utils/automation.js b/client/utils/automation.js index 208091473..65fe2c581 100644 --- a/client/utils/automation.js +++ b/client/utils/automation.js @@ -19,10 +19,13 @@ export const isSupportedByResponseCollector = ctx => { // Check if the AT and browser are supported by the automation const isNvdaWithSupportedBrowser = atKey === 'nvda' && (browserKey === 'chrome' || browserKey === 'firefox'); - const isVoiceOverMacWithSafari = - atKey === 'voiceover_macos' && browserKey === 'safari_macos'; + const isVoiceOverMacWithSupportedBrowser = + atKey === 'voiceover_macos' && + (browserKey === 'safari_macos' || + browserKey === 'chrome' || + browserKey === 'firefox'); - if (!isNvdaWithSupportedBrowser && !isVoiceOverMacWithSafari) { + if (!isNvdaWithSupportedBrowser && !isVoiceOverMacWithSupportedBrowser) { return false; } @@ -89,7 +92,12 @@ export const getBotUsernameFromAtBrowser = (at, browser) => { ) { return 'NVDA Bot'; } - if (at?.key === 'voiceover_macos' && browser?.key === 'safari_macos') { + if ( + at?.key === 'voiceover_macos' && + (browser?.key === 'safari_macos' || + browser?.key === 'chrome' || + browser?.key === 'firefox') + ) { return 'VoiceOver Bot'; } }; diff --git a/package.json b/package.json index 8b246dd76..4aaa848be 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "aria-at-app", - "version": "1.5.0", + "version": "1.6.0", "description": "Run ARIA-AT tests and report results", "main": "server/index.js", "private": true, diff --git a/server/migrations/20240724212706-updateMetrics.js b/server/migrations/20240724212706-updateMetrics.js new file mode 100644 index 000000000..008c4482f --- /dev/null +++ b/server/migrations/20240724212706-updateMetrics.js @@ -0,0 +1,64 @@ +'use strict'; + +const { getMetrics } = require('shared'); +const populateData = require('../services/PopulatedData/populateData'); +const runnableTestsResolver = require('../resolvers/TestPlanReport/runnableTestsResolver'); +const finalizedTestResultsResolver = require('../resolvers/TestPlanReport/finalizedTestResultsResolver'); +const { + updateTestPlanReportById +} = require('../models/services/TestPlanReportService'); +const getGraphQLContext = require('../graphql-context'); + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + return queryInterface.sequelize.transaction(async transaction => { + const context = getGraphQLContext({ req: { transaction } }); + + const testPlanReports = await queryInterface.sequelize.query( + `select distinct on ("TestPlanReport".id) "TestPlanReport".id, metrics + from "TestPlanReport" + join public."TestPlanRun" testPlanRun on "TestPlanReport".id = testPlanRun."testPlanReportId" + where jsonb_array_length(testPlanRun."testResults") > 0;`, + { + type: Sequelize.QueryTypes.SELECT, + transaction + } + ); + + for (const testPlanReport of testPlanReports) { + const { testPlanReport: testPlanReportPopulated } = await populateData( + { testPlanReportId: testPlanReport.id }, + { context } + ); + const runnableTests = runnableTestsResolver( + testPlanReportPopulated, + null, + context + ); + const finalizedTestResults = await finalizedTestResultsResolver( + testPlanReportPopulated, + null, + context + ); + const metrics = getMetrics({ + testPlanReport: { + ...testPlanReportPopulated, + finalizedTestResults, + runnableTests + } + }); + await updateTestPlanReportById({ + id: testPlanReportPopulated.id, + values: { + metrics: { + ...testPlanReportPopulated.metrics, + ...metrics + } + }, + transaction + }); + } + }); + } +}; diff --git a/server/resolvers/TestPlanReport/finalizedTestResultsResolver.js b/server/resolvers/TestPlanReport/finalizedTestResultsResolver.js index c8b6527a8..2c5546168 100644 --- a/server/resolvers/TestPlanReport/finalizedTestResultsResolver.js +++ b/server/resolvers/TestPlanReport/finalizedTestResultsResolver.js @@ -8,6 +8,9 @@ const finalizedTestResultsResolver = async (testPlanReport, _, context) => { // Return the primary test plan run, otherwise pick the first TestPlanRun found. const testPlanRun = testPlanReport.testPlanRuns.find(({ isPrimary }) => isPrimary) || + testPlanReport.testPlanRuns.find(testPlanRun => + testPlanRun.testResults?.some(testResult => !!testResult.completedAt) + ) || testPlanReport.testPlanRuns[0]; return testResultsResolver( diff --git a/server/services/GithubWorkflowService.js b/server/services/GithubWorkflowService.js index c72d76561..f4679c8ac 100644 --- a/server/services/GithubWorkflowService.js +++ b/server/services/GithubWorkflowService.js @@ -154,6 +154,7 @@ const createGithubWorkflow = async ({ job, directory, gitSha, atVersion }) => { // due to limitations on Github workflow runners // See https://github.com/w3c/aria-at-app/issues/1143 for more info inputs.macos_version = atVersion?.name?.split('.')[0]; + inputs.browser = browser; } const axiosConfig = { method: 'POST', diff --git a/shared/calculations.js b/shared/calculations.js new file mode 100644 index 000000000..02c870a94 --- /dev/null +++ b/shared/calculations.js @@ -0,0 +1,26 @@ +/** + * @param {number} value + * @param {number} total + * @param {boolean} ignoreError - to account for cases where having a NaN is "expected" + * @returns {number} + */ +const calculatePercentage = (value, total, { ignoreError = true } = {}) => { + if (!ignoreError && total === 0) { + throw new Error("Unable to divide. 'total' cannot be 0."); + } + return (value / total) * 100; +}; + +const trimDecimals = (number, decimals = 0) => { + if (decimals === undefined || decimals <= 0) { + return Math.floor(number); + } else { + let factor = Math.pow(10, decimals); + return Math.floor(number * factor) / factor; + } +}; + +module.exports = { + calculatePercentage, + trimDecimals +}; diff --git a/shared/getMetrics.js b/shared/getMetrics.js index 9a4bac2ca..9f00e52d6 100644 --- a/shared/getMetrics.js +++ b/shared/getMetrics.js @@ -1,4 +1,5 @@ const convertAssertionPriority = require('./convertAssertionPriority'); +const { calculatePercentage, trimDecimals } = require('./calculations'); const sum = arr => arr?.reduce((total, item) => total + item, 0) || 0; @@ -46,11 +47,32 @@ const countAssertions = ({ if (isClient) { all = scenarioResult?.[`${priority.toLowerCase()}AssertionResults`] || []; } else { - all = scenarioResult.assertionResults.filter( - a => + all = scenarioResult.assertionResults.filter(a => { + // Same as `server/resolvers/ScenarioResult/assertionResultsResolver.js` + if (a.assertion?.assertionExceptions?.length) { + const scenarioSettings = scenarioResult.scenario.settings; + const scenarioCommandId = + scenarioResult.scenario.commandIds.join(' '); + + const foundException = a.assertion.assertionExceptions.find( + exception => + exception.settings === scenarioSettings && + exception.commandId === scenarioCommandId + ); + + if (foundException) { + return ( + convertAssertionPriority(foundException.priority) === + convertAssertionPriority(priority) + ); + } + } + + return ( convertAssertionPriority(a.assertion.priority) === convertAssertionPriority(priority) - ); + ); + }); } if (passedOnly) return all.filter(each => each.passed).length; return all.length; @@ -236,9 +258,11 @@ const getMetrics = ({ supportLevel = 'FULL'; } - const supportPercent = Math.round( - (mustAssertionsPassedCount / mustAssertionsCount) * 100 + const percentage = calculatePercentage( + mustAssertionsPassedCount + shouldAssertionsPassedCount, + mustAssertionsCount + shouldAssertionsCount ); + const supportPercent = trimDecimals(percentage); const assertionsPassedCount = mustAssertionsPassedCount + diff --git a/shared/index.js b/shared/index.js index 93bd09727..c936a769c 100644 --- a/shared/index.js +++ b/shared/index.js @@ -1,3 +1,5 @@ +const calculations = require('./calculations'); const convertAssertionPriority = require('./convertAssertionPriority'); const getMetrics = require('./getMetrics'); -module.exports = { convertAssertionPriority, getMetrics }; + +module.exports = { calculations, convertAssertionPriority, getMetrics }; diff --git a/shared/tests/__mocks__/testPlanReportForMetricsFromLiveData.json b/shared/tests/__mocks__/testPlanReportForMetricsFromLiveData.json new file mode 100644 index 000000000..e6b9d6d9d --- /dev/null +++ b/shared/tests/__mocks__/testPlanReportForMetricsFromLiveData.json @@ -0,0 +1,3053 @@ +{ + "id": "242", + "runnableTests": [ + { + "id": "NjRhMeyIyIjoiNzUzMjIifQzQ0OG" + }, + { + "id": "MjE0ZeyIyIjoiNzUzMjIifQDE2YW" + }, + { + "id": "MmUxYeyIyIjoiNzUzMjIifQWNmOW" + }, + { + "id": "Y2EzOeyIyIjoiNzUzMjIifQTJkM2" + }, + { + "id": "ZTAxOeyIyIjoiNzUzMjIifQDgwZj" + }, + { + "id": "NjMzYeyIyIjoiNzUzMjIifQjc1YW" + }, + { + "id": "OTRiZeyIyIjoiNzUzMjIifQDhkOG" + }, + { + "id": "OWIyNeyIyIjoiNzUzMjIifQ2U4ZW" + }, + { + "id": "NzJiNeyIyIjoiNzUzMjIifQjc1ND" + }, + { + "id": "MjE3NeyIyIjoiNzUzMjIifQmQ3ZW" + }, + { + "id": "ZjZjMeyIyIjoiNzUzMjIifQGFiOD" + }, + { + "id": "NTgxYeyIyIjoiNzUzMjIifQTg1N2" + }, + { + "id": "ZTMzNeyIyIjoiNzUzMjIifQTIzYW" + }, + { + "id": "YzE1MeyIyIjoiNzUzMjIifQTg5MT" + }, + { + "id": "MGY5MeyIyIjoiNzUzMjIifQmM5Ym" + } + ], + "finalizedTestResults": [ + { + "id": "ODZiYeyIxMiI6NTU1fQjgyZm", + "scenarioResults": [ + { + "id": "ZjFlZeyIxMyI6Ik9EWmlZZXlJeE1pSTZOVFUxZlFqZ3labSJ9jQ1M2", + "scenario": { + "commands": [ + { + "id": "f" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "OTlkNeyIxMyI6Ik9EWmlZZXlJeE1pSTZOVFUxZlFqZ3labSJ9DM4Zm", + "scenario": { + "commands": [ + { + "id": "a" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "M2NkNeyIxMyI6Ik9EWmlZZXlJeE1pSTZOVFUxZlFqZ3labSJ9TVjYj", + "scenario": { + "commands": [ + { + "id": "down" + }, + { + "id": "down" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "ODhjZeyIxMyI6Ik9EWmlZZXlJeE1pSTZOVFUxZlFqZ3labSJ9DM3OG", + "scenario": { + "commands": [ + { + "id": "tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "NzdmZeyIxMyI6Ik9EWmlZZXlJeE1pSTZOVFUxZlFqZ3labSJ9jI1Nj", + "scenario": { + "commands": [ + { + "id": "tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "MDVkZeyIxMiI6NTU1fQDY0Mm", + "scenarioResults": [ + { + "id": "Y2NiOeyIxMyI6Ik1EVmtaZXlJeE1pSTZOVFUxZlFEWTBNbSJ9TRmZj", + "scenario": { + "commands": [ + { + "id": "shift+f" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "YzcxNeyIxMyI6Ik1EVmtaZXlJeE1pSTZOVFUxZlFEWTBNbSJ9DdkMz", + "scenario": { + "commands": [ + { + "id": "shift+a" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "OTVjMeyIxMyI6Ik1EVmtaZXlJeE1pSTZOVFUxZlFEWTBNbSJ9DA3OT", + "scenario": { + "commands": [ + { + "id": "up" + }, + { + "id": "up" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "YTY5OeyIxMyI6Ik1EVmtaZXlJeE1pSTZOVFUxZlFEWTBNbSJ9DY3YT", + "scenario": { + "commands": [ + { + "id": "shift+tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "ZDEyYeyIxMyI6Ik1EVmtaZXlJeE1pSTZOVFUxZlFEWTBNbSJ92EwZD", + "scenario": { + "commands": [ + { + "id": "shift+tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "ODk1NeyIxMiI6NTU1fQjYyMG", + "scenarioResults": [ + { + "id": "YWJiOeyIxMyI6Ik9EazFOZXlJeE1pSTZOVFUxZlFqWXlNRyJ9GU5Nz", + "scenario": { + "commands": [ + { + "id": "f" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "MDI0MeyIxMyI6Ik9EazFOZXlJeE1pSTZOVFUxZlFqWXlNRyJ9DE3OD", + "scenario": { + "commands": [ + { + "id": "a" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "NDJkMeyIxMyI6Ik9EazFOZXlJeE1pSTZOVFUxZlFqWXlNRyJ9jMxNj", + "scenario": { + "commands": [ + { + "id": "down" + }, + { + "id": "down" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "ZWRhYeyIxMyI6Ik9EazFOZXlJeE1pSTZOVFUxZlFqWXlNRyJ9TdjMG", + "scenario": { + "commands": [ + { + "id": "tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "NWJlYeyIxMyI6Ik9EazFOZXlJeE1pSTZOVFUxZlFqWXlNRyJ9zcyYW", + "scenario": { + "commands": [ + { + "id": "tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "Mzc5OeyIxMiI6NTU1fQTkzZD", + "scenarioResults": [ + { + "id": "MDM3ZeyIxMyI6Ik16YzVPZXlJeE1pSTZOVFUxZlFUa3paRCJ9jlkNT", + "scenario": { + "commands": [ + { + "id": "shift+f" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "MmY5MeyIxMyI6Ik16YzVPZXlJeE1pSTZOVFUxZlFUa3paRCJ92ExMG", + "scenario": { + "commands": [ + { + "id": "shift+a" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "OGNlOeyIxMyI6Ik16YzVPZXlJeE1pSTZOVFUxZlFUa3paRCJ9TdiNT", + "scenario": { + "commands": [ + { + "id": "up" + }, + { + "id": "up" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "NWNmYeyIxMyI6Ik16YzVPZXlJeE1pSTZOVFUxZlFUa3paRCJ9zVkOT", + "scenario": { + "commands": [ + { + "id": "shift+tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "NWE0MeyIxMyI6Ik16YzVPZXlJeE1pSTZOVFUxZlFUa3paRCJ9GUzND", + "scenario": { + "commands": [ + { + "id": "shift+tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "ZjIyMeyIxMiI6NTU1fQzJkOD", + "scenarioResults": [ + { + "id": "NTBiOeyIxMyI6IlpqSXlNZXlJeE1pSTZOVFUxZlF6SmtPRCJ9TQ4Nj", + "scenario": { + "commands": [ + { + "id": "shift+u" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [], + "mayAssertionResults": [ + { + "passed": false + } + ], + "unexpectedBehaviors": [] + }, + { + "id": "YWE4NeyIxMyI6IlpqSXlNZXlJeE1pSTZOVFUxZlF6SmtPRCJ9TY2YT", + "scenario": { + "commands": [ + { + "id": "up" + }, + { + "id": "up" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "NjY3ZeyIxMyI6IlpqSXlNZXlJeE1pSTZOVFUxZlF6SmtPRCJ9mQ3ZT", + "scenario": { + "commands": [ + { + "id": "shift+tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [], + "mayAssertionResults": [ + { + "passed": false + } + ], + "unexpectedBehaviors": [] + }, + { + "id": "YTVjYeyIxMyI6IlpqSXlNZXlJeE1pSTZOVFUxZlF6SmtPRCJ9mQxNT", + "scenario": { + "commands": [ + { + "id": "shift+tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [], + "mayAssertionResults": [ + { + "passed": false + } + ], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "ODdhOeyIxMiI6NTU1fQDI5NT", + "scenarioResults": [ + { + "id": "ZTIyNeyIxMyI6Ik9EZGhPZXlJeE1pSTZOVFUxZlFESTVOVCJ9mI2OT", + "scenario": { + "commands": [ + { + "id": "u" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [], + "mayAssertionResults": [ + { + "passed": false + } + ], + "unexpectedBehaviors": [] + }, + { + "id": "YzFjYeyIxMyI6Ik9EZGhPZXlJeE1pSTZOVFUxZlFESTVOVCJ9mM3NT", + "scenario": { + "commands": [ + { + "id": "down" + }, + { + "id": "down" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "M2U2ZeyIxMyI6Ik9EZGhPZXlJeE1pSTZOVFUxZlFESTVOVCJ9TdjNG", + "scenario": { + "commands": [ + { + "id": "tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [], + "mayAssertionResults": [ + { + "passed": false + } + ], + "unexpectedBehaviors": [] + }, + { + "id": "NzgxNeyIxMyI6Ik9EZGhPZXlJeE1pSTZOVFUxZlFESTVOVCJ92RiOG", + "scenario": { + "commands": [ + { + "id": "tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [], + "mayAssertionResults": [ + { + "passed": false + } + ], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "MDBiNeyIxMiI6NTU1fQDMwMD", + "scenarioResults": [ + { + "id": "MzgxZeyIxMyI6Ik1EQmlOZXlJeE1pSTZOVFUxZlFETXdNRCJ9WMyOT", + "scenario": { + "commands": [ + { + "id": "f" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "NTFlZeyIxMyI6Ik1EQmlOZXlJeE1pSTZOVFUxZlFETXdNRCJ9TRlZj", + "scenario": { + "commands": [ + { + "id": "a" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "ZmJhMeyIxMyI6Ik1EQmlOZXlJeE1pSTZOVFUxZlFETXdNRCJ92VkMD", + "scenario": { + "commands": [ + { + "id": "down" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "MDdhZeyIxMiI6NTU1fQDBlNz", + "scenarioResults": [ + { + "id": "NGU1NeyIxMyI6Ik1EZGhaZXlJeE1pSTZOVFUxZlFEQmxOeiJ9zcxY2", + "scenario": { + "commands": [ + { + "id": "shift+f" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "ODY2ZeyIxMyI6Ik1EZGhaZXlJeE1pSTZOVFUxZlFEQmxOeiJ9GMwMW", + "scenario": { + "commands": [ + { + "id": "shift+a" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "MzQ1OeyIxMyI6Ik1EZGhaZXlJeE1pSTZOVFUxZlFEQmxOeiJ9TQyNT", + "scenario": { + "commands": [ + { + "id": "up" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "NTBlNeyIxMiI6NTU1fQjY0Ym", + "scenarioResults": [ + { + "id": "NDlkYeyIxMyI6Ik5UQmxOZXlJeE1pSTZOVFUxZlFqWTBZbSJ9WI3Yz", + "scenario": { + "commands": [ + { + "id": "f" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "YjA0NeyIxMyI6Ik5UQmxOZXlJeE1pSTZOVFUxZlFqWTBZbSJ9jRlMm", + "scenario": { + "commands": [ + { + "id": "a" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "MDQ4ZeyIxMyI6Ik5UQmxOZXlJeE1pSTZOVFUxZlFqWTBZbSJ9jUxYm", + "scenario": { + "commands": [ + { + "id": "down" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "NzkwYeyIxMiI6NTU1fQmUzYj", + "scenarioResults": [ + { + "id": "OWM4ZeyIxMyI6Ik56a3dZZXlJeE1pSTZOVFUxZlFtVXpZaiJ9WJjNz", + "scenario": { + "commands": [ + { + "id": "shift+f" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "YTRjMeyIxMyI6Ik56a3dZZXlJeE1pSTZOVFUxZlFtVXpZaiJ9Tg4MD", + "scenario": { + "commands": [ + { + "id": "shift+a" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "MTZmZeyIxMyI6Ik56a3dZZXlJeE1pSTZOVFUxZlFtVXpZaiJ9jdjYW", + "scenario": { + "commands": [ + { + "id": "up" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "YmIzMeyIxMiI6NTU1fQWVlOT", + "scenarioResults": [ + { + "id": "ZjIwMeyIxMyI6IlltSXpNZXlJeE1pSTZOVFUxZlFXVmxPVCJ9DU1MT", + "scenario": { + "commands": [ + { + "id": "space" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + } + ], + "shouldAssertionResults": [], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "OTFjYeyIxMyI6IlltSXpNZXlJeE1pSTZOVFUxZlFXVmxPVCJ9zI1NG", + "scenario": { + "commands": [ + { + "id": "enter" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "mustAssertionResults": [ + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "MzFiYeyIxMyI6IlltSXpNZXlJeE1pSTZOVFUxZlFXVmxPVCJ9jQ3Yj", + "scenario": { + "commands": [ + { + "id": "space" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + } + ], + "shouldAssertionResults": [], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "NTZkMeyIxMiI6NTU1fQzJhY2", + "scenarioResults": [ + { + "id": "ZmE0ZeyIxMyI6Ik5UWmtNZXlJeE1pSTZOVFUxZlF6SmhZMiJ9jAxZD", + "scenario": { + "commands": [ + { + "id": "down" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "ZjE3OeyIxMyI6Ik5UWmtNZXlJeE1pSTZOVFUxZlF6SmhZMiJ9WQxZj", + "scenario": { + "commands": [ + { + "id": "right" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "ZTQ3NeyIxMiI6NTU1fQzNjND", + "scenarioResults": [ + { + "id": "YzJmOeyIxMyI6IlpUUTNOZXlJeE1pSTZOVFUxZlF6TmpORCJ9WQzYz", + "scenario": { + "commands": [ + { + "id": "up" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "NzJiZeyIxMyI6IlpUUTNOZXlJeE1pSTZOVFUxZlF6TmpORCJ9GNlNj", + "scenario": { + "commands": [ + { + "id": "left" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "M2U1MeyIxMiI6NTU1fQzIyNz", + "scenarioResults": [ + { + "id": "YzExMeyIxMyI6Ik0yVTFNZXlJeE1pSTZOVFUxZlF6SXlOeiJ92YwNm", + "scenario": { + "commands": [ + { + "id": "ins+tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "Yjc1YeyIxMyI6Ik0yVTFNZXlJeE1pSTZOVFUxZlF6SXlOeiJ92VjYT", + "scenario": { + "commands": [ + { + "id": "ins+up" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "NDdmZeyIxMyI6Ik0yVTFNZXlJeE1pSTZOVFUxZlF6SXlOeiJ9jg3Yz", + "scenario": { + "commands": [ + { + "id": "ins+tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "OGEwOeyIxMyI6Ik0yVTFNZXlJeE1pSTZOVFUxZlF6SXlOeiJ9WI3Yz", + "scenario": { + "commands": [ + { + "id": "ins+up" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + }, + { + "id": "YTFmZeyIxMiI6NTU1fQWFiZm", + "scenarioResults": [ + { + "id": "MDhiNeyIxMyI6IllURm1aZXlJeE1pSTZOVFUxZlFXRmlabSJ9TY4Nm", + "scenario": { + "commands": [ + { + "id": "ins+tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "ZGFmZeyIxMyI6IllURm1aZXlJeE1pSTZOVFUxZlFXRmlabSJ9Dk5Nj", + "scenario": { + "commands": [ + { + "id": "ins+up" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "MDM0YeyIxMyI6IllURm1aZXlJeE1pSTZOVFUxZlFXRmlabSJ9mU4ND", + "scenario": { + "commands": [ + { + "id": "ins+tab" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + }, + { + "id": "OWYzZeyIxMyI6IllURm1aZXlJeE1pSTZOVFUxZlFXRmlabSJ9jM0YT", + "scenario": { + "commands": [ + { + "id": "ins+up" + } + ] + }, + "assertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mustAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": true + } + ], + "shouldAssertionResults": [ + { + "passed": true + }, + { + "passed": true + }, + { + "passed": false + }, + { + "passed": false + } + ], + "mayAssertionResults": [], + "unexpectedBehaviors": [] + } + ] + } + ] +} diff --git a/shared/tests/calculations.test.js b/shared/tests/calculations.test.js new file mode 100644 index 000000000..07a8b7e9d --- /dev/null +++ b/shared/tests/calculations.test.js @@ -0,0 +1,59 @@ +const { calculatePercentage, trimDecimals } = require('../calculations'); + +describe('Verify calculatePercentage usage', () => { + it('should throw error if total value is 0', () => { + expect(() => { + calculatePercentage(25, 0, { ignoreError: false }); + }).toThrow(); + }); + + it('should return proper whole number', () => { + const value = 25; + const total = 100; + const result = calculatePercentage(value, total); + + expect(result).toBe(25); + }); + + it('should return precise value', () => { + const value = 205; + const total = 206; + const result = calculatePercentage(value, total); + + expect(result).toBe(99.51456310679612); + }); +}); + +describe('Verify trimDecimals usage', () => { + it('trims decimal places down to 2', () => { + const decimals = 2; + const value = 76.25845533342; + const result = trimDecimals(value, decimals); + + expect(result).toBe(76.25); + }); + + it('trims decimal places down to 1', () => { + const decimals = 1; + const value = 76.25845533342; + const result = trimDecimals(value, decimals); + + expect(result).toBe(76.2); + }); + + it('trims decimal places down to whole number if not provided with decimals', () => { + const decimals = 0; + const value = 76.25845533342; + const result = trimDecimals(value, decimals); + + expect(result).toBe(76); + }); + + it('returns whole number if given whole number with requested decimals', () => { + const decimals = 4; + const value = 76; + const result = trimDecimals(value, decimals); + + expect(result).toBe(76); + }); +}); diff --git a/shared/tests/getMetrics.test.js b/shared/tests/getMetrics.test.js new file mode 100644 index 000000000..10ac9f2fc --- /dev/null +++ b/shared/tests/getMetrics.test.js @@ -0,0 +1,303 @@ +const getMetrics = require('../getMetrics'); +const { calculatePercentage, trimDecimals } = require('../calculations'); +// eslint-disable-next-line jest/no-mocks-import +const testPlanReport = require('./__mocks__/testPlanReportForMetricsFromLiveData.json'); + +const generateRandomNumber = (max, min = 1) => { + return Math.floor(Math.random() * (max - min + 1)) + min; +}; + +const generateTestPlanReport = () => { + let testPlanReport = { + id: generateRandomNumber(200) + }; + + let runnableTests = []; + let finalizedTestResults = []; + + const testsCount = generateRandomNumber(12); + let testsPassedCount = 0; + let mustAssertionsCount = 0; + let mustAssertionsPassedCount = 0; + let mustAssertionsFailedCount = 0; + let shouldAssertionsCount = 0; + let shouldAssertionsPassedCount = 0; + let shouldAssertionsFailedCount = 0; + let mayAssertionsCount = 0; + let mayAssertionsPassedCount = 0; + let mayAssertionsFailedCount = 0; + let commandsCount = 0; + let unexpectedBehaviorCount = 0; + let severeImpactFailedAssertionCount = 0; + let moderateImpactFailedAssertionCount = 0; + + for (let runnableTestId = 1; runnableTestId <= testsCount; runnableTestId++) { + runnableTests.push({ id: runnableTestId }); + const finalizedTestResultId = runnableTestId; + + let isTestPassed = true; + + let scenarioResults = []; + const scenariosCount = generateRandomNumber(5); + for ( + let scenarioResultId = 1; + scenarioResultId <= scenariosCount; + scenarioResultId++ + ) { + let scenario = { + commands: [] + }; + let mustAssertionResults = []; + let shouldAssertionResults = []; + let mayAssertionResults = []; + let unexpectedBehaviors = []; + + const commandsLength = generateRandomNumber(6); + for (let commandId = 1; commandId <= commandsLength; commandId++) { + let command = { id: commandId }; + scenario.commands.push(command); + } + + const mustLength = generateRandomNumber(4, 1); + for (let mustIndex = 0; mustIndex < mustLength; mustIndex++) { + mustAssertionsCount++; + // To increase the chance of having at least 1 passed MUST assertion + const passed = Math.random() < 0.9; + if (passed) mustAssertionsPassedCount++; + else { + mustAssertionsFailedCount++; + isTestPassed = false; + } + + const assertion = { passed }; + mustAssertionResults.push(assertion); + } + + const shouldLength = generateRandomNumber(4, 0); + for (let shouldIndex = 0; shouldIndex < shouldLength; shouldIndex++) { + shouldAssertionsCount++; + const passed = Math.random() < 0.5; + if (passed) shouldAssertionsPassedCount++; + else { + shouldAssertionsFailedCount++; + isTestPassed = false; + } + + const assertion = { passed }; + shouldAssertionResults.push(assertion); + } + + const mayLength = generateRandomNumber(3, 0); + for (let mayIndex = 1; mayIndex <= mayLength; mayIndex++) { + mayAssertionsCount++; + const passed = Math.random() < 0.5; + if (passed) mayAssertionsPassedCount++; + else { + mayAssertionsFailedCount++; + isTestPassed = false; + } + + const assertion = { passed }; + mayAssertionResults.push(assertion); + } + + const unexpectedBehaviorsLength = generateRandomNumber(2, 0); + unexpectedBehaviorCount += unexpectedBehaviorsLength; + for ( + let unexpectedBehaviorIndex = 0; + unexpectedBehaviorIndex < unexpectedBehaviorsLength; + unexpectedBehaviorIndex++ + ) { + // otherwise, do moderate + const doSevere = Math.random() < 0.5; + const impact = doSevere ? 'SEVERE' : 'MODERATE'; + + const unexpectedBehavior = { impact }; + unexpectedBehaviors.push(unexpectedBehavior); + } + + if (unexpectedBehaviors.some(({ impact }) => impact === 'SEVERE')) + severeImpactFailedAssertionCount++; + if (unexpectedBehaviors.some(({ impact }) => impact === 'MODERATE')) + moderateImpactFailedAssertionCount++; + + const assertionResults = [ + ...mustAssertionResults, + ...shouldAssertionResults, + ...mayAssertionResults + ]; + + const scenarioResult = { + id: scenarioResultId, + scenario, + assertionResults, + mustAssertionResults, + shouldAssertionResults, + mayAssertionResults, + unexpectedBehaviors + }; + scenarioResults.push(scenarioResult); + } + + // Scenarios being equivalent to the amount of commands + commandsCount += scenarioResults.length; + if (isTestPassed) testsPassedCount++; + + const finalizedTestResult = { id: finalizedTestResultId, scenarioResults }; + finalizedTestResults.push(finalizedTestResult); + } + + testPlanReport = { ...testPlanReport, runnableTests, finalizedTestResults }; + + return { + testPlanReport, + testsCount, + testsPassedCount, + commandsCount, + mustAssertionsCount, + mustAssertionsPassedCount, + mustAssertionsFailedCount, + shouldAssertionsCount, + shouldAssertionsPassedCount, + shouldAssertionsFailedCount, + mayAssertionsCount, + mayAssertionsPassedCount, + mayAssertionsFailedCount, + unexpectedBehaviorCount, + severeImpactFailedAssertionCount, + moderateImpactFailedAssertionCount + }; +}; + +describe('getMetrics', () => { + it('returns expected metrics object for generated data', () => { + let { + testPlanReport, + testsCount, + testsPassedCount, + commandsCount, + mustAssertionsCount, + mustAssertionsPassedCount, + mustAssertionsFailedCount, + shouldAssertionsCount, + shouldAssertionsPassedCount, + shouldAssertionsFailedCount, + mayAssertionsCount, + mayAssertionsPassedCount, + mayAssertionsFailedCount, + unexpectedBehaviorCount, + severeImpactFailedAssertionCount, + moderateImpactFailedAssertionCount + } = generateTestPlanReport(); + const metrics = getMetrics({ testPlanReport }); + + const testsFailedCount = testsCount - testsPassedCount; + + const severeImpactPassedAssertionCount = + commandsCount - severeImpactFailedAssertionCount; + const moderateImpactPassedAssertionCount = + commandsCount - moderateImpactFailedAssertionCount; + + mustAssertionsCount += commandsCount; + mustAssertionsPassedCount += severeImpactPassedAssertionCount; + mustAssertionsFailedCount += severeImpactFailedAssertionCount; + + shouldAssertionsCount += commandsCount; + shouldAssertionsPassedCount += moderateImpactPassedAssertionCount; + shouldAssertionsFailedCount += moderateImpactFailedAssertionCount; + + const supportLevel = + unexpectedBehaviorCount > 0 || mustAssertionsFailedCount > 0 + ? 'FAILING' + : shouldAssertionsFailedCount > 0 + ? 'ALL_REQUIRED' + : 'FULL'; + + const percentage = calculatePercentage( + mustAssertionsPassedCount + shouldAssertionsPassedCount, + mustAssertionsCount + shouldAssertionsCount + ); + const supportPercent = trimDecimals(percentage); + + expect(metrics).toEqual( + expect.objectContaining({ + assertionsPassedCount: + mustAssertionsPassedCount + + shouldAssertionsPassedCount + + mayAssertionsPassedCount, + assertionsFailedCount: + mustAssertionsFailedCount + + shouldAssertionsFailedCount + + mayAssertionsFailedCount, + mustAssertionsPassedCount, + mustAssertionsCount, + mustAssertionsFailedCount, + shouldAssertionsPassedCount, + shouldAssertionsCount, + shouldAssertionsFailedCount, + mayAssertionsPassedCount, + mayAssertionsCount, + mayAssertionsFailedCount, + testsPassedCount, + testsCount, + testsFailedCount, + unexpectedBehaviorCount, + severeImpactPassedAssertionCount, + severeImpactFailedAssertionCount, + moderateImpactPassedAssertionCount, + moderateImpactFailedAssertionCount, + commandsCount, + mustFormatted: `${mustAssertionsPassedCount} of ${mustAssertionsCount} passed`, + shouldFormatted: + shouldAssertionsCount === 0 + ? false + : `${shouldAssertionsPassedCount} of ${shouldAssertionsCount} passed`, + mayFormatted: + mayAssertionsCount === 0 + ? false + : `${mayAssertionsPassedCount} of ${mayAssertionsCount} supported`, + unexpectedBehaviorsFormatted: + unexpectedBehaviorCount === 0 + ? false + : `${unexpectedBehaviorCount} found`, + supportLevel, + supportPercent + }) + ); + }); + + it('returns expected metrics object for testPlanReport with client data', () => { + const metrics = getMetrics({ testPlanReport }); + + expect(metrics).toEqual( + expect.objectContaining({ + assertionsPassedCount: 328, + assertionsFailedCount: 90, + mustAssertionsPassedCount: 206, + mustAssertionsCount: 206, + mustAssertionsFailedCount: 0, + shouldAssertionsPassedCount: 122, + shouldAssertionsCount: 206, + shouldAssertionsFailedCount: 84, + mayAssertionsPassedCount: 0, + mayAssertionsCount: 6, + mayAssertionsFailedCount: 6, + testsPassedCount: 2, + testsCount: 15, + testsFailedCount: 13, + unexpectedBehaviorCount: 0, + severeImpactPassedAssertionCount: 55, + severeImpactFailedAssertionCount: 0, + moderateImpactPassedAssertionCount: 55, + moderateImpactFailedAssertionCount: 0, + commandsCount: 55, + mustFormatted: '206 of 206 passed', + shouldFormatted: '122 of 206 passed', + mayFormatted: '0 of 6 supported', + unexpectedBehaviorsFormatted: false, + supportLevel: 'ALL_REQUIRED', + supportPercent: 79 + }) + ); + }); +});