Merge pull request #1179 from w3c/releases

Includes changes recently included in the [releases branch](https://github.com/w3c/aria-at-app/tree/releases) through #1178.

[Latest CHANGELOG.md update: v1.6.0](https://github.com/w3c/aria-at-app/blob/releases/CHANGELOG.md#160-2024-07-29)

CHANGELOG.md

@@ -1,3 +1,20 @@
+## [1.6.0](https://github.com/w3c/aria-at-app/compare/v1.5.0...v1.6.0) (2024-07-29)
+
+
+### Features
+
+* permit automated testing for firefox and chrome with voiceover ([#1170](https://github.com/w3c/aria-at-app/issues/1170)) ([9c73ab5](https://github.com/w3c/aria-at-app/commit/9c73ab516ac8f6bfbc0138191351ce84d680c732))
+
+
+### Bug Fixes
+
+* Update metrics calculations and related UI components ([#1172](https://github.com/w3c/aria-at-app/issues/1172)) ([e4242bc](https://github.com/w3c/aria-at-app/commit/e4242bcb38eec20eb50794f96c0bc8ae9a5aba97))
+
+
+### Reverts
+
+* revert accidental commit directly to develop ([d1c409c](https://github.com/w3c/aria-at-app/commit/d1c409c6f33005467f426771593c2c7eb00eef9b))
+
 ## [1.5.0](https://github.com/w3c/aria-at-app/compare/v1.4.0...v1.5.0) (2024-07-22)
 
 

client/components/AddTestToQueueWithConfirmation/index.jsx

@@ -166,7 +166,8 @@ function AddTestToQueueWithConfirmation({
           } catch (e) {
             console.error(e);
           }
-        }
+        },
+        testId: 'add-run-later'
       });
 
       if (!alreadyHasBotInTestPlanReport) {
@@ -186,7 +187,8 @@ function AddTestToQueueWithConfirmation({
             } catch (e) {
               console.error(e);
             }
-          }
+          },
+          testId: 'add-run-bot'
         });
       }
     }

client/components/CandidateReview/TestPlans/index.jsx

@@ -20,6 +20,7 @@ import {
 import ClippedProgressBar from '@components/common/ClippedProgressBar';
 import { convertDateToString } from '@client/utils/formatter';
 import './TestPlans.css';
+import { calculations } from 'shared';
 
 const FullHeightContainer = styled(Container)`
   min-height: calc(100vh - 64px);
@@ -486,13 +487,12 @@ const TestPlans = ({ testPlanVersions }) => {
                         obj.mustAssertionsFailedCount,
                       0
                     ),
-                    totalSupportPercent:
-                      Math.round(
-                        allMetrics.reduce(
-                          (acc, obj) => acc + obj.supportPercent,
-                          0
-                        ) / allMetrics.length
-                      ) || 0
+                    totalSupportPercent: calculations.trimDecimals(
+                      allMetrics.reduce(
+                        (acc, obj) => acc + obj.supportPercent,
+                        0
+                      ) / allMetrics.length
+                    )
                   };
 
                   // Make sure issues are unique
@@ -550,10 +550,10 @@ const TestPlans = ({ testPlanVersions }) => {
                         <CenteredTd>
                           <Link
                             to={`/candidate-test-plan/${testPlanVersion.id}/${atId}`}
+                            aria-label={`${metrics.totalSupportPercent}%`}
                           >
                             <ClippedProgressBar
                               progress={metrics.totalSupportPercent}
-                              label={`${metrics.totalSupportPercent}% completed`}
                               clipped
                             />
                           </Link>

client/components/Reports/SummarizeTestPlanReports.jsx

@@ -132,12 +132,10 @@ const SummarizeTestPlanReports = ({ testPlanVersions }) => {
                             `/report/${testPlanVersion.id}` +
                             `/targets/${testPlanReport.id}`
                           }
+                          aria-label={`${metrics.supportPercent}%`}
                         >
                           <ClippedProgressBar
                             progress={metrics.supportPercent}
-                            label={`${getTestPlanTargetTitle(
-                              testPlanTarget
-                            )}, ${metrics.supportPercent}% completed`}
                           />
                         </Link>
                       </td>

client/components/common/BasicModal/index.jsx

@@ -61,6 +61,7 @@ const BasicModal = ({
           variant={action.variant ?? 'primary'}
           onClick={action.onClick}
           className={action.className ?? ''}
+          data-testid={action.testId ?? ''}
         >
           {action.label ?? 'Continue'}
         </Button>
@@ -138,6 +139,7 @@ BasicModal.propTypes = {
       onClick: PropTypes.func,
       variant: PropTypes.string,
       className: PropTypes.string,
+      testId: PropTypes.string,
       component: PropTypes.elementType,
       props: PropTypes.object
     })

client/components/common/ClippedProgressBar/index.jsx

@@ -2,16 +2,11 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import './ClippedProgressBar.css';
 
-const ProgressBar = ({
-  progress = 0,
-  label = '',
-  clipped = true,
-  decorative
-}) => {
+const ProgressBar = ({ progress = 0, clipped = true, decorative }) => {
   return (
     <>
       {clipped ? (
-        <div className="progress clipped" aria-label={label}>
+        <div className="progress clipped">
           <div
             className="front"
             style={{
@@ -23,7 +18,7 @@ const ProgressBar = ({
           <div className="back">{decorative ? null : `${progress}%`}</div>
         </div>
       ) : (
-        <div className="progress" aria-label={label}>
+        <div className="progress">
           <div
             className="progress-bar bg-info"
             style={{
@@ -40,7 +35,6 @@ const ProgressBar = ({
 
 ProgressBar.propTypes = {
   progress: PropTypes.number,
-  label: PropTypes.string,
   clipped: PropTypes.bool,
   decorative: PropTypes.bool
 };

client/tests/AddTestToQueueWithConfirmation.test.jsx

@@ -111,6 +111,7 @@ describe('AddTestToQueueWithConfirmation', () => {
 
   test('calls mutation on button click with correct variables', async () => {
     fireEvent.click(getByTestId('add-button'));
+    fireEvent.click(await getByTestId('add-run-later'));
 
     await waitFor(() => {
       expect(mutationMock).toHaveBeenCalled();

client/utils/automation.js

@@ -19,10 +19,13 @@ export const isSupportedByResponseCollector = ctx => {
   // Check if the AT and browser are supported by the automation
   const isNvdaWithSupportedBrowser =
     atKey === 'nvda' && (browserKey === 'chrome' || browserKey === 'firefox');
-  const isVoiceOverMacWithSafari =
-    atKey === 'voiceover_macos' && browserKey === 'safari_macos';
+  const isVoiceOverMacWithSupportedBrowser =
+    atKey === 'voiceover_macos' &&
+    (browserKey === 'safari_macos' ||
+      browserKey === 'chrome' ||
+      browserKey === 'firefox');
 
-  if (!isNvdaWithSupportedBrowser && !isVoiceOverMacWithSafari) {
+  if (!isNvdaWithSupportedBrowser && !isVoiceOverMacWithSupportedBrowser) {
     return false;
   }
 
@@ -89,7 +92,12 @@ export const getBotUsernameFromAtBrowser = (at, browser) => {
   ) {
     return 'NVDA Bot';
   }
-  if (at?.key === 'voiceover_macos' && browser?.key === 'safari_macos') {
+  if (
+    at?.key === 'voiceover_macos' &&
+    (browser?.key === 'safari_macos' ||
+      browser?.key === 'chrome' ||
+      browser?.key === 'firefox')
+  ) {
     return 'VoiceOver Bot';
   }
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "aria-at-app",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Run ARIA-AT tests and report results",
   "main": "server/index.js",
   "private": true,

server/migrations/20240724212706-updateMetrics.js

@@ -0,0 +1,64 @@
+'use strict';
+
+const { getMetrics } = require('shared');
+const populateData = require('../services/PopulatedData/populateData');
+const runnableTestsResolver = require('../resolvers/TestPlanReport/runnableTestsResolver');
+const finalizedTestResultsResolver = require('../resolvers/TestPlanReport/finalizedTestResultsResolver');
+const {
+  updateTestPlanReportById
+} = require('../models/services/TestPlanReportService');
+const getGraphQLContext = require('../graphql-context');
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    return queryInterface.sequelize.transaction(async transaction => {
+      const context = getGraphQLContext({ req: { transaction } });
+
+      const testPlanReports = await queryInterface.sequelize.query(
+        `select distinct on ("TestPlanReport".id) "TestPlanReport".id, metrics
+         from "TestPlanReport"
+                join public."TestPlanRun" testPlanRun on "TestPlanReport".id = testPlanRun."testPlanReportId"
+         where jsonb_array_length(testPlanRun."testResults") > 0;`,
+        {
+          type: Sequelize.QueryTypes.SELECT,
+          transaction
+        }
+      );
+
+      for (const testPlanReport of testPlanReports) {
+        const { testPlanReport: testPlanReportPopulated } = await populateData(
+          { testPlanReportId: testPlanReport.id },
+          { context }
+        );
+        const runnableTests = runnableTestsResolver(
+          testPlanReportPopulated,
+          null,
+          context
+        );
+        const finalizedTestResults = await finalizedTestResultsResolver(
+          testPlanReportPopulated,
+          null,
+          context
+        );
+        const metrics = getMetrics({
+          testPlanReport: {
+            ...testPlanReportPopulated,
+            finalizedTestResults,
+            runnableTests
+          }
+        });
+        await updateTestPlanReportById({
+          id: testPlanReportPopulated.id,
+          values: {
+            metrics: {
+              ...testPlanReportPopulated.metrics,
+              ...metrics
+            }
+          },
+          transaction
+        });
+      }
+    });
+  }
+};

server/resolvers/TestPlanReport/finalizedTestResultsResolver.js

@@ -8,6 +8,9 @@ const finalizedTestResultsResolver = async (testPlanReport, _, context) => {
   // Return the primary test plan run, otherwise pick the first TestPlanRun found.
   const testPlanRun =
     testPlanReport.testPlanRuns.find(({ isPrimary }) => isPrimary) ||
+    testPlanReport.testPlanRuns.find(testPlanRun =>
+      testPlanRun.testResults?.some(testResult => !!testResult.completedAt)
+    ) ||
     testPlanReport.testPlanRuns[0];
 
   return testResultsResolver(

server/services/GithubWorkflowService.js

@@ -154,6 +154,7 @@ const createGithubWorkflow = async ({ job, directory, gitSha, atVersion }) => {
     // due to limitations on Github workflow runners
     // See https://github.com/w3c/aria-at-app/issues/1143 for more info
     inputs.macos_version = atVersion?.name?.split('.')[0];
+    inputs.browser = browser;
   }
   const axiosConfig = {
     method: 'POST',

shared/calculations.js

@@ -0,0 +1,26 @@
+/**
+ * @param {number} value
+ * @param {number} total
+ * @param {boolean} ignoreError - to account for cases where having a NaN is "expected"
+ * @returns {number}
+ */
+const calculatePercentage = (value, total, { ignoreError = true } = {}) => {
+  if (!ignoreError && total === 0) {
+    throw new Error("Unable to divide. 'total' cannot be 0.");
+  }
+  return (value / total) * 100;
+};
+
+const trimDecimals = (number, decimals = 0) => {
+  if (decimals === undefined || decimals <= 0) {
+    return Math.floor(number);
+  } else {
+    let factor = Math.pow(10, decimals);
+    return Math.floor(number * factor) / factor;
+  }
+};
+
+module.exports = {
+  calculatePercentage,
+  trimDecimals
+};

shared/getMetrics.js

@@ -1,4 +1,5 @@
 const convertAssertionPriority = require('./convertAssertionPriority');
+const { calculatePercentage, trimDecimals } = require('./calculations');
 
 const sum = arr => arr?.reduce((total, item) => total + item, 0) || 0;
 
@@ -46,11 +47,32 @@ const countAssertions = ({
     if (isClient) {
       all = scenarioResult?.[`${priority.toLowerCase()}AssertionResults`] || [];
     } else {
-      all = scenarioResult.assertionResults.filter(
-        a =>
+      all = scenarioResult.assertionResults.filter(a => {
+        // Same as `server/resolvers/ScenarioResult/assertionResultsResolver.js`
+        if (a.assertion?.assertionExceptions?.length) {
+          const scenarioSettings = scenarioResult.scenario.settings;
+          const scenarioCommandId =
+            scenarioResult.scenario.commandIds.join(' ');
+
+          const foundException = a.assertion.assertionExceptions.find(
+            exception =>
+              exception.settings === scenarioSettings &&
+              exception.commandId === scenarioCommandId
+          );
+
+          if (foundException) {
+            return (
+              convertAssertionPriority(foundException.priority) ===
+              convertAssertionPriority(priority)
+            );
+          }
+        }
+
+        return (
           convertAssertionPriority(a.assertion.priority) ===
           convertAssertionPriority(priority)
-      );
+        );
+      });
     }
     if (passedOnly) return all.filter(each => each.passed).length;
     return all.length;
@@ -236,9 +258,11 @@ const getMetrics = ({
     supportLevel = 'FULL';
   }
 
-  const supportPercent = Math.round(
-    (mustAssertionsPassedCount / mustAssertionsCount) * 100
+  const percentage = calculatePercentage(
+    mustAssertionsPassedCount + shouldAssertionsPassedCount,
+    mustAssertionsCount + shouldAssertionsCount
   );
+  const supportPercent = trimDecimals(percentage);
 
   const assertionsPassedCount =
     mustAssertionsPassedCount +

shared/index.js

@@ -1,3 +1,5 @@
+const calculations = require('./calculations');
 const convertAssertionPriority = require('./convertAssertionPriority');
 const getMetrics = require('./getMetrics');
-module.exports = { convertAssertionPriority, getMetrics };
+
+module.exports = { calculations, convertAssertionPriority, getMetrics };