From e6583b1767a8e9254dacbf2a30836bd28ce4c5a6 Mon Sep 17 00:00:00 2001 From: alflennik Date: Mon, 6 May 2024 12:43:25 -0400 Subject: [PATCH 01/19] docs: Add a step to make sure changelog is properly generated --- deploy/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 821c4cbe5..137e72dcf 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -49,6 +49,7 @@ can view logging from ansible with `sudo -i cat /var/log/messages`. To deploy this project to server: +1. Merge from development to the releases branch as detailed in [docs/release.md](../docs/release.md). 1. Obtain an authorized key and add it to your keychain. This is needed for deploys to Staging and Production. - The shared key is named `aria-at-bocoup`. - Place it in the ~/.ssh directory. @@ -63,11 +64,11 @@ To deploy this project to server: - Run `ssh root@bestla.w3.internal` (aria-at-staging.w3.org's server) and `ssh root@fenrir.w3.internal` (aria-at.w3.org's server) to verify that you can connect to the servers. - The RSA key fingerprint for `bestla.w3.internal` is `SHA256:F16aX2Wx4e39jbHhqEkeH8iRgY41C3WgxvAgvh7PQZ0`. - The RSA key fingerprint for `fenrir.w3.internal` is `SHA256:cF6u/K00P2ELEVbIazVVqqMz5q+Sbh4+Jog/VmXZomg`. -2. Bocoup maintains its own instance of the app on its internal infrastructure for quick and easy testing. Note that you must be a Bocouper to deploy to this environment. Follow the steps below to verify you are able to connect. +1. Bocoup maintains its own instance of the app on its internal infrastructure for quick and easy testing. Note that you must be a Bocouper to deploy to this environment. Follow the steps below to verify you are able to connect. - Run `ssh aria-at-app-sandbox.bocoup.com` and confirm you can connect. - Confirm that `sudo su` successfully switches you to the root user. You will need to enter the sudo password you chose during your Bocoup onboarding. This password will be required when deploying to the Sandbox. -3. Obtain a copy of the `ansible-vault-password.txt` file in LastPass and place it in the directory which contains this document. -4. Install [Ansible](https://www.ansible.com/) version 2.11. Instructions for macOS are as follows: +1. Obtain a copy of the `ansible-vault-password.txt` file in LastPass and place it in the directory which contains this document. +1. Install [Ansible](https://www.ansible.com/) version 2.11. Instructions for macOS are as follows: - Install Ansible at the specific 2.11 version: `python3 -m pip install --user ansible-core==2.11.1` - Add the following line to your `~/.zshrc` file, changing the path below to match where Python installs Ansible for you: ``` @@ -76,7 +77,7 @@ To deploy this project to server: - Run `source ~/.zshrc` to refresh your shell. - Install `ansible.posix` to make use of the [ansible.posix.synchronize](https://docs.ansible.com/ansible/latest/collections/ansible/posix/synchronize_module.html#ansible-posix-synchronize-module-a-wrapper-around-rsync-to-make-common-tasks-in-your-playbooks-quick-and-easy) module: `ansible-galaxy collection install ansible.posix` - Run `ansible --version` to verify your ansible is on version 2.11. -5. Execute the following command from the deploy directory: +1. Execute the following command from the deploy directory: - Sandbox: ``` ansible-playbook provision.yml --inventory inventory/sandbox.yml From b8cefcdae71963ee2fe8ac6bc6c65bbe8ef00c7e Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Mon, 6 May 2024 09:56:37 -0700 Subject: [PATCH 02/19] feat: Redesign support table to match APG styling, addresses #212 (#1056) * redesign warning * Fix position and semantics * FIx header level * Fix position and heading level * Align warning arrow * Fix warning positioning, font, and color --- server/handlebars/embed/public/style.css | 284 +++++++++++++---------- server/handlebars/embed/views/main.hbs | 2 +- 2 files changed, 166 insertions(+), 120 deletions(-) diff --git a/server/handlebars/embed/public/style.css b/server/handlebars/embed/public/style.css index 9778cd559..02117c5b6 100644 --- a/server/handlebars/embed/public/style.css +++ b/server/handlebars/embed/public/style.css @@ -1,37 +1,39 @@ +@import url('https://fonts.googleapis.com/css2?family=Montserrat:ital,wght@0,100;0,300;0,600;0,800;1,700&family=Noto+Sans:ital,wght@0,100..900;1,100..900&family=Roboto:wght@100;400&display=swap'); + body { margin: 0; /* Parent should use the posted message and set an explicit height */ overflow: hidden; -} - -#main { + } + + #main { width: 100%; -} - -a { + } + + a { color: #036; -} -a:focus-visible { + } + a:focus-visible { outline: 2px solid #3a86d1; -} - -h3#report-title { + } + + h3#report-title { margin-top: 0; margin-bottom: 0.5rem; - + font-family: Arial, Helvetica, sans-serif; font-size: 18px; -} - -#copied-message { + } + + #copied-message { align-self: center; -} - -#at-version { + } + + #at-version { font-weight: lighter; -} - -#no-data-content-container { + } + + #no-data-content-container { border-left: 1.5px solid #c0c0c0; border-right: 1.5px solid #c0c0c0; border-bottom: 1.5px solid #c0c0c0; @@ -41,105 +43,148 @@ h3#report-title { font-size: 14px; margin-bottom: 1em; text-align: center; -} - -#embed-report-phase-container { + } + + #embed-report-phase-container { margin-bottom: 1em; -} -#embed-report-phase-container summary:focus-visible { + } + #embed-report-phase-container summary:focus-visible { outline-offset: -2px; outline: 2px solid #3a86d1; -} - -#candidate-title { - border: 1.5px solid #ffc4a2; + } + + #embed-report-phase-container summary::after { + width: 30px; + height: 30px; + border-radius: 3px; + display: block; + content: ''; + background-position: center center; + background-repeat: no-repeat; + position: absolute; + right: 0.5em; + top: 1.5em; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='6.861' viewBox='0 0 12 6.861'%3E%3Cpath id='Icon_ionic-ios-arrow-down' data-name='Icon ionic-ios-arrow-down' d='M12.19,16.039,16.727,11.5a.854.854,0,0,1,1.211,0,.865.865,0,0,1,0,1.215L12.8,17.858a.856.856,0,0,1-1.183.025L6.438,12.717A.858.858,0,1,1,7.649,11.5Z' transform='translate(18.188 18.108) rotate(180)' fill='%2360470c'/%3E%3C/svg%3E%0A"); + background-color: #fce1a4; + transform: rotate(180deg); + } + + #embed-report-phase-container[open] summary::after { + transform: rotate(0deg); + } + + /* section: CANDIDATE WARNING STYLING */ + summary { + display: block; + } + + #candidate-title { + background-color:#fcebc3; + border: 1.5px solid #fce1a4; border-top-left-radius: 3px; border-top-right-radius: 3px; - background-color: #ffd8c1; padding: 0.5em 0.5em 0.5em 1em; cursor: pointer; -} - -#candidate-title > span { + --space-between-elements: 1em; + --left-right-padding: 55px; + } + + #candidate-title > span { + position: relative; + padding-left: var(--left-right-padding); + padding-right: var(--left-right-padding); + } + + #candidate-title > span > h4 { font-family: Arial, Helvetica, sans-serif; - font-size: 12px; - font-weight: bold; - color: white; - background-color: #c25401; - padding: 8px 14px; - border-radius: 15px; + font-size: 1em; + color: #60470C; display: inline-block; -} - -#candidate-title.recommended { + } + + #candidate-title.recommended { border: 1.5px solid #7ac498; background-color: #e9fbe9; -} -#candidate-title.recommended > span { + } + #candidate-title.recommended > span { background-color: #115b11; -} - -#candidate-content-container { - border-left: 1.5px solid #c0c0c0; - border-right: 1.5px solid #c0c0c0; - border-bottom: 1.5px solid #c0c0c0; + } + + #candidate-content-container { border-bottom-left-radius: 3px; border-bottom-right-radius: 3px; padding: 1em 0.5em 0.5em 1em; - font-family: Arial, Helvetica, sans-serif; - font-size: 14px; -} - -#candidate-content-container > ol > li:not(:last-child) { + font-family: Noto Sans, Trebuchet MS, Helvetica Neue, Arial, sans-serif; + font-size: 1rem; + color: #60470C; + background-color: #fdefce; + border-left: 1px solid #fce1a4; + border-right: 1px solid #fce1a4; + border-bottom: 1px solid #fce1a4; + } + + #candidate-content-container > ol > li:not(:last-child) { margin-bottom: 3px; -} - -.no-data-cell { + } + + #candidate-title > ::before { + width: 32px; + height: 32px; + display: block; + content: ''; + background-repeat: no-repeat; + position: absolute; + left: 5px; + top: -7px; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='30' height='31' viewBox='0 0 30 31'%3E%3Cg id='Group_1' data-name='Group 1' transform='translate(-11 -292)'%3E%3Cellipse id='Ellipse_1' data-name='Ellipse 1' cx='15' cy='15.5' rx='15' ry='15.5' transform='translate(11 292)' fill='%23d9ae4d'/%3E%3Cpath id='Path_3' data-name='Path 3' d='M4.524-17.368a4.507,4.507,0,0,0-1.56.312.278.278,0,0,0-.13.286V-14.9c0,2.184.416,8.294.468,9.022a.172.172,0,0,0,.182.156h2.08c.156,0,.182-.078.182-.156.078-.78.52-6.916.52-9.1v-1.872a.326.326,0,0,0-.156-.286A5.112,5.112,0,0,0,4.524-17.368Zm0,17.576A1.7,1.7,0,0,0,6.4-1.534,1.741,1.741,0,0,0,4.524-3.328,1.77,1.77,0,0,0,2.652-1.534,1.7,1.7,0,0,0,4.524.208Z' transform='translate(22 316)' fill='%23fff'/%3E%3C/g%3E%3C/svg%3E%0A"); + } + + .no-data-cell { display: block; color: #72777f; font-style: italic; text-align: center; -} - -.responsive-table { + } + + .responsive-table { overflow-x: auto; width: 100%; margin-bottom: 1em; -} - -table { + } + + table { border-collapse: collapse; border-top: 1.5px solid #c0c0c0; width: 100%; -} - -table td, -th { + } + + table td, + th { padding: 1rem; border: 1.5px solid #c0c0c0; -} - -th, -td { + } + + th, + td { font-family: Arial, Helvetica, sans-serif; padding: 15px; border: 1.5px solid #c0c0c0; -} - -th { + } + + th { background-color: #f2f2f2; -} - -table tbody tr th { + } + + table tbody tr th { text-align: left; -} - -table tbody tr td { + } + + table tbody tr td { padding: 20px; text-align: center; -} - -.meter { + } + + .meter { box-sizing: content-box; height: 15px; /* Can be anything */ position: relative; @@ -147,18 +192,18 @@ table tbody tr td { border-radius: 25px; margin-bottom: 10px; overflow: hidden; -} - -.meter > span { + } + + .meter > span { display: block; height: 100%; float: left; background-color: #175a6a; border-radius: 25px; position: relative; -} - -.button { + } + + .button { border: none; padding: 10px 20px; font-family: Arial, Helvetica, sans-serif; @@ -167,56 +212,57 @@ table tbody tr td { cursor: pointer; border-radius: 3px; border: 3px solid #175a6a; -} - -#view-report-button { + } + + #view-report-button { color: white; background-color: #175a6a; margin-right: 6px; -} -#view-report-button:hover { + } + #view-report-button:hover { background-color: #024e5c; border-color: #024e5c; -} -#view-report-button:focus-visible { + } + #view-report-button:focus-visible { margin-left: 4px; padding: 0 8px 0 12px; -} -#embed-button { + } + #embed-button { color: #175a6a; background-color: white; margin-left: 4px; -} -#embed-button:hover { + } + #embed-button:hover { background-color: #f4f4f4; -} - -#view-report-button, -#embed-button { + } + + #view-report-button, + #embed-button { margin-bottom: 1em; padding: 0 12px; line-height: 36px; -} -#view-report-button:focus-visible, -#embed-button:focus-visible { + } + #view-report-button:focus-visible, + #embed-button:focus-visible { outline-offset: 2px; outline: 2px solid #3a86d1; -} -#view-report-button svg, -#embed-button svg { + } + #view-report-button svg, + #embed-button svg { width: 24px; margin-right: 8px; float: left; position: relative; top: 6px; -} - -#embed-button-wrap { + } + + #embed-button-wrap { display: inline-block; -} - -#copied-message { + } + + #copied-message { font-family: Arial, Helvetica, sans-serif; margin: 5px; display: inline-block; -} + } + \ No newline at end of file diff --git a/server/handlebars/embed/views/main.hbs b/server/handlebars/embed/views/main.hbs index ca8459d4c..932bb6ba2 100644 --- a/server/handlebars/embed/views/main.hbs +++ b/server/handlebars/embed/views/main.hbs @@ -9,7 +9,7 @@ {{#unless dataEmpty}} {{#if (isCandidate phase)}}
- Warning! Unapproved Report +

Warning! Unapproved Report
The information in this report is generated from candidate tests developed and run by the ARIA-AT Project. Candidate ARIA-AT tests are in review by assistive technology developers and lack consensus regarding: From 01c818f5f4ee0310c85d3b6e30585cb6a594c7c2 Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Mon, 6 May 2024 12:08:34 -0700 Subject: [PATCH 03/19] fix: Correct sorting for the tester assignment menu, addresses #993 (#1067) * Fix tester menu sorting * Use localeCompare * Sort initial tester assign dropdown * Remove debugging code --------- Co-authored-by: Howard Edwards Co-authored-by: alflennik --- client/components/TestQueue/index.jsx | 4 +++- client/components/TestQueueRow/index.jsx | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client/components/TestQueue/index.jsx b/client/components/TestQueue/index.jsx index da9f8a3fd..80b3745f1 100644 --- a/client/components/TestQueue/index.jsx +++ b/client/components/TestQueue/index.jsx @@ -114,7 +114,9 @@ const TestQueue = () => { + a.username.localeCompare(b.username) + )} testPlanReportData={testPlanReport} latestTestPlanVersions={ latestTestPlanVersions diff --git a/client/components/TestQueueRow/index.jsx b/client/components/TestQueueRow/index.jsx index 0c449fb90..208c32673 100644 --- a/client/components/TestQueueRow/index.jsx +++ b/client/components/TestQueueRow/index.jsx @@ -241,7 +241,7 @@ const TestQueueRow = ({ {draftTestPlanRuns .slice() // because array was frozen .sort((a, b) => - a.tester.username < b.tester.username ? -1 : 1 + a.tester.username.localeCompare(b.tester.username) ) .map(({ tester }) => { return ( From c51bd9c9d82e1e891c25b08c3777eb787766438a Mon Sep 17 00:00:00 2001 From: Howard Edwards Date: Tue, 7 May 2024 09:53:14 -0400 Subject: [PATCH 04/19] Update deploy/README.md --- deploy/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/README.md b/deploy/README.md index 137e72dcf..41b7a707a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -47,7 +47,7 @@ can view logging from ansible with `sudo -i cat /var/log/messages`. ## Deployment -To deploy this project to server: +To deploy this project to server: 1. Merge from development to the releases branch as detailed in [docs/release.md](../docs/release.md). 1. Obtain an authorized key and add it to your keychain. This is needed for deploys to Staging and Production. From 8429aa6a5510a45580b94358faa0af23e4444e3d Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Wed, 8 May 2024 06:11:00 -0700 Subject: [PATCH 05/19] fix: Focus first item in menu dropdowns (#1089) Address #992 --- client/components/TestQueue/AssignTesterDropdown/index.jsx | 2 +- client/components/TestQueueRow/index.jsx | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/client/components/TestQueue/AssignTesterDropdown/index.jsx b/client/components/TestQueue/AssignTesterDropdown/index.jsx index a98df94f1..a21e5b226 100644 --- a/client/components/TestQueue/AssignTesterDropdown/index.jsx +++ b/client/components/TestQueue/AssignTesterDropdown/index.jsx @@ -115,7 +115,7 @@ const AssignTesterDropdown = ({ }; return ( - + { return ( - + - + Date: Wed, 8 May 2024 19:20:50 -0400 Subject: [PATCH 06/19] Collection Job - Per-test status updates (#1000) * Per-Test status updates - basic queries and endpoints * updates for tests - new urls and checking testStatus * Add test for test-status-only update * more testing for per-test-status crud * add tests for old urls/format * Working queries and updates to get BotRunTestStatusList to work * more updates for test queue page view * update tests to pass * ensure recursive query fails * update to new urls * Remove back-compat URLs * testing tests on github * remaining tests * correct test mock --- .../components/BotRunTestStatusList/index.js | 152 +++++++----------- .../BotRunTestStatusList/queries.js | 15 +- .../common/ReportStatusDot/index.jsx | 13 +- client/tests/BotRunTestStatusList.test.jsx | 132 ++++++++------- server/controllers/AutomationController.js | 137 ++++++++++------ server/graphql-schema.js | 23 +++ ...240404171101-addCollectionJobTestStatus.js | 58 +++++++ server/models/CollectionJob.js | 6 + server/models/CollectionJobTestStatus.js | 56 +++++++ .../models/services/CollectionJobService.js | 87 +++++++++- server/models/services/helpers.js | 8 +- server/resolvers/CollectionJob/index.js | 3 + .../CollectionJob/testPlanRunResolver.js | 10 ++ .../CollectionJob/testStatusResolver.js | 15 ++ .../cancelCollectionJobResolver.js | 8 +- .../TestPlanRun/collectionJobResolver.js | 19 +++ server/resolvers/TestPlanRun/index.js | 3 +- server/resolvers/index.js | 2 + server/routes/automation.js | 9 +- server/services/GithubWorkflowService.js | 4 +- .../integration/automation-scheduler.test.js | 148 +++++++++++++++-- server/tests/integration/graphql.test.js | 32 +++- .../util/mock-automation-scheduler-server.js | 74 ++++++--- server/util/enums.js | 8 +- 24 files changed, 742 insertions(+), 280 deletions(-) create mode 100644 server/migrations/20240404171101-addCollectionJobTestStatus.js create mode 100644 server/models/CollectionJobTestStatus.js create mode 100644 server/resolvers/CollectionJob/index.js create mode 100644 server/resolvers/CollectionJob/testPlanRunResolver.js create mode 100644 server/resolvers/CollectionJob/testStatusResolver.js create mode 100644 server/resolvers/TestPlanRun/collectionJobResolver.js diff --git a/client/components/BotRunTestStatusList/index.js b/client/components/BotRunTestStatusList/index.js index 4fb00a729..6d1873c50 100644 --- a/client/components/BotRunTestStatusList/index.js +++ b/client/components/BotRunTestStatusList/index.js @@ -1,13 +1,9 @@ -import React, { useEffect, useMemo, useRef, useState } from 'react'; +import React, { useMemo } from 'react'; import PropTypes from 'prop-types'; -import { - COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY, - TEST_PLAN_RUNS_TEST_RESULTS_QUERY -} from './queries'; -import { useLazyQuery, useQuery } from '@apollo/client'; +import { TEST_PLAN_RUNS_TEST_RESULTS_QUERY } from './queries'; +import { useQuery } from '@apollo/client'; import styled from '@emotion/styled'; import ReportStatusDot from '../common/ReportStatusDot'; -import { isBot } from '../../utils/automation'; const BotRunTestStatusUnorderedList = styled.ul` list-style-type: none; @@ -35,7 +31,9 @@ const BotRunTestStatusUnorderedList = styled.ul` const testCountString = (count, status) => `${count} Test${count === 1 ? '' : 's'} ${status}`; -const BotRunTestStatusList = ({ testPlanReportId, runnableTestsLength }) => { +const pollInterval = 2000; + +const BotRunTestStatusList = ({ testPlanReportId }) => { const { data: testPlanRunsQueryResult, startPolling, @@ -43,113 +41,83 @@ const BotRunTestStatusList = ({ testPlanReportId, runnableTestsLength }) => { } = useQuery(TEST_PLAN_RUNS_TEST_RESULTS_QUERY, { variables: { testPlanReportId }, fetchPolicy: 'cache-and-network', - pollInterval: 2000 + pollInterval }); - const [getCollectionJobStatus, { data: collectionJobStatusQueryResult }] = - useLazyQuery(COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY, { - fetchPolicy: 'cache-and-network' - }); - - const [collectedData, setCollectedData] = useState([]); - const requestedTestRunIds = useRef(new Set()); - - const botTestPlanRuns = useMemo(() => { - if (!testPlanRunsQueryResult?.testPlanRuns) { - return []; - } - return testPlanRunsQueryResult.testPlanRuns.filter(testPlanRun => - isBot(testPlanRun.tester) - ); - }, [testPlanRunsQueryResult?.testPlanRuns]); - - useEffect(() => { - const ids = botTestPlanRuns.map(run => run.id); - for (const id of ids) { - if (!requestedTestRunIds.current.has(id)) { - requestedTestRunIds.current.add(id); - getCollectionJobStatus({ - variables: { testPlanRunId: id } - }); - } - } - }, [botTestPlanRuns]); - - useEffect(() => { - if (collectionJobStatusQueryResult?.collectionJobByTestPlanRunId) { - const { status } = - collectionJobStatusQueryResult.collectionJobByTestPlanRunId; - setCollectedData(prev => [...prev, status]); - } - }, [collectionJobStatusQueryResult?.collectionJobByTestPlanRunId]); - - const [numTestsCompleted, numTestsQueued, numTestsCancelled] = - useMemo(() => { - const res = [0, 0, 0]; - if ( - botTestPlanRuns && - botTestPlanRuns.length && - collectedData.length === botTestPlanRuns.length - ) { - for (let i = 0; i < botTestPlanRuns.length; i++) { - const status = collectedData[i]; - res[0] += botTestPlanRuns[i].testResults.length; - switch (status) { - case 'COMPLETED': - case 'RUNNING': - case 'QUEUED': - res[1] += - runnableTestsLength - - botTestPlanRuns[i].testResults.length; - break; - case 'CANCELLED': - res[2] += - runnableTestsLength - - botTestPlanRuns[i].testResults.length; - break; - default: - break; + const { COMPLETED, ERROR, RUNNING, CANCELLED, QUEUED } = useMemo(() => { + const counter = { + COMPLETED: 0, + ERROR: 0, + RUNNING: 0, + CANCELLED: 0, + QUEUED: 0 + }; + let anyPossibleUpdates = false; + if (testPlanRunsQueryResult?.testPlanRuns) { + for (const { + collectionJob + } of testPlanRunsQueryResult.testPlanRuns) { + if (collectionJob?.testStatus) { + for (const { status } of collectionJob.testStatus) { + counter[status]++; + if (status === 'QUEUED' || status === 'RUNNING') { + anyPossibleUpdates = true; + } } } - if ( - res[0] + res[2] === - runnableTestsLength * botTestPlanRuns.length - ) { - stopPolling(); - } } - return res; - }, [testPlanRunsQueryResult, collectedData, stopPolling, startPolling]); + // it's possible that we got incomplete data on first fetch and + // stopped the polling, so restart the polling if we detect any + // possible future updates, otherwise stop. + if (anyPossibleUpdates) { + startPolling(pollInterval); + } else { + stopPolling(); + } + } + return counter; + }, [testPlanRunsQueryResult, stopPolling, startPolling]); if ( - !botTestPlanRuns || - botTestPlanRuns.length === 0 || - !collectedData || - !(collectedData.length === botTestPlanRuns.length) + !testPlanRunsQueryResult || + testPlanRunsQueryResult.testPlanRuns.length === 0 ) { return null; } return ( + {RUNNING > 0 && ( +
  • + + {testCountString(RUNNING, 'Running')} +
    • + )} + {ERROR > 0 && ( +
  • + + {testCountString(ERROR, 'Error')} +
    • + )}
  • - {testCountString(numTestsCompleted, 'Completed')} + {testCountString(COMPLETED, 'Completed')}
  • - {testCountString(numTestsQueued, 'Queued')} -
    • -
  • - - {testCountString(numTestsCancelled, 'Cancelled')} + {testCountString(QUEUED, 'Queued')}
    • + {CANCELLED > 0 && ( +
  • + + {testCountString(CANCELLED, 'Cancelled')} +
    • + )}     ); }; BotRunTestStatusList.propTypes = { - testPlanReportId: PropTypes.string.isRequired, - runnableTestsLength: PropTypes.number.isRequired + testPlanReportId: PropTypes.string.isRequired }; export default BotRunTestStatusList; diff --git a/client/components/BotRunTestStatusList/queries.js b/client/components/BotRunTestStatusList/queries.js index 048acc1aa..cd99fb2bc 100644 --- a/client/components/BotRunTestStatusList/queries.js +++ b/client/components/BotRunTestStatusList/queries.js @@ -16,15 +16,12 @@ export const TEST_PLAN_RUNS_TEST_RESULTS_QUERY = gql` } } } - } - } -`; - -export const COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY = gql` - query CollectionJobStatusByTestPlanRunId($testPlanRunId: ID!) { - collectionJobByTestPlanRunId(testPlanRunId: $testPlanRunId) { - id - status + collectionJob { + status + testStatus { + status + } + } } } `; diff --git a/client/components/common/ReportStatusDot/index.jsx b/client/components/common/ReportStatusDot/index.jsx index d369e91ad..0c44c6521 100644 --- a/client/components/common/ReportStatusDot/index.jsx +++ b/client/components/common/ReportStatusDot/index.jsx @@ -17,6 +17,15 @@ const ReportStatusDot = styled.span` background: #7c7c7c; } + &.tests-running { + border: 2px solid #1e8f37; + background: #d2d5d9; + } + + &.tests-error { + background: #e3261f; + } + &.tests-queued, &.reports-in-progress { background: #3876e8; @@ -27,7 +36,9 @@ const ReportStatusDot = styled.span` background: #2ba51c; } - &.tests-cancelled, + &.tests-cancelled { + background: #a231ff; + } &.reports-missing { background: #ce1b4c; } diff --git a/client/tests/BotRunTestStatusList.test.jsx b/client/tests/BotRunTestStatusList.test.jsx index 66b93e03d..2aec3b4c8 100644 --- a/client/tests/BotRunTestStatusList.test.jsx +++ b/client/tests/BotRunTestStatusList.test.jsx @@ -5,13 +5,11 @@ import React from 'react'; import { render, waitFor } from '@testing-library/react'; import { MockedProvider } from '@apollo/client/testing'; import BotRunTestStatusList from '../components/BotRunTestStatusList'; -import { - TEST_PLAN_RUNS_TEST_RESULTS_QUERY, - COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY -} from '../components/BotRunTestStatusList/queries'; +import { TEST_PLAN_RUNS_TEST_RESULTS_QUERY } from '../components/BotRunTestStatusList/queries'; import '@testing-library/jest-dom/extend-expect'; +import { COLLECTION_JOB_STATUS } from '../../server/util/enums'; -const getMocks = (testPlanRuns, collectionJobStatuses) => { +const getMocks = testPlanRuns => { const testPlanRunMock = { request: { query: TEST_PLAN_RUNS_TEST_RESULTS_QUERY, @@ -20,22 +18,7 @@ const getMocks = (testPlanRuns, collectionJobStatuses) => { result: { data: { testPlanRuns } } }; - const collectionJobStatusMocks = testPlanRuns.map((testRun, index) => ({ - request: { - query: COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY, - variables: { testPlanRunId: testRun.id } - }, - result: { - data: { - collectionJobByTestPlanRunId: { - status: collectionJobStatuses[index], - id: testRun.id - } - } - } - })); - - return [testPlanRunMock, ...collectionJobStatusMocks]; + return [testPlanRunMock]; }; test('correctly displays statuses for single COMPLETED test run', async () => { @@ -43,27 +26,31 @@ test('correctly displays statuses for single COMPLETED test run', async () => { { id: '0', testResults: new Array(3).fill(null), - tester: { username: 'bot' } + tester: { username: 'bot' }, + collectionJob: { + status: COLLECTION_JOB_STATUS.COMPLETED, + testStatus: [ + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.COMPLETED } + ] + } } ]; - const collectionJobStatuses = ['COMPLETED']; + const mocks = getMocks(testPlanRuns); - const mocks = getMocks(testPlanRuns, collectionJobStatuses); - - const { getByText } = render( + const screen = render( - + ); + const { getByText } = screen; + await waitFor(() => { expect(getByText('3 Tests Completed')).toBeInTheDocument(); expect(getByText('0 Tests Queued')).toBeInTheDocument(); - expect(getByText('0 Tests Cancelled')).toBeInTheDocument(); }); }); @@ -72,32 +59,34 @@ test('correctly ignores test results from a human-submitted test plan run', asyn { id: '0', testResults: new Array(2).fill(null), - tester: { username: 'bot' } + tester: { username: 'bot' }, + collectionJob: { + status: COLLECTION_JOB_STATUS.COMPLETED, + testStatus: [ + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.COMPLETED } + ] + } }, { id: '1', testResults: new Array(2).fill(null), - tester: { username: 'human' } + tester: { username: 'human' }, + collectionJob: null } ]; - const collectionJobStatuses = ['COMPLETED', 'COMPLETED']; - - const mocks = getMocks(testPlanRuns, collectionJobStatuses); + const mocks = getMocks(testPlanRuns); const { getByText } = render( - + ); await waitFor(async () => { expect(getByText('2 Tests Completed')).toBeInTheDocument(); expect(getByText('0 Tests Queued')).toBeInTheDocument(); - expect(getByText('0 Tests Cancelled')).toBeInTheDocument(); }); }); @@ -106,20 +95,23 @@ test('correctly displays statuses for CANCELLED test run', async () => { { id: '0', testResults: new Array(2).fill(null), - tester: { username: 'bot' } + tester: { username: 'bot' }, + collectionJob: { + status: COLLECTION_JOB_STATUS.CANCELLED, + testStatus: [ + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.CANCELLED } + ] + } } ]; - const collectionJobStatuses = ['CANCELLED']; - - const mocks = getMocks(testPlanRuns, collectionJobStatuses); + const mocks = getMocks(testPlanRuns); const { getByText } = render( - + ); @@ -135,35 +127,36 @@ test('correctly displays statuses for multiple RUNNING and QUEUED test runs', as { id: '0', testResults: new Array(2).fill(null), - tester: { username: 'bot' } + tester: { username: 'bot' }, + collectionJob: { + status: COLLECTION_JOB_STATUS.RUNNING, + testStatus: [ + { status: COLLECTION_JOB_STATUS.RUNNING }, + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.QUEUED } + ] + } }, { id: '1', testResults: new Array(2).fill(null), - tester: { username: 'bot' } - }, - { - id: '2', - testResults: [null], - tester: { username: 'bot' } - }, - { - id: '3', - testResults: new Array(2).fill(null), - tester: { username: 'human' } + tester: { username: 'bot' }, + collectionJob: { + status: COLLECTION_JOB_STATUS.CANCELLED, + testStatus: [ + { status: COLLECTION_JOB_STATUS.CANCELLED }, + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.CANCELLED } + ] + } } ]; - const collectionJobStatuses = ['RUNNING', 'RUNNING', 'CANCELLED']; - - const mocks = getMocks(testPlanRuns, collectionJobStatuses); + const mocks = getMocks(testPlanRuns); const { getByText } = render( - + ); @@ -171,9 +164,10 @@ test('correctly displays statuses for multiple RUNNING and QUEUED test runs', as // Wait for the component to update // Imperfect but prevents needing to detect loading removal await setTimeout(() => { + expect(getByText('1 Test Running')).toBeInTheDocument(); expect(getByText('2 Tests Completed')).toBeInTheDocument(); expect(getByText('1 Test Queued')).toBeInTheDocument(); - expect(getByText('1 Test Cancelled')).toBeInTheDocument(); + expect(getByText('2 Tests Cancelled')).toBeInTheDocument(); }, 500); }); }); diff --git a/server/controllers/AutomationController.js b/server/controllers/AutomationController.js index 110351019..39f2ba930 100644 --- a/server/controllers/AutomationController.js +++ b/server/controllers/AutomationController.js @@ -1,7 +1,8 @@ const axios = require('axios'); const { getCollectionJobById, - updateCollectionJobById + updateCollectionJobById, + updateCollectionJobTestStatusByQuery } = require('../models/services/CollectionJobService'); const { findOrCreateTestResult @@ -17,14 +18,14 @@ const { findOrCreateBrowserVersion } = require('../models/services/BrowserService'); const { HttpQueryError } = require('apollo-server-core'); -const { COLLECTION_JOB_STATUS } = require('../util/enums'); +const { COLLECTION_JOB_STATUS, isJobStatusFinal } = require('../util/enums'); const populateData = require('../services/PopulatedData/populateData'); const { getFinalizedTestResults } = require('../models/services/TestResultReadService'); const http = require('http'); const { NO_OUTPUT_STRING } = require('../util/constants'); -const getTests = require('../models/services/TestsService'); +const runnableTestsResolver = require('../resolvers/TestPlanReport/runnableTestsResolver'); const getGraphQLContext = require('../graphql-context'); const httpAgent = new http.Agent({ family: 4 }); @@ -99,6 +100,19 @@ const updateJobStatus = async (req, res) => { ...(externalLogsUrl != null && { externalLogsUrl }) }; + // When new status is considered "final" ('COMPLETED' or 'ERROR' or 'CANCELLED') + // update any CollectionJobTestStatus children still 'QUEUED' to be 'CANCELLED' + if (isJobStatusFinal(status)) { + await updateCollectionJobTestStatusByQuery({ + where: { + collectionJobId: req.params.jobID, + status: COLLECTION_JOB_STATUS.QUEUED + }, + values: { status: COLLECTION_JOB_STATUS.CANCELLED }, + transaction: req.transaction + }); + } + const graphqlResponse = await updateCollectionJobById({ id: req.params.jobID, values: updatePayload, @@ -137,32 +151,25 @@ const getApprovedFinalizedTestResults = async (testPlanRun, context) => { return getFinalizedTestResults({ testPlanReport, context }); }; +const getTestByRowNumber = async ({ testPlanRun, testRowNumber, context }) => { + const tests = await runnableTestsResolver( + testPlanRun.testPlanReport, + null, + context + ); + return tests.find( + test => parseInt(test.rowNumber, 10) === parseInt(testRowNumber, 10) + ); +}; + const updateOrCreateTestResultWithResponses = async ({ - testRowIdentifier, + testId, testPlanRun, responses, atVersionId, browserVersionId, context }) => { - const allTestsForTestPlanVersion = await getTests( - testPlanRun.testPlanReport.testPlanVersion - ); - - const isV2 = - testPlanRun.testPlanReport.testPlanVersion.metadata - .testFormatVersion === 2; - - const testId = allTestsForTestPlanVersion.find( - test => - (!isV2 || test.at?.name === 'NVDA') && - parseInt(test.rowNumber, 10) === testRowIdentifier - )?.id; - - if (testId === undefined) { - throwNoTestFoundError(testRowIdentifier); - } - const { testResult } = await findOrCreateTestResult({ testId, testPlanRunId: testPlanRun.id, @@ -239,20 +246,20 @@ const updateOrCreateTestResultWithResponses = async ({ }; const updateJobResults = async (req, res) => { - const id = req.params.jobID; + const { jobID: id, testRowNumber } = req.params; const context = getGraphQLContext({ req }); const { transaction } = context; const { - testCsvRow, - presentationNumber, responses, + status, capabilities: { atName, atVersion: atVersionName, browserName, browserVersion: browserVersionName - } + } = {} } = req.body; + const job = await getCollectionJobById({ id, transaction }); if (!job) { throwNoJobFoundError(id); @@ -263,35 +270,65 @@ const updateJobResults = async (req, res) => { `Job with id ${id} is not running, cannot update results` ); } + if (status && !Object.values(COLLECTION_JOB_STATUS).includes(status)) { + throw new HttpQueryError(400, `Invalid status: ${status}`, true); + } + const { testPlanRun } = job; - /* TODO: Change this to use a better key based lookup system after gh-958 */ - const [at] = await getAts({ search: atName, transaction }); - const [browser] = await getBrowsers({ search: browserName, transaction }); - - const [atVersion, browserVersion] = await Promise.all([ - findOrCreateAtVersion({ - where: { atId: at.id, name: atVersionName }, - transaction - }), - findOrCreateBrowserVersion({ - where: { browserId: browser.id, name: browserVersionName }, - transaction + const testId = ( + await getTestByRowNumber({ + testPlanRun, + testRowNumber, + context }) - ]); + )?.id; + + if (testId === undefined) { + throwNoTestFoundError(testRowNumber); + } - const processedResponses = convertEmptyStringsToNoOutputMessages(responses); + // status only update, or responses were provided (default to complete) + if (status || responses) { + await updateCollectionJobTestStatusByQuery({ + where: { collectionJobId: id, testId }, + // default to completed if not specified (when results are present) + values: { status: status ?? COLLECTION_JOB_STATUS.COMPLETED }, + transaction: req.transaction + }); + } - // v1 tests store testCsvRow in rowNumber, v2 tests store presentationNumber in rowNumber - const testRowIdentifier = presentationNumber ?? testCsvRow; + // responses were provided + if (responses) { + /* TODO: Change this to use a better key based lookup system after gh-958 */ + const [at] = await getAts({ search: atName, transaction }); + const [browser] = await getBrowsers({ + search: browserName, + transaction + }); - await updateOrCreateTestResultWithResponses({ - testRowIdentifier, - responses: processedResponses, - testPlanRun: job.testPlanRun, - atVersionId: atVersion.id, - browserVersionId: browserVersion.id, - context - }); + const [atVersion, browserVersion] = await Promise.all([ + findOrCreateAtVersion({ + where: { atId: at.id, name: atVersionName }, + transaction + }), + findOrCreateBrowserVersion({ + where: { browserId: browser.id, name: browserVersionName }, + transaction + }) + ]); + + const processedResponses = + convertEmptyStringsToNoOutputMessages(responses); + + await updateOrCreateTestResultWithResponses({ + testId, + responses: processedResponses, + testPlanRun, + atVersionId: atVersion.id, + browserVersionId: browserVersion.id, + context + }); + } res.json({ success: true }); }; diff --git a/server/graphql-schema.js b/server/graphql-schema.js index 006f34ef7..6c132b920 100644 --- a/server/graphql-schema.js +++ b/server/graphql-schema.js @@ -102,6 +102,25 @@ const graphqlSchema = gql` The URL where the logs for the job can be found. """ externalLogsUrl: String + """ + An array of individual test status for every runnable test in the Job. + """ + testStatus: [CollectionJobTestStatus] + } + + """ + A status for a specific Test on a specific CollectionJob. + """ + type CollectionJobTestStatus { + """ + The test this status reflects. + """ + test: Test! + """ + The status of the test, which can be "QUEUED", "RUNNING", "COMPLETED", + "ERROR", or "CANCELLED" + """ + status: CollectionJobStatus! } type Browser { @@ -837,6 +856,10 @@ const graphqlSchema = gql` Whether the TestPlanRun was initiated by the Response Collection System """ initiatedByAutomation: Boolean! + """ + The CollectionJob related to this testPlanRun + """ + collectionJob: CollectionJob } """ diff --git a/server/migrations/20240404171101-addCollectionJobTestStatus.js b/server/migrations/20240404171101-addCollectionJobTestStatus.js new file mode 100644 index 000000000..7abf682ce --- /dev/null +++ b/server/migrations/20240404171101-addCollectionJobTestStatus.js @@ -0,0 +1,58 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + return queryInterface.sequelize.transaction(async transaction => { + await queryInterface.createTable( + 'CollectionJobTestStatus', + { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true + }, + testId: { + type: Sequelize.STRING, + allowNull: false + }, + collectionJobId: { + type: Sequelize.INTEGER, + allowNull: false, + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + references: { + model: 'CollectionJob', + key: 'id' + } + }, + status: { + type: Sequelize.STRING, + allowNull: false, + defaultValue: 'QUEUED' + } + }, + { transaction } + ); + await queryInterface.addConstraint('CollectionJobTestStatus', { + type: 'unique', + name: 'CollectionJob_Test_unique', + fields: ['collectionJobId', 'testId'], + transaction + }); + }); + }, + + async down(queryInterface) { + return queryInterface.sequelize.transaction(async transaction => { + await queryInterface.removeConstraint( + 'CollectionJobTestStatus', + 'CollectionJob_Test_unique', + { transaction } + ); + await queryInterface.dropTable('CollectionJobTestStatus', { + transaction + }); + }); + } +}; diff --git a/server/models/CollectionJob.js b/server/models/CollectionJob.js index c66499a80..dc57b229d 100644 --- a/server/models/CollectionJob.js +++ b/server/models/CollectionJob.js @@ -45,6 +45,12 @@ module.exports = function (sequelize, DataTypes) { sourceKey: 'testPlanRunId', as: 'testPlanRun' }); + + Model.hasMany(models.CollectionJobTestStatus, { + as: 'testStatus', + foreignKey: 'collectionJobId', + sourceKey: 'id' + }); }; Model.QUEUED = COLLECTION_JOB_STATUS.QUEUED; diff --git a/server/models/CollectionJobTestStatus.js b/server/models/CollectionJobTestStatus.js new file mode 100644 index 000000000..88427a1dd --- /dev/null +++ b/server/models/CollectionJobTestStatus.js @@ -0,0 +1,56 @@ +const { COLLECTION_JOB_STATUS } = require('../util/enums'); + +const MODEL_NAME = 'CollectionJobTestStatus'; + +module.exports = function (sequelize, DataTypes) { + const Model = sequelize.define( + MODEL_NAME, + { + id: { + type: DataTypes.INTEGER, + allowNull: false, + primaryKey: true, + autoIncrement: true + }, + collectionJobId: { + type: DataTypes.INTEGER, + allowNull: false, + references: { + model: 'CollectionJob', + key: 'id' + } + }, + testId: { + type: DataTypes.STRING, + allowNull: null + }, + status: { + type: DataTypes.STRING, + allowNull: false, + defaultValue: COLLECTION_JOB_STATUS.QUEUED + } + }, + { + timestamps: false, + tableName: MODEL_NAME + } + ); + + Model.associate = function (models) { + Model.belongsTo(models.CollectionJob, { + foreignKey: 'collectionJobId', + targetKey: 'id', + as: 'collectionJob', + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }); + }; + + Model.QUEUED = COLLECTION_JOB_STATUS.QUEUED; + Model.RUNNING = COLLECTION_JOB_STATUS.RUNNING; + Model.COMPLETED = COLLECTION_JOB_STATUS.COMPLETED; + Model.CANCELLED = COLLECTION_JOB_STATUS.CANCELLED; + Model.ERROR = COLLECTION_JOB_STATUS.ERROR; + + return Model; +}; diff --git a/server/models/services/CollectionJobService.js b/server/models/services/CollectionJobService.js index a73dbd20e..bbe756bd9 100644 --- a/server/models/services/CollectionJobService.js +++ b/server/models/services/CollectionJobService.js @@ -1,5 +1,5 @@ const ModelService = require('./ModelService'); -const { CollectionJob } = require('../'); +const { CollectionJob, CollectionJobTestStatus } = require('../'); const { COLLECTION_JOB_ATTRIBUTES, TEST_PLAN_ATTRIBUTES, @@ -8,7 +8,8 @@ const { TEST_PLAN_VERSION_ATTRIBUTES, AT_ATTRIBUTES, BROWSER_ATTRIBUTES, - USER_ATTRIBUTES + USER_ATTRIBUTES, + COLLECTION_JOB_TEST_STATUS_ATTRIBUTES } = require('./helpers'); const { COLLECTION_JOB_STATUS } = require('../../util/enums'); const { Op } = require('sequelize'); @@ -175,10 +176,21 @@ const userAssociation = userAttributes => ({ attributes: userAttributes }); +/** + * @param {string[]} collectionJobTestStatusAttributes - attributes to be returned in the result + * @returns {{association: string, attributes: string[]}} + */ +const collectionJobTestStatusAssociation = + collectionJobTestStatusAttributes => ({ + association: 'testStatus', + attributes: collectionJobTestStatusAttributes + }); + /** * @param {object} options * @param {object} options.values - CollectionJob to be created - * @param {string[]} options.collectionJobAttributes - TestPlanRun attributes to be returned in the result + * @param {string[]} options.collectionJobAttributes - CollectionJob attributes to be returned in the result + * @param {string[]} options.collectionJobTestStatusAttributes - CollectionJobTestStatus attributes to be returned in the result * @param {string[]} options.testPlanReportAttributes - TestPlanReport attributes to be returned in the result * @param {string[]} options.testPlanVersionAttributes - TestPlanVersion attributes to be returned in the result * @param {string[]} options.testPlanAttributes - TestPlanVersion attributes to be returned in the result @@ -195,6 +207,7 @@ const createCollectionJob = async ({ testPlanReportId }, collectionJobAttributes = COLLECTION_JOB_ATTRIBUTES, + collectionJobTestStatusAttributes = COLLECTION_JOB_TEST_STATUS_ATTRIBUTES, testPlanReportAttributes = TEST_PLAN_REPORT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, testPlanVersionAttributes = TEST_PLAN_VERSION_ATTRIBUTES, @@ -222,10 +235,29 @@ const createCollectionJob = async ({ transaction }); + // create QUEUED status entries for each test in the test plan run + const context = getGraphQLContext({ req: { transaction } }); + const tests = await runnableTestsResolver( + testPlanRun.testPlanReport, + null, + context + ); + await ModelService.bulkCreate(CollectionJobTestStatus, { + valuesList: tests.map(test => ({ + testId: test.id, + collectionJobId: collectionJobResult.id, + status: COLLECTION_JOB_STATUS.QUEUED + })), + transaction + }); + return ModelService.getById(CollectionJob, { id: collectionJobResult.id, attributes: collectionJobAttributes, include: [ + collectionJobTestStatusAssociation( + collectionJobTestStatusAttributes + ), testPlanRunAssociation( testPlanRunAttributes, userAttributes, @@ -243,7 +275,8 @@ const createCollectionJob = async ({ /** * @param {object} options * @param {string} options.id - id for the CollectionJob - * @param {string[]} options.collectionJobAttributes - TestPlanRun attributes to be returned in the result + * @param {string[]} options.collectionJobAttributes - CollectionJob attributes to be returned in the result + * @param {string[]} options.collectionJobTestStatusAttributes - CollectionJobTestStatus attributes to be returned in the result * @param {string[]} options.testPlanReportAttributes - TestPlanReport attributes to be returned in the result * @param {string[]} options.testPlanVersionAttributes - TestPlanVersion attributes to be returned in the result * @param {string[]} options.testPlanAttributes - TestPlanVersion attributes to be returned in the result @@ -256,6 +289,7 @@ const createCollectionJob = async ({ const getCollectionJobById = async ({ id, collectionJobAttributes = COLLECTION_JOB_ATTRIBUTES, + collectionJobTestStatusAttributes = COLLECTION_JOB_TEST_STATUS_ATTRIBUTES, testPlanReportAttributes = TEST_PLAN_REPORT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, testPlanVersionAttributes = TEST_PLAN_VERSION_ATTRIBUTES, @@ -269,6 +303,9 @@ const getCollectionJobById = async ({ id, attributes: collectionJobAttributes, include: [ + collectionJobTestStatusAssociation( + collectionJobTestStatusAttributes + ), testPlanRunAssociation( testPlanRunAttributes, userAttributes, @@ -287,7 +324,8 @@ const getCollectionJobById = async ({ * @param {object} options * @param {string|any} options.search - use this to combine with {@param filter} to be passed to Sequelize's where clause * @param {object} options.where - use this define conditions to be passed to Sequelize's where clause - * @param {string[]} options.collectionJobAttributes - Browser attributes to be returned in the result + * @param {string[]} options.collectionJobAttributes - CollectionJob attributes to be returned in the result + * @param {string[]} options.collectionJobTestStatusAttributes - CollectionJobTestStatus attributes to be returned in the result * @param {string[]} options.testPlanReportAttributes - TestPlanReport attributes to be returned in the result * @param {string[]} options.testPlanVersionAttributes - TestPlanVersion attributes to be returned in the result * @param {string[]} options.testPlanAttributes - TestPlanVersion attributes to be returned in the result @@ -306,6 +344,7 @@ const getCollectionJobs = async ({ search, where = {}, collectionJobAttributes = COLLECTION_JOB_ATTRIBUTES, + collectionJobTestStatusAttributes = COLLECTION_JOB_TEST_STATUS_ATTRIBUTES, testPlanReportAttributes = TEST_PLAN_REPORT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, testPlanVersionAttributes = TEST_PLAN_VERSION_ATTRIBUTES, @@ -324,6 +363,9 @@ const getCollectionJobs = async ({ where, attributes: collectionJobAttributes, include: [ + collectionJobTestStatusAssociation( + collectionJobTestStatusAttributes + ), testPlanRunAssociation( testPlanRunAttributes, userAttributes, @@ -382,7 +424,8 @@ const triggerWorkflow = async (job, testIds, { transaction }) => { * @param {object} options * @param {string} options.id - id of the CollectionJob to be updated * @param {object} options.values - values to be used to update columns for the record being referenced for {@param id} - * @param {string[]} options.collectionJobAttributes - Browser attributes to be returned in the result + * @param {string[]} options.collectionJobAttributes - CollectionJob attributes to be returned in the result + * @param {string[]} options.collectionJobTestStatusAttributes - CollectionJobTestStatus attributes to be returned in the result * @param {string[]} options.testPlanReportAttributes - TestPlanReport attributes to be returned in the result * @param {string[]} options.testPlanVersionAttributes - TestPlanVersion attributes to be returned in the result * @param {string[]} options.testPlanAttributes - TestPlanVersion attributes to be returned in the result @@ -396,6 +439,7 @@ const updateCollectionJobById = async ({ id, values = {}, collectionJobAttributes = COLLECTION_JOB_ATTRIBUTES, + collectionJobTestStatusAttributes = COLLECTION_JOB_TEST_STATUS_ATTRIBUTES, testPlanReportAttributes = TEST_PLAN_REPORT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, testPlanVersionAttributes = TEST_PLAN_VERSION_ATTRIBUTES, @@ -415,6 +459,9 @@ const updateCollectionJobById = async ({ id, attributes: collectionJobAttributes, include: [ + collectionJobTestStatusAssociation( + collectionJobTestStatusAttributes + ), testPlanRunAssociation( testPlanRunAttributes, userAttributes, @@ -592,6 +639,30 @@ const restartCollectionJob = async ({ id }, { transaction }) => { return triggerWorkflow(job, [], { transaction }); }; +/** + * CollectionJobTestStatus updates + */ + +/** + * Update CollectionJobTestStatus entries in bulk via query. + * @param {object} options + * @param {object} options.where - values of the CollectionJobTestStatus record to be updated + * @param {object} options.values - values to be used to update columns for the record being referenced + * @param {*} options.transaction - Sequelize transaction + * @returns {Promise<*>} + */ +const updateCollectionJobTestStatusByQuery = ({ + where, + values = {}, + transaction +}) => { + return ModelService.update(CollectionJobTestStatus, { + values, + where, + transaction + }); +}; + module.exports = { // Basic CRUD createCollectionJob, @@ -603,5 +674,7 @@ module.exports = { scheduleCollectionJob, restartCollectionJob, cancelCollectionJob, - retryCanceledCollections + retryCanceledCollections, + // Basic CRUD for CollectionJobTestStatus + updateCollectionJobTestStatusByQuery }; diff --git a/server/models/services/helpers.js b/server/models/services/helpers.js index 32c75b7cb..f7f188c71 100644 --- a/server/models/services/helpers.js +++ b/server/models/services/helpers.js @@ -11,7 +11,8 @@ const { User, UserRoles, UserAts, - CollectionJob + CollectionJob, + CollectionJobTestStatus } = require('../index'); /** @@ -39,5 +40,8 @@ module.exports = { USER_ATTRIBUTES: getSequelizeModelAttributes(User), USER_ROLES_ATTRIBUTES: getSequelizeModelAttributes(UserRoles), USER_ATS_ATTRIBUTES: getSequelizeModelAttributes(UserAts), - COLLECTION_JOB_ATTRIBUTES: getSequelizeModelAttributes(CollectionJob) + COLLECTION_JOB_ATTRIBUTES: getSequelizeModelAttributes(CollectionJob), + COLLECTION_JOB_TEST_STATUS_ATTRIBUTES: getSequelizeModelAttributes( + CollectionJobTestStatus + ) }; diff --git a/server/resolvers/CollectionJob/index.js b/server/resolvers/CollectionJob/index.js new file mode 100644 index 000000000..64347cf91 --- /dev/null +++ b/server/resolvers/CollectionJob/index.js @@ -0,0 +1,3 @@ +const testStatus = require('./testStatusResolver'); +const testPlanRun = require('./testPlanRunResolver'); +module.exports = { testStatus, testPlanRun }; diff --git a/server/resolvers/CollectionJob/testPlanRunResolver.js b/server/resolvers/CollectionJob/testPlanRunResolver.js new file mode 100644 index 000000000..62014f8bd --- /dev/null +++ b/server/resolvers/CollectionJob/testPlanRunResolver.js @@ -0,0 +1,10 @@ +// eslint-disable-next-line no-unused-vars +const testPlanRunResolver = (collectionJob, _, context) => { + if (collectionJob.__testPlanRunChild) { + throw new Error( + 'can not request TestPlanRun.collectionJob.testPlanRun' + ); + } + return collectionJob.testPlanRun; +}; +module.exports = testPlanRunResolver; diff --git a/server/resolvers/CollectionJob/testStatusResolver.js b/server/resolvers/CollectionJob/testStatusResolver.js new file mode 100644 index 000000000..1c46a5d2b --- /dev/null +++ b/server/resolvers/CollectionJob/testStatusResolver.js @@ -0,0 +1,15 @@ +// eslint-disable-next-line no-unused-vars +const testStatusResolver = (collectionJob, _, context) => { + // resolve testStatus.test for each test + // testPlanRun "might" be null if the testPlanRun had been deleted + const testPlanTests = + collectionJob.testPlanRun?.testPlanReport.testPlanVersion.tests ?? []; + const tests = new Map(testPlanTests.map(test => [test.id, test])); + + return collectionJob.testStatus.map(status => ({ + ...status.dataValues, + // if not found, at least return the test id + test: tests.get(status.testId) ?? { id: status.testId } + })); +}; +module.exports = testStatusResolver; diff --git a/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js b/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js index ea53c36e9..c7240ce50 100644 --- a/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js +++ b/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js @@ -2,7 +2,8 @@ const { AuthenticationError } = require('apollo-server'); const { updateCollectionJobById, - getCollectionJobById + getCollectionJobById, + updateCollectionJobTestStatusByQuery } = require('../../models/services/CollectionJobService'); const { COLLECTION_JOB_STATUS } = require('../../util/enums'); @@ -36,6 +37,11 @@ const cancelCollectionJobResolver = async ( if (collectionJob.status === COLLECTION_JOB_STATUS.COMPLETED) { return collectionJob; } else { + await updateCollectionJobTestStatusByQuery({ + where: { collectionJobId, status: COLLECTION_JOB_STATUS.QUEUED }, + values: { status: COLLECTION_JOB_STATUS.CANCELLED }, + transaction + }); return updateCollectionJobById({ id: collectionJobId, values: { status: COLLECTION_JOB_STATUS.CANCELLED }, diff --git a/server/resolvers/TestPlanRun/collectionJobResolver.js b/server/resolvers/TestPlanRun/collectionJobResolver.js new file mode 100644 index 000000000..ecfb6552c --- /dev/null +++ b/server/resolvers/TestPlanRun/collectionJobResolver.js @@ -0,0 +1,19 @@ +const collectionJobByTestPlanRunIdResolver = require('../collectionJobByTestPlanRunIdResolver'); + +const collectionJobResolver = async ( + testPlanRun, + args, // eslint-disable-line no-unused-vars + context // eslint-disable-line no-unused-vars +) => { + const collectionJob = await collectionJobByTestPlanRunIdResolver( + null, + { testPlanRunId: testPlanRun.id }, + context + ); + if (collectionJob) { + return { ...collectionJob.dataValues, __testPlanRunChild: true }; + } + return collectionJob; +}; + +module.exports = collectionJobResolver; diff --git a/server/resolvers/TestPlanRun/index.js b/server/resolvers/TestPlanRun/index.js index 74770a793..f542d26d4 100644 --- a/server/resolvers/TestPlanRun/index.js +++ b/server/resolvers/TestPlanRun/index.js @@ -1,7 +1,8 @@ const testResults = require('./testResultsResolver'); const testResultsLength = require('./testResultsLengthResolver'); - +const collectionJob = require('./collectionJobResolver'); module.exports = { + collectionJob, testResults, testResultsLength }; diff --git a/server/resolvers/index.js b/server/resolvers/index.js index 5e097b81d..5cf7dece6 100644 --- a/server/resolvers/index.js +++ b/server/resolvers/index.js @@ -41,6 +41,7 @@ const TestPlanRunOperations = require('./TestPlanRunOperations'); const TestResultOperations = require('./TestResultOperations'); const TestPlanVersionOperations = require('./TestPlanVersionOperations'); const CollectionJobOperations = require('./CollectionJobOperations'); +const CollectionJob = require('./CollectionJob'); const TestPlanRun = require('./TestPlanRun'); const Test = require('./Test'); const ScenarioResult = require('./ScenarioResult'); @@ -84,6 +85,7 @@ const resolvers = { AtOperations, AtVersionOperations, BrowserOperations, + CollectionJob, User, TestPlan, TestPlanVersion, diff --git a/server/routes/automation.js b/server/routes/automation.js index 460516af7..5d6540e64 100644 --- a/server/routes/automation.js +++ b/server/routes/automation.js @@ -10,9 +10,12 @@ const { handleError } = require('../middleware/handleError'); const router = Router(); -router.post('/:jobID/update', verifyAutomationScheduler, updateJobStatus); - -router.post('/:jobID/result', verifyAutomationScheduler, updateJobResults); +router.post('/:jobID', verifyAutomationScheduler, updateJobStatus); +router.post( + '/:jobID/test/:testRowNumber', + verifyAutomationScheduler, + updateJobResults +); router.use(handleError); diff --git a/server/services/GithubWorkflowService.js b/server/services/GithubWorkflowService.js index 9d21765d7..2fe571563 100644 --- a/server/services/GithubWorkflowService.js +++ b/server/services/GithubWorkflowService.js @@ -128,8 +128,8 @@ const createGithubWorkflow = async ({ job, directory, gitSha }) => { ); const browser = job.testPlanRun.testPlanReport.browser.name.toLowerCase(); const inputs = { - callback_url: `https://${callbackUrlHostname}/api/jobs/${job.id}/result`, - status_url: `https://${callbackUrlHostname}/api/jobs/${job.id}/update`, + callback_url: `https://${callbackUrlHostname}/api/jobs/${job.id}/test/:testRowNumber`, + status_url: `https://${callbackUrlHostname}/api/jobs/${job.id}`, callback_header: `x-automation-secret:${process.env.AUTOMATION_SCHEDULER_SECRET}`, work_dir: `tests/${directory}`, test_pattern: '{reference/**,test-*-nvda.*}', diff --git a/server/tests/integration/automation-scheduler.test.js b/server/tests/integration/automation-scheduler.test.js index 09f5b209b..0dcd800da 100644 --- a/server/tests/integration/automation-scheduler.test.js +++ b/server/tests/integration/automation-scheduler.test.js @@ -12,6 +12,7 @@ const markAsFinalResolver = require('../../resolvers/TestPlanReportOperations/ma const AtLoader = require('../../models/loaders/AtLoader'); const BrowserLoader = require('../../models/loaders/BrowserLoader'); const getGraphQLContext = require('../../graphql-context'); +const { COLLECTION_JOB_STATUS } = require('../../util/enums'); let mockAutomationSchedulerServer; let apiServer; @@ -138,6 +139,10 @@ const getTestCollectionJob = async (jobId, { transaction }) => } } } + testStatus { + test { id } + status + } } } `, @@ -166,6 +171,10 @@ const scheduleCollectionJobByMutation = async ({ transaction }) => } } } + testStatus { + test { id } + status + } } } `, @@ -288,7 +297,7 @@ describe('Automation controller', () => { }); }); - it('should cancel a job', async () => { + it('should cancel a job and all remaining tests', async () => { await dbCleaner(async transaction => { const { scheduleCollectionJob: job } = await scheduleCollectionJobByMutation({ transaction }); @@ -300,6 +309,9 @@ describe('Automation controller', () => { const { collectionJob: storedCollectionJob } = await getTestCollectionJob(job.id, { transaction }); expect(storedCollectionJob.status).toEqual('CANCELLED'); + for (const test of storedCollectionJob.testStatus) { + expect(test.status).toEqual('CANCELLED'); + } }); }); @@ -342,9 +354,7 @@ describe('Automation controller', () => { await dbCleaner(async transaction => { const { scheduleCollectionJob: job } = await scheduleCollectionJobByMutation({ transaction }); - const response = await sessionAgent.post( - `/api/jobs/${job.id}/update` - ); + const response = await sessionAgent.post(`/api/jobs/${job.id}`); expect(response.statusCode).toBe(403); expect(response.body).toEqual({ error: 'Unauthorized' @@ -357,7 +367,7 @@ describe('Automation controller', () => { const { scheduleCollectionJob: job } = await scheduleCollectionJobByMutation({ transaction }); const response = await sessionAgent - .post(`/api/jobs/${job.id}/update`) + .post(`/api/jobs/${job.id}`) .send({ status: 'INVALID' }) .set( 'x-automation-secret', @@ -372,7 +382,7 @@ describe('Automation controller', () => { it('should fail to update a job status for a non-existent jobId', async () => { const response = await sessionAgent - .post(`/api/jobs/${444}/update`) + .post(`/api/jobs/${444}`) .send({ status: 'RUNNING' }) .set( 'x-automation-secret', @@ -389,7 +399,7 @@ describe('Automation controller', () => { const { scheduleCollectionJob: job } = await scheduleCollectionJobByMutation({ transaction }); const response = await sessionAgent - .post(`/api/jobs/${job.id}/update`) + .post(`/api/jobs/${job.id}`) .send({ status: 'RUNNING' }) .set( 'x-automation-secret', @@ -421,7 +431,7 @@ describe('Automation controller', () => { const { scheduleCollectionJob: job } = await scheduleCollectionJobByMutation({ transaction }); const response = await sessionAgent - .post(`/api/jobs/${job.id}/update`) + .post(`/api/jobs/${job.id}`) .send({ status: 'CANCELLED', externalLogsUrl: 'https://www.aol.com/' @@ -462,7 +472,7 @@ describe('Automation controller', () => { transaction }); await sessionAgent - .post(`/api/jobs/${job.id}/update`) + .post(`/api/jobs/${job.id}`) .send({ status: 'RUNNING' }) .set( 'x-automation-secret', @@ -492,9 +502,8 @@ describe('Automation controller', () => { scenario => scenario.atId === at.id ).length; const response = await sessionAgent - .post(`/api/jobs/${job.id}/result`) + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) .send({ - testCsvRow: selectedTestRowNumber, capabilities: { atName: at.name, atVersion: at.atVersions[0].name, @@ -543,6 +552,118 @@ describe('Automation controller', () => { ); }); }); + // also marks status for test as COMPLETED + const { collectionJob: storedCollectionJob } = + await getTestCollectionJob(job.id, { transaction }); + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('RUNNING'); + const testStatus = storedCollectionJob.testStatus.find( + status => status.test.id === selectedTest.id + ); + expect(testStatus.status).toEqual(COLLECTION_JOB_STATUS.COMPLETED); + }); + }); + + it('should properly handle per-test status updates without capabilities present', async () => { + await apiServer.sessionAgentDbCleaner(async transaction => { + const { scheduleCollectionJob: job } = + await scheduleCollectionJobByMutation({ transaction }); + const collectionJob = await getCollectionJobById({ + id: job.id, + transaction + }); + // flag overall job as RUNNING + const externalLogsUrl = 'https://example.com/test/log/url'; + await sessionAgent + .post(`/api/jobs/${job.id}`) + .send({ status: 'RUNNING', externalLogsUrl }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + + const { tests } = + collectionJob.testPlanRun.testPlanReport.testPlanVersion; + const selectedTestIndex = 0; + const selectedTestRowNumber = 1; + const selectedTest = tests[selectedTestIndex]; + let response = await sessionAgent + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) + .send({ + status: COLLECTION_JOB_STATUS.RUNNING + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + let { collectionJob: storedCollectionJob } = + await getTestCollectionJob(job.id, { transaction }); + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('RUNNING'); + expect(storedCollectionJob.externalLogsUrl).toEqual( + externalLogsUrl + ); + let foundStatus = false; + for (const testStatus of storedCollectionJob.testStatus) { + let expectedStatus = COLLECTION_JOB_STATUS.QUEUED; + if (testStatus.test.id === selectedTest.id) { + foundStatus = true; + expectedStatus = COLLECTION_JOB_STATUS.RUNNING; + } + expect(testStatus.status).toEqual(expectedStatus); + } + expect(foundStatus).toEqual(true); + + // check that putting this test into ERROR and sending an overall + // collection job ERROR will properly update things + response = await sessionAgent + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) + .send({ + status: COLLECTION_JOB_STATUS.ERROR + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + response = await sessionAgent + .post(`/api/jobs/${job.id}`) + .send({ + // avoiding sending externalLogsUrl here to test that when + // missing it is not overwritten/emptied. + status: COLLECTION_JOB_STATUS.ERROR + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + storedCollectionJob = ( + await getTestCollectionJob(job.id, { transaction }) + ).collectionJob; + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('ERROR'); + expect(storedCollectionJob.externalLogsUrl).toEqual( + externalLogsUrl + ); + foundStatus = false; + for (const testStatus of storedCollectionJob.testStatus) { + let expectedStatus = COLLECTION_JOB_STATUS.CANCELLED; + if (testStatus.test.id === selectedTest.id) { + foundStatus = true; + expectedStatus = COLLECTION_JOB_STATUS.ERROR; + } + expect(testStatus.status).toEqual(expectedStatus); + } + expect(foundStatus).toEqual(true); }); }); @@ -588,7 +709,7 @@ describe('Automation controller', () => { transaction }); await sessionAgent - .post(`/api/jobs/${job.id}/update`) + .post(`/api/jobs/${job.id}`) .send({ status: 'RUNNING' }) .set( 'x-automation-secret', @@ -597,9 +718,8 @@ describe('Automation controller', () => { .set('x-transaction-id', transaction.id); const response = await sessionAgent - .post(`/api/jobs/${job.id}/result`) + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) .send({ - testCsvRow: selectedTestRowNumber, capabilities: { atName: at.name, atVersion: atVersion.name, diff --git a/server/tests/integration/graphql.test.js b/server/tests/integration/graphql.test.js index 14dcb22e2..8152a5f68 100644 --- a/server/tests/integration/graphql.test.js +++ b/server/tests/integration/graphql.test.js @@ -140,7 +140,8 @@ describe('graphql', () => { // 'TestResult' 'Issue', 'Vendor', - 'scheduleCollectionJob' + 'scheduleCollectionJob', + 'CollectionJobTestStatus' ]; const excludedTypeNameAndField = [ // Items formatted like this: @@ -159,6 +160,7 @@ describe('graphql', () => { ['Command', 'atOperatingMode'], // TODO: Include when v2 test format CI tests are done ['CollectionJob', 'testPlanRun'], ['CollectionJob', 'externalLogsUrl'], + ['CollectionJob', 'testStatus'], // These interact with Response Scheduler API // which is mocked in other tests. ['Mutation', 'scheduleCollectionJob'], @@ -516,10 +518,11 @@ describe('graphql', () => { releasedAt } } - testPlanRun(id: 3) { + testPlanRun(id: 1) { __typename id initiatedByAutomation + collectionJob { id } testPlanReport { id } @@ -793,6 +796,31 @@ describe('graphql', () => { ); }); + // esure recursive query of collectionJob<>testPlanRun fails at some depth + await expect( + typeAwareQuery( + gql` + query { + collectionJob(id: 1) { + id + testPlanRun { + id + collectionJob { + id + testPlanRun { + id + } + } + } + } + } + `, + { + transaction: false + } + ) + ).rejects.toBeDefined(); + expect(() => { const missingTypes = checkForMissingTypes(); if (missingTypes.length) { diff --git a/server/tests/util/mock-automation-scheduler-server.js b/server/tests/util/mock-automation-scheduler-server.js index eeec2dd23..ca9c65ec1 100644 --- a/server/tests/util/mock-automation-scheduler-server.js +++ b/server/tests/util/mock-automation-scheduler-server.js @@ -12,6 +12,9 @@ const { } = require('../../middleware/transactionMiddleware'); const { query } = require('../util/graphql-test-utilities'); +// 0 = no chance of test errors, 1 = always errors +const TEST_ERROR_CHANCE = 0; + const setupMockAutomationSchedulerServer = async () => { const app = express(); app.use(express.json()); @@ -26,9 +29,22 @@ const setupMockAutomationSchedulerServer = async () => { shutdownManager = new GracefulShutdownManager(listener); }); + const timeout = ms => + new Promise(resolve => setTimeout(() => resolve(), ms)); + const simulateJobStatusUpdate = async (jobId, newStatus) => { await axios.post( - `${process.env.APP_SERVER}/api/jobs/${jobId}/update`, + `${process.env.APP_SERVER}/api/jobs/${jobId}`, + { + status: newStatus + }, + axiosConfig + ); + }; + + const simulateTestStatusUpdate = async (jobId, testId, newStatus) => { + await axios.post( + `${process.env.APP_SERVER}/api/jobs/${jobId}/test/${testId}`, { status: newStatus }, @@ -66,22 +82,35 @@ const setupMockAutomationSchedulerServer = async () => { responses }; - testResult[isV2 ? 'presentationNumber' : 'testCsvRow'] = - currentTest.rowNumber; try { - await axios.post( - `${process.env.APP_SERVER}/api/jobs/${jobId}/result`, - testResult, - axiosConfig + await simulateTestStatusUpdate( + jobId, + currentTest.rowNumber, + COLLECTION_JOB_STATUS.RUNNING ); - } catch (e) { - // Likely just means the test was cancelled - return; - } + await timeout(Math.random() * 2000); + + if (Math.random() < TEST_ERROR_CHANCE) { + await simulateTestStatusUpdate( + jobId, + currentTest.rowNumber, + COLLECTION_JOB_STATUS.ERROR + ); + return simulateJobStatusUpdate( + jobId, + COLLECTION_JOB_STATUS.ERROR + ); + } else { + await axios.post( + `${process.env.APP_SERVER}/api/jobs/${jobId}/test/${currentTest.rowNumber}`, + testResult, + axiosConfig + ); + } - if (currentTestIndex < tests.length - 1) { - setTimeout(() => { - simulateResultCompletion( + if (currentTestIndex < tests.length - 1) { + await timeout(Math.random() * 5000); + return simulateResultCompletion( tests, atName, atVersionName, @@ -91,19 +120,12 @@ const setupMockAutomationSchedulerServer = async () => { currentTestIndex + 1, isV2 ); - }, Math.random() * 5000); - } else { - setTimeout( - () => - simulateJobStatusUpdate( - jobId, - COLLECTION_JOB_STATUS.COMPLETED - ), - 1000 - ); + } else { + simulateJobStatusUpdate(jobId, COLLECTION_JOB_STATUS.COMPLETED); + } + } catch (error) { + console.error('Error simulating collection job', error); } - - return testResult; }; app.post('/jobs/new', async (req, res) => { diff --git a/server/util/enums.js b/server/util/enums.js index ce07982d2..56ab4c579 100644 --- a/server/util/enums.js +++ b/server/util/enums.js @@ -6,6 +6,12 @@ const COLLECTION_JOB_STATUS = { CANCELLED: 'CANCELLED' }; +const isJobStatusFinal = status => + status === COLLECTION_JOB_STATUS.COMPLETED || + status === COLLECTION_JOB_STATUS.CANCELLED || + status === COLLECTION_JOB_STATUS.ERROR; + module.exports = { - COLLECTION_JOB_STATUS + COLLECTION_JOB_STATUS, + isJobStatusFinal }; From f68f2ef21423bda4d2c596a0c94b777d1e418e3b Mon Sep 17 00:00:00 2001 From: "Mx. Corey Frang" Date: Thu, 9 May 2024 11:01:48 -0400 Subject: [PATCH 07/19] add isBot to schema and database --- server/graphql-schema.js | 4 ++++ server/migrations/20240509102300-addIsBot.js | 23 ++++++++++++++++++++ server/models/User.js | 5 +++++ server/seeders/20231218191524-addNVDABot.js | 1 + server/tests/integration/graphql.test.js | 1 + server/tests/models/User.spec.js | 2 +- server/util/responseCollectionUser.js | 1 + 7 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 server/migrations/20240509102300-addIsBot.js diff --git a/server/graphql-schema.js b/server/graphql-schema.js index 6c132b920..2085d08bd 100644 --- a/server/graphql-schema.js +++ b/server/graphql-schema.js @@ -50,6 +50,10 @@ const graphqlSchema = gql` List of types of actions the user can complete. """ roles: [Role]! + """ + Whether the user is an automation bot user. + """ + isBot: Boolean! # TODO: Either use the recorded data somewhere or eliminate the field. """ The ATs the user has indicated they are able to test. diff --git a/server/migrations/20240509102300-addIsBot.js b/server/migrations/20240509102300-addIsBot.js new file mode 100644 index 000000000..684e0cd89 --- /dev/null +++ b/server/migrations/20240509102300-addIsBot.js @@ -0,0 +1,23 @@ +module.exports = { + up: async (queryInterface, Sequelize) => { + return queryInterface.sequelize.transaction(async transaction => { + await queryInterface.addColumn( + 'User', + 'isBot', + { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: false + }, + { transaction } + ); + await queryInterface.sequelize.query( + `UPDATE "User" SET "isBot" = true WHERE "username" LIKE '% Bot'`, + { transaction } + ); + }); + }, + down: async queryInterface => { + await queryInterface.removeColumn('User', 'isBot'); + } +}; diff --git a/server/models/User.js b/server/models/User.js index cbd2ba3e9..a2459ace3 100644 --- a/server/models/User.js +++ b/server/models/User.js @@ -15,6 +15,11 @@ module.exports = function (sequelize, DataTypes) { allowNull: false, unique: true }, + isBot: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false + }, createdAt: { type: DataTypes.DATE }, updatedAt: { type: DataTypes.DATE } }, diff --git a/server/seeders/20231218191524-addNVDABot.js b/server/seeders/20231218191524-addNVDABot.js index 59e5fc10a..52f60747d 100644 --- a/server/seeders/20231218191524-addNVDABot.js +++ b/server/seeders/20231218191524-addNVDABot.js @@ -10,6 +10,7 @@ module.exports = { { id: responseCollectionUser.id, // Specified ID for NVDA Bot username: responseCollectionUser.username, + isBot: responseCollectionUser.isBot, createdAt: new Date(), updatedAt: new Date() } diff --git a/server/tests/integration/graphql.test.js b/server/tests/integration/graphql.test.js index 8152a5f68..cfc0ffbe6 100644 --- a/server/tests/integration/graphql.test.js +++ b/server/tests/integration/graphql.test.js @@ -246,6 +246,7 @@ describe('graphql', () => { __typename username roles + isBot } me { __typename diff --git a/server/tests/models/User.spec.js b/server/tests/models/User.spec.js index e704f96a7..efa71d252 100644 --- a/server/tests/models/User.spec.js +++ b/server/tests/models/User.spec.js @@ -19,7 +19,7 @@ describe('UserModel Schema Checks', () => { describe('properties', () => { // A3 - ['username', 'createdAt', 'updatedAt'].forEach( + ['username', 'isBot', 'createdAt', 'updatedAt'].forEach( checkPropertyExists(modelInstance) ); }); diff --git a/server/util/responseCollectionUser.js b/server/util/responseCollectionUser.js index 939d456bb..e1babb3f2 100644 --- a/server/util/responseCollectionUser.js +++ b/server/util/responseCollectionUser.js @@ -1,6 +1,7 @@ // TODO: Defaults to NVDA, offer AT-specific bot once support is added const responseCollectionUser = { username: 'NVDA Bot', + isBot: true, id: 9999 }; From 3c535804a5c2bce4416c89c84ab4e0bcca48135b Mon Sep 17 00:00:00 2001 From: Howard Edwards Date: Thu, 9 May 2024 08:08:31 -0700 Subject: [PATCH 08/19] feat: Add workflow to automate deployment to aria-at.w3.org on a push to the main branch (#1050) --- .github/workflows/deploy-production.yml | 30 +++++++++++++++++++++++++ .github/workflows/deploy-staging.yml | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/deploy-production.yml diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml new file mode 100644 index 000000000..4a15cf1b8 --- /dev/null +++ b/.github/workflows/deploy-production.yml @@ -0,0 +1,30 @@ +name: Deploy to production (aria-at.w3.org) + +on: + push: + branches: + - main + +jobs: + deploy-production: + runs-on: ubuntu-latest + + steps: + - name: Install SSH key for deploying + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.DEPLOY_SSH_PRIVATE_KEY }} + known_hosts: ${{ secrets.DEPLOY_KNOWN_HOSTS_PRODUCTION }} + config: ${{ secrets.DEPLOY_SSH_CONFIG }} + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install ansible and deploy to production + run: | + python -m pip install --user ansible-core==2.11.1 + cd deploy + echo ${{ secrets.ANSIBLE_VAULT_PASSWORD }} > ansible-vault-password.txt + ansible-vault view --vault-password-file ansible-vault-password.txt files/jwt-signing-key.pem.enc > ../jwt-signing-key.pem + ansible-galaxy collection install ansible.posix + ansible-playbook provision.yml -e ansible_python_interpreter=/usr/bin/python3 --inventory inventory/production.yml diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index dcee62a67..a88d06939 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -1,4 +1,4 @@ -name: Deploy to staging +name: Deploy to staging (aria-at-staging.w3.org) on: push: From c786a3ef5f360c77169fce32e75a0cc27a01d9f1 Mon Sep 17 00:00:00 2001 From: "Mx. Corey Frang" Date: Thu, 9 May 2024 12:29:51 -0400 Subject: [PATCH 09/19] isBot on client side --- .../components/ManageBotRunDialog/index.jsx | 3 +- .../TestQueue/AssignTesterDropdown/index.jsx | 11 ++---- client/components/TestQueue/queries.js | 5 +++ .../index.js | 9 ++--- client/components/TestQueueRow/index.jsx | 5 +-- client/components/TestRun/TestNavigator.jsx | 3 +- client/components/TestRun/index.jsx | 9 ++--- client/components/TestRun/queries.js | 6 +++ client/tests/AssignTesterDropdown.test.jsx | 6 +-- .../TestQueuePageAdminNotPopulatedMock.js | 9 +++-- .../TestQueuePageAdminPopulatedMock.js | 37 ++++++++++++++----- .../TestQueuePageTesterNotPopulatedMock.js | 3 ++ .../TestQueuePageTesterPopulatedMock.js | 15 +++++--- client/utils/automation.js | 2 - 14 files changed, 77 insertions(+), 46 deletions(-) diff --git a/client/components/ManageBotRunDialog/index.jsx b/client/components/ManageBotRunDialog/index.jsx index 4706dcf22..b2cd54584 100644 --- a/client/components/ManageBotRunDialog/index.jsx +++ b/client/components/ManageBotRunDialog/index.jsx @@ -10,7 +10,6 @@ import { } from './queries'; import DeleteButton from '../common/DeleteButton'; import BotRunTestStatusList from '../BotRunTestStatusList'; -import { isBot } from '../../utils/automation'; import './ManageBotRunDialog.css'; import MarkBotRunFinishedButton from './MarkBotRunFinishedButton'; @@ -59,7 +58,7 @@ const ManageBotRunDialog = ({ () => testers.filter( t => - !isBot(t) && + !t.isBot && !testPlanReportAssignedTestersQuery?.testPlanReport.draftTestPlanRuns.some( d => d.tester.id === t.id ) diff --git a/client/components/TestQueue/AssignTesterDropdown/index.jsx b/client/components/TestQueue/AssignTesterDropdown/index.jsx index a21e5b226..98463d388 100644 --- a/client/components/TestQueue/AssignTesterDropdown/index.jsx +++ b/client/components/TestQueue/AssignTesterDropdown/index.jsx @@ -16,10 +16,7 @@ import { import { useMutation, useQuery } from '@apollo/client'; import { LoadingStatus, useTriggerLoad } from '../../common/LoadingStatus'; import { SCHEDULE_COLLECTION_JOB_MUTATION } from '../../AddTestToQueueWithConfirmation/queries'; -import { - isBot, - isSupportedByResponseCollector -} from '../../../utils/automation'; +import { isSupportedByResponseCollector } from '../../../utils/automation'; import './AssignTesterDropdown.css'; @@ -77,7 +74,7 @@ const AssignTesterDropdown = ({ }); }, `Updating Test Plan Assignees. Deleting Test Plan Run for ${tester.username}`); } else { - if (isBot(tester)) { + if (tester.isBot) { await triggerLoad(async () => { await scheduleCollection({ variables: { @@ -131,12 +128,12 @@ const AssignTesterDropdown = ({ const testerIsAssigned = isTesterAssigned(username); const classname = [ testerIsAssigned ? 'assigned' : 'not-assigned', - isBot(tester) ? 'bot' : 'human' + tester.isBot ? 'bot' : 'human' ].join(' '); let icon; if (testerIsAssigned) { icon = faCheck; - } else if (isBot(tester)) { + } else if (tester.isBot) { const supportedByBot = isSupportedByResponseCollector( testPlanReportAtBrowserQuery?.testPlanReport diff --git a/client/components/TestQueue/queries.js b/client/components/TestQueue/queries.js index 0f5ca2375..51d57a5f8 100644 --- a/client/components/TestQueue/queries.js +++ b/client/components/TestQueue/queries.js @@ -11,6 +11,7 @@ export const TEST_QUEUE_PAGE_QUERY = gql` id username roles + isBot } ats { id @@ -77,6 +78,7 @@ export const TEST_QUEUE_PAGE_QUERY = gql` tester { id username + isBot } testResultsLength } @@ -128,6 +130,7 @@ export const TEST_PLAN_REPORT_QUERY = gql` tester { id username + isBot } testResults { id @@ -267,6 +270,7 @@ export const ASSIGN_TESTER_MUTATION = gql` tester { id username + isBot } } } @@ -304,6 +308,7 @@ export const REMOVE_TESTER_MUTATION = gql` tester { id username + isBot } } } diff --git a/client/components/TestQueueCompletionStatusListItem/index.js b/client/components/TestQueueCompletionStatusListItem/index.js index ba3099eea..1ade96353 100644 --- a/client/components/TestQueueCompletionStatusListItem/index.js +++ b/client/components/TestQueueCompletionStatusListItem/index.js @@ -3,7 +3,6 @@ import PropTypes from 'prop-types'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { faRobot } from '@fortawesome/free-solid-svg-icons'; import BotTestCompletionStatus from './BotTestCompletionStatus'; -import { isBot } from '../../utils/automation'; import PreviouslyAutomatedTestCompletionStatus from './PreviouslyAutomatedTestCompletionStatus'; const TestQueueCompletionStatusListItem = ({ @@ -12,14 +11,13 @@ const TestQueueCompletionStatusListItem = ({ id }) => { const { testResultsLength, tester } = testPlanRun; - const testerIsBot = useMemo(() => isBot(tester), [tester]); const testPlanRunPreviouslyAutomated = useMemo( () => testPlanRun.initiatedByAutomation, [testPlanRun] ); const renderTesterInfo = () => { - if (testerIsBot) { + if (tester.isBot) { return ( @@ -44,7 +42,7 @@ const TestQueueCompletionStatusListItem = ({ }; const renderTestCompletionStatus = () => { - if (testerIsBot) { + if (tester.isBot) { return ( { - const botTestPlanRun = draftTestPlanRuns.find(({ tester }) => - isBot(tester) + const botTestPlanRun = draftTestPlanRuns.find( + ({ tester: { isBot } }) => isBot ); if (isAdmin && !isLoading) { diff --git a/client/components/TestRun/TestNavigator.jsx b/client/components/TestRun/TestNavigator.jsx index 68e99ba36..e1aba2b0c 100644 --- a/client/components/TestRun/TestNavigator.jsx +++ b/client/components/TestRun/TestNavigator.jsx @@ -8,7 +8,6 @@ import { import { Col } from 'react-bootstrap'; import React, { useMemo } from 'react'; import '@fortawesome/fontawesome-svg-core/styles.css'; -import { isBot } from '../../utils/automation'; import { COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY } from './queries'; import { useQuery } from '@apollo/client'; @@ -24,7 +23,7 @@ const TestNavigator = ({ handleTestClick = () => {}, testPlanRun = null }) => { - const isBotCompletedTest = isBot(testPlanRun?.tester); + const isBotCompletedTest = testPlanRun?.tester?.isBot; const { data: collectionJobQuery } = useQuery( COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY, diff --git a/client/components/TestRun/index.jsx b/client/components/TestRun/index.jsx index 957c65cda..0f44ed085 100644 --- a/client/components/TestRun/index.jsx +++ b/client/components/TestRun/index.jsx @@ -42,7 +42,6 @@ import './TestRun.css'; import ReviewConflicts from '../ReviewConflicts'; import createIssueLink from '../../utils/createIssueLink'; import { convertDateToString } from '../../utils/formatter'; -import { isBot } from '../../utils/automation'; const TestRun = () => { const params = useParams(); @@ -895,7 +894,7 @@ const TestRun = () => { const renderTestsCompletedInfoBox = () => { let isReviewingBot = false; if (openAsUserId) { - isReviewingBot = isBot(openAsUser); + isReviewingBot = openAsUser.isBot; } let content; @@ -1037,7 +1036,7 @@ const TestRun = () => { href={issueLink} /> - {isBot(openAsUser) && externalLogsUrl ? ( + {openAsUser?.isBot && externalLogsUrl ? (
  • { submitButtonRef={ testRendererSubmitButtonRef } - isReviewingBot={isBot(openAsUser)} + isReviewingBot={openAsUser?.isBot} isSubmitted={isTestSubmitClicked} isEdit={isTestEditClicked} setIsRendererReady={setIsRendererReady} @@ -1222,7 +1221,7 @@ const TestRun = () => { let openAsUserHeading = null; if (openAsUserId) { - if (isBot(openAsUser)) { + if (openAsUser.isBot) { openAsUserHeading = (
    Reviewing tests of{' '} diff --git a/client/components/TestRun/queries.js b/client/components/TestRun/queries.js index e8d485dbe..138b77abf 100644 --- a/client/components/TestRun/queries.js +++ b/client/components/TestRun/queries.js @@ -8,6 +8,7 @@ export const TEST_RUN_PAGE_QUERY = gql` tester { id username + isBot } testResults { id @@ -179,6 +180,7 @@ export const TEST_RUN_PAGE_QUERY = gql` users { id username + isBot } } `; @@ -211,6 +213,7 @@ export const TEST_RUN_PAGE_ANON_QUERY = gql` id tester { username + isBot } } scenarioResult { @@ -304,6 +307,7 @@ export const FIND_OR_CREATE_TEST_RESULT_MUTATION = gql` tester { id username + isBot } testResults { id @@ -498,6 +502,7 @@ export const FIND_OR_CREATE_TEST_RESULT_MUTATION = gql` testPlanRun { id tester { + isBot username } } @@ -596,6 +601,7 @@ export const SAVE_TEST_RESULT_MUTATION = gql` tester { id username + isBot } testResults { id diff --git a/client/tests/AssignTesterDropdown.test.jsx b/client/tests/AssignTesterDropdown.test.jsx index 8852eb6ad..bb8dce235 100644 --- a/client/tests/AssignTesterDropdown.test.jsx +++ b/client/tests/AssignTesterDropdown.test.jsx @@ -28,9 +28,9 @@ jest.mock('@apollo/client', () => { }); const mockPossibleTesters = [ - { id: '1', username: 'bee' }, - { id: '2', username: 'puppy' }, - { id: '3', username: 'NVDA Bot' } + { id: '1', username: 'bee', isBot: false }, + { id: '2', username: 'puppy', isBot: false }, + { id: '3', username: 'NVDA Bot', isBot: true } ]; const mockProps = { diff --git a/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminNotPopulatedMock.js b/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminNotPopulatedMock.js index f5243b0df..71e44308a 100644 --- a/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminNotPopulatedMock.js +++ b/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminNotPopulatedMock.js @@ -17,17 +17,20 @@ export default testQueuePageQuery => [ { id: '1', username: 'foo-bar', - roles: ['ADMIN', 'TESTER'] + roles: ['ADMIN', 'TESTER'], + isBot: false }, { id: '4', username: 'bar-foo', - roles: ['TESTER'] + roles: ['TESTER'], + isBot: false }, { id: '5', username: 'boo-far', - roles: ['TESTER'] + roles: ['TESTER'], + isBot: false } ], testPlanVersions: [], diff --git a/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminPopulatedMock.js b/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminPopulatedMock.js index fad26ed4d..53008087a 100644 --- a/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminPopulatedMock.js +++ b/client/tests/__mocks__/GraphQLMocks/TestQueuePageAdminPopulatedMock.js @@ -8,7 +8,8 @@ export default testQueuePageQuery => [ me: { id: '101', username: 'alflennik', - roles: ['ADMIN', 'TESTER'] + roles: ['ADMIN', 'TESTER'], + isBot: false }, ats: [ { @@ -173,13 +174,20 @@ export default testQueuePageQuery => [ { id: '1', username: 'esmeralda-baggins', - roles: ['TESTER', 'ADMIN'] + roles: ['TESTER', 'ADMIN'], + isBot: false + }, + { + id: '2', + username: 'tom-proudfeet', + roles: ['TESTER'], + isBot: false }, - { id: '2', username: 'tom-proudfeet', roles: ['TESTER'] }, { id: '101', username: 'alflennik', - roles: ['TESTER', 'ADMIN'] + roles: ['TESTER', 'ADMIN'], + isBot: false } ], testPlanVersions: [ @@ -243,7 +251,8 @@ export default testQueuePageQuery => [ id: '1', tester: { id: '1', - username: 'esmeralda-baggins' + username: 'esmeralda-baggins', + isBot: false }, testResultsLength: 0, initiatedByAutomation: false @@ -272,7 +281,8 @@ export default testQueuePageQuery => [ id: '1', tester: { id: '1', - username: 'esmeralda-baggins' + username: 'esmeralda-baggins', + isBot: false }, testResultsLength: 0, initiatedByAutomation: false @@ -299,13 +309,21 @@ export default testQueuePageQuery => [ draftTestPlanRuns: [ { id: '3', - tester: { id: '2', username: 'tom-proudfeet' }, + tester: { + id: '2', + username: 'tom-proudfeet', + isBot: false + }, testResultsLength: 3, initiatedByAutomation: false }, { id: '101', - tester: { id: '101', username: 'alflennik' }, + tester: { + id: '101', + username: 'alflennik', + isBot: false + }, testResultsLength: 1, initiatedByAutomation: false }, @@ -313,7 +331,8 @@ export default testQueuePageQuery => [ id: '2', tester: { id: '1', - username: 'esmeralda-baggins' + username: 'esmeralda-baggins', + isBot: false }, testResultsLength: 3, initiatedByAutomation: false diff --git a/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterNotPopulatedMock.js b/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterNotPopulatedMock.js index fe03d4e0e..266235ed0 100644 --- a/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterNotPopulatedMock.js +++ b/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterNotPopulatedMock.js @@ -18,18 +18,21 @@ export default testQueuePageQuery => [ id: '1', username: 'foo-bar', roles: ['ADMIN', 'TESTER'], + isBot: false, __typename: 'User' }, { id: '4', username: 'bar-foo', roles: ['TESTER'], + isBot: false, __typename: 'User' }, { id: '5', username: 'boo-far', roles: ['TESTER'], + isBot: false, __typename: 'User' } ], diff --git a/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterPopulatedMock.js b/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterPopulatedMock.js index 31f3a1808..bb3f94a8b 100644 --- a/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterPopulatedMock.js +++ b/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterPopulatedMock.js @@ -174,17 +174,20 @@ export default testQueuePageQuery => [ { id: '1', username: 'foo-bar', - roles: ['ADMIN', 'TESTER'] + roles: ['ADMIN', 'TESTER'], + isBot: false }, { id: '4', username: 'bar-foo', - roles: ['TESTER'] + roles: ['TESTER'], + isBot: false }, { id: '5', username: 'boo-far', - roles: ['TESTER'] + roles: ['TESTER'], + isBot: false } ], testPlanVersions: [ @@ -256,7 +259,8 @@ export default testQueuePageQuery => [ id: '18', tester: { id: '1', - username: 'foo-bar' + username: 'foo-bar', + isBot: false }, testResultsLength: 0, initiatedByAutomation: false @@ -265,7 +269,8 @@ export default testQueuePageQuery => [ id: '19', tester: { id: '4', - username: 'bar-foo' + username: 'bar-foo', + isBot: false }, testResultsLength: 0, initiatedByAutomation: false diff --git a/client/utils/automation.js b/client/utils/automation.js index 51cbf084f..4406116ed 100644 --- a/client/utils/automation.js +++ b/client/utils/automation.js @@ -8,8 +8,6 @@ export const isSupportedByResponseCollector = ctx => { ); }; -export const isBot = user => user?.username?.toLowerCase().slice(-3) === 'bot'; - // TODO: Stub, support for more bot users should be added export const getBotUsernameFromAtBrowser = (at, browser) => { if (at?.name === 'NVDA' && browser?.name === 'Chrome') { From 93760e7bab5784b1eced71826d42c26b1394f77c Mon Sep 17 00:00:00 2001 From: "Mx. Corey Frang" Date: Thu, 16 May 2024 10:25:21 -0400 Subject: [PATCH 10/19] Fix missing isBot in mock data --- .../__mocks__/GraphQLMocks/TestQueuePageTesterPopulatedMock.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterPopulatedMock.js b/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterPopulatedMock.js index bb3f94a8b..7e621452c 100644 --- a/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterPopulatedMock.js +++ b/client/tests/__mocks__/GraphQLMocks/TestQueuePageTesterPopulatedMock.js @@ -307,7 +307,8 @@ export default testQueuePageQuery => [ id: '20', tester: { id: '5', - username: 'boo-far' + username: 'boo-far', + isBot: false }, testResultsLength: 0, initiatedByAutomation: false From 2af0b88f751ed13c925c2204f4a485fc5f924be4 Mon Sep 17 00:00:00 2001 From: Erika Miguel Date: Thu, 16 May 2024 11:52:05 -0400 Subject: [PATCH 11/19] Fix: Add support in import test script for importing test harness (#1064) * Add support in import test script for importing test harness * Support older version commits * Updating logging messages * Remove commands.json from server resources and update commands to commandsV1 * Adding note in local development * Remove client resources from cache --- .gitignore | 2 + client/resources/aria-at-harness.mjs | 675 ------ client/resources/aria-at-test-io-format.mjs | 1878 ----------------- client/resources/aria-at-test-run.mjs | 1336 ------------ client/resources/aria-at-test-window.mjs | 73 - client/resources/at-commands.mjs | 273 --- client/resources/commands.json | 115 - client/resources/keys.json | 117 - client/resources/keys.mjs | 133 -- client/resources/support.json | 410 ---- client/resources/types/aria-at-test-result.js | 39 - client/resources/types/aria-at-test-run.js | 100 - client/resources/vrender.mjs | 987 --------- docs/local-development.md | 2 + .../20211116172219-commandSequences.js | 2 +- server/resolvers/helpers/retrieveCommands.js | 4 +- .../{commands.json => commandsV1.json} | 0 server/scripts/import-tests/index.js | 39 +- 18 files changed, 45 insertions(+), 6140 deletions(-) delete mode 100644 client/resources/aria-at-harness.mjs delete mode 100644 client/resources/aria-at-test-io-format.mjs delete mode 100644 client/resources/aria-at-test-run.mjs delete mode 100644 client/resources/aria-at-test-window.mjs delete mode 100644 client/resources/at-commands.mjs delete mode 100644 client/resources/commands.json delete mode 100644 client/resources/keys.json delete mode 100644 client/resources/keys.mjs delete mode 100644 client/resources/support.json delete mode 100644 client/resources/types/aria-at-test-result.js delete mode 100644 client/resources/types/aria-at-test-run.js delete mode 100644 client/resources/vrender.mjs rename server/resources/{commands.json => commandsV1.json} (100%) diff --git a/.gitignore b/.gitignore index 50447732f..50eb2d5dc 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,5 @@ server/migrations/test_plan_target_id.csv # Private Key files (installed by deploy) jwt-signing-key.pem + +client/resources diff --git a/client/resources/aria-at-harness.mjs b/client/resources/aria-at-harness.mjs deleted file mode 100644 index bc52077be..000000000 --- a/client/resources/aria-at-harness.mjs +++ /dev/null @@ -1,675 +0,0 @@ -import { - element, - fragment, - property, - attribute, - className, - style, - focus, - render, -} from './vrender.mjs'; -import { - AssertionResultMap, - userCloseWindow, - userOpenWindow, - WhitespaceStyleMap, - UnexpectedBehaviorImpactMap, -} from './aria-at-test-run.mjs'; -import { TestRunExport, TestRunInputOutput } from './aria-at-test-io-format.mjs'; -import { TestWindow } from './aria-at-test-window.mjs'; - -const PAGE_STYLES = ` - table { - border-collapse: collapse; - margin-bottom: 1em; - } - - table, td, th { - border: 1px solid black; - } - - td { - padding: .5em; - } - - table.record-results tr:first-child { - font-weight: bold; - } - - textarea { - width: 100% - } - - fieldset.problem-select { - margin-top: 1em; - margin-left: 1em; - } - - div.problem-option-container.enabled { - margin-bottom: 0.5em; - } - - div.problem-option-container:last-child { - margin-bottom: 0; - } - - fieldset.assertions { - margin-bottom: 1em; - } - - label.assertion { - display: block; - } - - .required:not(.highlight-required) { - display: none; - } - - .required-other:not(.highlight-required) { - display: none; - } - - .required.highlight-required { - color: red; - } - - fieldset.highlight-required { - border-color: red; - } - - fieldset .highlight-required { - color: red; - } - - .off-screen { - position: absolute !important; - height: 1px; - width: 1px; - overflow: hidden; - clip: rect(1px, 1px, 1px, 1px); - white-space: nowrap; - } -`; - -let testRunIO = new TestRunInputOutput(); -testRunIO.setTitleInputFromTitle(document.title); -testRunIO.setUnexpectedInputFromBuiltin(); -testRunIO.setScriptsInputFromMap(typeof scripts === 'object' ? scripts : {}); - -/** - * @param {SupportJSON} newSupport - * @param {CommandsJSON} newCommandsData - * @param {AllCommandsJSON} allCommands - */ -export function initialize(newSupport, newCommandsData, allCommands) { - testRunIO.setSupportInputFromJSON(newSupport); - testRunIO.setAllCommandsInputFromJSON(allCommands); - testRunIO.setConfigInputFromQueryParamsAndSupport( - Array.from(new URL(document.location).searchParams) - ); - testRunIO.setKeysInputFromBuiltinAndConfig(); - testRunIO.setCommandsInputFromJSONAndConfigKeys(newCommandsData); -} - -/** - * @param {BehaviorJSON} atBehavior - */ -export function verifyATBehavior(atBehavior) { - if (testRunIO.behaviorInput !== null) { - throw new Error('Test files should only contain one verifyATBehavior call.'); - } - - testRunIO.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected(atBehavior); -} - -export async function loadCollectedTestAsync(testRoot, testFileName) { - const collectedTestResponse = await fetch(`${testRoot}/${testFileName}`); - const collectedTestJson = await collectedTestResponse.json(); - - // v2 commands.json - const commandsJsonResponse = await fetch('../commands.json'); - if (commandsJsonResponse.ok) { - const commandsJson = await commandsJsonResponse.json(); - testRunIO.setAllCommandsInputFromJSON(commandsJson); - } - - await testRunIO.setInputsFromCollectedTestAsync(collectedTestJson, testRoot); - testRunIO.setConfigInputFromQueryParamsAndSupport([ - ['at', collectedTestJson.target.at.key], - ...Array.from(new URL(document.location).searchParams), - ]); - - displayInstructionsForBehaviorTest(); -} - -export function displayTestPageAndInstructions(testPage) { - if (document.readyState !== 'complete') { - window.setTimeout(() => { - displayTestPageAndInstructions(testPage); - }, 100); - return; - } - - testRunIO.setPageUriInputFromPageUri(testPage); - - document.querySelector('html').setAttribute('lang', 'en'); - var style = document.createElement('style'); - style.innerHTML = PAGE_STYLES; - document.head.appendChild(style); - - displayInstructionsForBehaviorTest(); -} - -function displayInstructionsForBehaviorTest() { - const windowManager = new TestWindow({ - ...testRunIO.testWindowOptions(), - hooks: { - windowOpened() { - app.dispatch(userOpenWindow()); - }, - windowClosed() { - app.dispatch(userCloseWindow()); - }, - }, - }); - - // First, execute necesary set up script in test page if the test page is open from a previous behavior test - windowManager.prepare(); - - const app = new TestRunExport({ - hooks: { - openTestPage() { - windowManager.open(); - }, - closeTestPage() { - windowManager.close(); - }, - postResults: () => postResults(testRunIO.resultJSON(app.state)), - }, - state: testRunIO.testRunState(), - resultsJSON: state => testRunIO.resultJSON(state), - }); - app.observe(() => { - render(document.body, renderVirtualTestPage(app.testPageAndResults())); - }); - render(document.body, renderVirtualTestPage(app.testPageAndResults())); - - // if test is loaded in iFrame - if (window.parent && window.parent.postMessage) { - // results can be submitted by parent posting a message to the - // iFrame with a data.type property of 'submit' - window.addEventListener('message', function (message) { - if (!validateMessage(message, 'submit')) return; - app.hooks.submit(); - }); - - // send message to parent that test has loaded - window.parent.postMessage( - { - type: 'loaded', - data: { - testPageUri: windowManager.pageUri, - }, - }, - '*' - ); - } -} - -function validateMessage(message, type) { - if (window.location.origin !== message.origin) { - return false; - } - if (!message.data || typeof message.data !== 'object') { - return false; - } - if (message.data.type !== type) { - return false; - } - return true; -} - -/** - * @param {resultsJSON} resultsJSON - */ -function postResults(resultsJSON) { - // send message to parent if test is loaded in iFrame - if (window.parent && window.parent.postMessage) { - window.parent.postMessage( - { - type: 'results', - data: resultsJSON, - }, - '*' - ); - } -} - -function bind(fn, ...args) { - return (...moreArgs) => fn(...args, ...moreArgs); -} - -const a = bind(element, 'a'); -const br = bind(element, 'br'); -const button = bind(element, 'button'); -const div = bind(element, 'div'); -const em = bind(element, 'em'); -const kbd = bind(element, 'kbd'); -const fieldset = bind(element, 'fieldset'); -const h1 = bind(element, 'h1'); -const h2 = bind(element, 'h2'); -const h3 = bind(element, 'h3'); -const hr = bind(element, 'hr'); -const input = bind(element, 'input'); -const label = bind(element, 'label'); -const select = bind(element, 'select'); -const option = bind(element, 'option'); -const legend = bind(element, 'legend'); -const li = bind(element, 'li'); -const ol = bind(element, 'ol'); -const p = bind(element, 'p'); -const script = bind(element, 'script'); -const section = bind(element, 'section'); -const span = bind(element, 'span'); -const table = bind(element, 'table'); -const td = bind(element, 'td'); -const textarea = bind(element, 'textarea'); -const th = bind(element, 'th'); -const tr = bind(element, 'tr'); -const ul = bind(element, 'ul'); - -const forInput = bind(attribute, 'for'); -const href = bind(attribute, 'href'); -const id = bind(attribute, 'id'); -const name = bind(attribute, 'name'); -const tabIndex = bind(attribute, 'tabindex'); -const textContent = bind(attribute, 'textContent'); -const type = bind(attribute, 'type'); -const ariaLabel = bind(attribute, 'aria-label'); -const ariaHidden = bind(attribute, 'aria-hidden'); - -const value = bind(property, 'value'); -const checked = bind(property, 'checked'); -const disabled = bind(property, 'disabled'); - -/** @type {(cb: (ev: MouseEvent) => void) => any} */ -const onclick = bind(property, 'onclick'); -/** @type {(cb: (ev: InputEvent) => void) => any} */ -const onchange = bind(property, 'onchange'); -/** @type {(cb: (ev: KeyboardEvent) => void) => any} */ -const onkeydown = bind(property, 'onkeydown'); - -/** - * @param {Description} value - */ -function rich(value) { - if (typeof value === 'string') { - return value; - } else if (Array.isArray(value)) { - return fragment(...value.map(rich)); - } else if (value.kbd) { - return kbd.bind(value.kbd)(rich(value.kbd)); - } else { - if ('whitespace' in value) { - if (value.whitespace === WhitespaceStyleMap.LINE_BREAK) { - return br(); - } - return null; - } - return (value.href ? a.bind(null, href(value.href)) : span)( - className([ - value.offScreen ? 'off-screen' : '', - value.required ? 'required' : '', - value.highlightRequired ? 'highlight-required' : '', - ]), - rich(value.description) - ); - } -} - -/** - * @param {TestPageAndResultsDocument} doc - */ -function renderVirtualTestPage(doc) { - return fragment( - 'instructions' in doc - ? div( - section( - id('errors'), - style({ display: doc.errors && doc.errors.visible ? 'block' : 'none' }), - h2(doc.errors ? doc.errors.header : ''), - ul( - ...(doc.errors && doc.errors.errors ? doc.errors.errors.map(error => li(error)) : []) - ), - hr() - ), - section(id('instructions'), renderVirtualInstructionDocument(doc.instructions)), - section(id('record-results')) - ) - : null, - 'results' in doc ? renderVirtualResultsTable(doc.results) : null, - doc.resultsJSON - ? script( - type('text/json'), - id('__ariaatharness__results__'), - textContent(JSON.stringify(doc.resultsJSON)) - ) - : null - ); -} - -/** - * @param doc {InstructionDocument} - */ -function renderVirtualInstructionDocument(doc) { - function compose(...fns) { - return around => fns.reduceRight((carry, fn) => fn(carry), around); - } - - const map = (ary, el) => ary.map(item => el(item)); - - return div( - instructionHeader(doc.instructions), - - instructCommands(doc.instructions.instructions), - - instructAssertions(doc.instructions.assertions), - - button( - disabled(!doc.instructions.openTestPage.enabled), - onclick(doc.instructions.openTestPage.click), - rich(doc.instructions.openTestPage.button) - ), - - resultHeader(doc.results.header), - - section(...doc.results.commands.map(commandResult)), - - doc.submit ? button(onclick(doc.submit.click), rich(doc.submit.button)) : null - ); - - /** - * @param {InstructionDocumentResultsHeader} param0 - */ - function resultHeader({ header, description }) { - return fragment(h2(rich(header)), p(rich(description))); - } - - /** - * @param {InstructionDocumentResultsCommand} command - * @param {number} commandIndex - */ - function commandResult(command, commandIndex) { - return fragment( - h3(rich(command.header)), - p( - label(rich(command.atOutput.description)), - textarea( - value(command.atOutput.value), - focus(command.atOutput.focus), - onchange(ev => - command.atOutput.change(/** @type {HTMLInputElement} */ (ev.currentTarget).value) - ) - ) - ), - fieldset( - className(['assertions']), - legend(rich(command.assertionsHeader.descriptionHeader)), - ...command.assertions.map(bind(commandResultAssertion, commandIndex)) - ), - ...[command.unexpectedBehaviors].map(bind(commandResultUnexpectedBehavior, commandIndex)) - ); - } - - /** - * @param {number} commandIndex - * @param {InstructionDocumentResultsCommandsUnexpected} unexpected - */ - function commandResultUnexpectedBehavior(commandIndex, unexpected) { - return fieldset( - id(`cmd-${commandIndex}-problem`), - rich(unexpected.description), - div( - radioChoice( - `problem-${commandIndex}-true`, - `problem-${commandIndex}`, - unexpected.passChoice - ) - ), - div( - radioChoice( - `problem-${commandIndex}-false`, - `problem-${commandIndex}`, - unexpected.failChoice - ) - ), - fieldset( - className(['problem-select']), - id(`cmd-${commandIndex}-problem-checkboxes`), - legend(rich(unexpected.failChoice.options.header)), - ...unexpected.failChoice.options.options.map(failOption => { - const failOptionId = failOption.description - .toLowerCase() - .replace(/[.,]/g, '') - .replace(/\s+/g, '-'); - - const undesirableBehaviorCheckbox = div( - input( - type('checkbox'), - value(failOption.description), - id(`${failOptionId}-${commandIndex}-checkbox`), - className([`undesirable-${commandIndex}`]), - disabled(!failOption.enabled), - checked(failOption.checked), - focus(failOption.focus), - onchange(ev => - failOption.change(/** @type {HTMLInputElement} */ (ev.currentTarget).checked) - ), - onkeydown(ev => { - if (failOption.keydown(ev.key)) { - ev.stopPropagation(); - ev.preventDefault(); - } - }) - ), - label( - id(`${failOptionId}-${commandIndex}-label`), - forInput(`${failOptionId}-${commandIndex}-checkbox`), - rich(`${failOption.description} behavior occurred`) - ) - ); - - const impactSelect = div( - className([!failOption.checked && 'off-screen']), - ariaHidden(!failOption.checked), - label(forInput(`${failOptionId}-${commandIndex}-impact`), rich('Impact:')), - select( - id(`${failOptionId}-${commandIndex}-impact`), - ariaLabel(`Impact for ${failOption.description}`), - option(UnexpectedBehaviorImpactMap.MODERATE), - option(UnexpectedBehaviorImpactMap.SEVERE), - disabled(!failOption.checked), - onchange(ev => - failOption.impactchange(/** @type {HTMLInputElement} */ (ev.currentTarget).value) - ) - ) - ); - - const detailsTextInput = div( - className([!failOption.checked && 'off-screen']), - ariaHidden(!failOption.checked), - label( - forInput(`${failOptionId}-${commandIndex}-details`), - rich(failOption.more.description) - ), - input( - type('text'), - id(`${failOptionId}-${commandIndex}-details`), - ariaLabel(`Details for ${failOption.description}`), - className(['undesirable-other-input']), - disabled(!failOption.more.enabled), - value(failOption.more.value), - onchange(ev => - failOption.more.change(/** @type {HTMLInputElement} */ (ev.currentTarget).value) - ) - ) - ); - - return div( - className(['problem-option-container', failOption.checked && 'enabled']), - undesirableBehaviorCheckbox, - impactSelect, - detailsTextInput - ); - }) - ) - ); - } - - /** - * @param {number} commandIndex - * @param {InstructionDocumentResultsCommandsAssertion} assertion - * @param {number} assertionIndex - */ - function commandResultAssertion(commandIndex, assertion, assertionIndex) { - return label( - className(['assertion']), - input( - type('checkbox'), - id(`cmd-${commandIndex}-${assertionIndex}`), - checked(assertion.passed === AssertionResultMap.PASS), - onclick(assertion.click) - ), - rich(assertion.description) - ); - } - - /** - * @param {string} idKey - * @param {string} nameKey - * @param {InstructionDocumentAssertionChoice} choice - */ - function radioChoice(idKey, nameKey, choice) { - return fragment( - input( - type('radio'), - id(idKey), - name(nameKey), - checked(choice.checked), - focus(choice.focus), - onclick(choice.click) - ), - label(id(`${idKey}-label`), forInput(`${idKey}`), rich(choice.label)) - ); - } - - /** - * @param {InstructionDocumentInstructionsInstructions} param0 - * @returns - */ - function instructCommands({ - header, - instructions, - strongInstructions: boldInstructions, - commands, - }) { - return fragment( - h2(rich(header)), - ol( - ...map(instructions, compose(li, rich)), - ...map(boldInstructions, compose(li, em, rich)), - li(rich(commands.description), ul(...map(commands.commands, compose(li, em, rich)))) - ) - ); - } - - /** - * @param {InstructionDocumentInstructions} param0 - */ - function instructionHeader({ header, description }) { - return fragment( - h1(id('behavior-header'), tabIndex('0'), focus(header.focus), rich(header.header)), - p(rich(description)) - ); - } - - /** - * @param {InstructionDocumentInstructionsAssertions} param0 - */ - function instructAssertions({ header, description, assertions }) { - return fragment( - h2(rich(header)), - p(rich(description)), - ol(...map(assertions, compose(li, em, rich))) - ); - } -} - -/** - * @param {ResultsTableDocument} results - */ -function renderVirtualResultsTable(results) { - return fragment( - h1(rich(results.header)), - h2(id('overallstatus'), rich(results.status.header)), - - table( - (({ description, support, details }) => tr(th(description), th(support), th(details)))( - results.table.headers - ), - results.table.commands.map( - ({ - description, - support, - details: { output, passingAssertions, failingAssertions, unexpectedBehaviors }, - }) => - fragment( - tr( - td(rich(description)), - td(rich(support)), - td( - p(rich(output)), - commandDetailsList(passingAssertions), - commandDetailsList(failingAssertions), - commandDetailsList(unexpectedBehaviors) - ) - ) - ) - ) - ) - ); - - /** - * @param {object} list - * @param {Description} list.description - * @param {Description[]} list.items - */ - function commandDetailsList({ description, items }) { - return div(description, ul(...items.map(description => li(rich(description))))); - } -} - -/** @typedef {import('./aria-at-test-io-format.mjs').SupportJSON} SupportJSON */ -/** @typedef {import('./aria-at-test-io-format.mjs').AllCommandsJSON} AllCommandsJSON */ -/** @typedef {import('./aria-at-test-io-format.mjs').CommandsJSON} CommandsJSON */ -/** @typedef {import('./aria-at-test-io-format.mjs').BehaviorJSON} BehaviorJSON */ - -/** @typedef {import('./aria-at-test-run.mjs').TestRunState} TestRunState */ - -/** @typedef {import('./aria-at-test-run.mjs').Description} Description */ - -/** @typedef {import('./aria-at-test-run.mjs').InstructionDocument} InstructionDocument */ -/** @typedef {import('./aria-at-test-run.mjs').InstructionDocumentInstructions} InstructionDocumentInstructions */ -/** @typedef {import('./aria-at-test-run.mjs').InstructionDocumentInstructionsAssertions} InstructionDocumentInstructionsAssertions */ -/** @typedef {import('./aria-at-test-run.mjs').InstructionDocumentResultsHeader} InstructionDocumentResultsHeader */ -/** @typedef {import('./aria-at-test-run.mjs').InstructionDocumentResultsCommand} InstructionDocumentResultsCommand */ -/** @typedef {import('./aria-at-test-run.mjs').InstructionDocumentResultsCommandsUnexpected} InstructionDocumentResultsCommandsUnexpected */ -/** @typedef {import("./aria-at-test-run.mjs").InstructionDocumentResultsCommandsAssertion} InstructionDocumentResultsCommandsAssertion */ -/** @typedef {import("./aria-at-test-run.mjs").InstructionDocumentAssertionChoice} InstructionDocumentAssertionChoice */ -/** @typedef {import("./aria-at-test-run.mjs").InstructionDocumentInstructionsInstructions} InstructionDocumentInstructionsInstructions */ - -/** @typedef {import('./aria-at-test-run.mjs').ResultsTableDocument} ResultsTableDocument */ - -/** - * @typedef {import('./aria-at-test-io-format.mjs').TestPageAndResultsDocument} TestPageAndResultsDocument - */ diff --git a/client/resources/aria-at-test-io-format.mjs b/client/resources/aria-at-test-io-format.mjs deleted file mode 100644 index 8121c4133..000000000 --- a/client/resources/aria-at-test-io-format.mjs +++ /dev/null @@ -1,1878 +0,0 @@ -/// -/// -/// - -import { - AssertionResultMap, - CommonResultMap, - createEnumMap, - HasUnexpectedBehaviorMap, - TestRun, - UserActionMap, -} from './aria-at-test-run.mjs'; -import * as keysModule from './keys.mjs'; - -const UNEXPECTED_BEHAVIORS = [ - 'Output is excessively verbose, e.g., includes redundant and/or irrelevant speech', - 'Reading cursor position changed in an unexpected manner', - 'Screen reader became extremely sluggish', - 'Screen reader crashed', - 'Browser crashed', -]; - -/** Depends on ConfigInput. */ -class KeysInput { - /** - * @param {object} value - * @param {string} value.origin - * @param {{[KEY_ID: string]: string}} value.keys - * @param {ATJSON} value.at - * @param {{[atMode in ATMode]: string}} value.modeInstructions - * @private - */ - constructor(value) { - this.errors = []; - - /** @private */ - this._value = value; - } - - origin() { - return this._value.origin; - } - - /** - * @param {string} keyId - * @returns {string} - */ - keysForCommand(keyId) { - return this._value.keys[keyId]; - } - - /** - * @param {ATMode} atMode - */ - modeInstructions(atMode) { - if (this._value.modeInstructions[atMode]) { - return this._value.modeInstructions[atMode]; - } - return ''; - } - - /** - * @param {object} data - * @param {ConfigInput} data.configInput - */ - static fromBuiltinAndConfig({ configInput }) { - const keys = keysModule; - const atKey = configInput.at().key; - - invariant( - ['jaws', 'nvda', 'voiceover_macos'].includes(atKey), - '%s is one of "jaws", "nvda", or "voiceover_macos"', - atKey - ); - - return new KeysInput({ - origin: 'resources/keys.mjs', - keys, - at: atKey, - modeInstructions: { - reading: { - jaws: `Verify the Virtual Cursor is active by pressing ${keys.ALT_DELETE}. If it is not, exit Forms Mode to activate the Virtual Cursor by pressing ${keys.ESC}.`, - nvda: `Ensure NVDA is in browse mode by pressing ${keys.ESC}. Note: This command has no effect if NVDA is already in browse mode.`, - voiceover_macos: `Toggle Quick Nav ON by pressing the ${keys.LEFT} and ${keys.RIGHT} keys at the same time.`, - }[atKey], - interaction: { - jaws: `Verify the PC Cursor is active by pressing ${keys.ALT_DELETE}. If it is not, turn off the Virtual Cursor by pressing ${keys.INS_Z}.`, - nvda: `If NVDA did not make the focus mode sound when the test page loaded, press ${keys.INS_SPACE} to turn focus mode on.`, - voiceover_macos: `Toggle Quick Nav OFF by pressing the ${keys.LEFT} and ${keys.RIGHT} keys at the same time.`, - }[atKey], - }, - }); - } - - /** @param {AriaATFile.CollectedTest} collectedTest */ - static fromCollectedTest(collectedTest) { - return new KeysInput({ - origin: 'test.collected.json', - keys: collectedTest.commands.reduce((carry, { keypresses }) => { - return keypresses.reduce((carry, { id, keystroke }) => { - carry[id] = keystroke; - return carry; - }, carry); - }, {}), - at: collectedTest.target.at.key, - modeInstructions: collectedTest.instructions.mode, - }); - } -} - -class SupportInput { - /** - * @param {SupportJSON} value - * @private - */ - constructor(value) { - this.errors = []; - - /** @private */ - this._value = value; - } - - defaultAT() { - return this._value.ats[0]; - } - - /** - * @param {string} atKey - * @returns {ATJSON | undefined} - */ - findAT(atKey) { - const lowercaseATKey = atKey.toLowerCase(); - return this._value.ats.find(({ key }) => key === lowercaseATKey); - } - - /** - * @param {SupportJSON} json - */ - static fromJSON(json) { - return new SupportInput(json); - } - - /** - * @param {AriaATFile.CollectedTest} collectedTest - */ - static fromCollectedTest(collectedTest) { - return new SupportInput({ - ats: [ - typeof collectedTest.target.at.raw === 'object' - ? collectedTest.target.at.raw - : { key: collectedTest.target.at.key, name: collectedTest.target.at.name }, - ], - applies_to: {}, - examples: [], - }); - } -} - -class AllCommandsInput { - /** - * @param {AllCommandsJSON} value - * @private - */ - constructor(value) { - this.errors = []; - - /** @private */ - this._value = value; - - /** @private */ - this._flattened = this.flattenObject(this._value); - } - - flattenObject(obj, parentKey) { - const flattened = {}; - - for (const key in obj) { - if (typeof obj[key] === 'object') { - const subObject = this.flattenObject(obj[key], parentKey + key + '.'); - Object.assign(flattened, subObject); - } else { - flattened[parentKey + key] = obj[key]; - } - } - - return flattened; - } - - findValueByKey(keyToFind) { - const keys = Object.keys(this._flattened); - - // Need to specially handle VO modifier key combination - if (keyToFind === 'vo') - return this.findValuesByKeys([this._flattened['modifierAliases.vo']])[0]; - - if (keyToFind.includes('modifiers.') || keyToFind.includes('keys.')) { - const parts = keyToFind.split('.'); - const keyToCheck = parts[parts.length - 1]; // value after the '.' - - if (this._flattened[keyToFind]) - return { - value: this._flattened[keyToFind], - key: keyToCheck, - }; - - return null; - } - - for (const key of keys) { - const parts = key.split('.'); - const parentKey = parts[0]; - const keyToCheck = parts[parts.length - 1]; // value after the '.' - - if (keyToCheck === keyToFind) { - if (parentKey === 'modifierAliases') { - return this.findValueByKey(`modifiers.${this._flattened[key]}`); - } else if (parentKey === 'keyAliases') { - return this.findValueByKey(`keys.${this._flattened[key]}`); - } - - return { - value: this._flattened[key], - key: keyToCheck, - }; - } - } - - // Return null if the key is not found - return null; - } - - findValuesByKeys(keysToFind = []) { - const result = []; - - const patternSepWithReplacement = (keyToFind, pattern, replacement) => { - if (keyToFind.includes(pattern)) { - let value = ''; - let validKeys = true; - const keys = keyToFind.split(pattern); - - for (const key of keys) { - const keyResult = this.findValueByKey(key); - if (keyResult) - value = value ? `${value}${replacement}${keyResult.value}` : keyResult.value; - else validKeys = false; - } - if (validKeys) return { value, key: keyToFind }; - } - - return null; - }; - - const patternSepHandler = keyToFind => { - let value = ''; - - if (keyToFind.includes(' ') && keyToFind.includes('+')) { - const keys = keyToFind.split(' '); - for (let [index, key] of keys.entries()) { - const keyToFindResult = this.findValueByKey(key); - if (keyToFindResult) keys[index] = keyToFindResult.value; - if (key.includes('+')) keys[index] = patternSepWithReplacement(key, '+', '+').value; - } - value = keys.join(' then '); - - return { value, key: keyToFind }; - } else if (keyToFind.includes(' ')) - return patternSepWithReplacement(keyToFind, ' ', ' then '); - else if (keyToFind.includes('+')) return patternSepWithReplacement(keyToFind, '+', '+'); - }; - - for (const keyToFind of keysToFind) { - if (keyToFind.includes(' ') || keyToFind.includes('+')) { - result.push(patternSepHandler(keyToFind)); - } else { - const keyToFindResult = this.findValueByKey(keyToFind); - if (keyToFindResult) result.push(keyToFindResult); - } - } - - return result; - } - - static fromJSON(json) { - return new AllCommandsInput(json); - } -} - -/** Depends on ConfigInput and KeysInput. */ -class CommandsInput { - /** - * @param {object} value - * @param {CommandsJSON} value.commands - * @param {ATJSON} value.at - * @param {KeysInput} keysInput - * @param {AllCommandsInput} allCommandsInput - * @private - */ - constructor(value, keysInput, allCommandsInput) { - this.errors = []; - - /** @private */ - this._value = value; - - /** @private */ - this._keysInput = keysInput; - - this._allCommandsInput = allCommandsInput; - } - - /** - * @param {object} config - * @param {string} config.task - * @param {ATMode} mode - * @returns {string[]} - */ - getCommands({ task }, mode) { - if (mode === 'reading' || mode === 'interaction') { - const v1Commands = this.getCommandsV1(task, mode); - return { - commands: v1Commands, - commandsAndSettings: v1Commands.map(command => ({ command })), - }; - } else { - return this.getCommandsV2({ task }, mode); - } - } - - getCommandsV1(task, mode) { - const assistiveTech = this._value.at; - - if (!this._value.commands[task]) { - throw new Error( - `Task "${task}" does not exist, please add to at-commands or correct your spelling.` - ); - } else if (!this._value.commands[task][mode]) { - throw new Error( - `Mode "${mode}" instructions for task "${task}" does not exist, please add to at-commands or correct your spelling.` - ); - } - - let commandsData = this._value.commands[task][mode][assistiveTech.key] || []; - let commands = []; - - for (let c of commandsData) { - let innerCommands = []; - let commandSequence = c[0].split(','); - for (let command of commandSequence) { - command = this._keysInput.keysForCommand(command); - if (typeof command === 'undefined') { - throw new Error( - `Key instruction identifier "${c}" for AT "${assistiveTech.name}", mode "${mode}", task "${task}" is not an available identified. Update you commands.json file to the correct identifier or add your identifier to resources/keys.mjs.` - ); - } - - let furtherInstruction = c[1]; - command = furtherInstruction ? `${command} ${furtherInstruction}` : command; - innerCommands.push(command); - } - commands.push(innerCommands.join(', then ')); - } - - return commands; - } - - getCommandsV2({ task }, mode) { - const assistiveTech = this._value.at; - let commandsAndSettings = []; - let commands = []; - - // Mode could be in the format of mode1_mode2 - // If they are from the same AT, this needs to return the function in the format of [ [[commands], settings], [[commands], settings], ... ] - for (const _atMode of mode.split('_')) { - if (assistiveTech.settings[_atMode] || _atMode === 'defaultMode') { - const [atMode] = deriveModeWithTextAndInstructions(_atMode, assistiveTech); - - if (!this._value.commands[task]) { - throw new Error( - `Task "${task}" does not exist, please add to at-commands or correct your spelling.` - ); - } else if (!this._value.commands[task][atMode]) { - throw new Error( - `Mode "${atMode}" instructions for task "${task}" does not exist, please add to at-commands or correct your spelling.` - ); - } - - let commandsData = this._value.commands[task][atMode][assistiveTech.key] || []; - for (let commandSequence of commandsData) { - for (const commandWithPresentationNumber of commandSequence) { - const [commandId, presentationNumber] = commandWithPresentationNumber.split('|'); - - let command; - const foundCommandKV = this._allCommandsInput.findValuesByKeys([commandId]); - if (!foundCommandKV.length) command = undefined; - else { - const { value } = this._allCommandsInput.findValuesByKeys([commandId])[0]; - command = value; - } - - if (typeof command === 'undefined') { - throw new Error( - `Key instruction identifier "${commandSequence}" for AT "${assistiveTech.name}", mode "${atMode}", task "${task}" is not an available identified. Update your commands.json file to the correct identifier or add your identifier to resources/keys.mjs.` - ); - } - - commands.push(command); - commandsAndSettings.push({ - command, - commandId, - presentationNumber: Number(presentationNumber), - settings: _atMode, - settingsText: assistiveTech.settings?.[_atMode]?.screenText || 'default mode active', - settingsInstructions: assistiveTech.settings?.[_atMode]?.instructions || [ - assistiveTech.defaultConfigurationInstructionsHTML, - ], - }); - } - } - } - } - - return { commands, commandsAndSettings }; - } - - /** - * @param {CommandsJSON} json - * @param {object} data - * @param {ConfigInput} data.configInput - * @param {KeysInput} data.keysInput - */ - static fromJSONAndConfigKeys(json, { configInput, keysInput, allCommandsInput }) { - return new CommandsInput({ commands: json, at: configInput.at() }, keysInput, allCommandsInput); - } - - /** - * @param {AriaATFile.CollectedTest} collectedTest - * @param {object} data - * @param {KeysInput} data.keysInput - */ - static fromCollectedTestKeys(collectedTest, { keysInput, allCommandsInput }) { - let settingsForTest = {}; - - // For v2 test format - const settings = collectedTest.target.at.settings; - if (settings) { - for (const _atMode of settings.split('_')) { - settingsForTest[_atMode] = { - // Use settings attribute to verify in filter if available - [collectedTest.target.at.key]: collectedTest.commands - .filter(({ settings }) => (settings ? settings === _atMode : true)) - .map(({ id, extraInstruction }) => (extraInstruction ? [id, extraInstruction] : [id])), - }; - } - } else { - settingsForTest = { - [collectedTest.target.mode]: { - [collectedTest.target.at.key]: collectedTest.commands.map(({ id, extraInstruction }) => - extraInstruction ? [id, extraInstruction] : [id] - ), - }, - }; - } - - return new CommandsInput( - { - commands: { - [collectedTest.info.task || collectedTest.info.testId]: settingsForTest, - }, - at: - typeof collectedTest.target.at.raw === 'object' - ? collectedTest.target.at.raw - : collectedTest.target.at, - }, - keysInput, - allCommandsInput - ); - } -} - -/** - * Depends on SupportInput. - */ -class ConfigInput { - /** - * @param {string[]} errors - * @param {object} value - * @param {ATJSON} value.at - * @param {boolean} value.displaySubmitButton - * @param {boolean} value.renderResultsAfterSubmit - * @param {"SubmitResultsJSON" | "TestResultJSON"} value.resultFormat - * @param {AriaATTestResult.JSON | null} value.resultJSON - * @private - */ - constructor(errors, value) { - this.errors = errors; - - /** @private */ - this._value = value; - } - - at() { - return this._value.at; - } - - displaySubmitButton() { - return this._value.displaySubmitButton; - } - - renderResultsAfterSubmit() { - return this._value.renderResultsAfterSubmit; - } - - resultFormat() { - return this._value.resultFormat; - } - - resultJSON() { - return this._value.resultJSON; - } - - /** - * @param {ConfigQueryParams} queryParams - * @param {object} data - * @param {SupportInput} data.supportInput - */ - static fromQueryParamsAndSupport(queryParams, { supportInput }) { - const errors = []; - - let at = supportInput.defaultAT(); - let displaySubmitButton = true; - let renderResultsAfterSubmit = true; - let resultFormat = 'SubmitResultsJSON'; - let resultJSON = null; - - for (const [key, value] of queryParams) { - if (key === 'at') { - const requestedAT = value; - const knownAt = supportInput.findAT(requestedAT); - if (knownAt) { - at = knownAt; - } else { - errors.push( - `Harness does not have commands for the requested assistive technology ('${requestedAT}'), showing commands for assistive technology '${at.name}' instead. To test '${requestedAT}', please contribute command mappings to this project.` - ); - } - } else if (key === 'showResults') { - displaySubmitButton = decodeBooleanParam(value, displaySubmitButton); - } else if (key === 'showSubmitButton') { - renderResultsAfterSubmit = decodeBooleanParam(value, renderResultsAfterSubmit); - } else if (key === 'resultFormat') { - if (value !== 'SubmitResultsJSON' && value !== 'TestResultJSON') { - errors.push( - `resultFormat can be 'SubmitResultsJSON' or 'TestResultJSON'. '${value}' is not supported.` - ); - continue; - } - resultFormat = value; - } else if (key === 'resultJSON') { - try { - resultJSON = JSON.parse(value); - } catch (error) { - errors.push(`Failed to parse resultJSON: ${error.message}`); - } - } - } - - if (resultJSON && resultFormat !== 'TestResultJSON') { - errors.push(`resultJSON requires resultFormat to be set to 'TestResultJSON'.`); - resultJSON = null; - } - - return new ConfigInput(errors, { - at, - displaySubmitButton, - renderResultsAfterSubmit, - resultFormat, - resultJSON, - }); - - /** - * @param {string} param - * @param {boolean} defaultValue - * @returns {boolean} - */ - function decodeBooleanParam(param, defaultValue) { - if (param === 'true') { - return true; - } else if (param === 'false') { - return false; - } - return defaultValue; - } - } -} - -class ScriptsInput { - /** - * @param {object} value - * @param {SetupScripts} value.scripts - * @private - */ - constructor(value) { - this.errors = []; - - /** @private */ - this._value = value; - } - - scripts() { - return this._value.scripts; - } - - /** - * @param {SetupScripts} scripts - */ - static fromScriptsMap(scripts) { - return new ScriptsInput({ scripts }); - } - - /** - * @param {{source: string}} script - * @private - */ - static scriptsFromSource(script) { - return { [script.name]: new Function('testPageDocument', script.source) }; - } - - /** - * @param {{modulePath: string}} script - * @param {string} dataUrl - * @private - */ - static async scriptsFromModuleAsync(script, dataUrl) { - return await import(`${dataUrl}/${script.modulePath}`); - } - - /** - * @param {{jsonpPath: string}} script - * @param {string} dataUrl - * @private - */ - static async scriptsFromJsonpAsync(script, dataUrl) { - return await Promise.race([ - new Promise(resolve => { - window.scriptsJsonpLoaded = resolve; - const scriptTag = document.createElement('script'); - scriptTag.src = script.jsonpPath; - document.body.appendChild(scriptTag); - }), - new Promise((_, reject) => - setTimeout(() => reject(new Error('Loading scripts timeout error')), 10000) - ), - ]); - } - - /** - * @param {AriaATFile.CollectedTest} collectedAsync - * @param {string} dataUrl url to directory where CollectedTest was loaded from - */ - static async fromCollectedTestAsync({ target: { setupScript } }, dataUrl) { - if (!setupScript) { - return new ScriptsInput({ scripts: {} }); - } - try { - return new ScriptsInput({ scripts: ScriptsInput.scriptsFromSource(setupScript) }); - } catch (error) { - try { - return new ScriptsInput({ - scripts: await ScriptsInput.scriptsFromModuleAsync(setupScript, dataUrl), - }); - } catch (error2) { - try { - return new ScriptsInput({ - scripts: await ScriptsInput.scriptsFromJsonpAsync(setupScript, dataUrl), - }); - } catch (error3) { - throw new Error( - [error, error2, error3].map(error => error.stack || error.message).join('\n\n') - ); - } - } - } - } -} - -class UnexpectedInput { - /** - * @param {object} value - * @param {BehaviorUnexpectedItem[]} value.behaviors - * @private - */ - constructor(value) { - this.errors = []; - - this._value = value; - } - - behaviors() { - return this._value.behaviors; - } - - static fromBuiltin() { - return new UnexpectedInput({ - behaviors: [ - ...UNEXPECTED_BEHAVIORS.map(description => ({ description })), - { description: 'Other' }, - ], - }); - } -} - -class TitleInput { - /** - * @param {object} value - * @param {string} value.title - * @private - */ - constructor(value) { - this.errors = []; - - /** @private */ - this._value = value; - } - - title() { - return this._value.title; - } - - /** @param {string} title */ - static fromTitle(title) { - return new TitleInput({ - title, - }); - } -} - -/** Depends on CommandsInput, ConfigInput, KeysInput, TitleInput, and UnexpectedInput. */ -class BehaviorInput { - /** - * @param {object} value - * @param {Behavior} value.behavior - * @private - */ - constructor(value) { - this.errors = []; - - /** @private */ - this._value = value; - } - - behavior() { - return this._value.behavior; - } - - /** - * @param {BehaviorJSON} json - * @param {object} data - * @param {KeysInput} data.keysInput - * @param {CommandsInput} data.commandsInput - * @param {ConfigInput} data.configInput - * @param {UnexpectedInput} data.unexpectedInput - * @param {TitleInput} data.titleInput - */ - static fromJSONCommandsConfigKeysTitleUnexpected( - json, - { commandsInput, configInput, keysInput, titleInput, unexpectedInput } - ) { - const mode = Array.isArray(json.mode) ? json.mode[0] : json.mode; - const at = configInput.at(); - - const { commandsAndSettings } = commandsInput.getCommands({ task: json.task }, mode); - - // Use to determine assertionExceptions - const commandsInfo = json.commandsInfo?.[at.key]; - - return new BehaviorInput({ - behavior: { - description: titleInput.title(), - task: json.task, - mode, - modeInstructions: keysInput.modeInstructions(mode), - appliesTo: json.applies_to, - specificUserInstruction: json.specific_user_instruction, - setupScriptDescription: json.setup_script_description, - setupTestPage: json.setupTestPage, - assertionResponseQuestion: json.assertionResponseQuestion, - commands: commandsAndSettings.map(cs => { - const foundCommandInfo = commandsInfo?.find( - c => - cs.commandId === c.command && - cs.presentationNumber === c.presentationNumber && - cs.settings === c.settings - ); - if (!foundCommandInfo || !foundCommandInfo.assertionExceptions) return cs; - - // Only works for v2 - let assertionExceptions = json.output_assertions.map(each => each.assertionId); - foundCommandInfo.assertionExceptions.split(' ').forEach(each => { - let [priority, assertionId] = each.split(':'); - const index = assertionExceptions.findIndex(each => each === assertionId); - - priority = Number(priority); - assertionExceptions[index] = priority; - }); - // Preserve default priority or update with exception - assertionExceptions = assertionExceptions.map((each, index) => - isNaN(each) ? json.output_assertions[index].priority : each - ); - - return { ...cs, assertionExceptions }; - }), - assertions: (json.output_assertions ? json.output_assertions : []).map(assertion => { - // Tuple array [ priorityNumber, assertionText ] - if (Array.isArray(assertion)) { - return { - priority: Number(assertion[0]), - assertion: assertion[1], - }; - } - - // { assertionId, priority, assertionStatement, assertionPhrase, refIds, tokenizedAssertionStatements, tokenizedAssertionPhrases } - return { - priority: assertion.priority, - assertion: - assertion.tokenizedAssertionStatements?.[at.key] || assertion.assertionStatement, - }; - }), - additionalAssertions: (json.additional_assertions - ? json.additional_assertions[at.key] || [] - : [] - ).map(assertionTuple => ({ - priority: Number(assertionTuple[0]), - assertion: assertionTuple[1], - })), - unexpectedBehaviors: unexpectedInput.behaviors(), - }, - }); - } - - /** - * @param {AriaATFile.CollectedTest} collectedTest - * @param {object} data - * @param {CommandsInput} data.commandsInput - * @param {KeysInput} data.keysInput - * @param {UnexpectedInput} data.unexpectedInput - */ - static fromCollectedTestCommandsKeysUnexpected( - { info, target, instructions, assertions, commands }, - { commandsInput, keysInput, unexpectedInput } - ) { - // v1:info.task, v2: info.testId | v1:target.mode, v2:target.at.settings - const { commandsAndSettings } = commandsInput.getCommands( - { task: info.task || info.testId }, - target.mode || target.at.settings - ); - - return new BehaviorInput({ - behavior: { - description: info.title, - task: info.task || info.testId, - mode: target.mode || target.at.settings, - modeInstructions: instructions.mode, - appliesTo: [target.at.name], - specificUserInstruction: instructions.raw || instructions.instructions, - setupScriptDescription: target.setupScript ? target.setupScript.description : '', - setupTestPage: target.setupScript ? target.setupScript.name : undefined, - commands: commandsAndSettings.map(cs => { - const foundCommandInfo = commands.find( - c => cs.commandId === c.id && cs.settings === c.settings - ); - if (!foundCommandInfo || !foundCommandInfo.assertionExceptions) return cs; - - // Only works for v2 - let assertionExceptions = assertions.map(each => each.assertionId); - foundCommandInfo.assertionExceptions.forEach(each => { - let { priority, assertionId } = each; - const index = assertionExceptions.findIndex(each => each === assertionId); - - priority = Number(priority); - assertionExceptions[index] = priority; - }); - // Preserve default priority or update with exception - assertionExceptions = assertionExceptions.map((each, index) => - isNaN(each) ? assertions[index].priority : each - ); - - return { ...cs, assertionExceptions }; - }), - assertions: assertions.map( - ({ priority, expectation, assertionStatement, tokenizedAssertionStatements }) => { - let assertion = tokenizedAssertionStatements - ? tokenizedAssertionStatements[target.at.key] - : null; - assertion = assertion || expectation || assertionStatement; - - return { - priority, - assertion, - }; - } - ), - additionalAssertions: [], - unexpectedBehaviors: unexpectedInput.behaviors(), - }, - }); - } -} - -class PageUriInput { - /** - * @param {object} value - * @param {string} value.pageUri - * @private - */ - constructor(value) { - this._errors = []; - this._value = value; - } - - pageUri() { - return this._value.pageUri; - } - - /** - * @param {string} pageUri - */ - static fromPageUri(pageUri) { - return new PageUriInput({ pageUri }); - } -} - -export class TestRunInputOutput { - constructor() { - /** @type {BehaviorInput} */ - this.behaviorInput = null; - /** @type {CommandsInput} */ - this.commandsInput = null; - /** @type {ConfigInput} */ - this.configInput = null; - /** @type {KeysInput} */ - this.keysInput = null; - /** @type {PageUriInput} */ - this.pageUriInput = null; - /** @type {ScriptsInput} */ - this.scriptsInput = null; - /** @type {SupportInput} */ - this.supportInput = null; - /** @type {AllCommandsInput} */ - this.allCommandsInput = null; - /** @type {TitleInput} */ - this.titleInput = null; - /** @type {UnexpectedInput} */ - this.unexpectedInput = null; - } - - /** @param {BehaviorInput} behaviorInput */ - setBehaviorInput(behaviorInput) { - this.behaviorInput = behaviorInput; - } - - /** @param {BehaviorJSON} behaviorJSON */ - setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected(behaviorJSON) { - invariant( - this.commandsInput !== null, - 'Call %s or %s before calling %s.', - this.setCommandsInput.name, - this.setCommandsInputFromJSONAndConfigKeys.name, - this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name - ); - invariant( - this.configInput !== null, - 'Call %s or %s before calling %s.', - this.setConfigInput.name, - this.setConfigInputFromQueryParamsAndSupport.name, - this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name - ); - invariant( - this.keysInput !== null, - 'Call %s or %s before calling %s.', - this.setKeysInput.name, - this.setKeysInputFromBuiltinAndConfig.name, - this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name - ); - invariant( - this.titleInput !== null, - 'Call %s or %s before calling %s.', - this.setTitleInput.name, - this.setTitleInputFromTitle.name, - this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name - ); - invariant( - this.unexpectedInput !== null, - 'Call %s or %s before calling %s.', - this.setUnexpectedInput.name, - this.setUnexpectedInputFromBuiltin.name, - this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name - ); - - this.setBehaviorInput( - BehaviorInput.fromJSONCommandsConfigKeysTitleUnexpected(behaviorJSON, { - commandsInput: this.commandsInput, - configInput: this.configInput, - keysInput: this.keysInput, - titleInput: this.titleInput, - unexpectedInput: this.unexpectedInput, - }) - ); - } - - /** - * Set all inputs but ConfigInput. - * @param {AriaATFile.CollectedTest} collectedTest - * @param {string} dataUrl url to directory where CollectedTest was loaded from - */ - async setInputsFromCollectedTestAsync(collectedTest, dataUrl) { - const pageUriInput = PageUriInput.fromPageUri(collectedTest.target.referencePage); - const titleInput = TitleInput.fromTitle(collectedTest.info.title); - const supportInput = SupportInput.fromCollectedTest(collectedTest); - const scriptsInput = await ScriptsInput.fromCollectedTestAsync(collectedTest, dataUrl); - - const unexpectedInput = UnexpectedInput.fromBuiltin(); - const keysInput = KeysInput.fromCollectedTest(collectedTest); - const allCommandsInput = this.allCommandsInput; - const commandsInput = CommandsInput.fromCollectedTestKeys(collectedTest, { - keysInput, - allCommandsInput, - }); - const behaviorInput = BehaviorInput.fromCollectedTestCommandsKeysUnexpected(collectedTest, { - commandsInput, - keysInput, - unexpectedInput, - }); - - this.setTitleInput(titleInput); - this.setPageUriInput(pageUriInput); - this.setSupportInput(supportInput); - this.setScriptsInput(scriptsInput); - - this.setUnexpectedInput(unexpectedInput); - this.setKeysInput(keysInput); - this.setCommandsInput(commandsInput); - this.setBehaviorInput(behaviorInput); - } - - /** @param {CommandsInput} commandsInput */ - setCommandsInput(commandsInput) { - this.commandsInput = commandsInput; - } - - /** @param {CommandsJSON} commandsJSON */ - setCommandsInputFromJSONAndConfigKeys(commandsJSON) { - invariant( - this.configInput !== null, - 'Call %s or %s before calling %s.', - this.setConfigInput.name, - this.setConfigInputFromQueryParamsAndSupport.name, - this.setCommandsInputFromJSONAndConfigKeys.name - ); - invariant( - this.keysInput !== null, - 'Call %s or %s before calling %s.', - this.setKeysInput.name, - this.setKeysInputFromBuiltinAndConfig.name, - this.setCommandsInputFromJSONAndConfigKeys.name - ); - - this.setCommandsInput( - CommandsInput.fromJSONAndConfigKeys(commandsJSON, { - configInput: this.configInput, - keysInput: this.keysInput, - allCommandsInput: this.allCommandsInput, - }) - ); - } - - /** @param {ConfigInput} configInput */ - setConfigInput(configInput) { - this.configInput = configInput; - } - - /** @param {ConfigQueryParams} queryParams */ - setConfigInputFromQueryParamsAndSupport(queryParams) { - invariant( - this.supportInput !== null, - 'Call %s or %s before calling %s.', - this.setSupportInput.name, - this.setSupportInputFromJSON.name, - this.setConfigInputFromQueryParamsAndSupport.name - ); - - this.setConfigInput( - ConfigInput.fromQueryParamsAndSupport(queryParams, { - supportInput: this.supportInput, - }) - ); - } - - /** @param {KeysInput} keysInput */ - setKeysInput(keysInput) { - this.keysInput = keysInput; - } - - setKeysInputFromBuiltinAndConfig() { - invariant( - this.configInput !== null, - 'Call %s or %s before calling %s.', - this.setConfigInput.name, - this.setConfigInputFromQueryParamsAndSupport.name, - this.setCommandsInputFromJSONAndConfigKeys.name - ); - - this.setKeysInput(KeysInput.fromBuiltinAndConfig({ configInput: this.configInput })); - } - - /** @param {PageUriInput} pageUriInput */ - setPageUriInput(pageUriInput) { - this.pageUriInput = pageUriInput; - } - - /** @param {string} pageUri */ - setPageUriInputFromPageUri(pageUri) { - this.setPageUriInput(PageUriInput.fromPageUri(pageUri)); - } - - /** @param {ScriptsInput} scriptsInput */ - setScriptsInput(scriptsInput) { - this.scriptsInput = scriptsInput; - } - - /** @param {SetupScripts} scriptsMap */ - setScriptsInputFromMap(scriptsMap) { - this.setScriptsInput(ScriptsInput.fromScriptsMap(scriptsMap)); - } - - /** @param {SupportInput} supportInput */ - setSupportInput(supportInput) { - this.supportInput = supportInput; - } - - /** @param {SupportJSON} supportJSON */ - setSupportInputFromJSON(supportJSON) { - this.setSupportInput(SupportInput.fromJSON(supportJSON)); - } - - /** @param {AllCommandsInput} allCommandsInput */ - setAllCommandsInput(allCommandsInput) { - this.allCommandsInput = allCommandsInput; - } - - /** @param {AllCommandsJSON} allCommandsJSON */ - setAllCommandsInputFromJSON(allCommandsJSON) { - this.setAllCommandsInput(AllCommandsInput.fromJSON(allCommandsJSON)); - } - - /** @param {TitleInput} titleInput */ - setTitleInput(titleInput) { - this.titleInput = titleInput; - } - - /** @param {string} title */ - setTitleInputFromTitle(title) { - this.setTitleInput(TitleInput.fromTitle(title)); - } - - /** @param {UnexpectedInput} unexpectedInput */ - setUnexpectedInput(unexpectedInput) { - this.unexpectedInput = unexpectedInput; - } - - setUnexpectedInputFromBuiltin() { - this.setUnexpectedInput(UnexpectedInput.fromBuiltin()); - } - - /** @returns {AriaATTestRun.State} */ - testRunState() { - invariant( - this.behaviorInput !== null, - 'Call %s or %s before calling %s.', - this.setBehaviorInput.name, - this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name, - this.testRunState.name - ); - invariant( - this.configInput !== null, - 'Call %s or %s before calling %s.', - this.setConfigInput.name, - this.setConfigInputFromQueryParamsAndSupport.name, - this.testRunState.name - ); - - const errors = [ - ...this.behaviorInput.errors, - ...this.commandsInput.errors, - ...this.configInput.errors, - ]; - const test = this.behaviorInput.behavior(); - const config = this.configInput; - - function unescapeHTML(input) { - const textarea = document.createElement('textarea'); - textarea.innerHTML = input; - return textarea.value; - } - - const [atMode, screenText, instructions] = deriveModeWithTextAndInstructions( - test.mode, - config.at() - ); - - let state = { - errors, - info: { - description: test.description, - task: test.task, - mode: screenText || atMode, - modeInstructions: Array.isArray(instructions) - ? unescapeHTML(`${instructions[0]} ${instructions[1]}`) - : test.modeInstructions, - userInstructions: test.specificUserInstruction.split('|'), - setupScriptDescription: test.setupScriptDescription, - }, - config: { - at: config.at(), - displaySubmitButton: config.displaySubmitButton(), - renderResultsAfterSubmit: config.renderResultsAfterSubmit(), - }, - currentUserAction: UserActionMap.LOAD_PAGE, - openTest: { - enabled: true, - }, - assertionResponseQuestion: test.assertionResponseQuestion, - commands: test.commands.map( - command => - /** @type {import("./aria-at-test-run.mjs").TestRunCommand} */ ({ - description: command.command, - commandSettings: { - command: command.command, - description: command.settings, - text: command.settingsText, - instructions: command.settingsInstructions, - assertionExceptions: command.assertionExceptions, - }, - atOutput: { - highlightRequired: false, - value: '', - }, - assertions: test.assertions.map(assertion => ({ - description: assertion.assertion, - highlightRequired: false, - priority: assertion.priority, - result: CommonResultMap.NOT_SET, - })), - additionalAssertions: test.additionalAssertions.map(assertion => ({ - description: assertion.assertion, - highlightRequired: false, - priority: assertion.priority, - result: CommonResultMap.NOT_SET, - })), - unexpected: { - highlightRequired: false, - hasUnexpected: HasUnexpectedBehaviorMap.NOT_SET, - tabbedBehavior: 0, - behaviors: test.unexpectedBehaviors.map(({ description }) => ({ - description, - checked: false, - impact: UnexpectedBehaviorImpactMap.MODERATE, - more: { highlightRequired: false, value: '' }, - })), - }, - }) - ), - }; - - if (this.configInput.resultJSON()) { - state = this.testRunStateFromTestResultJSON(this.configInput.resultJSON(), state); - } - - return state; - } - - testWindowOptions() { - invariant( - this.behaviorInput !== null, - 'Call %s or %s before calling %s.', - this.setBehaviorInput.name, - this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name, - this.testWindowOptions.name - ); - invariant( - this.pageUriInput !== null, - 'Call %s or %s before calling %s.', - this.setPageUriInput.name, - this.setPageUriInputFromPageUri.name, - this.testWindowOptions.name - ); - invariant( - this.scriptsInput !== null, - 'Call %s or %s before calling %s.', - this.setScriptsInput.name, - this.setScriptsInputFromMap.name, - this.testWindowOptions.name - ); - - return { - pageUri: this.pageUriInput.pageUri(), - setupScriptName: this.behaviorInput.behavior().setupTestPage, - scripts: this.scriptsInput.scripts(), - }; - } - - /** - * @param {AriaATTestRun.State} state - * @returns {import("./aria-at-harness.mjs").SubmitResultJSON} - */ - submitResultsJSON(state) { - invariant( - this.behaviorInput !== null, - 'Call %s or %s before calling %s.', - this.setBehaviorInput.name, - this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name, - this.submitResultsJSON.name - ); - - const behavior = this.behaviorInput.behavior(); - - /** @type {SubmitResultDetailsJSON} */ - const details = { - name: state.info.description, - task: state.info.task, - specific_user_instruction: behavior.specificUserInstruction, - summary: { - 1: { - pass: countAssertions( - ({ priority, result }) => priority === 1 && result === CommonResultMap.PASS - ), - fail: countAssertions( - ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS - ), - }, - 2: { - pass: countAssertions( - ({ priority, result }) => priority === 2 && result === CommonResultMap.PASS - ), - fail: countAssertions( - ({ priority, result }) => priority === 2 && result !== CommonResultMap.PASS - ), - }, - unexpectedCount: countUnexpectedBehaviors(({ checked }) => checked), - }, - commands: state.commands.map(command => ({ - command: command.description, - output: command.atOutput.value, - support: commandSupport(command), - assertions: [...command.assertions, ...command.additionalAssertions].map( - assertionToAssertion - ), - unexpected_behaviors: command.unexpected.behaviors - .filter(({ checked }) => checked) - .map(({ description, more }) => (more ? more.value : description)), - })), - }; - - /** @type {SubmitResultStatusJSON} */ - const status = state.commands - .map(commandSupport) - .some(support => support === CommandSupportJSONMap.FAILING) - ? StatusJSONMap.FAIL - : StatusJSONMap.PASS; - - return { - test: state.info.description, - details, - status, - }; - - function commandSupport(command) { - const allAssertions = [...command.assertions, ...command.additionalAssertions]; - return allAssertions.some( - ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS - ) || command.unexpected.behaviors.some(({ checked }) => checked) - ? CommandSupportJSONMap.FAILING - : allAssertions.some( - ({ priority, result }) => priority === 2 && result !== CommonResultMap.PASS - ) - ? CommandSupportJSONMap.ALL_REQUIRED - : CommandSupportJSONMap.FULL; - } - - /** - * @param {(assertion: TestRunAssertion | TestRunAdditionalAssertion) => boolean} filter - * @returns {number} - */ - function countAssertions(filter) { - return state.commands.reduce( - (carry, command) => - carry + [...command.assertions, ...command.additionalAssertions].filter(filter).length, - 0 - ); - } - - /** - * @param {(behavior: TestRunUnexpected) => boolean} filter - * @returns {number} - */ - function countUnexpectedBehaviors(filter) { - return state.commands.reduce( - (carry, command) => carry + command.unexpected.behaviors.filter(filter).length, - 0 - ); - } - - /** - * @param {TestRunAssertion | TestRunAdditionalAssertion} assertion - * @returns {SubmitResultAssertionsJSON} - */ - function assertionToAssertion(assertion) { - return assertion.result === CommonResultMap.PASS - ? { - assertion: assertion.description, - priority: assertion.priority.toString(), - pass: AssertionPassJSONMap.GOOD_OUTPUT, - } - : { - assertion: assertion.description, - priority: assertion.priority.toString(), - fail: - assertion.result === AssertionResultMap.FAIL_MISSING - ? AssertionFailJSONMap.NO_OUTPUT - : assertion.result === AssertionResultMap.FAIL_INCORRECT - ? AssertionFailJSONMap.INCORRECT_OUTPUT - : AssertionFailJSONMap.NO_SUPPORT, - }; - } - } - - /** - * Transform a test run state into a test result json for serialization. - * @param {AriaATTestRun.State} state - * @returns {AriaATTestResult.JSON} - */ - testResultJSON(state) { - return { - test: { - title: state.info.description, - at: { - id: state.config.at.key, - }, - atMode: state.info.mode, - }, - scenarioResults: state.commands.map(command => ({ - scenario: { - command: { - id: command.description, - }, - }, - output: command.atOutput.value, - assertionResults: command.assertions.map(assertion => ({ - assertion: { - priority: assertion.priority === 1 ? 'MUST' : 'SHOULD', - text: assertion.description, - }, - passed: assertion.result === 'pass', - failedReason: - assertion.result === 'failIncorrect' - ? 'INCORRECT_OUTPUT' - : assertion.result === 'failMissing' - ? 'NO_OUTPUT' - : null, - })), - unexpectedBehaviors: command.unexpected.behaviors - .map(behavior => - behavior.checked - ? { - text: behavior.description, - impact: behavior.impact, - details: behavior.more.value, - } - : null - ) - .filter(Boolean), - })), - }; - } - - /** - * @param {AriaATTestRun.State} state - * @returns {SubmitResultJSON | AriaATTestResult.JSON} - */ - resultJSON(state) { - // If ConfigInput is available and resultFormat is TestResultJSON return result in that format. - if (this.configInput !== null) { - const resultFormat = this.configInput.resultFormat(); - if (resultFormat === 'TestResultJSON') { - return this.testResultJSON(state); - } - } - - return this.submitResultsJSON(state); - } - - /** - * Set a default or given test run state with the recorded results json. Intermediate state not stored into - * testResult, like highlightRequired, is to the default. - * @param {AriaATTestResult.JSON} testResult - * @param {AriaATTestRun.State} [state] - * @returns {AriaATTestRun.State} - */ - testRunStateFromTestResultJSON(testResult, state = this.testRunState()) { - return { - ...state, - commands: state.commands.map((command, commandIndex) => { - const scenarioResult = testResult.scenarioResults[commandIndex]; - return { - ...command, - atOutput: { highlightRequired: false, value: scenarioResult.output }, - assertions: command.assertions.map((assertion, assertionIndex) => { - const assertionResult = scenarioResult.assertionResults[assertionIndex]; - return { - ...assertion, - highlightRequired: false, - result: assertionResult.passed - ? 'pass' - : assertionResult.failedReason === 'INCORRECT_OUTPUT' - ? 'failIncorrect' - : assertionResult.failedReason === 'NO_OUTPUT' - ? 'failMissing' - : 'fail', - }; - }), - unexpected: { - ...command.unexpected, - highlightRequired: false, - hasUnexpected: - scenarioResult.unexpectedBehaviors.length > 0 - ? 'hasUnexpected' - : 'doesNotHaveUnexpected', - tabbedBehavior: 0, - behaviors: command.unexpected.behaviors.map(behavior => { - const behaviorResult = scenarioResult.unexpectedBehaviors.find( - unexpectedResult => unexpectedResult.text === behavior.description - ); - return { - ...behavior, - checked: behaviorResult ? true : false, - more: behavior.more - ? { - highlightRequired: false, - impact: behaviorResult - ? behavior.impact - : UnexpectedBehaviorImpactMap.MODERATE, - value: behaviorResult ? behaviorResult.details : '', - } - : behavior.more, - }; - }), - }, - }; - }), - }; - } -} - -/** - * Extended TestRun that can access methods to turn the TestRun["state"] into - * the desired output format. - */ -export class TestRunExport extends TestRun { - /** - * @param {TestRunOptions & TestRunExportOptions} options - */ - constructor({ resultsJSON, ...parentOptions }) { - super(parentOptions); - - this.resultsJSON = resultsJSON; - } - - testPageAndResults() { - const testPage = this.testPage(); - if ('results' in testPage) { - return { - ...testPage, - resultsJSON: this.resultsJSON(this.state), - }; - } - return { - ...testPage, - resultsJSON: - this.state.currentUserAction === UserActionMap.CLOSE_TEST_WINDOW - ? this.resultsJSON(this.state) - : null, - }; - } -} - -/** - * @typedef SubmitResultDetailsCommandsAssertionsPass - * @property {string} assertion - * @property {string} priority - * @property {AssertionPassJSON} pass - */ - -/** - * Passing assertion values submitted from the tester result form. - * - * In the submitted json object the values contain spaces and are title cased. - * @typedef {EnumValues} AssertionPassJSON - */ - -const AssertionPassJSONMap = createEnumMap({ - GOOD_OUTPUT: 'Good Output', - PASS: 'Pass', -}); - -/** - * @typedef SubmitResultDetailsCommandsAssertionsFail - * @property {string} assertion - * @property {string} priority - * @property {AssertionFailJSON} fail - */ - -/** - * Failing assertion values from the tester result form as are submitted in the - * JSON result object. - * - * In the submitted json object the values contain spaces and are title cased. - * @typedef {EnumValues} AssertionFailJSON - */ - -const AssertionFailJSONMap = createEnumMap({ - NO_OUTPUT: 'No Output', - INCORRECT_OUTPUT: 'Incorrect Output', - NO_SUPPORT: 'No Support', - FAIL: 'Fail', -}); - -const UnexpectedBehaviorImpactMap = createEnumMap({ - MODERATE: 'Moderate', - SEVERE: 'Severe', -}); - -/** @typedef {SubmitResultDetailsCommandsAssertionsPass | SubmitResultDetailsCommandsAssertionsFail} SubmitResultAssertionsJSON */ - -/** - * Command result derived from priority 1 and 2 assertions. - * - * Support is "FAILING" is priority 1 assertions fail. Support is "ALL REQUIRED" - * if priority 2 assertions fail. - * - * In the submitted json object values may contain spaces and are in ALL CAPS. - * - * @typedef {EnumValues} CommandSupportJSON - */ - -const CommandSupportJSONMap = createEnumMap({ - FULL: 'FULL', - FAILING: 'FAILING', - ALL_REQUIRED: 'ALL REQUIRED', -}); - -/** - * Highest level status submitted from test result. - * - * In the submitted json object values are in ALL CAPS. - * - * @typedef {EnumValues} SubmitResultStatusJSON - */ - -const StatusJSONMap = createEnumMap({ - PASS: 'PASS', - FAIL: 'FAIL', -}); - -/** - * - * @param {ATMode} mode - * @param {ATJSON} at - * @returns {[ATMode, string, [string]]} - */ -function deriveModeWithTextAndInstructions(mode, at) { - let atMode = mode; - let screenText = ''; - let instructions = []; - - if (mode.includes('_')) { - const atModes = mode.split('_'); - for (const _atMode of atModes) { - if (at.settings[_atMode]) { - atMode = _atMode; - screenText = at.settings[_atMode].screenText; - instructions = at.settings[_atMode].instructions; - } - } - } else { - if (at.settings && at.settings[atMode]) { - screenText = at.settings[atMode]?.screenText; - instructions = at.settings[atMode]?.instructions; - } - } - - return [atMode, screenText, instructions]; -} - -/** - * @param {boolean} test - * @param {string} message - * @param {any[]} args - * @returns {asserts test} - */ -function invariant(test, message, ...args) { - if (!test) { - let index = 0; - throw new Error(message.replace(/%%|%\w/g, match => (match[0] !== '%%' ? args[index++] : '%'))); - } -} - -/** @typedef {ConstructorParameters[0]} TestRunOptions */ -/** - * @typedef TestRunExportOptions - * @property {(state: AriaATTestRun.State) => SubmitResultJSON} resultsJSON - */ - -/** - * @typedef ATJSON - * @property {string} name - * @property {string} key - * @property {string} defaultConfigurationInstructionsHTML - * @property {object} settings - */ - -/** - * @typedef SupportJSON - * @property {ATJSON[]} ats - * @property {object} applies_to - * @property {object[]} examples - * @property {string} examples[].directory - * @property {string} examples[].name - */ - -/** - * @typedef AllCommandsJSON - * @property {object} modifiers - * @property {object} modifierAliases - * @property {object} keys - * @property {object} keyAliases - */ - -/** - * @typedef {([string] | [string, string])[]} CommandATJSON - */ - -/** - * @typedef {{[atMode: string]: CommandATJSON}} CommandModeJSON - */ - -/** - * @typedef CommandJSON - * @property {CommandModeJSON} [reading] - * @property {CommandModeJSON} [interaction] - */ - -/** - * @typedef {{[commandDescription: string]: CommandJSON}} CommandsJSON - */ - -/** - * @typedef {["at" | "showSubmitButton" | "showResults" | string, string][]} ConfigQueryParams - */ - -/** @typedef {"reading" | "interaction" | "virtualCursor", "pcCursor", "browseMode" | "focusMode" | "quickNavOn" | "quickNavOff" | "defaultMode"} ATMode */ - -/** @typedef OutputAssertion - * @property {string} assertionId - * @property {Number} priority - * @property {string} assertionStatement - * @property {string} assertionPhrase - * @property {string} refIds - */ - -/** - * @typedef BehaviorJSON - * @property {string} setup_script_description - * @property {string} setupTestPage - * @property {string[]} applies_to - * @property {ATMode | ATMode[]} mode - * @property {string} task - * @property {string} specific_user_instruction - * @property {[string, string][] | [OutputAssertion]} [output_assertions] - * @property {{[atKey: string]: [number, string][]}} [additional_assertions] - */ - -/** - * @typedef BehaviorAssertion - * @property {number} priority - * @property {string} assertion - */ - -/** - * @typedef BehaviorUnexpectedItem - * @property {string} description - */ - -/** - * @typedef Behavior - * @property {string} description - * @property {string} task - * @property {ATMode} mode - * @property {string} modeInstructions - * @property {string[]} appliesTo - * @property {string} specificUserInstruction - * @property {string} setupScriptDescription - * @property {string} setupTestPage - * @property {string[]} commands - * @property {BehaviorAssertion[]} assertions - * @property {BehaviorAssertion[]} additionalAssertions - * @property {BehaviorUnexpectedItem[]} unexpectedBehaviors - */ - -/** @typedef {{[key: string]: (document: Document) => void}} SetupScripts */ - -/** - * @typedef SubmitResultJSON - * @property {string} test - * @property {SubmitResultDetailsJSON} details - * @property {SubmitResultStatusJSON} status - */ - -/** - * @typedef SubmitResultSummaryPriorityJSON - * @property {number} pass - * @property {number} fail - */ - -/** - * @typedef {{[key in "1" | "2"]: SubmitResultSummaryPriorityJSON}} SubmitResultSummaryPriorityMapJSON - */ - -/** - * @typedef SubmitResultSummaryPropsJSON - * @property {number} unexpectedCount - */ - -/** - * @typedef {SubmitResultSummaryPriorityMapJSON & SubmitResultSummaryPropsJSON} SubmitResultSummaryJSON - */ - -/** - * @typedef SubmitResultDetailsJSON - * @property {string} name - * @property {string} specific_user_instruction - * @property {string} task - * @property {object[]} commands - * @property {string} commands[].command - * @property {string} commands[].output - * @property {string[]} commands[].unexpected_behaviors - * @property {CommandSupportJSON} commands[].support - * @property {SubmitResultAssertionsJSON[]} commands[].assertions - * @property {SubmitResultSummaryJSON} summary - */ - -/** - * @typedef ResultJSONDocument - * @property {SubmitResultJSON | null} resultsJSON - */ - -/** - * @typedef {TestPageDocument & ResultJSONDocument} TestPageAndResultsDocument - */ - -/** - * @typedef {import('./aria-at-test-run.mjs').EnumValues} EnumValues - * @template T - */ - -/** @typedef {import('./aria-at-test-run.mjs').TestRunAssertion} TestRunAssertion */ -/** @typedef {import('./aria-at-test-run.mjs').TestRunAdditionalAssertion} TestRunAdditionalAssertion */ -/** @typedef {import('./aria-at-test-run.mjs').TestRunCommand} TestRunCommand */ -/** @typedef {import("./aria-at-test-run.mjs").TestRunUnexpectedBehavior} TestRunUnexpected */ - -/** @typedef {import('./aria-at-test-run.mjs').TestPageDocument} TestPageDocument */ diff --git a/client/resources/aria-at-test-run.mjs b/client/resources/aria-at-test-run.mjs deleted file mode 100644 index c45a3e179..000000000 --- a/client/resources/aria-at-test-run.mjs +++ /dev/null @@ -1,1336 +0,0 @@ -export class TestRun { - /** - * @param {object} param0 - * @param {Partial} [param0.hooks] - * @param {TestRunState} param0.state - */ - constructor({ hooks, state }) { - /** @type {TestRunState} */ - this.state = state; - - const bindDispatch = transform => arg => this.dispatch(transform(arg)); - /** @type {TestRunHooks} */ - this.hooks = { - closeTestPage: bindDispatch(userCloseWindow), - focusCommandUnexpectedBehavior: bindDispatch(userFocusCommandUnexpectedBehavior), - openTestPage: bindDispatch(userOpenWindow), - postResults: () => {}, - setCommandAdditionalAssertion: bindDispatch(userChangeCommandAdditionalAssertion), - setCommandAssertion: bindDispatch(userChangeCommandAssertion), - setCommandHasUnexpectedBehavior: bindDispatch(userChangeCommandHasUnexpectedBehavior), - setCommandUnexpectedBehavior: bindDispatch(userChangeCommandUnexpectedBehavior), - setCommandUnexpectedBehaviorImpact: bindDispatch(userChangeCommandUnexpectedBehaviorImpact), - setCommandUnexpectedBehaviorMore: bindDispatch(userChangeCommandUnexpectedBehaviorMore), - setCommandOutput: bindDispatch(userChangeCommandOutput), - submit: () => submitResult(this), - ...hooks, - }; - - this.observers = []; - - this.dispatch = this.dispatch.bind(this); - } - - /** - * @param {(state: TestRunState) => TestRunState} updateMethod - */ - dispatch(updateMethod) { - this.state = updateMethod(this.state); - this.observers.forEach(subscriber => subscriber(this)); - } - - /** - * @param {(app: TestRun) => void} subscriber - * @returns {() => void} - */ - observe(subscriber) { - this.observers.push(subscriber); - return () => { - const index = this.observers.indexOf(subscriber); - if (index > -1) { - this.observers.splice(index, 1); - } - }; - } - - testPage() { - return testPageDocument(this.state, this.hooks); - } - - instructions() { - return instructionDocument(this.state, this.hooks); - } - - resultsTable() { - return resultsTableDocument(this.state); - } -} - -/** - * @param {U} map - * @returns {Readonly} - * @template {string} T - * @template {{[key: string]: T}} U - */ -export function createEnumMap(map) { - return Object.freeze(map); -} - -export const WhitespaceStyleMap = createEnumMap({ - LINE_BREAK: 'lineBreak', -}); - -function bind(fn, ...args) { - return (...moreArgs) => fn(...args, ...moreArgs); -} - -/** - * @param {TestRunState} resultState - * @param {TestRunHooks} hooks - * @returns {InstructionDocument} - */ -export function instructionDocument(resultState, hooks) { - const mode = resultState.info.mode; - const modeInstructions = resultState.info.modeInstructions; - const userInstructions = resultState.info.userInstructions; - const lastInstruction = userInstructions[userInstructions.length - 1]; - const setupScriptDescription = resultState.info.setupScriptDescription - ? ` and runs a script that ${resultState.info.setupScriptDescription}.` - : resultState.info.setupScriptDescription; - // As a hack, special case mode instructions for VoiceOver for macOS until we - // support modeless tests. - const modePhrase = - resultState.config.at.name === 'VoiceOver for macOS' - ? 'Describe ' - : `With ${resultState.config.at.name} in ${mode} mode, describe `; - - // TODO: Wrap each command token in - const commands = resultState.commands.map(({ description }) => description); - const commandSettings = resultState.commands.map(({ commandSettings }) => commandSettings); - const assertions = resultState.commands[0].assertions.map(({ description }) => description); - const additionalAssertions = resultState.commands[0].additionalAssertions.map( - ({ description }) => description - ); - - let firstRequired = true; - function focusFirstRequired() { - if (firstRequired) { - firstRequired = false; - return true; - } - return false; - } - - function convertModeInstructionsToKbdArray(inputString) { - const container = document.createElement('div'); - container.innerHTML = inputString; - - const resultArray = []; - for (const node of container.childNodes) { - if (node.nodeName === 'KBD') { - // Handle elements - resultArray.push({ kbd: node.innerText.trim() }); - } else { - // Handle text nodes - resultArray.push(node.textContent); - } - } - - return resultArray.length ? resultArray : null; - } - - const convertedModeInstructions = - modeInstructions !== undefined && !modeInstructions.includes('undefined') - ? convertModeInstructionsToKbdArray(modeInstructions) - : null; - - let strongInstructions = [...userInstructions]; - if (convertedModeInstructions) - strongInstructions = [convertedModeInstructions, ...strongInstructions]; - - return { - errors: { - visible: resultState.errors && resultState.errors.length > 0 ? true : false, - header: 'Test cannot be performed due to error(s)!', - errors: resultState.errors || [], - }, - instructions: { - header: { - header: `Testing task: ${resultState.info.description}`, - focus: resultState.currentUserAction === UserActionMap.LOAD_PAGE, - }, - description: `${modePhrase} how ${resultState.config.at.name} behaves when performing task "${lastInstruction}"`, - instructions: { - header: 'Test instructions', - instructions: [ - [ - `Restore default settings for ${resultState.config.at.name}. For help, read `, - { - href: 'https://github.com/w3c/aria-at/wiki/Configuring-Screen-Readers-for-Testing', - description: 'Configuring Screen Readers for Testing', - }, - `.`, - ], - `Activate the "Open test page" button below, which opens the example to test in a new window${setupScriptDescription}`, - ], - strongInstructions: strongInstructions.filter(el => el), - commands: { - description: `Using the following commands, ${lastInstruction}`, - commands: commands.map((command, index) => { - const { description: settings, text: settingsText } = commandSettings[index]; - return `${command}${ - settingsText && settings !== 'defaultMode' ? ` (${settingsText})` : '' - }`; - }), - }, - }, - assertions: { - header: 'Success Criteria', - description: `To pass this test, ${resultState.config.at.name} needs to meet all the following assertions when each specified command is executed:`, - assertions, - }, - openTestPage: { - button: 'Open Test Page', - enabled: resultState.openTest.enabled, - click: hooks.openTestPage, - }, - }, - results: { - header: { - header: 'Record Results', - description: `${resultState.info.description}`, - }, - commands: commands.map(commandResult), - }, - submit: resultState.config.displaySubmitButton - ? { - button: 'Submit Results', - click: hooks.submit, - } - : null, - }; - - /** - * @param {string} command - * @param {number} commandIndex - * @returns {InstructionDocumentResultsCommand} - */ - function commandResult(command, commandIndex) { - const resultStateCommand = resultState.commands[commandIndex]; - const resultUnexpectedBehavior = resultStateCommand.unexpected; - - const { - commandSettings: { description: settings, text: settingsText, assertionExceptions }, - } = resultStateCommand; - - return { - header: `After '${command}'${ - settingsText && settings !== 'defaultMode' ? ` (${settingsText})` : '' - }`, - atOutput: { - description: [ - `${resultState.config.at.name} output after ${command}`, - { - required: true, - highlightRequired: resultStateCommand.atOutput.highlightRequired, - description: '(required)', - }, - ], - value: resultStateCommand.atOutput.value, - focus: - resultState.currentUserAction === 'validateResults' && - resultStateCommand.atOutput.highlightRequired && - focusFirstRequired(), - change: atOutput => hooks.setCommandOutput({ commandIndex, atOutput }), - }, - assertionsHeader: { - descriptionHeader: `${resultState.assertionResponseQuestion} ${command}${ - settingsText && settings !== 'defaultMode' ? ` (${settingsText})` : '' - }?`, - }, - assertions: [ - ...assertions - // Ignore assertion if level 0 priority exception found for assertion's command - .filter((each, index) => (assertionExceptions ? assertionExceptions[index] !== 0 : each)) - .map(each => - assertionResult( - commandIndex, - each, - assertions.findIndex(e => e === each) - ) - ), - ...additionalAssertions.map(bind(additionalAssertionResult, commandIndex)), - ], - unexpectedBehaviors: { - description: [ - 'Were there additional undesirable behaviors?', - { - required: true, - highlightRequired: resultStateCommand.unexpected.highlightRequired, - description: '(required)', - }, - ], - passChoice: { - label: 'No, there were no additional undesirable behaviors.', - checked: - resultUnexpectedBehavior.hasUnexpected === - HasUnexpectedBehaviorMap.DOES_NOT_HAVE_UNEXPECTED, - focus: - resultState.currentUserAction === 'validateResults' && - resultUnexpectedBehavior.highlightRequired && - resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.NOT_SET && - focusFirstRequired(), - click: () => - hooks.setCommandHasUnexpectedBehavior({ - commandIndex, - hasUnexpected: HasUnexpectedBehaviorMap.DOES_NOT_HAVE_UNEXPECTED, - }), - }, - failChoice: { - label: 'Yes, there were additional undesirable behaviors', - checked: - resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED, - focus: - resultState.currentUserAction === 'validateResults' && - resultUnexpectedBehavior.highlightRequired && - resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.NOT_SET && - focusFirstRequired(), - click: () => - hooks.setCommandHasUnexpectedBehavior({ - commandIndex, - hasUnexpected: HasUnexpectedBehaviorMap.HAS_UNEXPECTED, - }), - options: { - header: 'Undesirable behaviors', - options: resultUnexpectedBehavior.behaviors.map((behavior, unexpectedIndex) => { - return { - description: behavior.description, - impact: behavior.impact, - enabled: - resultUnexpectedBehavior.hasUnexpected === - HasUnexpectedBehaviorMap.HAS_UNEXPECTED, - tabbable: resultUnexpectedBehavior.tabbedBehavior === unexpectedIndex, - checked: behavior.checked, - focus: - typeof resultState.currentUserAction === 'object' && - resultState.currentUserAction.action === UserObjectActionMap.FOCUS_UNDESIRABLE - ? resultState.currentUserAction.commandIndex === commandIndex && - resultUnexpectedBehavior.tabbedBehavior === unexpectedIndex - : resultState.currentUserAction === UserActionMap.VALIDATE_RESULTS && - resultUnexpectedBehavior.hasUnexpected === - HasUnexpectedBehaviorMap.HAS_UNEXPECTED && - resultUnexpectedBehavior.behaviors.every(({ checked }) => !checked) && - focusFirstRequired(), - change: checked => - hooks.setCommandUnexpectedBehavior({ commandIndex, unexpectedIndex, checked }), - impactchange: impact => - hooks.setCommandUnexpectedBehaviorImpact({ - commandIndex, - unexpectedIndex, - impact, - }), - keydown: key => { - const increment = keyToFocusIncrement(key); - if (increment) { - hooks.focusCommandUnexpectedBehavior({ - commandIndex, - unexpectedIndex, - increment, - }); - return true; - } - return false; - }, - more: { - description: /** @type {Description[]} */ ([ - `Details:`, - { - required: true, - highlightRequired: behavior.more.highlightRequired, - description: '(required)', - }, - ]), - enabled: behavior.checked, - value: behavior.more.value, - focus: - resultState.currentUserAction === 'validateResults' && - behavior.more.highlightRequired && - focusFirstRequired(), - change: value => - hooks.setCommandUnexpectedBehaviorMore({ - commandIndex, - unexpectedIndex, - more: value, - }), - }, - }; - }), - }, - }, - }, - }; - } - - /** - * @param {number} commandIndex - * @param {string} assertion - * @param {number} assertionIndex - */ - function assertionResult(commandIndex, assertion, assertionIndex) { - const resultAssertion = resultState.commands[commandIndex].assertions[assertionIndex]; - return /** @type {InstructionDocumentResultsCommandsAssertion} */ ({ - description: [assertion], - passed: resultAssertion.result === AssertionResultMap.PASS, - click: () => - hooks.setCommandAssertion({ - commandIndex, - assertionIndex, - result: - resultAssertion.result === AssertionResultMap.PASS - ? AssertionResultMap.FAIL - : AssertionResultMap.PASS, - }), - }); - } - - /** - * @param {number} commandIndex - * @param {string} assertion - * @param {number} assertionIndex - */ - function additionalAssertionResult(commandIndex, assertion, assertionIndex) { - const resultAdditionalAssertion = - resultState.commands[commandIndex].additionalAssertions[assertionIndex]; - return /** @type {InstructionDocumentResultsCommandsAssertion} */ ({ - description: [assertion], - passed: resultAdditionalAssertion.result === CommonResultMap.PASS, - click: () => - hooks.setCommandAssertion({ - commandIndex, - assertionIndex, - result: - resultAdditionalAssertion.result === AssertionResultMap.PASS - ? AssertionResultMap.FAIL - : AssertionResultMap.PASS, - }), - }); - } -} - -/** - * @typedef {typeof UserActionMap[keyof typeof UserActionMap]} UserAction - */ - -export const UserActionMap = createEnumMap({ - LOAD_PAGE: 'loadPage', - OPEN_TEST_WINDOW: 'openTestWindow', - CLOSE_TEST_WINDOW: 'closeTestWindow', - VALIDATE_RESULTS: 'validateResults', - CHANGE_TEXT: 'changeText', - CHANGE_SELECTION: 'changeSelection', - SHOW_RESULTS: 'showResults', -}); - -/** - * @typedef {typeof UserObjectActionMap[keyof typeof UserObjectActionMap]} UserObjectAction - */ - -export const UserObjectActionMap = createEnumMap({ - FOCUS_UNDESIRABLE: 'focusUndesirable', -}); - -/** - * @typedef {UserAction | UserActionFocusUnexpected} TestRunUserAction - */ - -/** - * @typedef {EnumValues} HasUnexpectedBehavior - */ - -export const HasUnexpectedBehaviorMap = createEnumMap({ - NOT_SET: 'notSet', - HAS_UNEXPECTED: 'hasUnexpected', - DOES_NOT_HAVE_UNEXPECTED: 'doesNotHaveUnexpected', -}); - -export const CommonResultMap = createEnumMap({ - NOT_SET: 'notSet', - PASS: 'pass', -}); - -/** - * @typedef {EnumValues} AdditionalAssertionResult - */ - -export const AdditionalAssertionResultMap = createEnumMap({ - ...CommonResultMap, - FAIL_SUPPORT: 'failSupport', -}); - -/** - * @typedef {EnumValues} AssertionResult - */ - -export const AssertionResultMap = createEnumMap({ - ...CommonResultMap, - FAIL_MISSING: 'failMissing', - FAIL_INCORRECT: 'failIncorrect', - FAIL: 'fail', -}); - -/** - * @typedef {EnumValues} UnexpectedBehaviorImpact - */ - -export const UnexpectedBehaviorImpactMap = createEnumMap({ - MODERATE: 'Moderate', - SEVERE: 'Severe', -}); - -/** - * @param {object} props - * @param {number} props.commandIndex - * @param {string} props.atOutput - * @returns {(state: TestRunState) => TestRunState} - */ -export function userChangeCommandOutput({ commandIndex, atOutput }) { - return function (state) { - return { - ...state, - currentUserAction: UserActionMap.CHANGE_TEXT, - commands: state.commands.map((commandState, index) => - index !== commandIndex - ? commandState - : { - ...commandState, - atOutput: { - ...commandState.atOutput, - value: atOutput, - }, - } - ), - }; - }; -} - -/** - * @param {object} props - * @param {number} props.commandIndex - * @param {number} props.assertionIndex - * @param {AssertionResult} props.result - * @returns {(state: TestRunState) => TestRunState} - */ -export function userChangeCommandAssertion({ commandIndex, assertionIndex, result }) { - return function (state) { - return { - ...state, - currentUserAction: UserActionMap.CHANGE_SELECTION, - commands: state.commands.map((command, commandI) => - commandI !== commandIndex - ? command - : { - ...command, - assertions: command.assertions.map((assertion, assertionI) => - assertionI !== assertionIndex ? assertion : { ...assertion, result } - ), - } - ), - }; - }; -} - -/** - * @param {object} props - * @param {number} props.commandIndex - * @param {number} props.additionalAssertionIndex - * @param {AdditionalAssertionResult} props.result - * @returns {(state: TestRunState) => TestRunState} - */ -export function userChangeCommandAdditionalAssertion({ - commandIndex, - additionalAssertionIndex, - result, -}) { - return function (state) { - return { - ...state, - currentUserAction: UserActionMap.CHANGE_SELECTION, - commands: state.commands.map((command, commandI) => - commandI !== commandIndex - ? command - : { - ...command, - additionalAssertions: command.additionalAssertions.map((assertion, assertionI) => - assertionI !== additionalAssertionIndex ? assertion : { ...assertion, result } - ), - } - ), - }; - }; -} - -/** - * @param {object} props - * @param {number} props.commandIndex - * @param {HasUnexpectedBehavior} props.hasUnexpected - * @returns {(state: TestRunState) => TestRunState} - */ -export function userChangeCommandHasUnexpectedBehavior({ commandIndex, hasUnexpected }) { - return function (state) { - return { - ...state, - currentUserAction: UserActionMap.CHANGE_SELECTION, - commands: state.commands.map((command, commandI) => - commandI !== commandIndex - ? command - : { - ...command, - unexpected: { - ...command.unexpected, - hasUnexpected: hasUnexpected, - tabbedBehavior: hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED ? 0 : -1, - behaviors: command.unexpected.behaviors.map(behavior => ({ - ...behavior, - checked: false, - more: behavior.more ? { ...behavior.more, value: '' } : null, - })), - }, - } - ), - }; - }; -} - -/** - * @param {object} props - * @param {number} props.commandIndex - * @param {number} props.unexpectedIndex - * @param {boolean} props.checked - * @returns {(state: TestRunState) => TestRunState} - */ -export function userChangeCommandUnexpectedBehavior({ commandIndex, unexpectedIndex, checked }) { - return function (state) { - return { - ...state, - currentUserAction: UserActionMap.CHANGE_SELECTION, - commands: state.commands.map((command, commandI) => - commandI !== commandIndex - ? command - : { - ...command, - unexpected: { - ...command.unexpected, - behaviors: command.unexpected.behaviors.map((unexpected, unexpectedI) => - unexpectedI !== unexpectedIndex - ? unexpected - : { - ...unexpected, - checked, - } - ), - }, - } - ), - }; - }; -} - -/** - * @param {object} props - * @param {number} props.commandIndex - * @param {number} props.unexpectedIndex - * @param {string} props.impact - * @returns {(state: TestRunState) => TestRunState} - */ -export function userChangeCommandUnexpectedBehaviorImpact({ - commandIndex, - unexpectedIndex, - impact, -}) { - return function (state) { - return { - ...state, - currentUserAction: UserActionMap.CHANGE_TEXT, - commands: state.commands.map((command, commandI) => - commandI !== commandIndex - ? command - : /** @type {TestRunCommand} */ ({ - ...command, - unexpected: { - ...command.unexpected, - behaviors: command.unexpected.behaviors.map((unexpected, unexpectedI) => - unexpectedI !== unexpectedIndex - ? unexpected - : /** @type {TestRunUnexpectedBehavior} */ ({ - ...unexpected, - impact: impact, - }) - ), - }, - }) - ), - }; - }; -} - -/** - * @param {object} props - * @param {number} props.commandIndex - * @param {number} props.unexpectedIndex - * @param {string} props.more - * @returns {(state: TestRunState) => TestRunState} - */ -export function userChangeCommandUnexpectedBehaviorMore({ commandIndex, unexpectedIndex, more }) { - return function (state) { - return { - ...state, - currentUserAction: UserActionMap.CHANGE_TEXT, - commands: state.commands.map((command, commandI) => - commandI !== commandIndex - ? command - : /** @type {TestRunCommand} */ ({ - ...command, - unexpected: { - ...command.unexpected, - behaviors: command.unexpected.behaviors.map((unexpected, unexpectedI) => - unexpectedI !== unexpectedIndex - ? unexpected - : /** @type {TestRunUnexpectedBehavior} */ ({ - ...unexpected, - more: { - ...unexpected.more, - value: more, - }, - }) - ), - }, - }) - ), - }; - }; -} - -/** - * @param {string} key - * @returns {TestRunFocusIncrement} - */ -function keyToFocusIncrement(key) { - switch (key) { - case 'Up': - case 'ArrowUp': - case 'Left': - case 'ArrowLeft': - return 'previous'; - - case 'Down': - case 'ArrowDown': - case 'Right': - case 'ArrowRight': - return 'next'; - } -} - -/** - * @param {TestRunState} state - * @param {TestRunHooks} hooks - * @returns {TestPageDocument} - */ -function testPageDocument(state, hooks) { - if (state.currentUserAction === UserActionMap.SHOW_RESULTS) { - return { - results: resultsTableDocument(state), - }; - } - const instructions = instructionDocument(state, hooks); - return { - errors: instructions.errors, - instructions, - }; -} - -/** - * @param {TestRun} app - */ -function submitResult(app) { - app.dispatch(userValidateState()); - - if (isSomeFieldRequired(app.state)) { - return; - } - - app.hooks.postResults(); - - app.hooks.closeTestPage(); - - if (app.state.config.renderResultsAfterSubmit) { - app.dispatch(userShowResults()); - } -} - -export function userShowResults() { - return function (/** @type {TestRunState} */ state) { - return /** @type {TestRunState} */ ({ - ...state, - currentUserAction: UserActionMap.SHOW_RESULTS, - }); - }; -} - -/** - * @param {TestRunState} state - * @returns - */ -function isSomeFieldRequired(state) { - return state.commands.some( - command => - command.atOutput.value.trim() === '' || - command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.NOT_SET || - (command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED && - (command.unexpected.behaviors.every(({ checked }) => !checked) || - command.unexpected.behaviors.some( - behavior => behavior.checked && behavior.more && behavior.more.value.trim() === '' - ))) - ); -} - -/** - * @param {TestRunState} state - * @returns {ResultsTableDocument} - */ -function resultsTableDocument(state) { - return { - header: state.info.description, - status: { - header: [ - 'Test result: ', - state.commands.some( - ({ - assertions, - additionalAssertions, - unexpected, - commandSettings: { assertionExceptions }, - }) => - [ - // Ignore assertion if level 0 priority exception found for assertion's command - ...assertions.filter((each, index) => - assertionExceptions ? assertionExceptions[index] !== 0 : each - ), - ...additionalAssertions, - ].some(({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS) || - unexpected.behaviors.some(({ checked }) => checked) - ) - ? 'FAIL' - : 'PASS', - ], - }, - table: { - headers: { - description: 'Command', - support: 'Support', - details: 'Details', - }, - commands: state.commands.map(command => { - const { - commandSettings: { assertionExceptions }, - } = command; - const allAssertions = [ - // Ignore assertion if level 0 priority exception found for assertion's command - ...command.assertions.filter((each, index) => - assertionExceptions ? assertionExceptions[index] !== 0 : each - ), - ...command.additionalAssertions, - ]; - - let passingAssertions = ['No passing assertions']; - let failingAssertions = ['No failing assertions']; - let unexpectedBehaviors = ['None']; - - if (allAssertions.some(({ result }) => result === CommonResultMap.PASS)) { - passingAssertions = allAssertions - .filter(({ result }) => result === CommonResultMap.PASS) - .map(({ description }) => description); - } - if (allAssertions.some(({ result }) => result !== CommonResultMap.PASS)) { - failingAssertions = allAssertions - .filter(({ result }) => result !== CommonResultMap.PASS) - .map(({ description }) => description); - } - if (command.unexpected.behaviors.some(({ checked }) => checked)) { - unexpectedBehaviors = command.unexpected.behaviors - .filter(({ checked }) => checked) - .map(({ description, more, impact }) => { - let result = `${description} (`; - if (more) result = `${result}Details: ${more.value}, `; - result = `${result}Impact: ${impact})`; - return result; - }); - } - - return { - description: command.description, - support: - allAssertions.some( - ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS - ) || command.unexpected.behaviors.some(({ checked }) => checked) - ? 'FAILING' - : allAssertions.some( - ({ priority, result }) => priority === 2 && result !== CommonResultMap.PASS - ) - ? 'ALL_REQUIRED' - : 'FULL', - details: { - output: /** @type {Description} */ [ - 'Output:', - /** @type {DescriptionWhitespace} */ ({ whitespace: WhitespaceStyleMap.LINE_BREAK }), - ' ', - ...command.atOutput.value.split(/(\r\n|\r|\n)/g).map(output => - /\r\n|\r|\n/.test(output) - ? /** @type {DescriptionWhitespace} */ ({ - whitespace: WhitespaceStyleMap.LINE_BREAK, - }) - : output - ), - ], - passingAssertions: { - description: 'Passing Assertions:', - items: passingAssertions, - }, - failingAssertions: { - description: 'Failing Assertions:', - items: failingAssertions, - }, - unexpectedBehaviors: { - description: 'Other behaviors that create negative impact:', - items: unexpectedBehaviors, - }, - }, - }; - }), - }, - }; -} - -export function userOpenWindow() { - return (/** @type {TestRunState} */ state) => - /** @type {TestRunState} */ ({ - ...state, - currentUserAction: UserActionMap.OPEN_TEST_WINDOW, - openTest: { ...state.openTest, enabled: false }, - }); -} - -export function userCloseWindow() { - return (/** @type {TestRunState} */ state) => - /** @type {TestRunState} */ ({ - ...state, - currentUserAction: UserActionMap.CLOSE_TEST_WINDOW, - openTest: { ...state.openTest, enabled: true }, - }); -} - -/** - * @param {object} props - * @param {number} props.commandIndex - * @param {number} props.unexpectedIndex - * @param {TestRunFocusIncrement} props.increment - * @returns {(state: TestRunState) => TestRunState} - */ -export function userFocusCommandUnexpectedBehavior({ commandIndex, unexpectedIndex, increment }) { - return function (state) { - const unexpectedLength = state.commands[commandIndex].unexpected.behaviors.length; - const incrementValue = increment === 'next' ? 1 : -1; - const newUnexpectedIndex = - (unexpectedIndex + incrementValue + unexpectedLength) % unexpectedLength; - - return { - ...state, - currentUserAction: { - action: UserObjectActionMap.FOCUS_UNDESIRABLE, - commandIndex, - unexpectedIndex: newUnexpectedIndex, - }, - commands: state.commands.map((command, commandI) => { - const tabbed = command.unexpected.tabbedBehavior; - const unexpectedLength = command.unexpected.behaviors.length; - const newTabbed = - (tabbed + (increment === 'next' ? 1 : -1) + unexpectedLength) % unexpectedLength; - return commandI !== commandIndex - ? command - : { - ...command, - unexpected: { - ...command.unexpected, - tabbedBehavior: newTabbed, - }, - }; - }), - }; - }; -} - -/** - * @returns {(state: TestRunState) => TestRunState} - */ -export function userValidateState() { - return function (state) { - return { - ...state, - currentUserAction: UserActionMap.VALIDATE_RESULTS, - commands: state.commands.map(command => { - return { - ...command, - atOutput: { - ...command.atOutput, - highlightRequired: !command.atOutput.value.trim(), - }, - unexpected: { - ...command.unexpected, - highlightRequired: - command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.NOT_SET || - (command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED && - command.unexpected.behaviors.every(({ checked }) => !checked)), - behaviors: command.unexpected.behaviors.map(unexpected => { - return unexpected.more - ? { - ...unexpected, - more: { - ...unexpected.more, - highlightRequired: unexpected.checked && !unexpected.more.value.trim(), - }, - } - : unexpected; - }), - }, - }; - }), - }; - }; -} - -/** - * @typedef AT - * @property {string} name - * @property {string} key - */ - -/** - * @typedef Behavior - * @property {string} description - * @property {string} task - * @property {string} mode - * @property {string} modeInstructions - * @property {string[]} appliesTo - * @property {string} specificUserInstruction - * @property {string} setupScriptDescription - * @property {string} setupTestPage - * @property {string[]} commands - * @property {[string, string][]} outputAssertions - * @property {[number, string][]} additionalAssertions - */ - -/** - * @typedef {"previous" | "next"} TestRunFocusIncrement - */ - -/** - * @typedef {(action: (state: TestRunState) => TestRunState) => void} Dispatcher - */ - -/** - * @typedef InstructionDocumentButton - * @property {Description} button - * @property {boolean} [enabled] - * @property {() => void} click - */ - -/** - * @typedef InstructionDocumentAssertionChoiceOptionsOptionsMore - * @property {Description} description - * @property {string} value - * @property {boolean} enabled - * @property {boolean} [focus] - * @property {(value: string) => void} change - */ - -/** - * @typedef InstructionDocumentAssertionChoiceOptionsOption - * @property {Description} description - * @property {boolean} checked - * @property {boolean} enabled - * @property {boolean} tabbable - * @property {boolean} [focus] - * @property {(checked: boolean) => void} change - * @property {(key: string) => boolean} keydown - * @property {InstructionDocumentAssertionChoiceOptionsOptionsMore} [more] - */ - -/** - * @typedef InstructionDocumentAssertionChoiceOptions - * @property {Description} header - * @property {InstructionDocumentAssertionChoiceOptionsOption[]} options - */ - -/** - * @typedef InstructionDocumentAssertionChoice - * @property {Description} label - * @property {boolean} checked - * @property {boolean} [focus] - * @property {() => void} click - * @property {InstructionDocumentAssertionChoiceOptions} [options] - */ - -/** - * @typedef DescriptionRich - * @property {string} [href] - * @property {boolean} [required] - * @property {boolean} [highlightRequired] - * @property {boolean} [offScreen] - * @property {Description} description - */ - -/** - * @typedef DescriptionWhitespace - * @property {typeof WhitespaceStyleMap["LINE_BREAK"]} whitespace - */ - -/** @typedef {string | DescriptionRich | DescriptionWhitespace | DescriptionArray} Description */ - -/** @typedef {Description[]} DescriptionArray */ - -/** - * @typedef InstructionDocumentResultsCommandsAssertion - * @property {Description} description - * @property {Boolean} passed - * @property {boolean} [focus] - * @property {() => void} click - */ - -/** - * @typedef InstructionDocumentResultsCommandsAssertionsHeader - * @property {Description} descriptionHeader - */ - -/** - * @typedef InstructionDocumentResultsCommandsATOutput - * @property {Description} description - * @property {string} value - * @property {boolean} focus - * @property {(value: string) => void} change - */ - -/** - * @typedef InstructionDocumentResultsCommandsUnexpected - * @property {Description} description - * @property {InstructionDocumentAssertionChoice} passChoice - * @property {InstructionDocumentAssertionChoice} failChoice - */ - -/** - * @typedef InstructionDocumentResultsCommand - * @property {Description} header - * @property {InstructionDocumentResultsCommandsATOutput} atOutput - * @property {InstructionDocumentResultsCommandsAssertionsHeader} assertionsHeader - * @property {InstructionDocumentResultsCommandsAssertion[]} assertions - * @property {InstructionDocumentResultsCommandsUnexpected} unexpectedBehaviors - */ - -/** - * @typedef InstructionDocumentResultsHeader - * @property {Description} header - * @property {Description} description - */ - -/** - * @typedef InstructionDocumentResults - * @property {InstructionDocumentResultsHeader} header - * @property {InstructionDocumentResultsCommand[]} commands - */ - -/** - * @typedef InstructionDocumentInstructionsInstructionsCommands - * @property {Description} description - * @property {Description[]} commands - */ - -/** - * @typedef InstructionDocumentInstructionsInstructions - * @property {Description} header - * @property {Description[]} instructions - * @property {Description[]} strongInstructions - * @property {InstructionDocumentInstructionsInstructionsCommands} commands - */ - -/** - * @typedef InstructionDocumentErrors - * @property {boolean} visible - * @property {Description} header - * @property {Description[]} errors - */ - -/** - * @typedef InstructionDocumentInstructionsHeader - * @property {Description} header - * @property {boolean} focus - */ - -/** - * @typedef InstructionDocumentInstructionsAssertions - * @property {Description} header - * @property {Description} description - * @property {Description[]} assertions - */ - -/** - * @typedef InstructionDocumentInstructions - * @property {InstructionDocumentInstructionsHeader} header - * @property {Description} description - * @property {InstructionDocumentInstructionsInstructions} instructions - * @property {InstructionDocumentInstructionsAssertions} assertions - * @property {InstructionDocumentButton} openTestPage - */ - -/** - * @typedef InstructionDocument - * @property {InstructionDocumentErrors} errors - * @property {InstructionDocumentInstructions} instructions - * @property {InstructionDocumentResults} results - * @property {InstructionDocumentButton} submit - */ - -/** - * @typedef TestRunHooks - * @property {() => void} closeTestPage - * @property {(options: {commandIndex: number, unexpectedIndex: number, increment: TestRunFocusIncrement}) => void} focusCommandUnexpectedBehavior - * @property {() => void} openTestPage - * @property {() => void} postResults - * @property {(options: {commandIndex: number, additionalAssertionIndex: number, result: AdditionalAssertionResult}) => void} setCommandAdditionalAssertion - * @property {(options: {commandIndex: number, assertionIndex: number, result: AssertionResult}) => void} setCommandAssertion - * @property {(options: {commandIndex: number, hasUnexpected: HasUnexpectedBehavior}) => void } setCommandHasUnexpectedBehavior - * @property {(options: {commandIndex: number, atOutput: string}) => void} setCommandOutput - * @property {(options: {commandIndex: number, unexpectedIndex: number, checked}) => void } setCommandUnexpectedBehavior - * @property {(options: {commandIndex: number, unexpectedIndex: number, impact: string}) => void } setCommandUnexpectedBehaviorImpact - * @property {(options: {commandIndex: number, unexpectedIndex: number, more: string}) => void } setCommandUnexpectedBehaviorMore - * @property {() => void} submit - */ - -/** - * @typedef UserActionFocusUnexpected - * @property {typeof UserObjectActionMap["FOCUS_UNDESIRABLE"]} action - * @property {number} commandIndex - * @property {number} unexpectedIndex - */ - -/** - * @typedef {T[keyof T]} EnumValues - * @template {{[key: string]: string}} T - */ - -/** - * @typedef TestRunAssertion - * @property {string} description - * @property {boolean} highlightRequired - * @property {number} priority - * @property {AssertionResult} result - */ - -/** - * @typedef TestRunAdditionalAssertion - * @property {string} description - * @property {boolean} highlightRequired - * @property {number} priority - * @property {AdditionalAssertionResult} result - */ - -/** - * @typedef TestRunUnexpectedBehavior - * @property {string} description - * @property {boolean} checked - * @property {object} [more] - * @property {boolean} more.highlightRequired - * @property {string} more.value - * @property {string} impact - */ - -/** - * @typedef TestRunUnexpectedGroup - * @property {boolean} highlightRequired - * @property {HasUnexpectedBehavior} hasUnexpected - * @property {number} tabbedBehavior - * @property {TestRunUnexpectedBehavior[]} behaviors - */ - -/** - * @typedef TestRunCommand - * @property {string} description - * @property {object} atOutput - * @property {boolean} atOutput.highlightRequired - * @property {string} atOutput.value - * @property {TestRunAssertion[]} assertions - * @property {TestRunAdditionalAssertion[]} additionalAssertions - * @property {TestRunUnexpectedGroup} unexpected - */ - -/** - * @typedef TestRunState - * This state contains all the serializable values that are needed to render any - * of the documents (InstructionDocument, ResultsTableDocument, and - * TestPageDocument) from this module. - * - * @property {string[] | null} errors - * @property {object} info - * @property {string} info.description - * @property {string} info.task - * @property {ATMode} info.mode - * @property {string} info.modeInstructions - * @property {string[]} info.userInstructions - * @property {string} info.setupScriptDescription - * @property {object} config - * @property {AT} config.at - * @property {boolean} config.renderResultsAfterSubmit - * @property {boolean} config.displaySubmitButton - * @property {TestRunUserAction} currentUserAction - * @property {TestRunCommand[]} commands - * @property {object} openTest - * @property {boolean} openTest.enabled - */ - -/** - * @typedef ResultsTableDetailsList - * @property {Description} description - * @property {Description[]} items - */ - -/** - * @typedef ResultsTableDocument - * @property {string} header - * @property {object} status - * @property {Description} status.header - * @property {object} table - * @property {object} table.headers - * @property {string} table.headers.description - * @property {string} table.headers.support - * @property {string} table.headers.details - * @property {object[]} table.commands - * @property {string} table.commands[].description - * @property {Description} table.commands[].support - * @property {object} table.commands[].details - * @property {Description} table.commands[].details.output - * @property {ResultsTableDetailsList} table.commands[].details.passingAssertions - * @property {ResultsTableDetailsList} table.commands[].details.failingAssertions - * @property {ResultsTableDetailsList} table.commands[].details.unexpectedBehaviors - */ - -/** - * @typedef TestPageDocumentResults - * @property {ResultsTableDocument} results - */ - -/** - * @typedef TestPageDocumentInstructions - * @property {string[] | null} errors - * @property {InstructionDocument} instructions - */ - -/** @typedef {TestPageDocumentInstructions | TestPageDocumentResults} TestPageDocument */ - -/** @typedef {"reading" | "interaction"} ATMode */ diff --git a/client/resources/aria-at-test-window.mjs b/client/resources/aria-at-test-window.mjs deleted file mode 100644 index c60b8c828..000000000 --- a/client/resources/aria-at-test-window.mjs +++ /dev/null @@ -1,73 +0,0 @@ -export class TestWindow { - /** - * @param {object} options - * @param {Window | null} [options.window] - * @param {string} options.pageUri - * @param {TestWindowHooks} [options.hooks] - */ - constructor({ window = null, pageUri, hooks }) { - /** @type {Window | null} */ - this.window = window; - - /** @type {string} */ - this.pageUri = pageUri; - - /** @type {TestWindowHooks} */ - this.hooks = { - windowOpened: () => {}, - windowClosed: () => {}, - ...hooks, - }; - } - - open() { - this.window = window.open( - this.pageUri, - '_blank', - 'toolbar=0,location=0,menubar=0,width=800,height=800' - ); - - this.hooks.windowOpened(); - - this.prepare(); - } - - prepare() { - if (!this.window) { - return; - } - - if (this.window.closed) { - this.window = undefined; - this.hooks.windowClosed(); - return; - } - - if ( - this.window.location.origin !== window.location.origin || // make sure the origin is the same, and prevent this from firing on an 'about' page - this.window.document.readyState !== 'complete' - ) { - window.setTimeout(() => { - this.prepare(); - }, 100); - return; - } - - // If the window is closed, re-enable open popup button - this.window.onunload = () => { - window.setTimeout(() => this.prepare(), 100); - }; - } - - close() { - if (this.window) { - this.window.close(); - } - } -} - -/** - * @typedef TestWindowHooks - * @property {() => void} windowOpened - * @property {() => void} windowClosed - */ diff --git a/client/resources/at-commands.mjs b/client/resources/at-commands.mjs deleted file mode 100644 index 6bbb5b776..000000000 --- a/client/resources/at-commands.mjs +++ /dev/null @@ -1,273 +0,0 @@ -/** @deprecated See aria-at-test-io-format.mjs */ - -import * as keys from './keys.mjs'; - -/** - * Class for getting AT-specific instructions for a test against a design pattern. - * @deprecated See aria-at-test-io-format.mjs:CommandsInput - */ -export class commandsAPI { - /** - * Creates an API to get AT-specific instructions for a design pattern. - * @param {object} commands - A data structure which is a nested object with the following format: - * { - * task: { - * mode: { - * at: [ - * key-command (string corresponding to export in keys.mjs), - * optional additional instructions to list after key command (string), - * ] - * } - * } - * } - * @param {object} supportJson - The data object found in `tests/support.json` - * @param {object} commandsJson - The data object found in `tests/commands.json` - */ - constructor(commands, supportJson, commandsJson) { - if (!commands) { - throw new Error('You must initialize commandsAPI with a commands data object'); - } - - if (!supportJson) { - throw new Error('You must initialize commandsAPI with a supportJson data object'); - } - - if (!commandsJson) { - throw new Error('You must initialize commandsAPI with a commandsJson data object'); - } - - this.AT_COMMAND_MAP = commands; - - this.MODE_INSTRUCTIONS = { - reading: { - jaws: `Verify the Virtual Cursor is active by pressing ${keys.ALT_DELETE}. If it is not, exit Forms Mode to activate the Virtual Cursor by pressing ${keys.ESC}.`, - nvda: `Ensure NVDA is in browse mode by pressing ${keys.ESC}. Note: This command has no effect if NVDA is already in browse mode.`, - voiceover_macos: `Toggle Quick Nav ON by pressing the ${keys.LEFT} and ${keys.RIGHT} keys at the same time.`, - }, - interaction: { - jaws: `Verify the PC Cursor is active by pressing ${keys.ALT_DELETE}. If it is not, turn off the Virtual Cursor by pressing ${keys.INS_Z}.`, - nvda: `If NVDA did not make the focus mode sound when the test page loaded, press ${keys.INS_SPACE} to turn focus mode on.`, - voiceover_macos: `Toggle Quick Nav OFF by pressing the ${keys.LEFT} and ${keys.RIGHT} keys at the same time.`, - }, - }; - - this.supportJson = supportJson; - this.commandsJson = this.flattenObject(commandsJson); - } - - /** - * Get AT-specific instruction - * @param {string} mode - The mode of the screen reader, "reading" or "interaction" - * @param {string} task - The task of the test. - * @param {object} assistiveTech - The assistive technology. - * @return {Array} - A list of commands (strings) - */ - getATCommands(mode, task, assistiveTech) { - let commands = []; - - for (const _atMode of mode.split('_')) { - if (this.AT_COMMAND_MAP[task][_atMode][assistiveTech.key]) { - mode = _atMode; - - if (!this.AT_COMMAND_MAP[task]) { - throw new Error( - `Task "${task}" does not exist, please add to at-commands or correct your spelling.` - ); - } - - if (!this.AT_COMMAND_MAP[task][mode]) { - throw new Error( - `Mode "${mode}" instructions for task "${task}" does not exist, please add to at-commands or correct your spelling.` - ); - } - - let commandsData = this.AT_COMMAND_MAP[task][mode][assistiveTech.key] || []; - - // V1 - if (mode === 'reading' || mode === 'interaction') { - for (let c of commandsData) { - let innerCommands = []; - let commandSequence = c[0].split(','); - for (let command of commandSequence) { - command = keys[command]; - if (typeof command === 'undefined') { - throw new Error( - `Key instruction identifier "${c}" for AT "${assistiveTech.name}", mode "${mode}", task "${task}" is not an available identifier. Update your commands.json file to the correct identifier or add your identifier to resources/keys.mjs.` - ); - } - - let furtherInstruction = c[1]; - command = furtherInstruction ? `${command} ${furtherInstruction}` : command; - innerCommands.push(command); - } - commands.push(innerCommands.join(', then ')); - } - } else { - // V2 - for (let c of commandsData) { - const commandWithPresentationNumber = c[0]; - const [commandId, presentationNumber] = commandWithPresentationNumber.split('|'); - - const commandKVs = this.findValuesByKeys([commandId]); - if (!commandKVs.length) { - throw new Error( - `Key instruction identifier "${commandId}" for AT "${assistiveTech.name}", mode "${mode}", task "${task}" is not an available identifier. Update your commands.json file to the correct identifier or add your identifier to tests/commands.json.` - ); - } - - commands.push( - ...commandKVs.map(({ value, key }) => { - value = assistiveTech.settings[mode].screenText - ? `${value} (${assistiveTech.settings[mode].screenText})` - : value; - return { - value, - key, - settings: mode, - }; - }) - ); - } - } - } - } - - return commands; - } - - /** - * Get AT-specific mode switching instructions - * @param {string} mode - The mode of the screen reader, "reading" or "interaction" - * @param {string} assistiveTech - The assistive technology. - * @return {string} - Instructions for switching into the correct mode. - */ - getModeInstructions(mode, assistiveTech) { - if (this.MODE_INSTRUCTIONS[mode] && this.MODE_INSTRUCTIONS[mode][assistiveTech.key]) { - return this.MODE_INSTRUCTIONS[mode][assistiveTech.key]; - } - return ''; - } - - /** - * Get AT-specific instruction - * @param {string} at - an assitve technology with any capitalization - * @return {string} - if this API knows instructions for `at`, it will return the `at` with proper capitalization - */ - isKnownAT(at) { - return this.supportJson.ats.find(o => o.key === at.toLowerCase()); - } - - defaultConfigurationInstructions(at) { - return this.supportJson.ats.find(o => o.key === at.toLowerCase()) - .defaultConfigurationInstructionsHTML; - } - - flattenObject(obj, parentKey) { - const flattened = {}; - - for (const key in obj) { - if (typeof obj[key] === 'object') { - const subObject = this.flattenObject(obj[key], parentKey + key + '.'); - Object.assign(flattened, subObject); - } else { - flattened[parentKey + key] = obj[key]; - } - } - - return flattened; - } - - findValueByKey(keyToFind) { - const keys = Object.keys(this.commandsJson); - - // Need to specially handle VO modifier key combination - if (keyToFind === 'vo') - return this.findValuesByKeys([this.commandsJson['modifierAliases.vo']])[0]; - - if (keyToFind.includes('modifiers.') || keyToFind.includes('keys.')) { - const parts = keyToFind.split('.'); - const keyToCheck = parts[parts.length - 1]; // value after the '.' - - if (this.commandsJson[keyToFind]) - return { - value: this.commandsJson[keyToFind], - key: keyToCheck, - }; - - return null; - } - - for (const key of keys) { - const parts = key.split('.'); - const parentKey = parts[0]; - const keyToCheck = parts[parts.length - 1]; // value after the '.' - - if (keyToCheck === keyToFind) { - if (parentKey === 'modifierAliases') { - return this.findValueByKey(`modifiers.${this.commandsJson[key]}`); - } else if (parentKey === 'keyAliases') { - return this.findValueByKey(`keys.${this.commandsJson[key]}`); - } - - return { - value: this.commandsJson[key], - key: keyToCheck, - }; - } - } - - // Return null if the key is not found - return null; - } - - findValuesByKeys(keysToFind = []) { - const result = []; - - const patternSepWithReplacement = (keyToFind, pattern, replacement) => { - if (keyToFind.includes(pattern)) { - let value = ''; - let validKeys = true; - const keys = keyToFind.split(pattern); - - for (const key of keys) { - const keyResult = this.findValueByKey(key); - if (keyResult) - value = value ? `${value}${replacement}${keyResult.value}` : keyResult.value; - else validKeys = false; - } - if (validKeys) return { value, key: keyToFind }; - } - - return null; - }; - - const patternSepHandler = keyToFind => { - let value = ''; - - if (keyToFind.includes(' ') && keyToFind.includes('+')) { - const keys = keyToFind.split(' '); - for (let [index, key] of keys.entries()) { - const keyToFindResult = this.findValueByKey(key); - if (keyToFindResult) keys[index] = keyToFindResult.value; - if (key.includes('+')) keys[index] = patternSepWithReplacement(key, '+', '+').value; - } - value = keys.join(' then '); - - return { value, key: keyToFind }; - } else if (keyToFind.includes(' ')) - return patternSepWithReplacement(keyToFind, ' ', ' then '); - else if (keyToFind.includes('+')) return patternSepWithReplacement(keyToFind, '+', '+'); - }; - - for (const keyToFind of keysToFind) { - if (keyToFind.includes(' ') || keyToFind.includes('+')) { - result.push(patternSepHandler(keyToFind)); - } else { - const keyToFindResult = this.findValueByKey(keyToFind); - if (keyToFindResult) result.push(keyToFindResult); - } - } - - return result; - } -} diff --git a/client/resources/commands.json b/client/resources/commands.json deleted file mode 100644 index 9f5e3cf94..000000000 --- a/client/resources/commands.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "modifiers": { - "alt": "Alt", - "opt": "Option", - "shift": "Shift", - "ctrl": "Control", - "cmd": "Command", - "win": "Windows", - "ins": "Insert" - }, - "modifierAliases": { - "jaws": "ins", - "nvda": "ins", - "vo": "ctrl+opt" - }, - "keys": { - "a": "a", - "b": "b", - "c": "c", - "d": "d", - "e": "e", - "f": "f", - "g": "g", - "h": "h", - "i": "i", - "j": "j", - "k": "k", - "l": "l", - "m": "m", - "n": "n", - "o": "o", - "p": "p", - "q": "q", - "r": "r", - "s": "s", - "t": "t", - "u": "u", - "v": "v", - "w": "w", - "x": "x", - "y": "y", - "z": "z", - "1": "1", - "2": "2", - "3": "3", - "4": "4", - "5": "5", - "6": "6", - "7": "7", - "8": "8", - "9": "9", - "0": "0", - "dash": "Dash", - "equals": "Equals", - "grave": "Grave", - "leftBracket": "Left Bracket", - "rightBracket": "Right Bracket", - "backslash": "Backslash", - "semicolon": "Semicolon", - "apostrophe": "Apostrophe", - "comma": "Comma", - "period": "Period", - "slash": "Slash", - "esc": "Escape", - "backspace": "Backspace", - "tab": "Tab", - "capsLock": "Caps Lock", - "enter": "Enter", - "space": "Space", - "f1": "F1", - "f2": "F2", - "f3": "F3", - "f4": "F4", - "f5": "F5", - "f6": "F6", - "f7": "F7", - "f8": "F8", - "f9": "F9", - "f10": "F10", - "f11": "F11", - "f12": "F12", - "scrollLock": "Scroll Lock", - "pause": "Pause", - "home": "Home", - "end": "End", - "pageUp": "Page Up", - "pageDown": "Page Down", - "del": "Delete", - "left": "Left Arrow", - "right": "Right Arrow", - "up": "Up Arrow", - "down": "Down Arrow", - "numLock": "Num Lock", - "numpadSlash": "Numpad Slash", - "numpadAsterisk": "Numpad Asterisk", - "numpadMinus": "Numpad Minus", - "numpadPlus": "Numpad Plus", - "numpadEnter": "Numpad Enter", - "numpad1": "Numpad 1", - "numpad2": "Numpad 2", - "numpad3": "Numpad 3", - "numpad4": "Numpad 4", - "numpad5": "Numpad 5", - "numpad6": "Numpad 6", - "numpad7": "Numpad 7", - "numpad8": "Numpad 8", - "numpad9": "Numpad 9", - "numpad0": "Numpad 0", - "numpadPeriod": "Numpad Period" - }, - "keyAliases": { - "delete": "del", - "escape": "esc" - } -} \ No newline at end of file diff --git a/client/resources/keys.json b/client/resources/keys.json deleted file mode 100644 index 640724f58..000000000 --- a/client/resources/keys.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "CTRL_HOME": "Control+Home", - "CTRL_OPT_HOME": "Control+Option+Home", - "CTRL_END": "Control+End", - "CTRL_OPT_END": "Control+Option+End", - "CTRL_HOME_THEN_DOWN": "Control+Home followed by Down Arrow", - "DELETE": "Delete", - "ALT_DELETE": "Alt+Delete", - "ALT_DOWN": "Alt+Down", - "CTRL_ALT_DOWN": "Control+Alt+Down", - "ALT_UP": "Alt+Up", - "C_AND_SHIFT_C": "C / Shift+C", - "SHIFT_C": "Shift+C", - "CTRL_INS_X": "Control+Insert+X", - "OPT_DOWN": "Option+Down", - "OPT_UP": "Option+Up", - "CTRL_OPT_LEFT": "Ctrl+Option+Left", - "CTRL_ALT_LEFT": "Control+Alt+Left", - "CTRL_OPT_RIGHT": "Control+Option+Right", - "CTRL_ALT_RIGHT": "Control+Alt+Right", - "CTRL_OPT_UP": "Control+Option+Up", - "CTRL_OPT_DOWN": "Control+Option+Down", - "CTRL_OPT_RIGHT_AND_CTRL_OPT_LEFT": "Control+Option+Right / Ctrl+Option+Left", - "CTRL_OPT_A": "Control+Option+A", - "CTRL_OPT_CMD_J": "Control+Option+Command+J", - "CTRL_OPT_CMD_L": "Control+Option+Command+L", - "CTRL_OPT_CMD_P": "Control+Option+Command+P", - "SHIFT_CTRL_OPT_CMD_J": "Shift+Control+Option+Command+J", - "SHIFT_CTRL_OPT_CMD_L": "Shift+Control+Option+Command+L", - "CTRL_OPT_CMD_J_AND_SHIFT_CTRL_OPT_CMD_J": "Control+Option+Command+J / Shift+Control+Option+Command+J", - "CTRL_OPT_CMD_C_AND_SHIFT_CTRL_OPT_CMD_C": "Control+Option+Command+C / Shift+Control+Option+Command+C", - "CTRL_OPT_F3": "Control+Option+F3", - "CTRL_OPT_F4": "Control+Option+F4", - "CTRL_OPT_SPACE": "Control+Option+Space", - "CTRL_OPT_SPACE_THEN_CTRL_OPT_RIGHT": "Control+Option+Space followed by Control+Option+Right", - "CTRL_U": "Control+U", - "CMD": "Command", - "CMD_LEFT": "Command+Left", - "CMD_RIGHT": "Command+Right", - "CMD_DOWN": "Command+Down", - "CMD_UP": "Command+Up", - "DOWN": "Down Arrow", - "END": "End", - "ENTER": "Enter", - "E_AND_SHIFT_E": "E / Shift+E", - "ESC": "Escape", - "F_AND_SHIFT_F": "F / Shift+F", - "HOME": "Home", - "INS_DOWN_OR_CAPS_DOWN": "Insert+Down (or CapsLock+Down)", - "INS_F7_OR_CAPS_F7": "Insert+F7 (or CapsLock+F7)", - "INS_SPACE": "Insert+Space", - "INS_TAB": "Insert+Tab", - "INS_TAB_OR_CAPS_TAB": "Insert+Tab (or CapsLock+Tab)", - "INS_UP_OR_CAPS_I": "Insert+Up (or CapsLock+I)", - "INS_UP": "Insert+Up", - "INS_UP_OR_CAPS_UP": "Insert+Up (or CapsLock+Up)", - "INS_Z": "Insert+Z", - "LEFT_AND_RIGHT": "Left Arrow / Right Arrow", - "LEFT": "Left Arrow", - "NUMPAD_5": "Numpad 5", - "INS_NUMPAD_5": "Insert+Numpad 5 (or CapsLock+Numpad 5)", - "INS_NUMPAD_6": "Insert+Numpad 6 (or CapsLock+Numpad 6)", - "NUMPAD_PLUS": "Numpad Plus", - "RIGHT": "Right Arrow", - "SPACE": "Space", - "TAB": "Tab", - "SHIFT_TAB": "Shift+Tab", - "TAB_AND_SHIFT_TAB": "Tab / Shift+Tab", - "UP": "Up Arrow", - "CTRL_ALT_UP": "Control+Alt+Up", - "UP_AND_DOWN": "Up Arrow / Down Arrow", - "SHIFT_X": "Shift+X", - "X_AND_SHIFT_X": "X / Shift+X", - "A": "A", - "SHIFT_A": "Shift+A", - "B": "B", - "SHIFT_B": "Shift+B", - "C": "C", - "D": "D", - "E": "E", - "SHIFT_E": "Shift+E", - "F": "F", - "SHIFT_F": "Shift+F", - "G": "G", - "H": "H", - "I": "I", - "SHIFT_I": "Shift+I", - "J": "J", - "K": "K", - "SHIFT_K": "Shift+K", - "L": "L", - "SHIFT_L": "Shift+L", - "M": "M", - "N": "N", - "O": "O", - "P": "P", - "Q": "Q", - "R": "R", - "SHIFT_R": "Shift+R", - "S": "S", - "T": "T", - "SHIFT_T": "Shift+T", - "CTRL_OPT_CMD_T": "Control+Option+Command+T", - "T_THEN_DOWN": "T followed by Down Arrow", - "SHIFT_T_THEN_DOWN": "Shift+T followed by Down Arrow", - "U": "U", - "SHIFT_U": "Shift+U", - "V": "V", - "W": "W", - "X": "X", - "Y": "Y", - "CTRL_OPT_CMD_Y": "Control+Option+Command+Y", - "SHIFT_CTRL_OPT_CMD_Y": "Shift+Control+Option+Command+Y", - "Z": "Z", - "PAGE_DOWN": "Page Down", - "PAGE_UP": "Page Up" -} diff --git a/client/resources/keys.mjs b/client/resources/keys.mjs deleted file mode 100644 index bfeff2bde..000000000 --- a/client/resources/keys.mjs +++ /dev/null @@ -1,133 +0,0 @@ -// Keys -export const CTRL_HOME = "Control+Home"; -export const CTRL_OPT_HOME = "Control+Option+Home"; -export const CTRL_END = "Control+End"; -export const CTRL_OPT_END = "Control+Option+End"; -export const CTRL_HOME_THEN_DOWN = "Control+Home followed by Down Arrow"; -export const DELETE = "Delete"; -export const ALT_DELETE = "Alt+Delete"; -export const ALT_DOWN = "Alt+Down"; -export const CTRL_ALT_DOWN = "Control+Alt+Down"; -export const ALT_UP = "Alt+Up"; -export const C_AND_SHIFT_C = "C / Shift+C"; -export const SHIFT_C = "Shift+C"; -export const CTRL_INS_X = "Control+Insert+X"; -export const OPT_DOWN = "Option+Down"; -export const OPT_UP = "Option+Up"; -export const CTRL_OPT_LEFT = "Ctrl+Option+Left"; -export const CTRL_ALT_LEFT = "Control+Alt+Left"; -export const CTRL_OPT_RIGHT = "Control+Option+Right"; -export const CTRL_ALT_RIGHT = "Control+Alt+Right"; -export const CTRL_OPT_UP = "Control+Option+Up"; -export const CTRL_OPT_DOWN = "Control+Option+Down"; -export const CTRL_OPT_RIGHT_AND_CTRL_OPT_LEFT = "Control+Option+Right / Ctrl+Option+Left"; -export const CTRL_OPT_A = "Control+Option+A"; -export const CTRL_OPT_CMD_J = "Control+Option+Command+J"; -export const CTRL_OPT_CMD_L = "Control+Option+Command+L"; -export const CTRL_OPT_CMD_P = "Control+Option+Command+P"; -export const SHIFT_CTRL_OPT_CMD_J = "Shift+Control+Option+Command+J"; -export const SHIFT_CTRL_OPT_CMD_L = "Shift+Control+Option+Command+L"; -export const CTRL_OPT_CMD_J_AND_SHIFT_CTRL_OPT_CMD_J = "Control+Option+Command+J / Shift+Control+Option+Command+J"; -export const CTRL_OPT_CMD_C_AND_SHIFT_CTRL_OPT_CMD_C = "Control+Option+Command+C / Shift+Control+Option+Command+C"; -export const CTRL_OPT_F3 = "Control+Option+F3"; -export const CTRL_OPT_F4 = "Control+Option+F4"; -export const CTRL_OPT_SPACE = "Control+Option+Space"; -export const CTRL_OPT_SPACE_THEN_CTRL_OPT_RIGHT = "Control+Option+Space followed by Control+Option+Right"; -export const CTRL_U = "Control+U"; -export const CMD = "Command"; -export const CMD_LEFT = "Command+Left"; -export const CMD_RIGHT = "Command+Right"; -export const CMD_DOWN = "Command+Down"; -export const CMD_UP = "Command+Up"; -export const DOWN = "Down Arrow"; -export const END = "End"; -export const ENTER = "Enter"; -export const E_AND_SHIFT_E = "E / Shift+E"; -export const ESC = "Escape"; -export const F_AND_SHIFT_F = "F / Shift+F"; -export const HOME = "Home"; -export const INS_DOWN_OR_CAPS_DOWN = "Insert+Down (or CapsLock+Down)"; -export const INS_F7_OR_CAPS_F7 = "Insert+F7 (or CapsLock+F7)"; -export const INS_SPACE = "Insert+Space"; -export const INS_TAB = "Insert+Tab"; -export const INS_TAB_OR_CAPS_TAB = "Insert+Tab (or CapsLock+Tab)"; -export const INS_UP_OR_CAPS_I = "Insert+Up (or CapsLock+I)"; -export const INS_UP = "Insert+Up"; -export const INS_UP_OR_CAPS_UP = "Insert+Up (or CapsLock+Up)"; -export const INS_Z = "Insert+Z"; -export const LEFT_AND_RIGHT = "Left Arrow / Right Arrow"; -export const LEFT = "Left Arrow"; -export const NUMPAD_5 = "Numpad 5"; -export const INS_NUMPAD_5 = "Insert+Numpad 5 (or CapsLock+Numpad 5)"; -export const INS_NUMPAD_6 = "Insert+Numpad 6 (or CapsLock+Numpad 6)"; -export const NUMPAD_PLUS = "Numpad Plus"; -export const RIGHT = "Right Arrow"; -export const SPACE = "Space"; -export const TAB = "Tab"; -export const SHIFT_TAB = "Shift+Tab"; -export const TAB_AND_SHIFT_TAB = "Tab / Shift+Tab"; -export const UP = "Up Arrow"; -export const CTRL_ALT_UP = "Control+Alt+Up"; -export const UP_AND_DOWN = "Up Arrow / Down Arrow"; -export const SHIFT_X = "Shift+X"; -export const X_AND_SHIFT_X = "X / Shift+X"; -export const A = "A"; -export const SHIFT_A = "Shift+A"; -export const B = "B"; -export const SHIFT_B = "Shift+B"; -export const C = "C"; -export const D = "D"; -export const E = "E"; -export const SHIFT_E = "Shift+E"; -export const F = "F"; -export const SHIFT_F = "Shift+F"; -export const G = "G"; -export const H = "H"; -export const I = "I"; -export const SHIFT_I = "Shift+I"; -export const J = "J"; -export const K = "K"; -export const SHIFT_K = "Shift+K"; -export const L = "L"; -export const SHIFT_L = "Shift+L"; -export const M = "M"; -export const N = "N"; -export const O = "O"; -export const P = "P"; -export const Q = "Q"; -export const R = "R"; -export const SHIFT_R = "Shift+R"; -export const S = "S"; -export const T = "T"; -export const SHIFT_T = "Shift+T"; -export const CTRL_OPT_CMD_T = "Control+Option+Command+T"; -export const T_THEN_DOWN = "T followed by Down Arrow"; -export const SHIFT_T_THEN_DOWN = "Shift+T followed by Down Arrow"; -export const U = "U"; -export const SHIFT_U = "Shift+U"; -export const V = "V"; -export const W = "W"; -export const X = "X"; -export const Y = "Y"; -export const CTRL_OPT_CMD_Y = "Control+Option+Command+Y"; -export const SHIFT_CTRL_OPT_CMD_Y = "Shift+Control+Option+Command+Y"; -export const Z = "Z"; -export const PAGE_DOWN = "Page Down"; -export const PAGE_UP = "Page Up"; -export const SHIFT_D = "Shift+D"; -export const CTRL_OPT_CMD_G = "Control+Option+Command+G"; -export const CTRL_OPT_CMD_H = "Control+Option+Command+H"; -export const CTRL_OPT_CMD_X = "Control+Option+Command+X"; -export const SHIFT_CTRL_OPT_CMD_X = "Shift+Control+Option+Command+X"; -export const SHIFT_CTRL_OPT_CMD_G = "Shift+Control+Option+Command+G"; -export const SHIFT_CTRL_OPT_CMD_H = "Shift+Control+Option+Command+H"; -export const SHIFT_CTRL_OPT_CMD_P = "Shift+Control+Option+Command+P"; -export const SHIFT_G = "Shift+G"; -export const SHIFT_H = "Shift+H"; -export const ONE = "1"; -export const TWO = "2"; -export const SHIFT_ONE = "Shift+1"; -export const SHIFT_TWO = "Shift+2"; -export const SHIFT_P = "Shift+P"; -export const COMMA = "Comma"; -export const SHIFT_PERIOD = "Shift+Period"; diff --git a/client/resources/support.json b/client/resources/support.json deleted file mode 100644 index 4b91b7f71..000000000 --- a/client/resources/support.json +++ /dev/null @@ -1,410 +0,0 @@ -{ - "ats": [ - { - "name": "JAWS", - "key": "jaws", - "defaultConfigurationInstructionsHTML": "Configure JAWS with default settings. For help, read <a href="https://github.com/w3c/aria-at/wiki/Configuring-Screen-Readers-for-Testing">Configuring Screen Readers for Testing</a>.", - "assertionTokens": { - "screenReader": "JAWS", - "readingMode": "virtual cursor active", - "interactionMode": "PC cursor active" - }, - "settings": { - "virtualCursor": { - "screenText": "virtual cursor active", - "instructions": [ - "Press <kbd>Alt</kbd>+<kbd>Delete</kbd> to determine which cursor is active.", - "If the PC cursor is active, press <kbd>Escape</kbd> to activate the virtual cursor." - ] - }, - "pcCursor": { - "screenText": "PC cursor active", - "instructions": [ - "Press <kbd>Alt</kbd>+<kbd>Delete</kbd> to determine which cursor is active.", - "If the virtual cursor is active, press <kbd>Insert</kbd>+<kbd>z</kbd> to disable the virtual cursor." - ] - } - } - }, - { - "name": "NVDA", - "key": "nvda", - "defaultConfigurationInstructionsHTML": "Configure NVDA with default settings. For help, read <a href="https://github.com/w3c/aria-at/wiki/Configuring-Screen-Readers-for-Testing">Configuring Screen Readers for Testing</a>.", - "assertionTokens": { - "screenReader": "NVDA", - "readingMode": "browse mode", - "interactionMode": "focus mode" - }, - "settings": { - "browseMode": { - "screenText": "browse mode on", - "instructions": [ - "Press <kbd>Insert</kbd>+<kbd>Space</kbd>.", - "If NVDA made the focus mode sound, press <kbd>Insert</kbd>+<kbd>Space</kbd> again to turn browse mode back on." - ] - }, - "focusMode": { - "screenText": "focus mode on", - "instructions": [ - "Press <kbd>Insert</kbd>+<kbd>Space</kbd>.", - "If NVDA made the browse mode sound, press <kbd>Insert</kbd>+<kbd>Space</kbd> again to turn focus mode back on." - ] - } - } - }, - { - "name": "VoiceOver for macOS", - "key": "voiceover_macos", - "defaultConfigurationInstructionsHTML": "Configure VoiceOver with default settings. For help, read <a href="https://github.com/w3c/aria-at/wiki/Configuring-Screen-Readers-for-Testing">Configuring Screen Readers for Testing</a>.", - "settings": { - "quickNavOn": { - "screenText": "quick nav on", - "instructions": [ - "Simultaneously press <kbd>Left Arrow</kbd> and <kbd>Right Arrow</kbd>.", - "If VoiceOver said 'quick nav off', press <kbd>Left Arrow</kbd> and <kbd>Right Arrow</kbd> again to turn it back on." - ] - }, - "quickNavOff": { - "screenText": "quick nav off", - "instructions": [ - "Simultaneously press <kbd>Left Arrow</kbd> and <kbd>Right Arrow</kbd>.", - "If VoiceOver said 'quick nav on', press <kbd>Left Arrow</kbd> and <kbd>Right Arrow</kbd> again to turn it back off." - ] - } - } - } - ], - "applies_to": { - "Desktop Screen Readers": [ - "VoiceOver for macOS", - "NVDA", - "JAWS" - ], - "Screen Readers": [ - "VoiceOver for macOS", - "NVDA", - "JAWS" - ] - }, - "testPlanStrings": { - "ariaSpecsPreface": "Tested ARIA features:", - "openExampleInstruction": "Activate the "Open test page" button, which opens the example to test in a new window and runs a script that", - "commandListPreface": "Do this with each of the following commands or command sequences.", - "commandListSettingsPreface": "If any settings are specified in parentheses, ensure the settings are active before executing the command or command sequence.", - "settingInstructionsPreface": "To perform a task with", - "assertionResponseQuestion": "Which statements are true about the response to" - }, - "references": { - "aria": { - "baseUrl": "https://www.w3.org/TR/wai-aria/", - "linkText": "ARIA Specification", - "fragmentIds": { - "alert": "#alert", - "alertdialog": "#alertdialog", - "application": "#application", - "article": "#article", - "associationlist": "#associationlist", - "associationlistitemkey": "#associationlistitemkey", - "associationlistitemvalue": "#associationlistitemvalue", - "banner": "#banner", - "blockquote": "#blockquote", - "button": "#button", - "caption": "#caption", - "cell": "#cell", - "checkbox": "#checkbox", - "code": "#code", - "columnheader": "#columnheader", - "combobox": "#combobox", - "command": "#command", - "comment": "#comment", - "complementary": "#complementary", - "composite": "#composite", - "contentinfo": "#contentinfo", - "definition": "#definition", - "deletion": "#deletion", - "dialog": "#dialog", - "directory": "#directory", - "document": "#document", - "emphasis": "#emphasis", - "feed": "#feed", - "figure": "#figure", - "form": "#form", - "generic": "#generic", - "grid": "#grid", - "gridcell": "#gridcell", - "group": "#group", - "heading": "#heading", - "image": "#image", - "img": "#img", - "input": "#input", - "insertion": "#insertion", - "landmark": "#landmark", - "link": "#link", - "list": "#list", - "listbox": "#listbox", - "listitem": "#listitem", - "log": "#log", - "main": "#main", - "mark": "#mark", - "marquee": "#marquee", - "math": "#math", - "menu": "#menu", - "menubar": "#menubar", - "menuitem": "#menuitem", - "menuitemcheckbox": "#menuitemcheckbox", - "menuitemradio": "#menuitemradio", - "meter": "#meter", - "navigation": "#navigation", - "none": "#none", - "note": "#note", - "option": "#option", - "paragraph": "#paragraph", - "presentation": "#presentation", - "progressbar": "#progressbar", - "radio": "#radio", - "radiogroup": "#radiogroup", - "range": "#range", - "region": "#region", - "roletype": "#roletype", - "row": "#row", - "rowgroup": "#rowgroup", - "rowheader": "#rowheader", - "scrollbar": "#scrollbar", - "search": "#search", - "searchbox": "#searchbox", - "section": "#section", - "sectionhead": "#sectionhead", - "select": "#select", - "separator": "#separator", - "slider": "#slider", - "spinbutton": "#spinbutton", - "status": "#status", - "strong": "#strong", - "structure": "#structure", - "subscript": "#subscript", - "suggestion": "#suggestion", - "superscript": "#superscript", - "switch": "#switch", - "tab": "#tab", - "table": "#table", - "tablist": "#tablist", - "tabpanel": "#tabpanel", - "term": "#term", - "textbox": "#textbox", - "time": "#time", - "timer": "#timer", - "toolbar": "#toolbar", - "tooltip": "#tooltip", - "tree": "#tree", - "treegrid": "#treegrid", - "treeitem": "#treeitem", - "widget": "#widget", - "window": "#window", - "aria-activedescendant": "#aria-activedescendant", - "aria-atomic": "#aria-atomic", - "aria-autocomplete": "#aria-autocomplete", - "aria-braillelabel": "#aria-braillelabel", - "aria-brailleroledescription": "#aria-brailleroledescription", - "aria-busy": "#aria-busy", - "aria-checked": "#aria-checked", - "aria-colcount": "#aria-colcount", - "aria-colindex": "#aria-colindex", - "aria-colindextext": "#aria-colindextext", - "aria-colspan": "#aria-colspan", - "aria-controls": "#aria-controls", - "aria-current": "#aria-current", - "aria-describedby": "#aria-describedby", - "aria-description": "#aria-description", - "aria-details": "#aria-details", - "aria-disabled": "#aria-disabled", - "aria-errormessage": "#aria-errormessage", - "aria-expanded": "#aria-expanded", - "aria-flowto": "#aria-flowto", - "aria-haspopup": "#aria-haspopup", - "aria-hidden": "#aria-hidden", - "aria-invalid": "#aria-invalid", - "aria-keyshortcuts": "#aria-keyshortcuts", - "aria-label": "#aria-label", - "aria-labelledby": "#aria-labelledby", - "aria-level": "#aria-level", - "aria-live": "#aria-live", - "aria-modal": "#aria-modal", - "aria-multiline": "#aria-multiline", - "aria-multiselectable": "#aria-multiselectable", - "aria-orientation": "#aria-orientation", - "aria-owns": "#aria-owns", - "aria-placeholder": "#aria-placeholder", - "aria-posinset": "#aria-posinset", - "aria-pressed": "#aria-pressed", - "aria-readonly": "#aria-readonly", - "aria-relevant": "#aria-relevant", - "aria-required": "#aria-required", - "aria-roledescription": "#aria-roledescription", - "aria-rowcount": "#aria-rowcount", - "aria-rowindex": "#aria-rowindex", - "aria-rowindextext": "#aria-rowindextext", - "aria-rowspan": "#aria-rowspan", - "aria-selected": "#aria-selected", - "aria-setsize": "#aria-setsize", - "aria-sort": "#aria-sort", - "aria-valuemax": "#aria-valuemax", - "aria-valuemin": "#aria-valuemin", - "aria-valuenow": "#aria-valuenow", - "aria-valuetext": "#aria-valuetext" - } - }, - "htmlAam": { - "baseUrl": "https://www.w3.org/TR/html-aam-1.0/", - "linkText": "Accessibility API Mapping", - "fragmentIds": { - "a": "#el-a", - "aNoHref": "#el-a-no-href", - "abbr": "#el-abbr", - "address": "#el-address", - "area": "#el-area", - "areaNoHref": "#el-area-no-href", - "article": "#el-article", - "asideBodyOrMainScope": "#el-aside-ancestorbodymain", - "asideSectionScope": "#el-aside", - "audio": "#el-audio", - "autonomous custom element": "#el-autonomous-custom-element", - "b": "#el-b", - "base": "#el-base", - "bdi": "#el-bdi", - "bdo": "#el-bdo", - "blockquote": "#el-blockquote", - "body": "#el-body", - "br": "#el-br", - "button": "#el-button", - "canvas": "#el-canvas", - "caption": "#el-caption", - "cite": "#el-cite", - "code": "#el-code", - "col": "#el-col", - "colgroup": "#el-colgroup", - "data": "#el-data", - "datalist": "#el-datalist", - "dd": "#el-dd", - "del": "#el-del", - "details": "#el-details", - "dfn": "#el-dfn", - "dialog": "#el-dialog", - "div": "#el-div", - "dl": "#el-dl", - "dt": "#el-dt", - "em": "#el-em", - "embed": "#el-embed", - "fieldset": "#el-fieldset", - "figcaption": "#el-figcaption", - "figure": "#el-figure", - "footerBodyScope": "#el-footer-ancestorbody", - "footerMainScope": "#el-footer", - "form": "#el-form", - "formAssociatedCustomElement": "#el-form-associated-custom-element", - "heading": "#el-h1-h6", - "head": "#el-head", - "headerBodyScope": "#el-header-ancestorbody", - "headerMainScope": "#el-header", - "hgroup": "#el-hgroup", - "hr": "#el-hr", - "html": "#el-html", - "i": "#el-i", - "iframe": "#el-iframe", - "img": "#el-img", - "imgEmptyAlt": "#el-img-empty-alt", - "inputTypeButton": "#el-input-button", - "inputTypeCheckbox": "#el-input-checkbox", - "inputTypeColor": "#el-input-color", - "inputTypeDate": "#el-input-date", - "inputTypeDateTime": "#el-input-datetime-local", - "inputTypeEmail": "#el-input-email", - "inputTypeFile": "#el-input-file", - "inputTypeHidden": "#el-input-hidden", - "inputTypeImage": "#el-input-image", - "inputTypeMonth": "#el-input-month", - "inputTypeNumber": "#el-input-number", - "inputTypePassword": "#el-input-password", - "inputTypeRadio": "#el-input-radio", - "inputTypeRange": "#el-input-range", - "inputTypeReset": "#el-input-reset", - "inputTypeSearch": "#el-input-search", - "inputTypeSubmit": "#el-input-submit", - "inputTypeTelephone": "#el-input-tel", - "inputTypeText": "#el-input-text", - "inputTypeTextAutocomplete": "#el-input-textetc-autocomplete", - "inputTypeTime": "#el-input-time", - "inputTypeUrl": "#el-input-url", - "inputTypeWeek": "#el-input-week", - "ins": "#el-ins", - "kbd": "#el-kbd", - "label": "#el-label", - "legend": "#el-legend", - "li": "#el-li", - "link": "#el-link", - "main": "#el-main", - "map": "#el-map", - "mark": "#el-mark", - "math": "#el-math", - "menu": "#el-menu", - "meta": "#el-meta", - "meter": "#el-meter", - "nav": "#el-nav", - "noscript": "#el-noscript", - "object": "#el-object", - "ol": "#el-ol", - "optgroup": "#el-optgroup", - "option": "#el-option", - "output": "#el-output", - "p": "#el-p", - "param": "#el-param", - "picture": "#el-picture", - "pre": "#el-pre", - "progress": "#el-progress", - "q": "#el-q", - "rb": "#el-rb", - "rp": "#el-rp", - "rt": "#el-rt", - "rtc": "#el-rtc", - "ruby": "#el-ruby", - "s": "#el-s", - "samp": "#el-samp", - "script": "#el-script", - "search": "#el-search", - "section": "#el-section", - "select": "#el-select-listbox", - "selectSize1": "#el-select-combobox", - "slot": "#el-slot", - "small": "#el-small", - "source": "#el-source", - "span": "#el-span", - "strong": "#el-strong", - "style": "#el-style", - "sub": "#el-sub", - "summary": "#el-summary", - "sup": "#el-sup", - "svg": "#el-svg", - "table": "#el-table", - "tbody": "#el-tbody", - "td": "#el-td", - "tdGridcell": "#el-td-gridcell", - "template": "#el-template", - "textarea": "#el-textarea", - "tfoot": "#el-tfoot", - "th": "#el-th", - "thGridcell": "#el-th-gridcell", - "thColgroupHeader": "#el-th-columnheader", - "thRowgroupHeader": "#el-th-rowheader", - "thead": "#el-thead", - "time": "#el-time", - "title": "#el-title", - "tr": "#el-tr", - "track": "#el-track", - "u": "#el-u", - "ul": "#el-ul", - "var": "#el-var", - "video": "#el-video", - "wbr": "#el-wbr" - } - } - } -} diff --git a/client/resources/types/aria-at-test-result.js b/client/resources/types/aria-at-test-result.js deleted file mode 100644 index ec8eab55d..000000000 --- a/client/resources/types/aria-at-test-result.js +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Types of a format of a test result submitted to or received from aria-at-app. - * @namespace AriaATTestResult - */ - -/** - * @typedef {"MUST" - * | "SHOULD"} AriaATTestResult.AssertionPriorityJSON - */ - -/** - * @typedef {"INCORRECT_OUTPUT" - * | "NO_OUTPUT"} AriaATTestResult.AssertionFailedReasonJSON - */ - -/** - * @typedef AriaATTestResult.JSON - * @property {object} test - * @property {string} test.title - * @property {object} test.at - * @property {string} test.at.id - * @property {string} test.atMode - * @property {object[]} scenarioResults - * @property {object} scenarioResults[].scenario - * @property {object} scenarioResults[].scenario.command - * @property {string} scenarioResults[].scenario.command.id - * @property {string} scenarioResults[].output - * @property {object[]} scenarioResults[].assertionResults - * @property {object} scenarioResults[].assertionResults[].assertion - * @property {AriaATTestResult.AssertionPriorityJSON} scenarioResults[].assertionResults[].assertion.priority - * @property {string} scenarioResults[].assertionResults[].assertion.text - * @property {boolean} scenarioResults[].assertionResults[].passed - * @property {AriaATTestResult.AssertionFailedReasonJSON | null} [scenarioResults[].assertionResults[].failedReason] - * @property {object[]} scenarioResults[].unexpectedBehaviors - * @property {string} scenarioResults[].unexpectedBehaviors[].id - * @property {string} scenarioResults[].unexpectedBehaviors[].text - * @property {string} scenarioResults[].unexpectedBehaviors[].impact - * @property {string} scenarioResults[].unexpectedBehaviors[].details - */ diff --git a/client/resources/types/aria-at-test-run.js b/client/resources/types/aria-at-test-run.js deleted file mode 100644 index cdeceb90e..000000000 --- a/client/resources/types/aria-at-test-run.js +++ /dev/null @@ -1,100 +0,0 @@ -/** @namespace AriaATTestRun */ - -/** - * @typedef {"reading" - * | "interaction"} AriaATTestRun.ATMode - */ - -/** - * @typedef {"loadPage" - * | "openTestWindow" - * | "closeTestWindow" - * | "validateResults" - * | "changeText" - * | "changeSelection" - * | "showResults"} AriaATTestRun.UserActionName - */ - -/** - * @typedef {"focusUndesirable"} AriaATTestRun.UserActionObjectName - */ - -/** - * @typedef AriaATTestRun.UserActionFocusUnexpected - * @property {"focusUndesirable"} action - * @property {number} commandIndex - * @property {number} unexpectedIndex - */ - -/** - * @typedef {AriaATTestRun.UserActionName - * | AriaATTestRun.UserActionFocusUnexpected} AriaATTestRun.UserAction - */ - -/** - * @typedef {"notSet" - * | "pass" - * | "failMissing" - * | "failIncorrect"} AriaATTestRun.AssertionResult - */ - -/** - * @typedef {"notSet" - * | "pass" - * | "failSupport"} AriaATTestRun.AdditionalAssertionResult - */ - -/** - * @typedef {"notSet" - * | "hasUnexpected" - * | "doesNotHaveUnexpected"} AriaATTestRun.HasUnexpectedBehavior - */ - -/** - * @typedef AriaATTestRun.State - * This state contains all the serializable values that are needed to render any of the documents (InstructionDocument, - * ResultsTableDocument, and TestPageDocument) from the test-run module. - * - * @property {string[] | null} errors - * @property {object} info - * @property {string} info.description - * @property {string} info.task - * @property {AriaATTestRun.ATMode} info.mode - * @property {string} info.modeInstructions - * @property {string[]} info.userInstructions - * @property {string} info.setupScriptDescription - * @property {object} config - * @property {object} config.at - * @property {string} config.at.key - * @property {string} config.at.name - * @property {boolean} config.renderResultsAfterSubmit - * @property {boolean} config.displaySubmitButton - * @property {AriaATTestRun.UserAction} currentUserAction - * @property {object[]} commands - * @property {string} commands[].description - * @property {object} commands[].atOutput - * @property {boolean} commands[].atOutput.highlightRequired - * @property {string} commands[].atOutput.value - * @property {object[]} commands[].assertions - * @property {string} commands[].assertions[].description - * @property {boolean} commands[].assertions[].highlightRequired - * @property {number} commands[].assertions[].priority - * @property {AriaATTestRun.AssertionResult} commands[].assertions[].result - * @property {object[]} commands[].additionalAssertions - * @property {string} commands[].additionalAssertions[].description - * @property {boolean} commands[].additionalAssertions[].highlightRequired - * @property {number} commands[].additionalAssertions[].priority - * @property {AriaATTestRun.AdditionalAssertionResult} commands[].additionalAssertions[].result - * @property {object} commands[].unexpected - * @property {boolean} commands[].unexpected.highlightRequired - * @property {AriaATTestRun.HasUnexpectedBehavior} commands[].unexpected.hasUnexpected - * @property {number} commands[].unexpected.tabbedBehavior - * @property {object[]} commands[].unexpected.behaviors - * @property {string} commands[].unexpected.behaviors[].description - * @property {boolean} commands[].unexpected.behaviors[].checked - * @property {object} [commands[].unexpected.behaviors[].more] - * @property {boolean} commands[].unexpected.behaviors[].more.highlightRequired - * @property {string} commands[].unexpected.behaviors[].more.value - * @property {object} openTest - * @property {boolean} openTest.enabled - */ diff --git a/client/resources/vrender.mjs b/client/resources/vrender.mjs deleted file mode 100644 index f21b6b2dc..000000000 --- a/client/resources/vrender.mjs +++ /dev/null @@ -1,987 +0,0 @@ -const mounts = new WeakMap(); - -/** - * @param {HTMLElement} mount - * @param {NodeNode} newValue - */ -export function render(mount, newValue) { - let lastValue = mounts.get(mount); - if (!lastValue) { - lastValue = ElementType.init(newQueue(), null, null, element(mount.tagName)); - lastValue.ref = mount; - mounts.set(mount, lastValue); - } - const queue = newQueue(); - lastValue.type.diff(queue, lastValue, element(mount.tagName, newValue)); - runQueue(queue); - if (!newValue) { - mounts.delete(mount); - } -} - -/** - * @param {string} shape - * @param {...NodeNode} content - * @returns {ElementNode} - */ -export function element(shape, ...content) { - return { - type: ELEMENT_TYPE_NAME, - shape, - content: content.map(asNode), - }; -} - -/** - * @param {...NodeNode} content - * @returns {FragmentNode} - */ -export function fragment(...content) { - return { - type: FRAGMENT_TYPE_NAME, - shape: FRAGMENT_TYPE_NAME, - content: content.map(asNode), - }; -} - -/** - * @param {string} content - * @returns {TextNode} - */ -export function text(content) { - return { - type: TEXT_TYPE_NAME, - shape: TEXT_TYPE_NAME, - content, - }; -} - -/** - * @param {function} shape - * @param {...NodeNode} content - * @returns {ComponentNode} - */ -export function component(shape, ...content) { - return { - type: COMPONENT_TYPE_NAME, - shape, - content, - }; -} - -/** - * @param {{[key: string]: string}} styleMap - * @returns {MemberNode} - */ -export function style(styleMap) { - return attribute( - 'style', - Object.keys(styleMap) - .map(key => `${key}: ${styleMap[key]};`) - .join(' ') - ); -} - -/** - * @param {string[]} names - * @returns {MemberNode} - */ -export function className(names) { - return attribute('class', names.filter(Boolean).join(' ')); -} - -/** - * @param {string} name - * @param {string | boolean} value - * @returns {MemberNode} - */ -export function attribute(name, value) { - return { - type: ATTRIBUTE_TYPE_NAME, - name, - value, - }; -} - -/** - * @param {string} name - * @param {any} value - * @returns {MemberNode} - */ -export function property(name, value) { - return { - type: PROPERTY_TYPE_NAME, - name, - value, - }; -} - -/** - * @param {string} name - * @param {any} value - * @returns {MemberNode} - */ -export function meta(name, value) { - return { - type: META_TYPE_NAME, - name, - value, - }; -} - -const refMap = new WeakMap(); - -/** - * @param {{ref: HTMLElement | null}} value - * @returns {MemberNode} - */ -export function ref(value) { - let refHook = refMap.get(value); - if (!refHook) { - refHook = (/** @type {HTMLElement} */ element) => { - value.ref = element; - }; - refMap.set(value, refHook); - } - return { - type: REF_TYPE_NAME, - name: 'ref', - value: refHook, - }; -} - -const noop = function () {}; - -/** - * @param {boolean} shouldFocus - */ -export function focus(shouldFocus) { - return { - type: REF_TYPE_NAME, - name: 'focus', - value: shouldFocus ? element => element.focus() : noop, - }; -} - -function asNode(item) { - if (typeof item === 'string') { - return text(item); - } else if (Array.isArray(item)) { - return fragment(...item); - } else if (item === null || item === undefined) { - return fragment(); - } - return item; -} - -const ELEMENT_TYPE_NAME = 'element'; -const FRAGMENT_TYPE_NAME = 'fragment'; -const COMPONENT_TYPE_NAME = 'component'; -const TEXT_TYPE_NAME = 'text'; -const ATTRIBUTE_TYPE_NAME = 'attribute'; -const PROPERTY_TYPE_NAME = 'property'; -const REF_TYPE_NAME = 'ref'; -const META_TYPE_NAME = 'meta'; - -/** @type ElementStateType */ -const ElementType = { - name: ELEMENT_TYPE_NAME, - diffEntry: /** @type {DiffEntryFunction} */ (diffChildEntry), - init: /** @type {InitNodeFunction} */ ( - function (queue, parent, after, /** @type {ElementNode} */ node) { - const state = { - type: ElementType, - parent, - after, - shape: node.shape, - content: null, - ref: null, - refHooks: null, - rewriteChildIndex: 0, - children: null, - rewriteMemberIndex: 0, - members: null, - }; - enqueueChange(queue, addElement, state); - return state; - } - ), - diff: /** @type {DiffFunction} */ ( - function (queue, /** @type {ElementState} */ lastValue, /** @type {ElementNode} */ newValue) { - lastValue.rewriteMemberIndex = 0; - diffFragment(queue, lastValue, newValue); - if (lastValue.members !== null) { - const group = lastValue.members; - let index; - for (index = lastValue.rewriteMemberIndex; index < group.length; index++) { - const node = group[index]; - node.type.teardown(queue, node); - } - if (lastValue.rewriteMemberIndex === 0) { - lastValue.members = null; - } else { - group.length = lastValue.rewriteMemberIndex; - } - } - } - ), - teardown: /** @type {TeardownFunction} */ ( - function (queue, /** @type {ElementState} */ state) { - enqueueChange(queue, removeElement, state); - const { children } = state; - if (children !== null) { - for (let i = 0; i < children.length; i++) { - children[i].type.softTeardown(children[i]); - } - } - } - ), - softTeardown: /** @type {SoftTeardownFunction} */ (softTeardownElement), -}; - -/** @type {FragmentStateType} */ -const FragmentType = { - name: FRAGMENT_TYPE_NAME, - diffEntry: /** @type {DiffEntryFunction} */ (diffChildEntry), - init(queue, parent, after) { - return { - type: FragmentType, - parent, - after, - shape: FRAGMENT_TYPE_NAME, - content: null, - rewriteChildIndex: 0, - children: null, - }; - }, - diff: /** @type {DiffFunction} */ (diffFragment), - teardown: /** @type {TeardownFunction} */ ( - function (queue, /** @type {FragmentState} */ state) { - const { children } = state; - if (children !== null) { - for (let i = 0; i < children.length; i++) { - children[i].type.teardown(queue, children[i]); - } - } - } - ), - softTeardown: /** @type {SoftTeardownFunction} */ (softTeardownElement), -}; - -/** @type ComponentStateType */ -const ComponentType = { - name: COMPONENT_TYPE_NAME, - diffEntry: /** @type {DiffEntryFunction} */ (diffChildEntry), - init: /** @type {InitNodeFunction} */ ( - function (queue, parent, after, /** @type {ComponentNode} */ node) { - return { - type: ComponentType, - parent, - after, - shape: node.shape, - content: null, - rendered: null, - }; - } - ), - diff: /** @type {DiffFunction} */ ( - function (queue, /** @type {ComponentState} */ lastChild, /** @type {ComponentNode} */ node) { - /** @type {MemberNode} */ - const componentOptionsMeta = node.content.find(isOptions) || null; - const componentOptions = componentOptionsMeta ? componentOptionsMeta.value : null; - if (shallowEquals(lastChild.content, componentOptions) === false) { - lastChild.content = componentOptions; - queue.prepare.push(lastChild); - } - } - ), - teardown: /** @type {TeardownFunction} */ ( - function (queue, /** @type {ComponentState} */ state) { - if (state.rendered !== null) { - state.rendered.type.teardown(queue, state.rendered); - } - } - ), - softTeardown: /** @type {SoftTeardownFunction} */ ( - function (/** @type {ComponentState} */ state) { - if (state.rendered !== null) { - state.rendered.type.softTeardown(state.rendered); - } - } - ), -}; - -/** @type TextStateType */ -const TextType = { - name: TEXT_TYPE_NAME, - diffEntry: /** @type {DiffEntryFunction} */ (diffChildEntry), - init: /** @type {InitNodeFunction} */ ( - function (queue, parent, after, /** @type {TextNode} */ node) { - /** @type {TextState} */ - const state = { - type: TextType, - parent, - after, - shape: TEXT_TYPE_NAME, - content: node.content, - ref: null, - }; - enqueueChange(queue, addText, state); - return state; - } - ), - diff: /** @type {DiffFunction} */ ( - function (queue, /** @type {TextState} */ lastChild, /** @type {TextNode} */ node) { - if (lastChild.content !== node.content) { - lastChild.content = node.content; - enqueueChange(queue, changeText, lastChild); - } - } - ), - teardown: /** @type {TeardownFunction} */ ( - function (queue, /** @type {TextState} */ state) { - enqueueChange(queue, removeText, state); - } - ), - softTeardown() {}, -}; - -/** @type {MemberStateType} */ -const AttributeType = { - name: ATTRIBUTE_TYPE_NAME, - diffEntry: /** @type {DiffEntryFunction} */ (diffMemberEntry), - init(parent, name) { - return { - type: AttributeType, - parent, - name, - value: null, - }; - }, - diff: /** @type {DiffFunction} */ ( - function (queue, /** @type {MemberState} */ old, /** @type {MemberNode} */ memberNode) { - if (old.value !== memberNode.value) { - old.value = memberNode.value; - if (old.value === false) { - enqueueChange(queue, removeAttribute, old); - } else { - enqueueChange(queue, changeAttribute, old); - } - } - } - ), - teardown(queue, state) { - enqueueChange(queue, removeAttribute, state); - }, -}; - -/** @type {MemberStateType} */ -const PropertyType = { - name: PROPERTY_TYPE_NAME, - diffEntry: /** @type {DiffEntryFunction} */ (diffMemberEntry), - init(parent, name) { - return { - type: PropertyType, - parent, - name, - value: null, - }; - }, - diff: /** @type {DiffFunction} */ ( - function (queue, /** @type {MemberState} */ old, /** @type {MemberNode} */ memberNode) { - if (old.value !== memberNode.value) { - old.value = memberNode.value; - enqueueChange(queue, changeProperty, old); - } - } - ), - teardown(queue, state) { - state.value = undefined; - enqueueChange(queue, changeProperty, state); - }, -}; - -/** @type {MemberStateType} */ -const RefType = { - name: REF_TYPE_NAME, - diffEntry: /** @type {DiffEntryFunction} */ (diffMemberEntry), - init(parent, name) { - return { - type: RefType, - parent, - name, - value: null, - }; - }, - diff: /** @type {DiffFunction} */ ( - function (queue, /** @type {MemberState} */ state, /** @type {MemberNode} */ node) { - if (state.value !== node.value) { - state.value = node.value; - if (state.parent.refHooks === null) { - state.parent.refHooks = []; - } - if (state.parent.refHooks.indexOf(state) === -1) { - state.parent.refHooks.push(state); - } - enqueuePost(queue, updateRef, state); - } - } - ), - teardown(queue, state) { - const index = state.parent.refHooks.indexOf(state); - state.parent.refHooks.splice(index, 1); - if (state.parent.refHooks.length === 0) { - state.parent.refHooks = null; - } - enqueuePost(queue, unsetRef, state); - }, -}; - -const typeMap = { - [ELEMENT_TYPE_NAME]: ElementType, - [FRAGMENT_TYPE_NAME]: FragmentType, - [COMPONENT_TYPE_NAME]: ComponentType, - [TEXT_TYPE_NAME]: TextType, - [ATTRIBUTE_TYPE_NAME]: AttributeType, - [PROPERTY_TYPE_NAME]: PropertyType, - [REF_TYPE_NAME]: RefType, -}; - -/** @type DiffEntryFunction */ -function diffChildEntry( - queue, - parent, - /** @type NodeStateType */ metaType, - /** @type NodeNode */ element -) { - if (!parent.children) { - parent.children = []; - } - const index = parent.rewriteChildIndex++; - let state = parent.children[index]; - if (!state || state.shape !== element.shape) { - if (state) { - state.type.teardown(queue, state); - } - state = /** @type {NodeState} */ ( - metaType.init(queue, parent, parent.children[index - 1] || null, element) - ); - parent.children[index] = state; - - const sibling = parent.children[index + 1]; - if (sibling) { - sibling.after = state; - } - } - state.type.diff(queue, state, element); -} - -/** @type {DiffFunction} */ -function diffFragment( - queue, - /** @type {ElementState | FragmentState} */ lastValue, - /** @type {ElementNode | FragmentNode} */ newValue -) { - lastValue.rewriteChildIndex = 0; - const { content } = newValue; - for (let i = 0; i < content.length; i++) { - const node = content[i]; - const metaType = typeMap[node.type]; - metaType.diffEntry(queue, lastValue, metaType, node); - } - const children = lastValue.children; - if (children !== null) { - const childIndex = lastValue.rewriteChildIndex; - for (let i = childIndex; i < children.length; i++) { - const node = children[i]; - node.type.teardown(queue, node); - } - if (childIndex === 0) { - lastValue.children = null; - } else { - children.length = childIndex; - } - } -} - -/** @type {DiffEntryFunction} */ -function diffMemberEntry( - queue, - /** @type {ElementState} */ element, - /** @type {MemberStateType} */ nodeType, - /** @type {MemberNode} */ node -) { - if (element.members === null) { - element.members = []; - } - const group = element.members; - - const writeIndex = element.rewriteMemberIndex++; - let index; - for (index = writeIndex; index < group.length; index++) { - const item = group[index]; - if (item.type.name === node.type && item.name === node.name) { - break; - } - } - - let old = group[index]; - if (index !== writeIndex) { - group[index] = group[writeIndex]; - } - if (!old) { - old = nodeType.init(element, node.name); - group[writeIndex] = old; - } - old.type.diff(queue, old, node); -} - -/** @type {SoftTeardownFunction} */ -function softTeardownElement(/** @type {ElementState} */ state) { - const { children } = state; - if (children !== null) { - for (let i = 0; i < children.length; i++) { - children[i].type.softTeardown(children[i]); - } - } -} - -/** - * @param {Data} node - * @returns {node is MemberNode} - */ -function isOptions(node) { - return node.type === META_TYPE_NAME && node.name === 'options'; -} - -/** - * @param {Queue} queue - */ -function runQueue(queue) { - const { prepare, changes: apply, post } = queue; - for (let i = 0; i < prepare.length; i++) { - changeViewRender(prepare[i], queue); - } - for (let i = 0; i < apply.length; i += 2) { - apply[i](apply[i + 1]); - } - for (let i = 0; i < post.length; i += 2) { - post[i](post[i + 1]); - } -} - -/** - * @param {ComponentState} componentState - * @param {Queue} queue - */ -function changeViewRender(componentState, queue) { - const newRender = componentState.shape(componentState.content) || null; - if (newRender) { - const metaType = typeMap[newRender.type]; - let lastRender = componentState.rendered; - if (!lastRender || lastRender.shape !== newRender.shape) { - if (lastRender) { - lastRender.type.teardown(queue, lastRender); - } - lastRender = metaType.init(queue, componentState, null, newRender); - componentState.rendered = lastRender; - } - lastRender.type.diff(queue, lastRender, newRender); - } else if (componentState.rendered) { - componentState.rendered.type.teardown(queue, componentState.rendered); - componentState.rendered = null; - } -} - -/** - * @param {MemberState} state - */ -function changeProperty(state) { - state.parent.ref[state.name] = state.value; -} - -/** - * @param {MemberState} state - */ -function removeAttribute(state) { - state.parent.ref.removeAttribute(state.name); -} - -/** - * @param {MemberState} state - */ -function changeAttribute(state) { - state.parent.ref.setAttribute(state.name, state.value); -} - -/** - * @param {TextState} state - */ -function removeText(state) { - state.ref.parentNode.removeChild(state.ref); -} - -/** - * @param {TextState} state - */ -function changeText(state) { - state.ref.textContent = state.content; -} - -/** - * @param {TextState} state - */ -function addText(state) { - state.ref = document.createTextNode(state.content); - state.ref.textContent = state.content; - insertAfterSibling(state); -} - -/** - * @param {NodeState} after - * @returns {NodeState} - */ -function deepestDescendant(after) { - if (after === null) { - return after; - } else if (after.type === FragmentType) { - const { children } = /** @type {FragmentState} */ (after); - if (children !== null) { - return deepestDescendant(children[children.length - 1]); - } - } else if (after.type === ComponentType) { - const { rendered } = /** @type {ComponentState} */ (after); - if (rendered !== null) { - return deepestDescendant(rendered); - } - } - return after; -} - -/** - * @param {ElementState | TextState} element - */ -function insertAfterSibling(element) { - /** @type {NodeState} */ - let state = element; - let after = deepestDescendant(state.after); - do { - if (after === null) { - if (state.parent.type === ElementType) { - break; - } - state = state.parent; - after = state; - } - if (after.type !== ElementType && after.type !== TextType) { - after = deepestDescendant(after.after); - } - } while (after === null || (after.type !== ElementType && after.type !== TextType)); - - if (after !== null) { - const { ref } = /** @type {ElementState | TextState} */ (after); - ref.parentNode.insertBefore(element.ref, ref.nextSibling); - } else { - const { ref } = /** @type {ElementState} */ (state.parent); - if (ref.childNodes.length) { - ref.insertBefore(element.ref, ref.childNodes[0]); - } else { - ref.appendChild(element.ref); - } - } -} - -/** - * @param {ElementState} state - */ -function removeElement(state) { - state.ref.parentNode.removeChild(state.ref); - state.ref = null; -} - -/** - * @param {ElementState} state - */ -function addElement(state) { - state.ref = document.createElement(state.shape); - insertAfterSibling(state); -} - -/** - * @param {MemberState} state - */ -function updateRef(state) { - state.value(state.parent.ref); -} - -/** - * @param {MemberState} state - */ -function unsetRef(state) { - state.value(null); -} - -function shallowEquals(a, b) { - if (a === null || b === null) { - return a === b; - } - for (const key of Object.keys(a)) { - if (key in b) { - if (a[key] !== b[key]) { - return false; - } - } else { - return false; - } - } - for (const key of Object.keys(b)) { - if (!(key in a)) { - return false; - } - } - return true; -} - -/** - * @callback StateAction - * @param {S} state - * @template {State} S - */ - -/** - * @returns {Queue} - */ -function newQueue() { - return { prepare: [], changes: [], post: [] }; -} - -/** - * @param {Queue} queue - * @param {StateAction} fn - * @param {S} state - * @template {State} S - */ -function enqueueChange(queue, fn, state) { - queue.changes.push(fn, state); -} - -/** - * @param {Queue} queue - * @param {StateAction} fn - * @param {S} state - * @template {State} S - */ -function enqueuePost(queue, fn, state) { - queue.post.push(fn, state); -} - -/** - * @typedef Queue - * @property {Array} prepare - * @property {Array} changes - * @property {Array} post - */ - -/** - * @callback DiffEntryFunction - * @param {Queue} queue - * @param {ElementState | FragmentState} parent - * @param {StateType} nodeType - * @param {Data} node - * @returns {void} - */ - -/** - * @callback InitNodeFunction - * @param {Queue} queue - * @param {NodeState} parent - * @param {NodeState | null} after - * @param {NodeNode} node - * @returns {NodeState} - */ - -/** - * @callback DiffFunction - * @param {Queue} queue - * @param {State} state - * @param {Data} node - * @returns {void} - */ - -/** - * @callback TeardownFunction - * @param {Queue} queue - * @param {NodeState} state - * @returns {void} - */ - -/** - * @callback SoftTeardownFunction - * @param {NodeState} state - * @returns {void} - */ - -/** - * @typedef NodeStateTypeGeneric - * @property {Name} name - * @property {DiffEntryFunction} diffEntry - * @property {DiffFunction} diff - * @property {InitNodeFunction} init - * @property {TeardownFunction} teardown - * @property {SoftTeardownFunction} softTeardown - * @template {string | symbol} Name - */ - -/** - * @typedef NodeStateTypeCreate - * @property {Name} name - * @property {(queue: Queue, parent: ElementState | FragmentState, meta: StateType, node: Node) => void} diffEntry - * @property {(queue: Queue, state: S, node: Node) => void} diff - * @property {(queue: Queue, parent: ElementState | FragmentState, after: NodeState, node: Node) => void} init - * @property {(queue: Queue, state: S) => void} teardown - * @property {(state: S) => void} softTeardown - * @template {string | symbol} Name - * @template {State} S - * @template {Data} Node - */ - -/** @typedef {NodeStateTypeGeneric} ElementStateType */ -/** @typedef {NodeStateTypeGeneric} FragmentStateType */ -/** @typedef {NodeStateTypeGeneric} ComponentStateType */ -/** @typedef {NodeStateTypeGeneric} TextStateType */ - -/** @typedef {ElementStateType | FragmentStateType| ComponentStateType | TextStateType} NodeStateType */ - -/** - * @callback InitMemberFunction - * @param {ElementState} parent - * @param {string} name - * @returns {MemberState} - */ - -/** - * @callback TeardownMemberFunction - * @param {Queue} queue - * @param {MemberState} member - * @returns {void} - */ - -/** - * @typedef MemberStateType - * @property {string | symbol} name - * @property {DiffEntryFunction} diffEntry - * @property {DiffFunction} diff - * @property {InitMemberFunction} init - * @property {TeardownMemberFunction} teardown - */ - -/** - * @typedef {NodeStateType | MemberStateType} StateType - */ - -/** - * @typedef ElementState - * @property {ElementStateType} type - * @property {NodeState} parent - * @property {NodeState | null} after - * @property {string} shape - * @property {null} content - * @property {HTMLElement | null} ref - * @property {MemberState[] | null} refHooks - * @property {number} rewriteChildIndex - * @property {NodeState[] | null} children - * @property {number} rewriteMemberIndex - * @property {MemberState[] | null} members - */ - -/** - * @typedef FragmentState - * @property {FragmentStateType} type - * @property {NodeState} parent - * @property {NodeState | null} after - * @property {typeof FRAGMENT_TYPE_NAME} shape - * @property {null} content - * @property {number} rewriteChildIndex - * @property {NodeState[] | null} children - */ - -/** @typedef {ElementState | FragmentState} ParentState */ - -/** - * @typedef ComponentState - * @property {ComponentStateType} type - * @property {NodeState} parent - * @property {NodeState | null} after - * @property {function} shape - * @property {object} content - * @property {NodeState | null} rendered - */ - -/** - * @typedef TextState - * @property {TextStateType} type - * @property {NodeState} parent - * @property {NodeState | null} after - * @property {typeof TEXT_TYPE_NAME} shape - * @property {string} content - * @property {Text | null} ref - */ - -/** - * @typedef {ElementState | FragmentState | ComponentState | TextState} NodeState - */ - -/** - * @typedef MemberState - * @property {MemberStateType} type - * @property {ElementState} parent - * @property {string} name - * @property {*} value - */ - -/** @typedef {NodeState | MemberState} State */ - -/** @typedef {typeof ELEMENT_TYPE_NAME | typeof FRAGMENT_TYPE_NAME | typeof COMPONENT_TYPE_NAME | typeof TEXT_TYPE_NAME} NodeNodeType */ - -/** @typedef {typeof ATTRIBUTE_TYPE_NAME | typeof PROPERTY_TYPE_NAME | typeof REF_TYPE_NAME | typeof META_TYPE_NAME} MemberNodeType */ - -/** - * @typedef ElementNode - * @property {typeof ELEMENT_TYPE_NAME} type - * @property {string} shape - * @property {Data[]} content - */ - -/** - * @typedef FragmentNode - * @property {typeof FRAGMENT_TYPE_NAME} type - * @property {typeof FRAGMENT_TYPE_NAME} shape - * @property {Data[]} content - */ - -/** - * @typedef TextNode - * @property {typeof TEXT_TYPE_NAME} type - * @property {typeof TEXT_TYPE_NAME} shape - * @property {string} content - */ - -/** - * @typedef ComponentNode - * @property {typeof COMPONENT_TYPE_NAME} type - * @property {function} shape - * @property {Data[]} content - */ - -/** - * @typedef {ElementNode | FragmentNode | TextNode | ComponentNode} NodeNode - */ - -/** - * @typedef MemberNode - * @property {MemberNodeType} type - * @property {string} name - * @property {*} value - */ - -/** @typedef {NodeNode | MemberNode} Data */ diff --git a/docs/local-development.md b/docs/local-development.md index 33fb84e45..5b494a9d5 100644 --- a/docs/local-development.md +++ b/docs/local-development.md @@ -17,6 +17,8 @@ yarn install ``` 2. Set up local database using the instructions provided in [database.md](database.md). + - Note: You must run `yarn db-import-tests:dev` after setting up your database to import the latest test harness into + your project. 3. Run the server ``` yarn dev diff --git a/server/migrations/20211116172219-commandSequences.js b/server/migrations/20211116172219-commandSequences.js index 85d6243ac..610924c71 100644 --- a/server/migrations/20211116172219-commandSequences.js +++ b/server/migrations/20211116172219-commandSequences.js @@ -1,6 +1,6 @@ const { omit } = require('lodash'); const { TestPlanVersion } = require('../models'); -const commandList = require('../resources/commands.json'); +const commandList = require('../resources/commandsV1.json'); module.exports = { up: async queryInterface => { diff --git a/server/resolvers/helpers/retrieveCommands.js b/server/resolvers/helpers/retrieveCommands.js index a7c42515b..846a404de 100644 --- a/server/resolvers/helpers/retrieveCommands.js +++ b/server/resolvers/helpers/retrieveCommands.js @@ -1,4 +1,4 @@ -const commands = require('../../resources/commands.json'); +const commandsV1 = require('../../resources/commandsV1.json'); const commandsV2 = require('../../resources/commandsV2.json'); function findValueByKey(keyMappings, keyToFindText) { @@ -110,7 +110,7 @@ function findValuesByKeys(commandsMapping, keysToFind = []) { } const getCommandV1 = commandId => { - return commands.find(command => command.id === commandId); + return commandsV1.find(command => command.id === commandId); }; const getCommandV2 = commandId => { diff --git a/server/resources/commands.json b/server/resources/commandsV1.json similarity index 100% rename from server/resources/commands.json rename to server/resources/commandsV1.json diff --git a/server/scripts/import-tests/index.js b/server/scripts/import-tests/index.js index 17860e9c3..b432916ba 100644 --- a/server/scripts/import-tests/index.js +++ b/server/scripts/import-tests/index.js @@ -74,6 +74,8 @@ const importTestPlanVersions = async transaction => { }); console.log('`npm run build` output', buildOutput.stdout.toString()); + importHarness(); + const { support } = await updateJsons(); const ats = await At.findAll(); @@ -251,6 +253,41 @@ const readDirectoryGitInfo = directoryPath => { return { gitSha, gitMessage, gitCommitDate }; }; +const importHarness = () => { + const sourceFolder = path.resolve(`${testsDirectory}/resources`); + const targetFolder = path.resolve('../', 'client/resources'); + console.info(`Updating harness directory, ${targetFolder} ...`); + fse.rmSync(targetFolder, { recursive: true, force: true }); + + // Copy source folder + console.info('Importing latest harness files ...'); + fse.copySync(sourceFolder, targetFolder, { + filter: src => { + if (fse.lstatSync(src).isDirectory()) { + return true; + } + if (!src.includes('.html')) { + return true; + } + } + }); + + // Copy files + const commandsJson = 'commands.json'; + const supportJson = 'support.json'; + if (fse.existsSync(`${testsDirectory}/${commandsJson}`)) { + fse.copyFileSync( + `${testsDirectory}/${commandsJson}`, + `${targetFolder}/${commandsJson}` + ); + } + fse.copyFileSync( + `${testsDirectory}/${supportJson}`, + `${targetFolder}/${supportJson}` + ); + console.info('Harness files update complete.'); +}; + const getAppUrl = (directoryRelativePath, { gitSha, directoryPath }) => { return path.join( '/', @@ -318,7 +355,7 @@ const updateJsons = async () => { // Write commands for v1 format await fse.writeFile( - path.resolve(__dirname, '../../resources/commands.json'), + path.resolve(__dirname, '../../resources/commandsV1.json'), JSON.stringify(commands, null, 4) ); From a062f631cf8e117284ba4963ea05e8a67483d2fe Mon Sep 17 00:00:00 2001 From: Mx Corey Frang Date: Thu, 16 May 2024 12:40:40 -0400 Subject: [PATCH 12/19] Fix: status for RUNNING tests updates when job status becomes finalized (#1094) * Fix: status for RUNNING tests updates when job status becomes finalized * Update server/tests/integration/automation-scheduler.test.js Co-authored-by: Howard Edwards --------- Co-authored-by: Howard Edwards --- server/controllers/AutomationController.js | 15 +- .../cancelCollectionJobResolver.js | 5 + .../integration/automation-scheduler.test.js | 268 ++++++++++++++++++ 3 files changed, 287 insertions(+), 1 deletion(-) diff --git a/server/controllers/AutomationController.js b/server/controllers/AutomationController.js index 39f2ba930..dfae76172 100644 --- a/server/controllers/AutomationController.js +++ b/server/controllers/AutomationController.js @@ -101,8 +101,8 @@ const updateJobStatus = async (req, res) => { }; // When new status is considered "final" ('COMPLETED' or 'ERROR' or 'CANCELLED') - // update any CollectionJobTestStatus children still 'QUEUED' to be 'CANCELLED' if (isJobStatusFinal(status)) { + // update any CollectionJobTestStatus children still 'QUEUED' to be 'CANCELLED' await updateCollectionJobTestStatusByQuery({ where: { collectionJobId: req.params.jobID, @@ -111,6 +111,19 @@ const updateJobStatus = async (req, res) => { values: { status: COLLECTION_JOB_STATUS.CANCELLED }, transaction: req.transaction }); + // update any CollectionJobTestStatus children still 'RUNNING' to be 'ERROR' or 'CANCELLED' + let runningTestNewStatus = + status === COLLECTION_JOB_STATUS.ERROR + ? COLLECTION_JOB_STATUS.ERROR + : COLLECTION_JOB_STATUS.CANCELLED; + await updateCollectionJobTestStatusByQuery({ + where: { + collectionJobId: req.params.jobID, + status: COLLECTION_JOB_STATUS.RUNNING + }, + values: { status: runningTestNewStatus }, + transaction: req.transaction + }); } const graphqlResponse = await updateCollectionJobById({ diff --git a/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js b/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js index c7240ce50..3f6d61f99 100644 --- a/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js +++ b/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js @@ -42,6 +42,11 @@ const cancelCollectionJobResolver = async ( values: { status: COLLECTION_JOB_STATUS.CANCELLED }, transaction }); + await updateCollectionJobTestStatusByQuery({ + where: { collectionJobId, status: COLLECTION_JOB_STATUS.RUNNING }, + values: { status: COLLECTION_JOB_STATUS.CANCELLED }, + transaction + }); return updateCollectionJobById({ id: collectionJobId, values: { status: COLLECTION_JOB_STATUS.CANCELLED }, diff --git a/server/tests/integration/automation-scheduler.test.js b/server/tests/integration/automation-scheduler.test.js index 0dcd800da..22b070604 100644 --- a/server/tests/integration/automation-scheduler.test.js +++ b/server/tests/integration/automation-scheduler.test.js @@ -667,6 +667,274 @@ describe('Automation controller', () => { }); }); + it('should convert RUNNING test status to ERROR when job state becomes ERROR', async () => { + await apiServer.sessionAgentDbCleaner(async transaction => { + const { scheduleCollectionJob: job } = + await scheduleCollectionJobByMutation({ transaction }); + const collectionJob = await getCollectionJobById({ + id: job.id, + transaction + }); + // flag overall job as RUNNING + const externalLogsUrl = 'https://example.com/test/log/url'; + await sessionAgent + .post(`/api/jobs/${job.id}`) + .send({ status: 'RUNNING', externalLogsUrl }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + + const { tests } = + collectionJob.testPlanRun.testPlanReport.testPlanVersion; + const selectedTestIndex = 0; + const selectedTestRowNumber = 1; + const selectedTest = tests[selectedTestIndex]; + let response = await sessionAgent + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) + .send({ + status: COLLECTION_JOB_STATUS.RUNNING + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + let { collectionJob: storedCollectionJob } = + await getTestCollectionJob(job.id, { transaction }); + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('RUNNING'); + expect(storedCollectionJob.externalLogsUrl).toEqual( + externalLogsUrl + ); + let foundStatus = false; + for (const testStatus of storedCollectionJob.testStatus) { + let expectedStatus = COLLECTION_JOB_STATUS.QUEUED; + if (testStatus.test.id === selectedTest.id) { + foundStatus = true; + expectedStatus = COLLECTION_JOB_STATUS.RUNNING; + } + expect(testStatus.status).toEqual(expectedStatus); + } + expect(foundStatus).toEqual(true); + + // Leave our test RUNNING but ERROR the job + + response = await sessionAgent + .post(`/api/jobs/${job.id}`) + .send({ + // avoiding sending externalLogsUrl here to test that when + // missing it is not overwritten/emptied. + status: COLLECTION_JOB_STATUS.ERROR + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + storedCollectionJob = ( + await getTestCollectionJob(job.id, { transaction }) + ).collectionJob; + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('ERROR'); + expect(storedCollectionJob.externalLogsUrl).toEqual( + externalLogsUrl + ); + foundStatus = false; + for (const testStatus of storedCollectionJob.testStatus) { + let expectedStatus = COLLECTION_JOB_STATUS.CANCELLED; + if (testStatus.test.id === selectedTest.id) { + foundStatus = true; + expectedStatus = COLLECTION_JOB_STATUS.ERROR; + } + expect(testStatus.status).toEqual(expectedStatus); + } + expect(foundStatus).toEqual(true); + }); + }); + + it('should convert RUNNING test status to CANCELLED when job state becomes CANCELLED', async () => { + await apiServer.sessionAgentDbCleaner(async transaction => { + const { scheduleCollectionJob: job } = + await scheduleCollectionJobByMutation({ transaction }); + const collectionJob = await getCollectionJobById({ + id: job.id, + transaction + }); + // flag overall job as RUNNING + const externalLogsUrl = 'https://example.com/test/log/url'; + await sessionAgent + .post(`/api/jobs/${job.id}`) + .send({ status: 'RUNNING', externalLogsUrl }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + + const { tests } = + collectionJob.testPlanRun.testPlanReport.testPlanVersion; + const selectedTestIndex = 0; + const selectedTestRowNumber = 1; + const selectedTest = tests[selectedTestIndex]; + let response = await sessionAgent + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) + .send({ + status: COLLECTION_JOB_STATUS.RUNNING + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + let { collectionJob: storedCollectionJob } = + await getTestCollectionJob(job.id, { transaction }); + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('RUNNING'); + expect(storedCollectionJob.externalLogsUrl).toEqual( + externalLogsUrl + ); + let foundStatus = false; + for (const testStatus of storedCollectionJob.testStatus) { + let expectedStatus = COLLECTION_JOB_STATUS.QUEUED; + if (testStatus.test.id === selectedTest.id) { + foundStatus = true; + expectedStatus = COLLECTION_JOB_STATUS.RUNNING; + } + expect(testStatus.status).toEqual(expectedStatus); + } + expect(foundStatus).toEqual(true); + + // Leave our test RUNNING but CANCELLED the job + + response = await sessionAgent + .post(`/api/jobs/${job.id}`) + .send({ + // avoiding sending externalLogsUrl here to test that when + // missing it is not overwritten/emptied. + status: COLLECTION_JOB_STATUS.CANCELLED + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + storedCollectionJob = ( + await getTestCollectionJob(job.id, { transaction }) + ).collectionJob; + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('CANCELLED'); + expect(storedCollectionJob.externalLogsUrl).toEqual( + externalLogsUrl + ); + for (const testStatus of storedCollectionJob.testStatus) { + expect(testStatus.status).toEqual( + COLLECTION_JOB_STATUS.CANCELLED + ); + } + }); + }); + + it('should convert RUNNING test status to CANCELLED when job state becomes COMPLETED', async () => { + await apiServer.sessionAgentDbCleaner(async transaction => { + const { scheduleCollectionJob: job } = + await scheduleCollectionJobByMutation({ transaction }); + const collectionJob = await getCollectionJobById({ + id: job.id, + transaction + }); + // flag overall job as RUNNING + const externalLogsUrl = 'https://example.com/test/log/url'; + await sessionAgent + .post(`/api/jobs/${job.id}`) + .send({ status: 'RUNNING', externalLogsUrl }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + + const { tests } = + collectionJob.testPlanRun.testPlanReport.testPlanVersion; + const selectedTestIndex = 0; + const selectedTestRowNumber = 1; + const selectedTest = tests[selectedTestIndex]; + let response = await sessionAgent + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) + .send({ + status: COLLECTION_JOB_STATUS.RUNNING + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + let { collectionJob: storedCollectionJob } = + await getTestCollectionJob(job.id, { transaction }); + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('RUNNING'); + expect(storedCollectionJob.externalLogsUrl).toEqual( + externalLogsUrl + ); + let foundStatus = false; + for (const testStatus of storedCollectionJob.testStatus) { + let expectedStatus = COLLECTION_JOB_STATUS.QUEUED; + if (testStatus.test.id === selectedTest.id) { + foundStatus = true; + expectedStatus = COLLECTION_JOB_STATUS.RUNNING; + } + expect(testStatus.status).toEqual(expectedStatus); + } + expect(foundStatus).toEqual(true); + + // Leave our test RUNNING but COMPLETED the job + + response = await sessionAgent + .post(`/api/jobs/${job.id}`) + .send({ + // avoiding sending externalLogsUrl here to test that when + // missing it is not overwritten/emptied. + status: COLLECTION_JOB_STATUS.COMPLETED + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + storedCollectionJob = ( + await getTestCollectionJob(job.id, { transaction }) + ).collectionJob; + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('COMPLETED'); + expect(storedCollectionJob.externalLogsUrl).toEqual( + externalLogsUrl + ); + foundStatus = false; + for (const testStatus of storedCollectionJob.testStatus) { + let expectedStatus = COLLECTION_JOB_STATUS.CANCELLED; + if (testStatus.test.id === selectedTest.id) { + foundStatus = true; + expectedStatus = COLLECTION_JOB_STATUS.CANCELLED; + } + expect(testStatus.status).toEqual(expectedStatus); + } + expect(foundStatus).toEqual(true); + }); + }); + it('should copy assertion results when updating with results that match historical results', async () => { await apiServer.sessionAgentDbCleaner(async transaction => { const context = getGraphQLContext({ From b6a5f543a98d1846896cff0945c558fd782cda31 Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Mon, 20 May 2024 07:20:02 -0700 Subject: [PATCH 13/19] fix: Add type-ahead to assign tester dropdown (#1095) Address #991 --- .../TestQueue/AssignTesterDropdown/index.jsx | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/client/components/TestQueue/AssignTesterDropdown/index.jsx b/client/components/TestQueue/AssignTesterDropdown/index.jsx index 98463d388..7625a3988 100644 --- a/client/components/TestQueue/AssignTesterDropdown/index.jsx +++ b/client/components/TestQueue/AssignTesterDropdown/index.jsx @@ -110,9 +110,33 @@ const AssignTesterDropdown = ({ const clearAriaLiveRegion = () => { setAlertMessage(''); }; + + const handleKeyDown = event => { + const { key } = event; + if (key.match(/[0-9a-zA-Z]/)) { + const container = event.target.closest('[role=menu]'); + const matchingMenuItem = Array.from(container.children).find( + menuItem => { + return menuItem.innerText + .trim() + .toLowerCase() + .startsWith(key.toLowerCase()); + } + ); + + if (matchingMenuItem) { + matchingMenuItem.focus(); + } + } + }; + return ( - + Date: Mon, 20 May 2024 11:51:37 -0400 Subject: [PATCH 14/19] Update /deploy/../main.yml to account for client/resources being untracked (#1099) --- deploy/roles/application/tasks/main.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/deploy/roles/application/tasks/main.yml b/deploy/roles/application/tasks/main.yml index 796ffc519..2e791d276 100644 --- a/deploy/roles/application/tasks/main.yml +++ b/deploy/roles/application/tasks/main.yml @@ -28,6 +28,14 @@ when: deployment_mode != 'development' notify: "restart server" +- name: Make client resources folder writable for import harness + file: + path: '{{source_dir}}/client/resources' + mode: '0777' + recurse: yes + when: deployment_mode != 'development' + notify: "restart server" + - name: Link application code file: dest: '{{source_dir}}' @@ -76,6 +84,11 @@ args: chdir: '{{source_dir}}' +- name: Import latest tests and harness from w3c/aria-at + shell: ./deploy/scripts/export-and-exec.sh {{environment_config.dest}} node ./server/scripts/import-tests/index.js + args: + chdir: '{{source_dir}}' + - name: Build front end package command: ./deploy/scripts/export-and-exec.sh {{environment_config.dest}} yarn workspace client build become: yes From 9414180047e8e668151c22af147fb74a14fb0df6 Mon Sep 17 00:00:00 2001 From: Erika Miguel Date: Mon, 20 May 2024 12:53:51 -0400 Subject: [PATCH 15/19] fix: Update paths for import tests in main.yml deploy (#1100) --- deploy/roles/application/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/roles/application/tasks/main.yml b/deploy/roles/application/tasks/main.yml index 2e791d276..b04d8b51c 100644 --- a/deploy/roles/application/tasks/main.yml +++ b/deploy/roles/application/tasks/main.yml @@ -85,9 +85,9 @@ chdir: '{{source_dir}}' - name: Import latest tests and harness from w3c/aria-at - shell: ./deploy/scripts/export-and-exec.sh {{environment_config.dest}} node ./server/scripts/import-tests/index.js + shell: ../deploy/scripts/export-and-exec.sh {{environment_config.dest}} node ./scripts/import-tests/index.js args: - chdir: '{{source_dir}}' + chdir: '{{source_dir}}/server' - name: Build front end package command: ./deploy/scripts/export-and-exec.sh {{environment_config.dest}} yarn workspace client build From ae97530e2545c3812f1e1dc2b8a8556f30de6670 Mon Sep 17 00:00:00 2001 From: Erika Miguel Date: Mon, 20 May 2024 17:05:08 -0400 Subject: [PATCH 16/19] fix: Use aria-checked for AssignTesterDropdown (#1097) Address #977 * Use aria-checked for `AssignTesterDropdown` * Update dropdown item role to `menuitemcheckbox` and remove `aria-hidden` from username --- .../TestQueue/AssignTesterDropdown/index.jsx | 15 +++++---------- client/tests/AssignTesterDropdown.test.jsx | 4 ++-- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/client/components/TestQueue/AssignTesterDropdown/index.jsx b/client/components/TestQueue/AssignTesterDropdown/index.jsx index 7625a3988..a36a82b5d 100644 --- a/client/components/TestQueue/AssignTesterDropdown/index.jsx +++ b/client/components/TestQueue/AssignTesterDropdown/index.jsx @@ -169,10 +169,13 @@ const AssignTesterDropdown = ({ } return ( { const updatedIsAssigned = !testerIsAssigned; @@ -189,15 +192,7 @@ const AssignTesterDropdown = ({ }} > {icon && } - {`${username} ${ - testerIsAssigned - ? 'checked' - : 'unchecked' - }`} - diff --git a/client/tests/AssignTesterDropdown.test.jsx b/client/tests/AssignTesterDropdown.test.jsx index bb8dce235..032b98235 100644 --- a/client/tests/AssignTesterDropdown.test.jsx +++ b/client/tests/AssignTesterDropdown.test.jsx @@ -179,7 +179,7 @@ describe('AssignTesterDropdown', () => { fireEvent.click(button); const items = await screen.findAllByText(/bee/); - expect(items.length).toBe(2); // One for display, one for sr-only + expect(items.length).toBe(1); fireEvent.click(items[0]); await waitFor(async () => { @@ -205,7 +205,7 @@ describe('AssignTesterDropdown', () => { fireEvent.click(button); const items = await screen.findAllByText(/NVDA Bot/); - expect(items.length).toBe(2); // One for display, one for sr-only + expect(items.length).toBe(1); fireEvent.click(items[0]); await waitFor(() => { From 7b75cbe4f614fd8a47003cad194c13203abaf404 Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Tue, 21 May 2024 13:27:35 -0700 Subject: [PATCH 17/19] feat: Restructure APG Support Tables (#1053) Addresses https://github.com/w3c/aria-practices/issues/2971 with the following: * Renames the columns to "Must-Have Behaviors" and "Should-Have Behaviors" * Sorts the rows alphabetically by AT first, then Browser -- so JAWS + Chrome, JAWS + Firefox, ... NVDA + Chrome, ..., etc * Adds a column title for the first column of "Assistive Technology - Browser" * Removes the word "Supported" from each cell and places the percentage number to the right of the meter graphic so they are on the same line * Each AT-Browser row title is now a link that takes the user to the detailed report page for that combination --- server/apps/embed.js | 243 ++++------------- server/handlebars/embed/helpers/index.js | 69 ++--- server/handlebars/embed/public/script.js | 9 +- server/handlebars/embed/public/style.css | 320 ++++++++++++----------- server/handlebars/embed/views/main.hbs | 210 ++++++++------- server/resolvers/testPlanResolver.js | 3 +- server/tests/integration/embed.test.js | 9 +- 7 files changed, 359 insertions(+), 504 deletions(-) diff --git a/server/apps/embed.js b/server/apps/embed.js index dda139e0d..bb1f34c11 100644 --- a/server/apps/embed.js +++ b/server/apps/embed.js @@ -4,13 +4,11 @@ const { create } = require('express-handlebars'); const { gql } = require('apollo-server-core'); const apolloServer = require('../graphql-server'); const staleWhileRevalidate = require('../util/staleWhileRevalidate'); -const hash = require('object-hash'); const app = express(); const handlebarsPath = path.resolve(__dirname, '../handlebars/embed'); -// handlebars const hbs = create({ layoutsDir: path.resolve(handlebarsPath, 'views/layouts'), extname: 'hbs', @@ -30,10 +28,15 @@ app.set('views', path.resolve(handlebarsPath, 'views')); // stale data for however long it takes for the query to complete. const millisecondsUntilStale = 5000; -const queryReports = async () => { +const renderEmbed = async ({ + queryTitle, + testPlanDirectory, + protocol, + host +}) => { const { data, errors } = await apolloServer.executeOperation({ query: gql` - query { + query TestPlanQuery($testPlanDirectory: ID!) { ats { id name @@ -42,234 +45,98 @@ const queryReports = async () => { name } } - testPlanReports( - testPlanVersionPhases: [CANDIDATE, RECOMMENDED] - isFinal: true - ) { - id - metrics - at { - id - name - } - browser { - id - name - } - latestAtVersionReleasedAt { - id - name - releasedAt - } - testPlanVersion { + testPlan(id: $testPlanDirectory) { + testPlanVersions { id title phase - updatedAt - testPlan { + testPlanReports(isFinal: true) { id - } - tests { - ats { + metrics + at { + id + name + } + browser { + id + name + } + latestAtVersionReleasedAt { id name + releasedAt } } } } } - ` + `, + variables: { testPlanDirectory } }); if (errors) { throw new Error(errors); } - const reportsHashed = hash(data.testPlanReports); - - return { - allTestPlanReports: data.testPlanReports, - reportsHashed, - ats: data.ats - }; -}; - -// As of now, a full query for the complete list of reports is needed to build -// the embed for a single pattern. This caching allows that query to be reused -// between pattern embeds. -const queryReportsCached = staleWhileRevalidate(queryReports, { - millisecondsUntilStale -}); - -const getLatestReportsForPattern = ({ allTestPlanReports, pattern }) => { - let title; - - const testPlanReports = allTestPlanReports.filter(report => { - if (report.testPlanVersion.testPlan.id === pattern) { - title = report.testPlanVersion.title; - return true; - } - }); - - let allAts = new Set(); - let allBrowsers = new Set(); - let allAtVersionsByAt = {}; - let reportsByAt = {}; - let testPlanVersionIds = new Set(); - const uniqueReports = []; - let latestReports = []; - - testPlanReports.forEach(report => { - allAts.add(report.at.name); - allBrowsers.add(report.browser.name); + let testPlanVersion; - if (!allAtVersionsByAt[report.at.name]) - allAtVersionsByAt[report.at.name] = - report.latestAtVersionReleasedAt; - else if ( - new Date(report.latestAtVersionReleasedAt.releasedAt) > - new Date(allAtVersionsByAt[report.at.name].releasedAt) - ) { - allAtVersionsByAt[report.at.name] = - report.latestAtVersionReleasedAt; - } + const recommendedTestPlanVersion = data.testPlan?.testPlanVersions.find( + testPlanVersion => testPlanVersion.phase === 'RECOMMENDED' + ); - const sameAtAndBrowserReports = testPlanReports.filter( - r => - r.at.name === report.at.name && - r.browser.name === report.browser.name + if (data.testPlan && recommendedTestPlanVersion) { + testPlanVersion = recommendedTestPlanVersion; + } else if (data.testPlan) { + testPlanVersion = data.testPlan.testPlanVersions.find( + testPlanVersion => testPlanVersion.phase === 'CANDIDATE' ); + } - // Only add a group of reports with same - // AT and browser once - if ( - !uniqueReports.find(group => - group.some( - g => - g.at.name === report.at.name && - g.browser.name === report.browser.name - ) - ) - ) { - uniqueReports.push(sameAtAndBrowserReports); - } - - testPlanVersionIds.add(report.testPlanVersion.id); - }); - - uniqueReports.forEach(group => { - if (group.length <= 1) { - latestReports.push(group.pop()); - } else { - const latestReport = group - .sort( - (a, b) => - new Date(a.latestAtVersionReleasedAt.releasedAt) - - new Date(b.latestAtVersionReleasedAt.releasedAt) - ) - .pop(); - - latestReports.push(latestReport); + const testPlanReports = (testPlanVersion?.testPlanReports ?? []).sort( + (a, b) => { + if (a.at.name !== b.at.name) { + return a.at.name.localeCompare(b.at.name); + } + return a.browser.name.localeCompare(b.browser.name); } - }); - - allBrowsers = Array.from(allBrowsers).sort(); - testPlanVersionIds = Array.from(testPlanVersionIds); - - const allAtsAlphabetical = Array.from(allAts).sort((a, b) => - a.localeCompare(b) - ); - allAtsAlphabetical.forEach(at => { - reportsByAt[at] = latestReports - .filter(report => report.at.name === at) - .sort((a, b) => a.browser.name.localeCompare(b.browser.name)); - }); - - const hasAnyCandidateReports = Object.values(reportsByAt).find(atReports => - atReports.some(report => report.testPlanVersion.phase === 'CANDIDATE') - ); - let phase = hasAnyCandidateReports ? 'CANDIDATE' : 'RECOMMENDED'; - - return { - title, - allBrowsers, - allAtVersionsByAt, - testPlanVersionIds, - phase, - reportsByAt - }; -}; - -const renderEmbed = ({ - ats, - allTestPlanReports, - queryTitle, - pattern, - protocol, - host -}) => { - const { - title, - allBrowsers, - allAtVersionsByAt, - testPlanVersionIds, - phase, - reportsByAt - } = getLatestReportsForPattern({ pattern, allTestPlanReports }); - const allAtBrowserCombinations = Object.fromEntries( - ats.map(at => { - return [ - at.name, - at.browsers.map(browser => { - return browser.name; - }) - ]; - }) ); return hbs.renderView(path.resolve(handlebarsPath, 'views/main.hbs'), { layout: 'index', - dataEmpty: Object.keys(reportsByAt).length === 0, - allAtBrowserCombinations, - title: queryTitle || title || 'Pattern Not Found', - pattern, - phase, - allBrowsers, - allAtVersionsByAt, - reportsByAt, - completeReportLink: `${protocol}${host}/report/${testPlanVersionIds.join( - ',' - )}`, - embedLink: `${protocol}${host}/embed/reports/${pattern}` + dataEmpty: !testPlanVersion?.testPlanReports.length, + title: queryTitle || testPlanVersion?.title || 'Pattern Not Found', + phase: testPlanVersion?.phase, + testPlanVersionId: testPlanVersion?.id, + testPlanReports, + protocol, + host, + completeReportLink: `${protocol}${host}/report/${testPlanVersion?.id}`, + embedLink: `${protocol}${host}/embed/reports/${testPlanDirectory}` }); }; -// Limit the number of times the template is rendered +// staleWhileRevalidate() caching allows this page to handle very high traffic like +// it will see on the APG website. It works by immediately serving a recent +// version of the page and checks for updates in the background. const renderEmbedCached = staleWhileRevalidate(renderEmbed, { - getCacheKeyFromArguments: ({ reportsHashed, pattern }) => - reportsHashed + pattern, + getCacheKeyFromArguments: ({ testPlanDirectory }) => testPlanDirectory, millisecondsUntilStale }); -app.get('/reports/:pattern', async (req, res) => { +app.get('/reports/:testPlanDirectory', async (req, res) => { // In the instance where an editor doesn't want to display a certain title // as it has defined when importing into the ARIA-AT database for being too // verbose, etc. eg. `Link Example 1 (span element with text content)` // Usage: https://aria-at.w3.org/embed/reports/command-button?title=Link+Example+(span+element+with+text+content) const queryTitle = req.query.title; - const pattern = req.params.pattern; + const testPlanDirectory = req.params.testPlanDirectory; const host = req.headers.host; const protocol = /dev|vagrant/.test(process.env.ENVIRONMENT) ? 'http://' : 'https://'; - const { allTestPlanReports, reportsHashed, ats } = - await queryReportsCached(); const embedRendered = await renderEmbedCached({ - ats, - allTestPlanReports, - reportsHashed, queryTitle, - pattern, + testPlanDirectory, protocol, host }); diff --git a/server/handlebars/embed/helpers/index.js b/server/handlebars/embed/helpers/index.js index 12347364d..5886d2528 100644 --- a/server/handlebars/embed/helpers/index.js +++ b/server/handlebars/embed/helpers/index.js @@ -1,60 +1,29 @@ -let map = {}; - module.exports = { - isBrowser: function (a, b) { - return a === b; - }, - isInAllBrowsers: function (value, object) { - return object.allBrowsers.includes(value); + dataEmpty: function (object) { + return object.length === 0; }, + isCandidate: function (value) { return value === 'CANDIDATE'; }, - getAtVersion: function (object, key) { - return object.allAtVersionsByAt[key].name; + getMustSupportData: function (object) { + return Math.trunc( + (object.metrics.mustAssertionsPassedCount / + object.metrics.mustAssertionsCount) * + 100 + ); }, - combinationExists: function (object, atName, browserName) { - if (object.allAtBrowserCombinations[atName].includes(browserName)) { - return true; - } - return false; + isMustAssertionPriority: function (object) { + return object.metrics.mustAssertionsCount > 0; }, - elementExists: function (parentObject, childObject, at, key, last) { - const atBrowsers = childObject.map(o => o.browser.name); - - if (!map[parentObject.pattern]) { - map[parentObject.pattern] = {}; - } - - if (!(at in map[parentObject.pattern])) { - map[parentObject.pattern][at] = {}; - } - - const moreThanOneColumn = Object.values(childObject).length > 1; - - const conditional = - moreThanOneColumn && - (key in map[parentObject.pattern][at] || atBrowsers.includes(key)); - - // Cache columns that don't have data - if ( - !(key in map[parentObject.pattern][at]) && - !atBrowsers.includes(key) - ) { - map[parentObject.pattern][at][key] = true; - } - - // Don't write to the Safari column unless it's the last element - if (!last && key === 'Safari' && !atBrowsers.includes(key)) { - return true; - } else if (last && key === 'Safari' && !atBrowsers.includes(key)) { - return false; - } - - return conditional; + isShouldAssertionPriority: function (object) { + return object.metrics.shouldAssertionsCount > 0; }, - resetMap: function () { - map = {}; - return; + getShouldSupportData: function (object) { + return Math.trunc( + (object.metrics.shouldAssertionsPassedCount / + object.metrics.shouldAssertionsCount) * + 100 + ); } }; diff --git a/server/handlebars/embed/public/script.js b/server/handlebars/embed/public/script.js index e73f604c2..198aec4b9 100644 --- a/server/handlebars/embed/public/script.js +++ b/server/handlebars/embed/public/script.js @@ -1,6 +1,6 @@ -const iframeClass = `support-levels-${document - .querySelector('script[pattern]') - .getAttribute('pattern')}`; +const iframeClass = `support-levels-${ + document.location.href.match(/([^/]+)\/?$/)?.[1] +}`; const iframeCode = link => `