Support v2 Test Format in import script and page changes (#840)

* Update client/resources * Update import script to support v2 test format import * Update import script to include flattened commandsV2.json and add uniquely generated scenario and assertion IDs * Updating client and server files to support V2 test format * Remove console.log and add TODO * Include tokenized assertionStatement in TestPlanVersion.tests[x].assertions * Add retrieveCommands utility * Update candidate review and datamanagement page * Update Reports/queries to also include mayOptionalAssertionResults * Add support for AtMode and and referencing appropriate screen text in testsResolver * Remove console log * Fix edge case issue when saving test result and add priority standardizing utility * Add assertions checks for may assertions * Update getMetrics utils * Update test-import to account for single {at}-focused .collected file * Address file error * Fix tests * Update import branch * Fix edge case crash when viewing CandidateTestPlanRun * Update tests and prepare support for testing v2 format test plan versions * Update tests * Update workflow * Clarifying comment * Update runtest.yml to exclude v2 test format import * Update server/resolvers/TestPlanVersion/testsResolver.js Remove unnecessary `at.settings` check Co-authored-by: Stalgia Grigg <[email protected]> * Clearer differences between v1 and v2 test format tests being imported * Include typedef for RenderableContent * Update thrown error message when missing commands.json (v2) * Additional check for flattenObject * Fix bug found after rebase and update jsdoc on 'retrieveAttributes' * Updating Test Run page to support #743 * Update Test Run page instructions for v2 test format * Add react-html-parser; update InstructionsRenderer * Use testPlanVersion.metadata.testFormatVersion * Stop defaulting testFormatVersion to 1 * Stop using NumberedList * Update client/components/TestRenderer/utils.js Co-authored-by: Stalgia Grigg <[email protected]> * Define instructions variables * Prepend tests with 'Test {number}' on reports page * Address CG comments * Address PR feedback * Consistency * Passed/failed for assertion results in TestRenderer * Update test plan report conflict calculation to support pass/fail assertion results * Correct assertion legend, margin right for checkbox * Update inline snapshot for test queue * Remove failedReason * Update server/resolvers/TestResultOperations/saveTestResultCommon.js Co-authored-by: Howard Edwards <[email protected]> * Fix linting issue in saveTestResultCommon * Undo change to checkAssertionResult * Consistency * Do radio group and supporting docs * Revert "Do radio group and supporting docs" This reverts commit 579f545. * Revert aria-at import branch --------- Co-authored-by: Paul Clue <[email protected]> Co-authored-by: Stalgia Grigg <[email protected]>
w3c · Dec 5, 2023 · 6962cef · 6962cef
1 parent 703b01f
commit 6962cef
Show file tree

Hide file tree

Showing 54 changed files with 3,796 additions and 1,551 deletions.
diff --git a/.github/workflows/runtest.yml b/.github/workflows/runtest.yml
@@ -38,7 +38,7 @@ jobs:
           yarn sequelize:test db:seed:all
           yarn workspace server db-import-tests:test -c ${IMPORT_ARIA_AT_TESTS_COMMIT_1}
           yarn workspace server db-import-tests:test -c ${IMPORT_ARIA_AT_TESTS_COMMIT_2}
-          yarn workspace server db-import-tests:test
+          yarn workspace server db-import-tests:test -c ${IMPORT_ARIA_AT_TESTS_COMMIT_3}
           yarn workspace server db-populate-sample-data:test
       - name: test
         run: yarn test

diff --git a/client/components/CandidateReview/CandidateTestPlanRun/InstructionsRenderer.jsx b/client/components/CandidateReview/CandidateTestPlanRun/InstructionsRenderer.jsx
@@ -1,8 +1,9 @@
 import React, { useEffect, useState } from 'react';
 import PropTypes from 'prop-types';
-import nextId from 'react-id-generator';
 import styled from '@emotion/styled';
 import { Button } from 'react-bootstrap';
+import { unescape } from 'lodash';
+import { parseListContent } from '../../TestRenderer/utils';
 import {
     userCloseWindow,
     userOpenWindow
@@ -13,6 +14,8 @@ import {
 } from '../../../resources/aria-at-test-io-format.mjs';
 import { TestWindow } from '../../../resources/aria-at-test-window.mjs';
 import { evaluateAtNameKey } from '../../../utils/aria.js';
+import commandsJson from '../../../resources/commands.json';
+import supportJson from '../../../resources/support.json';
 
 const NumberedList = styled.ol`
     counter-reset: numbered-list;
@@ -41,20 +44,16 @@ const NumberedList = styled.ol`
     }
 `;
 
-const BulletList = styled.ul`
-    padding-inline-start: 2.5rem;
-    list-style-type: circle;
-
-    > li {
-        display: list-item;
-        list-style: circle;
-    }
-`;
-
-const InstructionsRenderer = ({ test, testPageUrl, at, headingLevel = 2 }) => {
+const InstructionsRenderer = ({
+    test,
+    testPageUrl,
+    at,
+    headingLevel = 2,
+    testFormatVersion
+}) => {
     const { renderableContent } = test;
     const [testRunExport, setTestRunExport] = useState();
-    const [instructionsContent, setInstructionsContent] = useState({
+    const [pageContent, setPageContent] = useState({
         instructions: {
             assertions: { assertions: [] },
             instructions: {
@@ -69,6 +68,7 @@ const InstructionsRenderer = ({ test, testPageUrl, at, headingLevel = 2 }) => {
         const testRunIO = new TestRunInputOutput();
         const configQueryParams = [['at', evaluateAtNameKey(at.name)]];
 
+        testRunIO.setAllCommandsInputFromJSON(commandsJson);
         await testRunIO.setInputsFromCollectedTestAsync(renderableContent);
         testRunIO.setConfigInputFromQueryParamsAndSupport(configQueryParams);
 
@@ -114,106 +114,68 @@ const InstructionsRenderer = ({ test, testPageUrl, at, headingLevel = 2 }) => {
 
     useEffect(() => {
         if (testRunExport) {
-            setInstructionsContent(testRunExport.instructions());
+            setPageContent(testRunExport.instructions());
         }
     }, [testRunExport]);
 
-    const parseRichContent = (instruction = []) => {
-        let content = null;
-        for (let value of instruction) {
-            if (typeof value === 'string') {
-                if (value === '.')
-                    content = (
-                        <>
-                            {content}
-                            {value}
-                        </>
-                    );
-                else
-                    content = content = (
-                        <>
-                            {content} {value}
-                        </>
-                    );
-            } else if ('href' in value) {
-                const { href, description } = value;
-                content = (
-                    <>
-                        {content} <a href={href}>{description}</a>
-                    </>
-                );
-            }
-        }
-        return content;
-    };
-
-    const parseListContent = (instructions = [], commandsContent = null) => {
-        return instructions.map((value, index) => {
-            if (typeof value === 'string')
-                return (
-                    <li key={nextId()}>
-                        {value}
-                        {commandsContent &&
-                            index === instructions.length - 1 && (
-                                <BulletList>{commandsContent}</BulletList>
-                            )}
-                    </li>
-                );
-            else if (Array.isArray(value))
-                return (
-                    <li key={nextId()}>
-                        {parseRichContent(value)}
-                        {commandsContent &&
-                            index === instructions.length - 1 && (
-                                <BulletList>{commandsContent}</BulletList>
-                            )}
-                    </li>
-                );
-        });
-    };
-
-    const allInstructions = [
-        ...instructionsContent.instructions.instructions.instructions,
-        ...instructionsContent.instructions.instructions.strongInstructions,
-        instructionsContent.instructions.instructions.commands.description
-    ];
-
-    const commands =
-        instructionsContent.instructions.instructions.commands.commands;
+    let allInstructions;
+    const isV2 = testFormatVersion === 2;
+
+    if (isV2) {
+        const commandSettingSpecified = renderableContent.commands.some(
+            ({ settings }) => settings && settings !== 'defaultMode'
+        );
+
+        const defaultInstructions =
+            renderableContent.target.at.raw
+                .defaultConfigurationInstructionsHTML;
+        const setupScriptDescription = `${supportJson.testPlanStrings.openExampleInstruction} ${renderableContent.target.setupScript.scriptDescription}`;
+        const testInstructions = renderableContent.instructions.instructions;
+        const settingsInstructions = `${
+            supportJson.testPlanStrings.commandListPreface
+        }${
+            commandSettingSpecified
+                ? ` ${supportJson.testPlanStrings.commandListSettingsPreface}`
+                : ''
+        }`;
+
+        allInstructions = [
+            defaultInstructions,
+            setupScriptDescription + '.',
+            testInstructions + ' ' + settingsInstructions
+        ].map(e => unescape(e));
+    } else {
+        allInstructions = [
+            ...pageContent.instructions.instructions.instructions,
+            ...pageContent.instructions.instructions.strongInstructions,
+            pageContent.instructions.instructions.commands.description
+        ];
+    }
 
+    const commands = pageContent.instructions.instructions.commands.commands;
     const commandsContent = parseListContent(commands);
+
     const allInstructionsContent = parseListContent(
         allInstructions,
         commandsContent
     );
 
-    const assertions = [
-        ...instructionsContent.instructions.assertions.assertions
-    ];
-
+    const assertions = [...pageContent.instructions.assertions.assertions];
     const assertionsContent = parseListContent(assertions);
 
     const Heading = `h${headingLevel}`;
 
     return (
         <>
-            <p>{instructionsContent.instructions.description}</p>
-            <Heading>
-                {instructionsContent.instructions.instructions.header}
-            </Heading>
             <NumberedList>{allInstructionsContent}</NumberedList>
-            <Heading>
-                {instructionsContent.instructions.assertions.header}
-            </Heading>
-            {instructionsContent.instructions.assertions.description}
+            <Heading>{pageContent.instructions.assertions.header}</Heading>
+            {pageContent.instructions.assertions.description}
             <NumberedList>{assertionsContent}</NumberedList>
             <Button
-                disabled={
-                    !instructionsContent.instructions.openTestPage.enabled
-                }
-                onClick={instructionsContent.instructions.openTestPage.click}
+                disabled={!pageContent.instructions.openTestPage.enabled}
+                onClick={pageContent.instructions.openTestPage.click}
             >
-                {instructionsContent.instructions.openTestPage.button}
+                {pageContent.instructions.openTestPage.button}
             </Button>
         </>
     );
@@ -225,6 +187,7 @@ InstructionsRenderer.propTypes = {
     at: PropTypes.shape({
         name: PropTypes.string.isRequired
     }),
+    testFormatVersion: PropTypes.number,
     headingLevel: PropTypes.number
 };
 

diff --git a/client/components/CandidateReview/CandidateTestPlanRun/index.jsx b/client/components/CandidateReview/CandidateTestPlanRun/index.jsx
@@ -498,6 +498,9 @@ const CandidateTestPlanRun = () => {
                         at={testPlanReport.at}
                         test={currentTest}
                         testPageUrl={testPlanReport.testPlanVersion.testPageUrl}
+                        testFormatVersion={
+                            testPlanVersion.metadata.testFormatVersion
+                        }
                     />,
                     ...testPlanReports.map(testPlanReport => {
                         const testResult =

diff --git a/client/components/CandidateReview/CandidateTestPlanRun/queries.js b/client/components/CandidateReview/CandidateTestPlanRun/queries.js
@@ -114,7 +114,6 @@ export const CANDIDATE_REPORTS_QUERY = gql`
                             text
                         }
                         passed
-                        failedReason
                     }
                     requiredAssertionResults: assertionResults(
                         priority: REQUIRED
@@ -123,7 +122,6 @@ export const CANDIDATE_REPORTS_QUERY = gql`
                             text
                         }
                         passed
-                        failedReason
                     }
                     optionalAssertionResults: assertionResults(
                         priority: OPTIONAL
@@ -132,7 +130,12 @@ export const CANDIDATE_REPORTS_QUERY = gql`
                             text
                         }
                         passed
-                        failedReason
+                    }
+                    mayAssertionResults: assertionResults(priority: MAY) {
+                        assertion {
+                            text
+                        }
+                        passed
                     }
                     unexpectedBehaviors {
                         id

diff --git a/client/components/Reports/SummarizeTestPlanReport.jsx b/client/components/Reports/SummarizeTestPlanReport.jsx
@@ -24,7 +24,7 @@ const ResultsContainer = styled.div`
     border-left: 1px solid #dee2e6;
     border-right: 1px solid #dee2e6;
     border-bottom: 1px solid #dee2e6;
-    margin-bottom: 2em;
+    margin-bottom: 0.5em;
 `;
 
 const getTestersRunHistory = (
@@ -176,7 +176,7 @@ const SummarizeTestPlanReport = ({ testPlanVersion, testPlanReports }) => {
                     </li>
                 ) : null}
             </ul>
-            {testPlanReport.finalizedTestResults.map(testResult => {
+            {testPlanReport.finalizedTestResults.map((testResult, index) => {
                 const test = testResult.test;
 
                 const reportLink = `https://aria-at.w3.org${location.pathname}#result-${testResult.id}`;
@@ -207,7 +207,8 @@ const SummarizeTestPlanReport = ({ testPlanVersion, testPlanReports }) => {
                     <Fragment key={testResult.id}>
                         <div className="test-result-heading">
                             <h2 id={`result-${testResult.id}`} tabIndex="-1">
-                                {test.title}&nbsp;({passedAssertionsCount}
+                                Test {index + 1}: {test.title}&nbsp;(
+                                {passedAssertionsCount}
                                 &nbsp;passed, {failedAssertionsCount} failed)
                                 <DisclaimerInfo phase={testPlanVersion.phase} />
                             </h2>
@@ -244,6 +245,10 @@ const SummarizeTestPlanReport = ({ testPlanVersion, testPlanReports }) => {
                                 key={`TestPlanResultsTable__${testResult.id}`}
                                 test={{ ...test, at }}
                                 testResult={testResult}
+                                optionalHeader={
+                                    <h3>Results for each command</h3>
+                                }
+                                commandHeadingLevel={4}
                             />
                         </ResultsContainer>
 
@@ -293,7 +298,6 @@ SummarizeTestPlanReport.propTypes = {
                                 PropTypes.shape({
                                     id: PropTypes.string.isRequired,
                                     passed: PropTypes.bool.isRequired,
-                                    failedReason: PropTypes.string,
                                     assertion: PropTypes.shape({
                                         text: PropTypes.string.isRequired
                                     }).isRequired