diff --git a/client/tests/e2e/TestRun.e2e.test.js b/client/tests/e2e/TestRun.e2e.test.js index 3ec3aeb06..227ed3928 100644 --- a/client/tests/e2e/TestRun.e2e.test.js +++ b/client/tests/e2e/TestRun.e2e.test.js @@ -108,22 +108,24 @@ describe('Test Run when not signed in', () => { }); describe('Test Run when signed in as tester', () => { - const assignSelfAndNavigateToRun = async page => { - const modalDialogSectionButtonSelector = - 'button#disclosure-btn-modal-dialog-0'; - const modalDialogTableSelector = - 'table[aria-label="Reports for Modal Dialog Example V24.06.07 in draft phase"]'; + const assignSelfAndNavigateToRun = async ( + page, + { + testPlanSectionButtonSelector = 'button#disclosure-btn-modal-dialog-0', + testPlanTableSelector = 'table[aria-label="Reports for Modal Dialog Example V24.06.07 in draft phase"]' + } = {} + ) => { const startTestingButtonSelector = 'a[role="button"] ::-p-text(Start Testing)'; // Expand Modal Dialog's V24.06.07 section - await page.waitForSelector(modalDialogSectionButtonSelector); - await page.click(modalDialogSectionButtonSelector); + await page.waitForSelector(testPlanSectionButtonSelector); + await page.click(testPlanSectionButtonSelector); // Wait for the table to render - await page.waitForSelector(modalDialogTableSelector); + await page.waitForSelector(testPlanTableSelector); - await page.$eval(modalDialogTableSelector, el => { + await page.$eval(testPlanTableSelector, el => { // First button found on table would be 'Assign Yourself' el.querySelector('button').click(); }); @@ -161,6 +163,72 @@ describe('Test Run when signed in as tester', () => { await page.waitForNetworkIdle(); }; + const handlePageSubmit = async (page, { expectConflicts = true } = {}) => { + await page.waitForSelector('h1 ::-p-text(Test 1)'); + + // Confirm that submission cannot happen with empty form + // Specificity with selector because there's a 2nd 'hidden' button coming + // from the harness which is what is actually called for the submit event + const submitResultsButtonSelector = + 'button[class="btn btn-primary"] ::-p-text(Submit Results)'; + await page.waitForSelector(submitResultsButtonSelector); + await page.click(submitResultsButtonSelector); + await page.waitForNetworkIdle(); + await page.waitForSelector('::-p-text((required))'); + + // Should refocus on topmost output textarea on page + const activeElementAfterEmptySubmit = await page.evaluate(() => { + return { + id: document.activeElement.id, + nodeName: document.activeElement.nodeName.toLowerCase() + }; + }); + + // Input output for valid submission + await page.evaluate(() => { + const yesRadios = document.querySelectorAll( + 'input[data-testid^="radio-yes-"]' + ); + const noRadios = document.querySelectorAll( + 'input[data-testid^="radio-no-"]' + ); + const noUndesiredRadios = document.querySelectorAll( + 'input[id^="problem-"][id$="-true"]' + ); + const noOutputCheckboxes = document.querySelectorAll( + 'input[id^="no-output-checkbox"]' + ); + + yesRadios.forEach((radio, index) => { + if (index % 2 === 0) { + radio.click(); + } else { + noRadios[index].click(); + } + }); + + noUndesiredRadios.forEach(radio => { + radio.click(); + }); + + noOutputCheckboxes.forEach(checkbox => { + checkbox.click(); + }); + }); + // Submit valid form + await page.click(submitResultsButtonSelector); + await page.waitForNetworkIdle(); + if (expectConflicts) + await page.waitForSelector( + '::-p-text(This test has conflicting results)' + ); + await page.waitForSelector('h2 ::-p-text(Test Results)'); + await page.waitForSelector('button ::-p-text(Edit Results)'); + + expect(activeElementAfterEmptySubmit.id).toBe('speechoutput-0'); + expect(activeElementAfterEmptySubmit.nodeName).toBe('textarea'); + }; + it('self assigns tester on Test Queue page and opens test run', async () => { await getPage({ role: 'tester', url: '/test-queue' }, async page => { await assignSelfAndNavigateToRun(page); @@ -324,72 +392,25 @@ describe('Test Run when signed in as tester', () => { }); it('inputs results and successfully submits', async () => { - await getPage({ role: 'tester', url: '/test-queue' }, async page => { - await assignSelfAndNavigateToRun(page); - - await page.waitForSelector('h1 ::-p-text(Test 1)'); - - // Confirm that submission cannot happen with empty form - // Specificity with selector because there's a 2nd 'hidden' button coming - // from the harness which is what is actually called for the submit event - const submitResultsButtonSelector = - 'button[class="btn btn-primary"] ::-p-text(Submit Results)'; - await page.waitForSelector(submitResultsButtonSelector); - await page.click(submitResultsButtonSelector); - await page.waitForNetworkIdle(); - await page.waitForSelector('::-p-text((required))'); - - // Should refocus on topmost output textarea on page - const activeElementAfterEmptySubmit = await page.evaluate(() => { - return { - id: document.activeElement.id, - nodeName: document.activeElement.nodeName.toLowerCase() - }; - }); - - // Input output for valid submission - await page.evaluate(() => { - const yesRadios = document.querySelectorAll( - 'input[data-testid^="radio-yes-"]' - ); - const noRadios = document.querySelectorAll( - 'input[data-testid^="radio-no-"]' - ); - const noUndesiredRadios = document.querySelectorAll( - 'input[id^="problem-"][id$="-true"]' - ); - const noOutputCheckboxes = document.querySelectorAll( - 'input[id^="no-output-checkbox"]' - ); - - yesRadios.forEach((radio, index) => { - if (index % 2 === 0) { - radio.click(); - } else { - noRadios[index].click(); - } - }); - - noUndesiredRadios.forEach(radio => { - radio.click(); - }); - - noOutputCheckboxes.forEach(checkbox => { - checkbox.click(); + await getPage( + { role: 'tester', url: '/test-queue' }, + async (page, { baseUrl }) => { + await assignSelfAndNavigateToRun(page); + await handlePageSubmit(page); + + // Do the same for Color Viewer Slider which has specially handled + // exclusions; + // Excluded in tests.csv and re-included in *-commands.csv + await page.goto(`${baseUrl}/test-queue`); + await assignSelfAndNavigateToRun(page, { + testPlanSectionButtonSelector: + 'button#disclosure-btn-horizontal-slider-0', + testPlanTableSelector: + 'table[aria-label="Reports for Color Viewer Slider V24.06.26 in draft phase"]' }); - }); - // Submit valid form - await page.click(submitResultsButtonSelector); - await page.waitForNetworkIdle(); - await page.waitForSelector( - '::-p-text(This test has conflicting results)' - ); - await page.waitForSelector('h2 ::-p-text(Test Results)'); - await page.waitForSelector('button ::-p-text(Edit Results)'); - - expect(activeElementAfterEmptySubmit.id).toBe('speechoutput-0'); - expect(activeElementAfterEmptySubmit.nodeName).toBe('textarea'); - }); + await handlePageSubmit(page, { expectConflicts: false }); + } + ); }); it('opens popup with content after clicking "Open Test Page" button', async () => { diff --git a/client/tests/e2e/snapshots/saved/_data-management.html b/client/tests/e2e/snapshots/saved/_data-management.html index 6950c45da..25cf970b0 100644 --- a/client/tests/e2e/snapshots/saved/_data-management.html +++ b/client/tests/e2e/snapshots/saved/_data-management.html @@ -266,7 +266,7 @@

Radio Group Example Using aria-activedescendant - + @@ -377,7 +377,7 @@

Test Plans Status Summary

type="button" aria-pressed="false" class="css-14jv778 btn btn-secondary"> - R&D Complete (32) + R&D Complete (31)
  • @@ -385,7 +385,7 @@

    Test Plans Status Summary

    type="button" aria-pressed="false" class="css-14jv778 btn btn-secondary"> - In Draft Review (5) + In Draft Review (6)
  • @@ -906,6 +906,64 @@

    Test Plans Status Summary

    None Yet + + Color Viewer Slider + + +
    + JAWS, NVDA and VoiceOver for macOS +
    + + +
    + Draft +

    Review Started Oct 2, 2024

    +
    + + N/A + +
    + V24.06.26 +
    +
    + + Not Started + None Yet + + Command Button ExampleTest Plans Status Summary Not Started None Yet - + Select Only Combobox ExampleTest Plans Status Summary Not Started None Yet - + Toggle ButtonTest Plans Status Summary Not Started None Yet - + Action Menu Button Example Using element.focus()Test Plans Status Summary Not Started None Yet - + Banner Landmark @@ -1286,7 +1344,7 @@

    Test Plans Status Summary

    Not Started None Yet - + Breadcrumb ExampleTest Plans Status Summary Not Started None Yet - + Checkbox Example (Two State)Test Plans Status Summary Not Started None Yet - - - Color Viewer Slider - - -
    - JAWS, NVDA and VoiceOver for macOS -
    - - -
    - R&D -

    Complete Jun 26, 2024

    -
    - - -
    - V24.06.26 -
    - - Not Started - Not Started - None Yet - + @@ -508,6 +509,226 @@

    +

    Color Viewer Slider

    +
    +

    + +

    +
    +
    + View tests in V24.06.26 +
    +
    +
    + + + + + + + + + + + + + + + + + + + +
    Assistive TechnologyBrowserTestersStatusActions
    +
    + VoiceOver for macOS 11.6 (20G165) or later +
    +     		+
    + Safari Any version +
    +     		+
    + + +
    + +     		+
    +
    +
    +
    +
    + 0% complete by esmeralda-baggins  +
    +     		+
    + +
    + +
    + +
    +
    +
    +
    +

    diff --git a/server/graphql-schema.js b/server/graphql-schema.js index 4ae520528..07295cb7d 100644 --- a/server/graphql-schema.js +++ b/server/graphql-schema.js @@ -644,19 +644,26 @@ const graphqlSchema = gql` Some assertions are more akin to recommendations or best practices, and, while we want to record whether they are passing or failing, we do not want to count the entire test as failing when they fail. + + These definitions for MUST, SHOULD and MAY are described at + https://github.com/w3c/aria-at/wiki/Glossary#assertion-priority in more + detail """ enum AssertionPriority { """ - All required assertions must pass for the test to pass. This should be - considered as 'MUST Behaviors'. + The assertion is an absolute requirement for the test to pass. This should + be considered as 'MUST Behaviors'. """ MUST """ - This assertion is not considered when deciding if a test is passing. - This should be considered as 'SHOULD Behaviors'. + This assertion is a strongly recommended requirement. This should be + considered as 'SHOULD Behaviors'. """ SHOULD - # TODO Define MAY + """ + This assertion is truly optional. This should be considered as 'MAY + Behaviors'. + """ MAY """ This assertion should not be included in the test and should not be diff --git a/server/resolvers/TestResultOperations/saveTestResultCommon.js b/server/resolvers/TestResultOperations/saveTestResultCommon.js index 3a6f908ba..c6312bbf4 100644 --- a/server/resolvers/TestResultOperations/saveTestResultCommon.js +++ b/server/resolvers/TestResultOperations/saveTestResultCommon.js @@ -60,7 +60,7 @@ const saveTestResultCommon = async ({ } if (isSubmit) { - assertTestResultIsValid(newTestResult); + assertTestResultIsValid(newTestResult, test.assertions); newTestResult.completedAt = new Date(); } else { newTestResult.completedAt = null; @@ -115,15 +115,22 @@ const saveTestResultCommon = async ({ return populateData({ testResultId }, { context }); }; -const assertTestResultIsValid = newTestResult => { +const assertTestResultIsValid = (newTestResult, assertions = []) => { let failed = false; const checkAssertionResult = assertionResult => { + const isExcluded = assertions.find( + assertion => + assertion.id === assertionResult.assertionId && + assertion.priority === 'EXCLUDE' + ); + if ( assertionResult.passed === null || assertionResult.passed === undefined ) { - failed = true; + if (isExcluded) assertionResult.passed = false; + else failed = true; } }; diff --git a/server/scripts/populate-test-data/pg_dump_test_data.sql b/server/scripts/populate-test-data/pg_dump_test_data.sql index 7c6ec522d..84db3ef54 100644 --- a/server/scripts/populate-test-data/pg_dump_test_data.sql +++ b/server/scripts/populate-test-data/pg_dump_test_data.sql @@ -84,13 +84,15 @@ INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinal INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "minimumAtVersionId", "browserId", "vendorReviewStatus") VALUES (18, get_test_plan_version_id(text 'Command Button Example', '2'), '2023-12-13 14:18:23.602-05', '2023-12-14', 3, 3, 3, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "atId", "minimumAtVersionId", "browserId", "vendorReviewStatus") VALUES (19, get_test_plan_version_id(text 'Modal Dialog Example', '2'), '2024-05-14 14:18:23.602-05', 2, 2, 2, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (20, get_test_plan_version_id(text 'Checkbox Example (Mixed-State)', '1'), '2021-05-14 14:18:23.602-05', '2022-08-07', 2, 4, 2, 'READY'); --- Fully recommended TestPlanVersion +-- Fully recommended TestPlanVersion [START] INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (21, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 1, 1, 1, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (22, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 1, 1, 2, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (23, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 2, 4, 1, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (24, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 2, 4, 2, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (25, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 3, 5, 2, 'READY'); INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "markedFinalAt", "atId", "exactAtVersionId", "browserId", "vendorReviewStatus") VALUES (26, get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'), '2024-09-20 14:18:23.602-05', '2024-09-21', 3, 5, 3, 'READY'); +--- Fully recommended TestPlanVersion [END] +INSERT INTO "TestPlanReport" (id, "testPlanVersionId", "createdAt", "atId", "minimumAtVersionId", "browserId", "vendorReviewStatus") VALUES (27, get_test_plan_version_id(text 'Color Viewer Slider', '2'), '2024-10-02 14:18:23.602-05', 3, 3, 3, 'READY'); -- -- Data for Name: TestPlanVersion; Type: TABLE DATA; Schema: public; Owner: atr @@ -104,6 +106,7 @@ UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2022-07 UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2023-12-14' WHERE id = get_test_plan_version_id(text 'Command Button Example', '2'); UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2024-05-14' WHERE id = get_test_plan_version_id(text 'Modal Dialog Example', '2'); UPDATE "TestPlanVersion" SET "phase" = 'RECOMMENDED', "draftPhaseReachedAt" = '2024-09-20', "candidatePhaseReachedAt" = '2024-09-22', "recommendedPhaseTargetDate" = '2025-03-21', "recommendedPhaseReachedAt" = '2024-09-23' WHERE id = get_test_plan_version_id(text 'Action Menu Button Example Using aria-activedescendant', '2'); +UPDATE "TestPlanVersion" SET "phase" = 'DRAFT', "draftPhaseReachedAt" = '2024-10-02' WHERE id = get_test_plan_version_id(text 'Color Viewer Slider', '2'); -- -- Data for Name: User; Type: TABLE DATA; Schema: public; Owner: atr @@ -154,6 +157,7 @@ INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults" INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (25, 2, 24, '[]'); INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (26, 2, 25, '[]'); INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (27, 2, 26, '[]'); +INSERT INTO "TestPlanRun" (id, "testerUserId", "testPlanReportId", "testResults") VALUES (28, 1, 27, '[]'); -- -- Data for Name: CollectionJob; Type: TABLE DATA; Schema: public; Owner: atr