diff --git a/client/components/TestRenderer/index.jsx b/client/components/TestRenderer/index.jsx index 8e1b8a0f1..d264de541 100644 --- a/client/components/TestRenderer/index.jsx +++ b/client/components/TestRenderer/index.jsx @@ -248,7 +248,7 @@ const TestRenderer = ({ } for (let i = 0; i < scenarioResults.length; i++) { - const { + let { output, assertionResults, unexpectedBehaviors, @@ -259,15 +259,23 @@ const TestRenderer = ({ if (output) commands[i].atOutput.value = output; commands[i].atOutput.highlightRequired = highlightRequired; + // Required because assertionResults can now be returned without an id if there is a 0-priority exception + // applied + assertionResults = assertionResults.filter(el => !!el.id); + for (let j = 0; j < assertionResults.length; j++) { - const assertionResult = assertionResults[j]; - const { highlightRequired } = assertionResult; + const { passed, highlightRequired, assertion } = + assertionResults[j]; - commands[i].assertions[j].result = assertionResult.passed + let assertionForCommandIndex = commands[i].assertions.findIndex( + ({ description }) => description === assertion?.text + ); + commands[i].assertions[assertionForCommandIndex].result = passed ? 'pass' : 'fail'; - - commands[i].assertions[j].highlightRequired = highlightRequired; + commands[i].assertions[ + assertionForCommandIndex + ].highlightRequired = highlightRequired; } if (unexpectedBehaviors && unexpectedBehaviors.length) { diff --git a/client/components/TestRun/index.jsx b/client/components/TestRun/index.jsx index c7d6dade1..84123b4fa 100644 --- a/client/components/TestRun/index.jsx +++ b/client/components/TestRun/index.jsx @@ -402,9 +402,12 @@ const TestRun = () => { // process assertion results for (let j = 0; j < assertions.length; j++) { - const { result, highlightRequired } = assertions[j]; + const { description, result, highlightRequired } = + assertions[j]; const assertionResult = { - ...scenarioResult.assertionResults[j], + ...scenarioResult.assertionResults.find( + ({ assertion: { text } }) => text === description + ), passed: result === 'pass' }; assertionResults.push( @@ -656,10 +659,18 @@ const TestRun = () => { id, output: output, unexpectedBehaviors: unexpectedBehaviors, - assertionResults: assertionResults.map(({ id, passed }) => ({ - id, - passed - })) + assertionResults: assertionResults + // All assertions are always being passed from the TestRenderer results, but + // when there is a 0-priority assertion exception, an id won't be provided, + // so do not include that result. + // This is due to the TestRenderer still requiring the position of the + // excluded assertion, but it can be removed at this point before being passed + // to the server + .filter(el => !!el.id) + .map(({ id, passed }) => ({ + id, + passed + })) }) ); diff --git a/client/components/TestRun/queries.js b/client/components/TestRun/queries.js index f6705d6f3..af3bc022c 100644 --- a/client/components/TestRun/queries.js +++ b/client/components/TestRun/queries.js @@ -523,6 +523,7 @@ export const FIND_OR_CREATE_TEST_RESULT_MUTATION = gql` id title phase + updatedAt gitSha testPageUrl testPlan { @@ -807,6 +808,7 @@ export const SAVE_TEST_RESULT_MUTATION = gql` id title phase + updatedAt gitSha testPageUrl testPlan { @@ -1091,6 +1093,7 @@ export const SUBMIT_TEST_RESULT_MUTATION = gql` id title phase + updatedAt gitSha testPageUrl testPlan { diff --git a/client/resources/aria-at-test-io-format.mjs b/client/resources/aria-at-test-io-format.mjs index d967c7392..22b60873b 100644 --- a/client/resources/aria-at-test-io-format.mjs +++ b/client/resources/aria-at-test-io-format.mjs @@ -405,6 +405,8 @@ class CommandsInput { commands.push(command); commandsAndSettings.push({ command, + commandId, + presentationNumber: Number(presentationNumber), settings: _atMode, settingsText: assistiveTech.settings?.[_atMode]?.screenText || 'default mode active', settingsInstructions: assistiveTech.settings?.[_atMode]?.instructions || [ @@ -767,6 +769,9 @@ class BehaviorInput { const { commandsAndSettings } = commandsInput.getCommands({ task: json.task }, mode); + // Use to determine assertionExceptions + const commandsInfo = json.commandsInfo?.[at.key]; + return new BehaviorInput({ behavior: { description: titleInput.title(), @@ -778,7 +783,31 @@ class BehaviorInput { setupScriptDescription: json.setup_script_description, setupTestPage: json.setupTestPage, assertionResponseQuestion: json.assertionResponseQuestion, - commands: commandsAndSettings, + commands: commandsAndSettings.map(cs => { + const foundCommandInfo = commandsInfo?.find( + c => + cs.commandId === c.command && + cs.presentationNumber === c.presentationNumber && + cs.settings === c.settings + ); + if (!foundCommandInfo || !foundCommandInfo.assertionExceptions) return cs; + + // Only works for v2 + let assertionExceptions = json.output_assertions.map(each => each.assertionId); + foundCommandInfo.assertionExceptions.split(' ').forEach(each => { + let [priority, assertionId] = each.split(':'); + const index = assertionExceptions.findIndex(each => each === assertionId); + + priority = Number(priority); + assertionExceptions[index] = priority; + }); + // Preserve default priority or update with exception + assertionExceptions = assertionExceptions.map((each, index) => + isNaN(each) ? json.output_assertions[index].priority : each + ); + + return { ...cs, assertionExceptions }; + }), assertions: (json.output_assertions ? json.output_assertions : []).map(assertion => { // Tuple array [ priorityNumber, assertionText ] if (Array.isArray(assertion)) { @@ -788,7 +817,7 @@ class BehaviorInput { }; } - // { assertionId, priority, assertionStatement, assertionPhrase, refIds, commandInfo, tokenizedAssertionStatements } + // { assertionId, priority, assertionStatement, assertionPhrase, refIds, tokenizedAssertionStatements } return { priority: assertion.priority, assertion: @@ -815,7 +844,7 @@ class BehaviorInput { * @param {UnexpectedInput} data.unexpectedInput */ static fromCollectedTestCommandsKeysUnexpected( - { info, target, instructions, assertions }, + { info, target, instructions, assertions, commands }, { commandsInput, keysInput, unexpectedInput } ) { // v1:info.task, v2: info.testId | v1:target.mode, v2:target.at.settings @@ -834,7 +863,28 @@ class BehaviorInput { specificUserInstruction: instructions.raw || instructions.instructions, setupScriptDescription: target.setupScript ? target.setupScript.description : '', setupTestPage: target.setupScript ? target.setupScript.name : undefined, - commands: commandsAndSettings, + commands: commandsAndSettings.map(cs => { + const foundCommandInfo = commands.find( + c => cs.commandId === c.id && cs.settings === c.settings + ); + if (!foundCommandInfo || !foundCommandInfo.assertionExceptions) return cs; + + // Only works for v2 + let assertionExceptions = assertions.map(each => each.assertionId); + foundCommandInfo.assertionExceptions.forEach(each => { + let { priority, assertionId } = each; + const index = assertionExceptions.findIndex(each => each === assertionId); + + priority = Number(priority); + assertionExceptions[index] = priority; + }); + // Preserve default priority or update with exception + assertionExceptions = assertionExceptions.map((each, index) => + isNaN(each) ? assertions[index].priority : each + ); + + return { ...cs, assertionExceptions }; + }), assertions: assertions.map( ({ priority, expectation, assertionStatement, tokenizedAssertionStatements }) => { let assertion = tokenizedAssertionStatements @@ -1187,6 +1237,7 @@ export class TestRunInputOutput { description: command.settings, text: command.settingsText, instructions: command.settingsInstructions, + assertionExceptions: command.assertionExceptions, }, atOutput: { highlightRequired: false, diff --git a/client/resources/aria-at-test-run.mjs b/client/resources/aria-at-test-run.mjs index 04c7fd4c4..d1882a3c6 100644 --- a/client/resources/aria-at-test-run.mjs +++ b/client/resources/aria-at-test-run.mjs @@ -219,7 +219,7 @@ export function instructionDocument(resultState, hooks) { const resultUnexpectedBehavior = resultStateCommand.unexpected; const { - commandSettings: { description: settings, text: settingsText }, + commandSettings: { description: settings, text: settingsText, assertionExceptions }, } = resultStateCommand; return { @@ -248,7 +248,16 @@ export function instructionDocument(resultState, hooks) { }?`, }, assertions: [ - ...assertions.map(bind(assertionResult, commandIndex)), + ...assertions + // Ignore assertion if level 0 priority exception found for assertion's command + .filter((each, index) => (assertionExceptions ? assertionExceptions[index] !== 0 : each)) + .map(each => + assertionResult( + commandIndex, + each, + assertions.findIndex(e => e === each) + ) + ), ...additionalAssertions.map(bind(additionalAssertionResult, commandIndex)), ], unexpectedBehaviors: { @@ -742,10 +751,20 @@ function resultsTableDocument(state) { header: [ 'Test result: ', state.commands.some( - ({ assertions, additionalAssertions, unexpected }) => - [...assertions, ...additionalAssertions].some( - ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS - ) || unexpected.behaviors.some(({ checked }) => checked) + ({ + assertions, + additionalAssertions, + unexpected, + commandSettings: { assertionExceptions }, + }) => + [ + // Ignore assertion if level 0 priority exception found for assertion's command + ...assertions.filter((each, index) => + assertionExceptions ? assertionExceptions[index] !== 0 : each + ), + ...additionalAssertions, + ].some(({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS) || + unexpected.behaviors.some(({ checked }) => checked) ) ? 'FAIL' : 'PASS', @@ -758,11 +777,20 @@ function resultsTableDocument(state) { details: 'Details', }, commands: state.commands.map(command => { - const allAssertions = [...command.assertions, ...command.additionalAssertions]; + const { + commandSettings: { assertionExceptions }, + } = command; + const allAssertions = [ + // Ignore assertion if level 0 priority exception found for assertion's command + ...command.assertions.filter((each, index) => + assertionExceptions ? assertionExceptions[index] !== 0 : each + ), + ...command.additionalAssertions, + ]; let passingAssertions = ['No passing assertions.']; let failingAssertions = ['No failing assertions.']; - let unexpectedBehaviors = ['No unexpect behaviors.']; + let unexpectedBehaviors = ['No unexpected behaviors.']; if (allAssertions.some(({ result }) => result === CommonResultMap.PASS)) { passingAssertions = allAssertions diff --git a/client/resources/at-commands.mjs b/client/resources/at-commands.mjs index c6bbfb5a1..6bbb5b776 100644 --- a/client/resources/at-commands.mjs +++ b/client/resources/at-commands.mjs @@ -123,6 +123,7 @@ export class commandsAPI { return { value, key, + settings: mode, }; }) ); diff --git a/server/graphql-schema.js b/server/graphql-schema.js index 0711ed5e1..381a141ae 100644 --- a/server/graphql-schema.js +++ b/server/graphql-schema.js @@ -532,6 +532,14 @@ const graphqlSchema = gql` A human-readable version of the command, such as "Control+Alt+Down" """ text: String! + """ + The AT mode this command may be getting ran in, such as quickNavOn, + browseMode, etc. + The same command can be ran during the same test, but in a different + mode. + """ + # TODO: Add link to list of known AT modes + atOperatingMode: String } """ diff --git a/server/migrations/20240111225130-includeMissingAssertionExceptions.js b/server/migrations/20240111225130-includeMissingAssertionExceptions.js new file mode 100644 index 000000000..4117189d5 --- /dev/null +++ b/server/migrations/20240111225130-includeMissingAssertionExceptions.js @@ -0,0 +1,47 @@ +'use strict'; + +const { regenerateResultsAndRecalculateHashes } = require('./utils'); +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + const updateV2TestsToIncludeEmptyAssertionExceptions = + async transaction => { + const testPlanVersions = await queryInterface.sequelize.query( + `SELECT id, tests FROM "TestPlanVersion" WHERE metadata->>'testFormatVersion' = '2'`, + { + type: Sequelize.QueryTypes.SELECT, + transaction + } + ); + await Promise.all( + testPlanVersions.map(async ({ id, tests }) => { + const updatedTests = JSON.stringify( + tests.map(test => { + test.assertions = test.assertions.map( + assertion => ({ + ...assertion, + assertionExceptions: [] + }) + ); + return test; + }) + ); + await queryInterface.sequelize.query( + `UPDATE "TestPlanVersion" SET tests = ? WHERE id = ?`, + { replacements: [updatedTests, id], transaction } + ); + }) + ); + }; + + return queryInterface.sequelize.transaction(async transaction => { + await updateV2TestsToIncludeEmptyAssertionExceptions(transaction); + + // Recalculate the hashes + await regenerateResultsAndRecalculateHashes( + queryInterface, + transaction + ); + }); + } +}; diff --git a/server/models/services/TestsService.js b/server/models/services/TestsService.js index 5823a756b..25d52e9a7 100644 --- a/server/models/services/TestsService.js +++ b/server/models/services/TestsService.js @@ -36,7 +36,8 @@ const getTests = parentRecord => { id: commandKVs[0].key, text: `${commandKVs[0].value}${ screenText ? ` (${screenText})` : '' - }` + }`, + atOperatingMode: scenario.settings }; } return { id: '', text: '' }; diff --git a/server/resolvers/ScenarioResult/assertionResultsResolver.js b/server/resolvers/ScenarioResult/assertionResultsResolver.js index 132a45f52..f537179d9 100644 --- a/server/resolvers/ScenarioResult/assertionResultsResolver.js +++ b/server/resolvers/ScenarioResult/assertionResultsResolver.js @@ -1,13 +1,33 @@ -const convertAssertionPriority = require('../helpers/convertAssertionPriority'); +const convertAssertionPriority = require('../../util/convertAssertionPriority'); const assertionResultsResolver = (scenarioResult, { priority }) => { if (!priority) return scenarioResult.assertionResults; - return scenarioResult.assertionResults.filter( - assertionResult => + return scenarioResult.assertionResults.filter(assertionResult => { + if (assertionResult.assertion?.assertionExceptions?.length) { + const scenarioSettings = scenarioResult.scenario.settings; + const scenarioCommandId = scenarioResult.scenario.commandIds[0]; + + const foundException = + assertionResult.assertion.assertionExceptions.find( + exception => + exception.settings === scenarioSettings && + exception.commandId === scenarioCommandId + ); + + if (foundException) { + return ( + convertAssertionPriority(foundException.priority) === + convertAssertionPriority(priority) + ); + } + } + + return ( convertAssertionPriority(assertionResult.assertion.priority) === convertAssertionPriority(priority) - ); + ); + }); }; module.exports = assertionResultsResolver; diff --git a/server/resolvers/TestPlanRunOperations/createTestResultSkeleton.js b/server/resolvers/TestPlanRunOperations/createTestResultSkeleton.js index ddc33eee8..30d6146f0 100644 --- a/server/resolvers/TestPlanRunOperations/createTestResultSkeleton.js +++ b/server/resolvers/TestPlanRunOperations/createTestResultSkeleton.js @@ -4,6 +4,25 @@ const { createAssertionResultId } = require('../../services/PopulatedData/locationOfDataId'); +/** + * Determine whether a given assertion belongs to a given scenario and includes + * at least one exception with a given priority. + * + * @param {Assertion} assertion + * @param {Scenario} scenario + * @param {string} priority + */ +const hasExceptionWithPriority = (assertion, scenario, priority) => { + return assertion.assertionExceptions?.some( + exception => + scenario.commands.find( + command => + command.id === exception.commandId && + command.atOperatingMode === exception.settings + ) && exception.priority === priority + ); +}; + const createTestResultSkeleton = ({ test, testPlanRun, @@ -30,16 +49,25 @@ const createTestResultSkeleton = ({ id: scenarioResultId, scenarioId: scenario.id, output: null, - assertionResults: test.assertions.map(assertion => { - return { + assertionResults: test.assertions + // Filter out assertionResults for the current scenario which were marked + // with a 0-priority exception + .filter( + assertion => + !hasExceptionWithPriority( + assertion, + scenario, + 'EXCLUDE' + ) + ) + .map(assertion => ({ id: createAssertionResultId( scenarioResultId, assertion.id ), assertionId: assertion.id, passed: null - }; - }), + })), unexpectedBehaviors: null }; }) diff --git a/server/resolvers/helpers/convertAssertionPriority.js b/server/resolvers/helpers/convertAssertionPriority.js deleted file mode 100644 index 9590a6c49..000000000 --- a/server/resolvers/helpers/convertAssertionPriority.js +++ /dev/null @@ -1,7 +0,0 @@ -const convertAssertionPriority = priority => { - if (priority === 'REQUIRED') return 'MUST'; - if (priority === 'OPTIONAL') return 'SHOULD'; - return priority; -}; - -module.exports = convertAssertionPriority; diff --git a/server/scripts/import-tests/index.js b/server/scripts/import-tests/index.js index feb320d55..66c61447b 100644 --- a/server/scripts/import-tests/index.js +++ b/server/scripts/import-tests/index.js @@ -27,6 +27,7 @@ const { const deepPickEqual = require('../../util/deepPickEqual'); const { hashTests } = require('../../util/aria'); const convertDateToString = require('../../util/convertDateToString'); +const convertAssertionPriority = require('../../util/convertAssertionPriority'); const args = require('minimist')(process.argv.slice(2), { alias: { @@ -465,6 +466,28 @@ const getTests = ({ tokenizedAssertionStatement || assertion.assertionStatement; result.assertionPhrase = assertion.assertionPhrase; + result.assertionExceptions = data.commands.flatMap( + command => { + return command.assertionExceptions + .filter( + exception => + exception.assertionId === + assertion.assertionId + ) + .map(({ priority: assertionPriority }) => { + let priority = + convertAssertionPriority( + assertionPriority + ); + + return { + priority, + commandId: command.id, + settings: command.settings + }; + }); + } + ); } return result; diff --git a/server/tests/integration/graphql.test.js b/server/tests/integration/graphql.test.js index aaaa5255f..2f8e59583 100644 --- a/server/tests/integration/graphql.test.js +++ b/server/tests/integration/graphql.test.js @@ -155,6 +155,7 @@ describe('graphql', () => { ['TestPlanVersion', 'recommendedPhaseTargetDate'], ['TestPlanVersion', 'deprecatedAt'], ['Test', 'viewers'], + ['Command', 'atOperatingMode'], // TODO: Include when v2 test format CI tests are done ['CollectionJob', 'testPlanRun'], ['CollectionJob', 'externalLogsUrl'], // These interact with Response Scheduler API @@ -305,6 +306,7 @@ describe('graphql', () => { __typename id text + atOperatingMode } } assertions { @@ -560,7 +562,6 @@ describe('graphql', () => { } ` ); - // console.info(queryResult); await dbCleaner(async () => { const { diff --git a/server/util/convertAssertionPriority.js b/server/util/convertAssertionPriority.js new file mode 100644 index 000000000..9265ae177 --- /dev/null +++ b/server/util/convertAssertionPriority.js @@ -0,0 +1,23 @@ +/** + * This should take any of the valid priority inputs and return the currently known string + * representation of that value. + * TODO: Eventually, this should only need to take 'number' types as priority once 'REQUIRED' and + * 'OPTIONAL' are no longer used + * + * @param {number|string} priority + * @returns {null|string} + */ +const convertAssertionPriority = priority => { + const validInputRegex = + /^(0|1|2|3|EXCLUDE|REQUIRED|MUST|OPTIONAL|SHOULD|MAY)$/; + + if (!validInputRegex.test(priority)) return null; + + if (priority === 0) return 'EXCLUDE'; + if (priority === 'REQUIRED' || priority === 1) return 'MUST'; + if (priority === 'OPTIONAL' || priority === 2) return 'SHOULD'; + if (priority === 3) return 'MAY'; + return priority; +}; + +module.exports = convertAssertionPriority; diff --git a/server/util/convertAssertionPriority.test.js b/server/util/convertAssertionPriority.test.js new file mode 100644 index 000000000..3a5799bb0 --- /dev/null +++ b/server/util/convertAssertionPriority.test.js @@ -0,0 +1,55 @@ +const convertAssertionPriority = require('./convertAssertionPriority'); + +describe('Verify expected values are returned when calling convertAssertionPriority', () => { + it('expects valid priority strings are returned for known priority inputs', () => { + const excludePriorityA = convertAssertionPriority(0); + const excludePriorityB = convertAssertionPriority('EXCLUDE'); + + const mustPriorityA = convertAssertionPriority(1); + const mustPriorityB = convertAssertionPriority('REQUIRED'); + const mustPriorityC = convertAssertionPriority('MUST'); + + const shouldPriorityA = convertAssertionPriority(2); + const shouldPriorityB = convertAssertionPriority('OPTIONAL'); + const shouldPriorityC = convertAssertionPriority('SHOULD'); + + const mayPriorityA = convertAssertionPriority(3); + const mayPriorityB = convertAssertionPriority('MAY'); + + expect(excludePriorityA).toEqual('EXCLUDE'); + expect(excludePriorityB).toEqual('EXCLUDE'); + + expect(mustPriorityA).toEqual('MUST'); + expect(mustPriorityB).toEqual('MUST'); + expect(mustPriorityC).toEqual('MUST'); + + expect(shouldPriorityA).toEqual('SHOULD'); + expect(shouldPriorityB).toEqual('SHOULD'); + expect(shouldPriorityC).toEqual('SHOULD'); + + expect(mayPriorityA).toEqual('MAY'); + expect(mayPriorityB).toEqual('MAY'); + }); + + it('expects null values are returned for unknown priority inputs', () => { + const invalidInputA = convertAssertionPriority(-1); + const invalidInputB = convertAssertionPriority(4); + const invalidInputC = convertAssertionPriority(12); + const invalidInputD = convertAssertionPriority('EXCLUDED'); + const invalidInputE = convertAssertionPriority('RANDOM STRING'); + + expect(invalidInputA).not.toEqual('MUST'); + expect(invalidInputA).toEqual(null); + + expect(invalidInputB).toEqual(null); + + expect(invalidInputC).not.toEqual('MUST'); + expect(invalidInputC).not.toEqual('SHOULD'); + expect(invalidInputC).toEqual(null); + + expect(invalidInputD).not.toEqual('EXCLUDE'); + expect(invalidInputD).toEqual(null); + + expect(invalidInputE).toEqual(null); + }); +}); diff --git a/server/util/getMetrics.js b/server/util/getMetrics.js index d9f1168d2..d5b7e9feb 100644 --- a/server/util/getMetrics.js +++ b/server/util/getMetrics.js @@ -1,4 +1,4 @@ -const convertAssertionPriority = require('../resolvers/helpers/convertAssertionPriority'); +const convertAssertionPriority = require('./convertAssertionPriority'); const sum = arr => arr.reduce((total, item) => total + item, 0);