From fbf0c5f71ea0ee19a1525d42b3ca0f28b8da09ac Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Wed, 25 Oct 2023 11:54:04 -0500 Subject: [PATCH 1/9] Hide mark as final button --- client/components/TestQueue/queries.js | 5 +++++ client/components/TestQueueRow/index.jsx | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/client/components/TestQueue/queries.js b/client/components/TestQueue/queries.js index c5e44af88..7b26b3458 100644 --- a/client/components/TestQueue/queries.js +++ b/client/components/TestQueue/queries.js @@ -49,6 +49,11 @@ export const TEST_QUEUE_PAGE_QUERY = gql` id name } + runnableTests { + id + title + renderedUrl + } testPlanVersion { id title diff --git a/client/components/TestQueueRow/index.jsx b/client/components/TestQueueRow/index.jsx index b8df4534b..f2d4e635c 100644 --- a/client/components/TestQueueRow/index.jsx +++ b/client/components/TestQueueRow/index.jsx @@ -23,6 +23,7 @@ import TestPlanUpdaterModal from '../TestPlanUpdater/TestPlanUpdaterModal'; import BasicThemedModal from '../common/BasicThemedModal'; import { LoadingStatus, useTriggerLoad } from '../common/LoadingStatus'; import './TestQueueRow.css'; +import { differenceBy } from 'lodash'; const TestQueueRow = ({ user = {}, @@ -414,6 +415,12 @@ const TestQueueRow = ({ 'completed' ].join('-'); + const completedAllTests = !testPlanReport.draftTestPlanRuns.find( + testPlanRun => + testPlanRun.testResultsLength !== + testPlanReport.runnableTests.length + ); + return ( @@ -494,7 +501,8 @@ const TestQueueRow = ({ !testPlanReport.conflictsLength && testPlanReport.draftTestPlanRuns.length && testPlanReport.draftTestPlanRuns[0] - .testResultsLength ? ( + .testResultsLength && + completedAllTests ? ( <> - {skippedTests.length ? ( - - {skippedTests.length} Tests Were Skipped - - ) : null} Date: Mon, 6 Nov 2023 15:07:56 -0500 Subject: [PATCH 4/9] Add migration for unfinished tests --- .../20231031162340-verifyNoSkippedTests.js | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 server/migrations/20231031162340-verifyNoSkippedTests.js diff --git a/server/migrations/20231031162340-verifyNoSkippedTests.js b/server/migrations/20231031162340-verifyNoSkippedTests.js new file mode 100644 index 000000000..b84d8466a --- /dev/null +++ b/server/migrations/20231031162340-verifyNoSkippedTests.js @@ -0,0 +1,67 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface /* , Sequelize */) { + return queryInterface.sequelize.transaction(async transaction => { + const [rows] = await queryInterface.sequelize.query( + ` + SELECT + "TestPlanReport".id, + "TestPlanReport"."atId", + "TestPlanRun"."testResults", + "TestPlanVersion".tests + FROM + "TestPlanReport" + INNER JOIN "TestPlanVersion" ON "TestPlanVersion".id = "TestPlanReport"."testPlanVersionId" + INNER JOIN "TestPlanRun" ON "TestPlanReport".id = "TestPlanRun"."testPlanReportId" + WHERE + "TestPlanReport"."markedFinalAt" IS NOT NULL + `, + { transaction } + ); + + const dataById = {}; + rows.forEach(({ id, atId, testResults, tests }) => { + if (dataById[id]) { + dataById[id].runLengths.push(testResults.length); + } else { + const runnableTestsLength = tests.filter(test => + test.atIds.includes(atId) + ).length; + dataById[id] = { + id, + runnableTestsLength, + runLengths: [testResults.length] + }; + } + }); + + for (const data of Object.values(dataById)) { + let isComplete = true; + + data.runLengths.forEach(runLength => { + if (runLength !== data.runnableTestsLength) { + isComplete = false; + } + }); + if (!isComplete) { + console.info(`Found incomplete report ${data.id}`); // eslint-disable-line + // Since final test plan reports can + // no longer have skipped tests, we should + // move them to the test queue where incomplete + // runs are still supported. + await queryInterface.sequelize.query( + ` + UPDATE "TestPlanReport" SET "markedFinalAt" = NULL WHERE id = ? + `, + { + replacements: [data.id], + transaction + } + ); + } + } + }); + } +}; From d5e12cc5909e1d99783531450bf8c8b48ae72b70 Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:09:18 -0500 Subject: [PATCH 5/9] Simplify necessary files --- .../finalizedTestResultsResolver.js | 27 +++----- server/resolvers/testPlanReportsResolver.js | 7 +- server/scripts/populate-test-data/index.js | 6 +- .../populateFakeTestResults.js | 69 +++++++++++++------ 4 files changed, 61 insertions(+), 48 deletions(-) diff --git a/server/resolvers/TestPlanReport/finalizedTestResultsResolver.js b/server/resolvers/TestPlanReport/finalizedTestResultsResolver.js index 89cb7ccdb..fae66da6c 100644 --- a/server/resolvers/TestPlanReport/finalizedTestResultsResolver.js +++ b/server/resolvers/TestPlanReport/finalizedTestResultsResolver.js @@ -1,31 +1,20 @@ const testResultsResolver = require('../TestPlanRun/testResultsResolver'); -const deepCustomMerge = require('../../util/deepCustomMerge'); -/** - * Completed test results sourced from all the report's runs. The runs must be - * merged because each run might have skipped different tests. - */ const finalizedTestResultsResolver = async (testPlanReport, _, context) => { if (!testPlanReport.testPlanRuns.length) { return null; } - let merged = []; + // Since conflicts are now resolved, all testPlanRuns are interchangeable. + const testPlanRun = testPlanReport.testPlanRuns[0]; - for (let i = 0; i < testPlanReport.testPlanRuns.length; i += 1) { - merged = deepCustomMerge( - merged, - testPlanReport.testPlanRuns[i].testResults.filter( - testResult => !!testResult.completedAt - ), - { - identifyArrayItem: item => - item.testId ?? item.scenarioId ?? item.assertionId - } - ); - } return testResultsResolver( - { testPlanReport, testResults: merged }, + { + testPlanReport, + testResults: testPlanRun.testResults.filter( + testResult => !!testResult.completedAt + ) + }, null, context ); diff --git a/server/resolvers/testPlanReportsResolver.js b/server/resolvers/testPlanReportsResolver.js index fcbcae476..f8c3b7d8c 100644 --- a/server/resolvers/testPlanReportsResolver.js +++ b/server/resolvers/testPlanReportsResolver.js @@ -30,12 +30,7 @@ const testPlanReportsResolver = async ( attributes: testPlanReportAttributes } = retrieveAttributes('testPlanReport', TEST_PLAN_REPORT_ATTRIBUTES, info); - const { attributes: testPlanRunAttributes } = retrieveAttributes( - 'draftTestPlanRuns', - TEST_PLAN_RUN_ATTRIBUTES, - info, - true - ); + const testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES; const { attributes: testPlanVersionAttributes } = retrieveAttributes( 'testPlanVersion', diff --git a/server/scripts/populate-test-data/index.js b/server/scripts/populate-test-data/index.js index 9c285042e..2fbb791d3 100644 --- a/server/scripts/populate-test-data/index.js +++ b/server/scripts/populate-test-data/index.js @@ -64,7 +64,7 @@ const populateTestDatabase = async () => { await populateFakeTestResults(4, [ 'completeAndPassing', - null, + 'completeAndPassing', 'completeAndFailingDueToIncorrectAssertions', 'completeAndFailingDueToNoOutputAssertions', 'completeAndFailingDueToUnexpectedBehaviors', @@ -78,7 +78,7 @@ const populateTestDatabase = async () => { await populateFakeTestResults(5, [ 'completeAndPassing', - null, + 'completeAndPassing', 'completeAndFailingDueToIncorrectAssertions', 'completeAndPassing', 'completeAndFailingDueToNoOutputAssertions', @@ -89,7 +89,7 @@ const populateTestDatabase = async () => { await populateFakeTestResults(6, [ 'completeAndPassing', - null, + 'completeAndPassing', 'completeAndFailingDueToIncorrectAssertions', 'completeAndPassing', 'completeAndFailingDueToIncorrectAssertions', diff --git a/server/scripts/populate-test-data/populateFakeTestResults.js b/server/scripts/populate-test-data/populateFakeTestResults.js index 0278cf0c0..fce144d21 100644 --- a/server/scripts/populate-test-data/populateFakeTestResults.js +++ b/server/scripts/populate-test-data/populateFakeTestResults.js @@ -6,7 +6,29 @@ const { const { query, mutate } = require('../../tests/util/graphql-test-utilities'); const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { + const { + populateData: { testPlanReport } + } = await query(gql` + query { + populateData(locationOfData: { testPlanRunId: ${testPlanRunId} }) { + testPlanReport { + isFinal + at { + id + } + browser { + id + } + runnableTests { + id + } + } + } + } + `); + let index = 0; + for (const fakeTestResultType of fakeTestResultTypes) { // eslint-disable-next-line no-console console.info( @@ -21,6 +43,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'completeAndPassing': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'passing', @@ -29,6 +52,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'completeAndFailingDueToIncorrectAssertions': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'failingDueToIncorrectAssertions', @@ -37,6 +61,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'completeAndFailingDueToNoOutputAssertions': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'failingDueToNoOutputAssertions', @@ -45,6 +70,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'completeAndFailingDueToUnexpectedBehaviors': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'failingDueToUnexpectedBehaviors', @@ -53,6 +79,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'completeAndFailingDueToMultiple': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'failingDueToMultiple', @@ -61,6 +88,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'incompleteAndEmpty': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'empty', @@ -69,6 +97,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'incompleteAndPassing': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'passing', @@ -77,6 +106,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'incompleteAndFailingDueToIncorrectAssertions': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'failingDueToIncorrectAssertions', @@ -85,6 +115,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'incompleteAndFailingDueToNoOutputAssertions': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'failingDueToNoOutputAssertions', @@ -93,6 +124,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'incompleteAndFailingDueToUnexpectedBehaviors': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'failingDueToUnexpectedBehaviors', @@ -101,6 +133,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { break; case 'incompleteAndFailingDueToMultiple': await getFake({ + testPlanReport, testPlanRunId, index, fakeTestResultType: 'failingDueToMultiple', @@ -115,34 +148,30 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { index += 1; } + + if ( + testPlanReport.isFinal && + testPlanReport.runnableTests.length - fakeTestResultTypes.length !== 0 + ) { + for (let i = index; i < testPlanReport.runnableTests.length; i += 1) { + await getFake({ + testPlanReport, + testPlanRunId, + index: i, + fakeTestResultType: 'passing', + submit: true + }); + } + } }; const getFake = async ({ + testPlanReport, testPlanRunId, index, fakeTestResultType, submit }) => { - const { - populateData: { testPlanReport } - } = await query(gql` - query { - populateData(locationOfData: { testPlanRunId: ${testPlanRunId} }) { - testPlanReport { - at { - id - } - browser { - id - } - runnableTests { - id - } - } - } - } - `); - const testId = testPlanReport.runnableTests[index].id; const atVersion = await getAtVersionByQuery( From 24620e78205c2e817f8b5466578cd56d40221137 Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Wed, 8 Nov 2023 15:19:42 -0500 Subject: [PATCH 6/9] Fix failing tests --- client/components/TestQueue/queries.js | 5 ----- client/components/TestQueueRow/index.jsx | 3 +-- .../scripts/populate-test-data/populateFakeTestResults.js | 6 +----- server/tests/integration/graphql.test.js | 4 ++-- 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/client/components/TestQueue/queries.js b/client/components/TestQueue/queries.js index 26b0d8d12..37f6b2d52 100644 --- a/client/components/TestQueue/queries.js +++ b/client/components/TestQueue/queries.js @@ -61,11 +61,6 @@ export const TEST_QUEUE_PAGE_QUERY = gql` id name } - runnableTests { - id - title - renderedUrl - } testPlanVersion { id title diff --git a/client/components/TestQueueRow/index.jsx b/client/components/TestQueueRow/index.jsx index f6729064b..8bb97baf0 100644 --- a/client/components/TestQueueRow/index.jsx +++ b/client/components/TestQueueRow/index.jsx @@ -416,8 +416,7 @@ const TestQueueRow = ({ const completedAllTests = !testPlanReport.draftTestPlanRuns.find( testPlanRun => - testPlanRun.testResultsLength !== - testPlanReport.runnableTests.length + testPlanRun.testResultsLength !== testPlanReport.runnableTestsLength ); return ( diff --git a/server/scripts/populate-test-data/populateFakeTestResults.js b/server/scripts/populate-test-data/populateFakeTestResults.js index fce144d21..d60dbb2df 100644 --- a/server/scripts/populate-test-data/populateFakeTestResults.js +++ b/server/scripts/populate-test-data/populateFakeTestResults.js @@ -12,7 +12,6 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { query { populateData(locationOfData: { testPlanRunId: ${testPlanRunId} }) { testPlanReport { - isFinal at { id } @@ -149,10 +148,7 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { index += 1; } - if ( - testPlanReport.isFinal && - testPlanReport.runnableTests.length - fakeTestResultTypes.length !== 0 - ) { + if (testPlanReport.runnableTests.length !== fakeTestResultTypes.length) { for (let i = index; i < testPlanReport.runnableTests.length; i += 1) { await getFake({ testPlanReport, diff --git a/server/tests/integration/graphql.test.js b/server/tests/integration/graphql.test.js index 3a7a3f8db..08414505b 100644 --- a/server/tests/integration/graphql.test.js +++ b/server/tests/integration/graphql.test.js @@ -595,13 +595,13 @@ describe('graphql', () => { } } } - markReportAsFinal: testPlanReport(id: 2) { + markReportAsFinal: testPlanReport(id: 8) { __typename markAsFinal { locationOfData } } - unmarkReportAsFinal: testPlanReport(id: 2) { + unmarkReportAsFinal: testPlanReport(id: 8) { __typename unmarkAsFinal { locationOfData From 5a6072f7f48e01d9c15db88a87e6ceb3d9c168cb Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Tue, 21 Nov 2023 09:53:23 -0500 Subject: [PATCH 7/9] Add code review fixes --- client/components/TestQueueRow/index.jsx | 4 ++-- .../migrations/20231031162340-verifyNoSkippedTests.js | 10 +++------- .../populate-test-data/populateFakeTestResults.js | 4 ++++ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/components/TestQueueRow/index.jsx b/client/components/TestQueueRow/index.jsx index 8bb97baf0..9e3b44790 100644 --- a/client/components/TestQueueRow/index.jsx +++ b/client/components/TestQueueRow/index.jsx @@ -414,9 +414,9 @@ const TestQueueRow = ({ 'completed' ].join('-'); - const completedAllTests = !testPlanReport.draftTestPlanRuns.find( + const completedAllTests = testPlanReport.draftTestPlanRuns.every( testPlanRun => - testPlanRun.testResultsLength !== testPlanReport.runnableTestsLength + testPlanRun.testResultsLength === testPlanReport.runnableTestsLength ); return ( diff --git a/server/migrations/20231031162340-verifyNoSkippedTests.js b/server/migrations/20231031162340-verifyNoSkippedTests.js index b84d8466a..c186aaa84 100644 --- a/server/migrations/20231031162340-verifyNoSkippedTests.js +++ b/server/migrations/20231031162340-verifyNoSkippedTests.js @@ -38,13 +38,9 @@ module.exports = { }); for (const data of Object.values(dataById)) { - let isComplete = true; - - data.runLengths.forEach(runLength => { - if (runLength !== data.runnableTestsLength) { - isComplete = false; - } - }); + const isComplete = data.runLengths.every( + runLength => runLength === data.runnableTestsLength + ); if (!isComplete) { console.info(`Found incomplete report ${data.id}`); // eslint-disable-line // Since final test plan reports can diff --git a/server/scripts/populate-test-data/populateFakeTestResults.js b/server/scripts/populate-test-data/populateFakeTestResults.js index d60dbb2df..a0c0e6d43 100644 --- a/server/scripts/populate-test-data/populateFakeTestResults.js +++ b/server/scripts/populate-test-data/populateFakeTestResults.js @@ -148,6 +148,10 @@ const populateFakeTestResults = async (testPlanRunId, fakeTestResultTypes) => { index += 1; } + // The following makes it so that fake tests are made for + // all possible tests that can be completed for any given + // test plan. Therefore, there will be no missing or skipped + // fake tests in the database. if (testPlanReport.runnableTests.length !== fakeTestResultTypes.length) { for (let i = index; i < testPlanReport.runnableTests.length; i += 1) { await getFake({ From 25db202bd3719e2b389584f0f8fd379744aeda92 Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Tue, 21 Nov 2023 12:11:09 -0500 Subject: [PATCH 8/9] Remove no-longer-necessary style class --- client/components/Reports/Reports.css | 31 --------------------------- client/components/TestRun/TestRun.css | 2 +- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/client/components/Reports/Reports.css b/client/components/Reports/Reports.css index e6c96b768..2a3768dbd 100644 --- a/client/components/Reports/Reports.css +++ b/client/components/Reports/Reports.css @@ -56,36 +56,6 @@ margin-top: 1em; } -.skipped-tests-heading { - background: #fefbea; - padding: 1.75em 1.75em 1em 1.75em; - border: 1px solid #eddfbe; - border-bottom: 0; - margin-top: 2em; -} - -.skipped-tests { - columns: 2; - line-height: 2; - border: 1px solid #eddfbe; - padding: 1.5em 3em; - background: #fdfcf6; -} - -.skipped-tests li { - margin-bottom: 0.5em; - border-bottom: 1px solid #eddfbe; - padding-bottom: 0.5em; - margin-right: 3em; - -webkit-column-break-inside: avoid; - page-break-inside: avoid; - break-inside: avoid; -} - -.skipped-tests li:last-child { - border-bottom: none; -} - .test-result-heading { display: flex; justify-content: space-between; @@ -100,7 +70,6 @@ margin-bottom: 0; } -.skipped-tests-heading h2, .test-result-heading h2 { margin-top: 0; } diff --git a/client/components/TestRun/TestRun.css b/client/components/TestRun/TestRun.css index ddaf9e32b..6a3b3f001 100644 --- a/client/components/TestRun/TestRun.css +++ b/client/components/TestRun/TestRun.css @@ -135,7 +135,7 @@ button.test-navigator-toggle:focus { left: 4px; } -.test-name-wrapper.skipped .progress-indicator { +.test-name-wrapper .progress-indicator { background: White; border: 2px dashed black; } From 9729dc827a6f15334b907e36715f77dbcf952cec Mon Sep 17 00:00:00 2001 From: Paul Clue <67766160+Paul-Clue@users.noreply.github.com> Date: Tue, 21 Nov 2023 18:27:49 -0500 Subject: [PATCH 9/9] Remove reference --- server/resolvers/testPlanReportsResolver.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/resolvers/testPlanReportsResolver.js b/server/resolvers/testPlanReportsResolver.js index f8c3b7d8c..43090c157 100644 --- a/server/resolvers/testPlanReportsResolver.js +++ b/server/resolvers/testPlanReportsResolver.js @@ -30,8 +30,6 @@ const testPlanReportsResolver = async ( attributes: testPlanReportAttributes } = retrieveAttributes('testPlanReport', TEST_PLAN_REPORT_ATTRIBUTES, info); - const testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES; - const { attributes: testPlanVersionAttributes } = retrieveAttributes( 'testPlanVersion', TEST_PLAN_VERSION_ATTRIBUTES, @@ -59,7 +57,7 @@ const testPlanReportsResolver = async ( null, where, testPlanReportAttributes, - testPlanRunAttributes, + TEST_PLAN_RUN_ATTRIBUTES, testPlanVersionAttributes, null, null,