From 61bf0e348639d91fb999415eca7459afbac241ba Mon Sep 17 00:00:00 2001 From: "reportportal.io" Date: Tue, 13 Sep 2022 17:49:03 +0000 Subject: [PATCH 1/4] 5.0.6 -> 5.0.7-SNAPSHOT --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index c20c645..eeefa9f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.6 +5.0.7-SNAPSHOT From aac59fa6a224b3fae89a26837d8fe899e17ad267 Mon Sep 17 00:00:00 2001 From: Ilya Hancharyk Date: Thu, 15 Sep 2022 18:22:21 +0300 Subject: [PATCH 2/4] EPMRPP-79577 || Suites finishing mechanism refactoring --- .../reporter/finishSuiteReporting.spec.ts | 38 +++++- .../reporter/finishTestItemReporting.spec.ts | 39 +++++- src/__tests__/reporter/logReporting.spec.ts | 46 ++++--- .../reporter/startSuiteTestReporting.spec.ts | 19 +-- src/__tests__/utils.spec.ts | 18 --- src/reporter.ts | 115 +++++++++--------- src/utils.ts | 7 -- 7 files changed, 163 insertions(+), 119 deletions(-) diff --git a/src/__tests__/reporter/finishSuiteReporting.spec.ts b/src/__tests__/reporter/finishSuiteReporting.spec.ts index 48844f0..0259ad9 100644 --- a/src/__tests__/reporter/finishSuiteReporting.spec.ts +++ b/src/__tests__/reporter/finishSuiteReporting.spec.ts @@ -28,12 +28,22 @@ describe('finish report suite', () => { reporter.launchId = 'tempLaunchId'; const testParams = { - title: 'test', + title: 'testTitle', parent: { - title: suiteName, + title: rootSuite, project: () => ({ name: rootSuite }), + allTests: () => [ + { title: 'testTitle', titlePath: () => ['', rootSuite, suiteName, 'testTitle'] }, + ], + parent: { + title: rootSuite, + project: () => ({ name: rootSuite }), + allTests: () => [ + { title: 'testTitle', titlePath: () => ['', rootSuite, suiteName, 'testTitle'] }, + ], + }, }, - titlePath: () => [rootSuite, suiteName, 'testTitle'], + titlePath: () => ['', rootSuite, suiteName, 'testTitle'], location: { file: `C:${path.sep}testProject${path.sep}tests${path.sep}example.js`, }, @@ -44,13 +54,29 @@ describe('finish report suite', () => { }; reporter.testItems = new Map([ - ['tempTestItemId', { id: 'tempTestItemId', name: 'test', playwrightProjectName: rootSuite }], + [ + 'tempTestItemId', + { id: 'tempTestItemId', name: 'testTitle', playwrightProjectName: rootSuite }, + ], ]); reporter.suites = new Map([ - [rootSuite, { id: 'rootsuiteId', name: rootSuite, rootSuiteLength: 1, rootSuite }], + [ + rootSuite, + { + id: 'rootsuiteId', + name: rootSuite, + testCount: 1, + descendants: [`${rootSuite}/${suiteName}/testTitle`], + }, + ], [ `${rootSuite}/${suiteName}`, - { id: 'parentSuiteId', name: suiteName, testsLength: 1, rootSuite }, + { + id: 'parentSuiteId', + name: suiteName, + testCount: 1, + descendants: [`${rootSuite}/${suiteName}/testTitle`], + }, ], ]); diff --git a/src/__tests__/reporter/finishTestItemReporting.spec.ts b/src/__tests__/reporter/finishTestItemReporting.spec.ts index b8d7a83..043ead1 100644 --- a/src/__tests__/reporter/finishTestItemReporting.spec.ts +++ b/src/__tests__/reporter/finishTestItemReporting.spec.ts @@ -28,11 +28,30 @@ describe('finish test reporting', () => { reporter.client = new RPClientMock(mockConfig); reporter.launchId = 'tempLaunchId'; reporter.testItems = new Map([ - ['tempTestItemId', { id: 'tempTestItemId', name: 'test', playwrightProjectName: rootSuite }], + [ + 'tempTestItemId', + { id: 'tempTestItemId', name: 'testTitle', playwrightProjectName: rootSuite }, + ], ]); reporter.suites = new Map([ - [rootSuite, { id: 'rootsuiteId', name: rootSuite, rootSuiteLength: 1, rootSuite }], - [`${rootSuite}/${suiteName}`, { id: 'suiteId', name: suiteName, testsLength: 1, rootSuite }], + [ + rootSuite, + { + id: 'rootsuiteId', + name: rootSuite, + testCount: 1, + descendants: [`${rootSuite}/${suiteName}/testTitle`], + }, + ], + [ + `${rootSuite}/${suiteName}`, + { + id: 'suiteId', + name: suiteName, + testCount: 1, + descendants: [`${rootSuite}/${suiteName}/testTitle`], + }, + ], ]); const attributes = [ { @@ -44,10 +63,20 @@ describe('finish test reporting', () => { const description = 'description'; const testParams = { - title: 'test', + title: 'testTitle', parent: { - title: suiteName, + title: rootSuite, project: () => ({ name: rootSuite }), + allTests: () => [ + { title: 'testTitle', titlePath: () => ['', rootSuite, suiteName, 'testTitle'] }, + ], + parent: { + title: rootSuite, + project: () => ({ name: rootSuite }), + allTests: () => [ + { title: 'testTitle', titlePath: () => ['', rootSuite, suiteName, 'testTitle'] }, + ], + }, }, titlePath: () => [rootSuite, suiteName, 'testTitle'], location: { diff --git a/src/__tests__/reporter/logReporting.spec.ts b/src/__tests__/reporter/logReporting.spec.ts index 0f5690b..6cb08fc 100644 --- a/src/__tests__/reporter/logReporting.spec.ts +++ b/src/__tests__/reporter/logReporting.spec.ts @@ -91,40 +91,56 @@ describe('logs reporting', () => { [ playwrightProjectName, { - id: tempTestItemId, + id: 'rootSuiteId', name: playwrightProjectName, - testsLength: 0, - rootSuite: playwrightProjectName, - rootSuiteLength: 1, + testCount: 1, + descendants: [`${playwrightProjectName}/${suiteName}/testTitle`], }, ], [ `${playwrightProjectName}/${suiteName}`, - { id: 'suiteId', name: suiteName, testsLength: 1, rootSuite: playwrightProjectName }, + { + id: 'suiteId', + name: suiteName, + testCount: 1, + descendants: [`${playwrightProjectName}/${suiteName}/testTitle`], + }, ], ]); reporter.testItems = new Map([ - [tempTestItemId, { id: tempTestItemId, name: 'test', playwrightProjectName }], + [tempTestItemId, { id: tempTestItemId, name: 'testTitle', playwrightProjectName }], ]); const testParams = { - title: 'test', + title: 'testTitle', parent: { title: playwrightProjectName, + location: 'tests/example.js', + tests: ['testTitle'], project: () => ({ name: playwrightProjectName }), + allTests: () => [ + { + title: 'testTitle', + titlePath: () => ['', playwrightProjectName, suiteName, 'testTitle'], + }, + ], + parent: { + title: suiteName, + location: 'tests/example.js', + project: () => ({ name: playwrightProjectName }), + allTests: () => [ + { + title: 'testTitle', + titlePath: () => ['', playwrightProjectName, suiteName, 'testTitle'], + }, + ], + }, }, location: { file: `C:${path.sep}testProject${path.sep}tests${path.sep}example.js`, line: 5, column: 3, }, - titlePath: () => [ - '', - playwrightProjectName, - 'tests/example.js', - 'rootDescribe', - 'parentDescribe', - 'testTitle', - ], + titlePath: () => ['', playwrightProjectName, suiteName, 'testTitle'], }; const result = { diff --git a/src/__tests__/reporter/startSuiteTestReporting.spec.ts b/src/__tests__/reporter/startSuiteTestReporting.spec.ts index 7203376..b44d4b8 100644 --- a/src/__tests__/reporter/startSuiteTestReporting.spec.ts +++ b/src/__tests__/reporter/startSuiteTestReporting.spec.ts @@ -34,14 +34,17 @@ describe('start reporting suite/test', () => { parent: { title: suiteName, location: 'tests/example.js', - tests: ['test'], project: () => ({ name: '' }), - allTests: () => ['test'], + allTests: () => [ + { title: 'testTitle', titlePath: () => ['', rootSuite, suiteName, 'testTitle'] }, + ], parent: { title: rootSuite, location: 'tests/example.js', project: () => ({ name: '' }), - allTests: () => ['test'], + allTests: () => [ + { title: 'testTitle', titlePath: () => ['', rootSuite, suiteName, 'testTitle'] }, + ], }, }, location: { @@ -61,9 +64,8 @@ describe('start reporting suite/test', () => { { id: 'tempTestItemId', name: rootSuite, - rootSuite, - rootSuiteLength: 1, - testsLength: 1, + testCount: 1, + descendants: [`${rootSuite}/${suiteName}/testTitle`], }, ], [ @@ -71,9 +73,8 @@ describe('start reporting suite/test', () => { { id: 'tempTestItemId', name: suiteName, - rootSuite, - rootSuiteLength: undefined, - testsLength: 1, + descendants: [`${rootSuite}/${suiteName}/testTitle`], + testCount: 1, }, ], ]); diff --git a/src/__tests__/utils.spec.ts b/src/__tests__/utils.spec.ts index 0c75f85..0d809ca 100644 --- a/src/__tests__/utils.spec.ts +++ b/src/__tests__/utils.spec.ts @@ -27,7 +27,6 @@ import { getAttachments, isErrorLog, convertToRpStatus, - getTestFilePath, } from '../utils'; import fs from 'fs'; import path from 'path'; @@ -287,21 +286,4 @@ describe('testing utils', () => { expect(status).not.toBe(STATUSES.FAILED); }); }); - describe('getTestFilePath', () => { - test('getTestFilePath should return test file path string', () => { - const mockedTest = { - title: 'first', - location: { - file: `C:${path.sep}project${path.sep}tests${path.sep}simpleTest.spec.ts`, - }, - titlePath: () => ['', 'project', 'tests/simpleTest.spec.ts', 'first'], - }; - - // @ts-ignore - const receivedValue = getTestFilePath(mockedTest, mockedTest.title); - const expectedValue = 'project/tests/simpleTest.spec.ts'; - - expect(receivedValue).toBe(expectedValue); - }); - }); }); diff --git a/src/reporter.ts b/src/reporter.ts index 91ad0fe..bbc0cac 100644 --- a/src/reporter.ts +++ b/src/reporter.ts @@ -39,7 +39,6 @@ import { getAttachments, getCodeRef, getSystemAttributes, - getTestFilePath, isErrorLog, isFalse, promiseErrorHandler, @@ -57,10 +56,9 @@ export interface TestItem { } interface Suite extends TestItem { - rootSuite?: string; logs?: LogRQ[]; - testsLength?: number; - rootSuiteLength?: number | undefined; + testCount?: number; + descendants?: string[]; } export class RPReporter implements Reporter { @@ -220,23 +218,8 @@ export class RPReporter implements Reporter { promiseErrorHandler(promise, 'Failed to send log'); } - finishSuites(testFileName?: string, rootSuiteName?: string): void { - let suitesToFinish: [string, Suite][]; - const suitesArray = Array.from(this.suites); - - const isTestsExistInRootSuite = this.suites.get(rootSuiteName).rootSuiteLength < 1; - - if (isTestsExistInRootSuite) { - suitesToFinish = testFileName - ? suitesArray.filter(([key]) => key.includes(rootSuiteName)) - : suitesArray; - } else { - suitesToFinish = testFileName - ? suitesArray.filter( - ([key, { testsLength }]) => key.includes(rootSuiteName) && testsLength < 1, - ) - : suitesArray; - } + finishSuites(): void { + const suitesToFinish = Array.from(this.suites).filter(([, { testCount }]) => testCount < 1); suitesToFinish.forEach(([key, { id, status, logs }]) => { if (logs) { @@ -274,10 +257,14 @@ export class RPReporter implements Reporter { this.launchId = tempId; } - findTestItem(testItems: Map, title: string, projectName?: string): Suite { - if (projectName !== undefined) { + findTestItem( + testItems: Map, + title: string, + playwrightProjectName?: string, + ): Suite { + if (playwrightProjectName !== undefined) { for (const [, value] of testItems) { - if (value.name === title && projectName === value.playwrightProjectName) { + if (value.name === title && playwrightProjectName === value.playwrightProjectName) { return value; } } @@ -306,7 +293,8 @@ export class RPReporter implements Reporter { const projectName = test.parent.project().name; for (let i = lastSuiteIndex; i >= 0; i--) { - const currentSuiteTitle = orderedSuites[i].title; + const currentSuite = orderedSuites[i]; + const currentSuiteTitle = currentSuite.title; const fullSuiteName = getCodeRef(test, currentSuiteTitle); if (this.suites.get(fullSuiteName)?.id) { @@ -332,22 +320,20 @@ export class RPReporter implements Reporter { const suiteObj = this.client.startTestItem(startSuiteObj, this.launchId, parentId); this.addRequestToPromisesQueue(suiteObj.promise, 'Failed to start suite.'); - let rootSuiteLength = - i === lastSuiteIndex ? orderedSuites[lastSuiteIndex].allTests().length : undefined; - - let testsLength = orderedSuites[i].allTests().length; + const allSuiteTests = currentSuite.allTests(); + const descendants = allSuiteTests.map((testCase) => getCodeRef(testCase, testCase.title)); + let testCount = allSuiteTests.length; if (test.retries) { - testsLength = testsLength * (test.retries + 1); - rootSuiteLength = rootSuiteLength * (test.retries + 1); + const possibleInvocations = test.retries + 1; + testCount = testCount * possibleInvocations; } this.suites.set(fullSuiteName, { id: suiteObj.tempId, name: currentSuiteTitle, - testsLength, - rootSuiteLength, - rootSuite: getCodeRef(test, orderedSuites[lastSuiteIndex].title), + testCount, + descendants, ...(status && { status }), ...(logs && { logs }), // TODO: may be send it on suite start }); @@ -483,36 +469,47 @@ export class RPReporter implements Reporter { this.addRequestToPromisesQueue(promise, 'Failed to finish test.'); this.testItems.delete(testItemId); + this.updateAncestorsTestCount(test, result); + const fullParentName = getCodeRef(test, test.parent.title); - const parentObj = this.suites.get(fullParentName); - const rootSuiteName = parentObj.rootSuite; - const rootSuite = this.suites.get(rootSuiteName); - - const decreaseIndex = - test.retries > 0 && (result.status === STATUSES.PASSED || result.status === STATUSES.SKIPPED) - ? test.retries + 1 - : 1; - - this.suites.set(rootSuiteName, { - ...rootSuite, - rootSuiteLength: rootSuite.rootSuiteLength - decreaseIndex, - }); - const testFilePath = getTestFilePath(test, test.title); + // if all children of the test parent have already finished, then finish all empty ancestors + if (this.suites.get(fullParentName).testCount < 1) { + this.finishSuites(); + } + } + + updateAncestorsTestCount(test: TestCase, result: TestResult): void { + // decrease by 1 by default as only one test case finished + let decreaseIndex = 1; + const nonRetriedResult = + result.status === STATUSES.PASSED || result.status === STATUSES.SKIPPED; + + // if test case has retries, and it will not be retried anymore + if (test.retries > 0 && nonRetriedResult) { + const possibleInvocations = test.retries + 1; + const possibleInvocationsLeft = possibleInvocations - test.results.length; + // we need to decrease also all the rest possible invocations as the test case will not be retried anymore + decreaseIndex = decreaseIndex + possibleInvocationsLeft; + } + + const fullTestName = getCodeRef(test, test.title); - Array.from(this.suites) - .filter(([key]) => key.includes(fullParentName) || key === testFilePath) - .map(([key, { testsLength }]) => { + this.suites.forEach((value, key) => { + const { descendants, testCount } = value; + + if (descendants.length && descendants.includes(fullTestName)) { + const newTestCount = testCount - decreaseIndex; this.suites.set(key, { - ...this.suites.get(key), - testsLength: testsLength - decreaseIndex, + ...value, + testCount: newTestCount, + descendants: + newTestCount < 1 + ? descendants.filter((testName) => testName !== fullTestName) + : descendants, }); - }); - - // if all children of the test parent have already finished, then finish the parent - if (this.suites.get(fullParentName).testsLength < 1) { - this.finishSuites(testFilePath, rootSuiteName); - } + } + }); } async onEnd(): Promise { diff --git a/src/utils.ts b/src/utils.ts index 775206a..9679a8f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -81,13 +81,6 @@ export const getCodeRef = ( .replace(new RegExp('\\'.concat(path.sep), 'g'), '/'); }; -export const getTestFilePath = (testItem: testItemPick, itemTitle: string): string => { - const codeRefArray = getCodeRef(testItem, itemTitle).split('/'); - const testFileName = path.parse(testItem.location.file).base; - const testFileNameIndex = codeRefArray.indexOf(testFileName); - return codeRefArray.slice(0, testFileNameIndex + 1).join('/'); -}; - export const sendEventToReporter = (type: string, data: any, suite?: string): void => { process.stdout.write(JSON.stringify({ type, data, suite })); }; From 1c0f065f0410ccdca71c0df25086340950e0cd36 Mon Sep 17 00:00:00 2001 From: Ilya Hancharyk Date: Thu, 15 Sep 2022 18:26:41 +0300 Subject: [PATCH 3/4] EPMRPP-79577 || Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47bf05..cb8f92a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +### Fixed +- Suites finishing algorithm bugfixes ## [5.0.6] - 2022-09-13 ### Fixed From 924bd3822c9a281e10f0ac593b06c64c92bb6c1a Mon Sep 17 00:00:00 2001 From: Ilya Hancharyk Date: Thu, 15 Sep 2022 18:35:17 +0300 Subject: [PATCH 4/4] Update readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 73890ad..b2c3dfb 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,10 @@ npm install --save-dev @reportportal/agent-js-playwright ``` ## Reporting +When organizing tests, specify titles for `test.describe` blocks, as this is necessary to build the correct structure of reports. + +It is also required to specify playwright project names in `playwright.config.ts` when running the same tests in different playwright projects. + ### Attachments Attachments can be easily added during test run via `testInfo.attachments` according to the Playwright [docs](https://playwright.dev/docs/api/class-testinfo#test-info-attachments).