Merge pull request #2219 from epam/feat/e2e_improvements

Feat/e2e improvements
epam · May 2, 2024 · 075a695 · 075a695
2 parents fd43dfb + 1e5e50f
commit 075a695
Show file tree

Hide file tree

Showing 17 changed files with 191 additions and 124 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -33,7 +33,9 @@ jobs:
                   cd server && yarn
 
             - name: 'Run e2e tests'
-              run: yarn --cwd uui-e2e-tests local-test-e2e
+              run: |
+                  yarn --cwd uui-e2e-tests local-check-issues
+                  yarn --cwd uui-e2e-tests local-test-e2e
 
             - name: 'Upload e2e HTML report'
               if: always()

diff --git a/uui-e2e-tests/.env.docker b/uui-e2e-tests/.env.docker
@@ -21,5 +21,3 @@ UUI_APP_BASE_URL=http://{{HOST_NAME}}:3793
 # - otherwise "docker" is used as fallback;
 #
 UUI_DOCKER_CONTAINER_ENGINE=
-
-UUI_REPORT_OBSOLETE_SCREENSHOTS=true
diff --git a/uui-e2e-tests/.env.local b/uui-e2e-tests/.env.local
@@ -2,5 +2,3 @@
 # Parameters for the case when tests are run without Docker (e.g. on CI)
 #
 UUI_APP_BASE_URL=http://localhost:5000
-
-UUI_REPORT_OBSOLETE_SCREENSHOTS=true
diff --git a/uui-e2e-tests/package.json b/uui-e2e-tests/package.json
@@ -7,14 +7,14 @@
   "private": true,
   "scripts": {
     "typecheck": "tsc --project ./tsconfig.json --skipLibCheck true --noEmit true",
-    "docker-test-e2e": "yarn typecheck && npx playwright test",
-    "docker-test-e2e-chromium": "yarn typecheck && npx playwright test --project=chromium",
+    "docker-test-e2e": "yarn typecheck && npx playwright test --pass-with-no-tests",
     "docker-test-e2e-update": "yarn docker-test-e2e --update-snapshots",
     "local-test-e2e-deps-install": "npx ../node_modules/playwright install --with-deps chromium webkit",
     "local-test-e2e": "yarn typecheck && npx ../node_modules/playwright test",
+    "local-check-issues": "cross-env UUI_TEST_PARAM_CHECK_ISSUES=true yarn local-test-e2e --list",
     "test-e2e": "ts-node scripts/cmd/cmdRunPwDocker.ts",
-    "test-e2e-chromium": "yarn test-e2e --projectChromium",
-    "test-e2e-update": "yarn test-e2e --updateSnapshots",
+    "test-e2e-chromium": "cross-env UUI_TEST_PARAM_PROJECT=chromium UUI_TEST_PARAM_ONLY_FAILED=false yarn test-e2e",
+    "test-e2e-update": "cross-env UUI_TEST_PARAM_UPDATE_SNAPSHOTS=true yarn test-e2e",
     "test-e2e-report": "npx ../node_modules/playwright show-report ./tests/.report/report",
     "start-server": "yarn --cwd ../app server"
   },

diff --git a/uui-e2e-tests/playwright.config.ts b/uui-e2e-tests/playwright.config.ts
@@ -1,10 +1,10 @@
+import path from 'node:path';
 import { defineConfig, devices, type TraceMode } from '@playwright/test';
 import { SHARED_DEVICE_CFG } from './src/constants';
-import { readEnvParams } from './scripts/cliUtils';
 import { readEnvFile } from './scripts/envFileUtils';
-import path from 'node:path';
+import { readUuiSpecificEnvVariables } from './scripts/envParamUtils';
 
-const { isCi, isDocker } = readEnvParams();
+const { isCi, isDocker, UUI_TEST_PARAM_PROJECT } = readUuiSpecificEnvVariables();
 const { UUI_APP_BASE_URL } = readEnvFile();
 
 const timeout = isCi ? 20000 : 50000;
@@ -23,6 +23,7 @@ export const screenshotsDirAbsPath = path.resolve(process.cwd(), 'tests/__screen
 const testMatch = `${parentDir}tests/*.e2e.ts`;
 const outputDir = `${parentDir}tests/.report/results`;
 const outputFolder = `${parentDir}tests/.report/report`;
+export const outputJsonFile = `${parentDir}tests/.report/report.json`;
 const snapshotPathTemplate = '{testFileDir}/__screenshots__/{platform}/{projectName}/{arg}{ext}';
 export const stylePath = `${parentDir}src/fixtures/screenshot.css`;
 
@@ -36,7 +37,10 @@ export default defineConfig({
     workers,
     outputDir,
     snapshotPathTemplate,
-    reporter: [['html', { outputFolder, open: (isDocker || isCi) ? 'never' : 'on-failure' }]],
+    reporter: [
+        ['html', { outputFolder, open: (isDocker || isCi) ? 'never' : 'on-failure' }],
+        ['json', { outputFile: outputJsonFile }],
+    ],
     use: {
         baseURL: server.baseUrl,
         trace,
@@ -55,7 +59,12 @@ export default defineConfig({
                 ...devices['Desktop Safari'],
             },
         },
-    ],
+    ].filter(({ name }) => {
+        if (UUI_TEST_PARAM_PROJECT) {
+            return name === UUI_TEST_PARAM_PROJECT;
+        }
+        return true;
+    }),
     webServer: server.startCmd ? {
         command: server.startCmd,
         url: server.baseUrl,

diff --git a/uui-e2e-tests/scripts/cliUtils.ts b/uui-e2e-tests/scripts/cliUtils.ts
@@ -1,18 +1,8 @@
-import { CLI_ARGS } from './constants';
 // @ts-ignore
 import spawn from 'cross-spawn';
 import { Logger } from '../src/utils/logger';
 
-export function readEnvParams() {
-    const { UUI_IS_DOCKER, CI, UUI_DOCKER_HOST_MACHINE_IP } = process.env;
-    return {
-        isDocker: UUI_IS_DOCKER === 'true',
-        isCi: !!CI,
-        UUI_DOCKER_HOST_MACHINE_IP,
-    };
-}
-
-export function hasCliArg(arg: typeof CLI_ARGS[keyof typeof CLI_ARGS]) {
+export function hasCliArg(arg: string) {
     const args = getAllCliArgs();
     return args.indexOf(arg) !== -1;
 }

diff --git a/uui-e2e-tests/scripts/cmd/cmdRunPwDocker.ts b/uui-e2e-tests/scripts/cmd/cmdRunPwDocker.ts
@@ -1,15 +1,20 @@
-import { spawnProcessSync, hasCliArg } from '../cliUtils';
+import { spawnProcessSync } from '../cliUtils';
 import {
-    CLI_ARGS,
     DOCKER_CONTAINER_NAME,
     DOCKER_FILES,
-    DOCKER_IMAGE_TAGS,
+    DOCKER_IMAGE_TAGS, ENV_UUI_PARAMS,
     YARN_TASKS,
 } from '../constants';
 import { currentMachineIpv4 } from '../ipUtils';
 import { getContainerEngineCmd } from '../containerEngineUtils';
+import { readUuiSpecificEnvVariables } from '../envParamUtils';
 
 const CONTAINER_ENGINE_CMD = getContainerEngineCmd();
+const {
+    UUI_TEST_PARAM_ONLY_FAILED,
+    UUI_TEST_PARAM_PROJECT,
+    UUI_TEST_PARAM_UPDATE_SNAPSHOTS,
+} = readUuiSpecificEnvVariables();
 
 main();
 
@@ -50,8 +55,7 @@ function main() {
             '--ipc',
             'host',
             ...getVolumesMapArgs(),
-            '-e',
-            `UUI_DOCKER_HOST_MACHINE_IP=${currentMachineIpv4}`,
+            ...getEnvParamsForDocker(),
             DOCKER_IMAGE_TAGS.TEST,
             npmTaskName,
         ],
@@ -60,14 +64,23 @@ function main() {
 }
 
 function resolvePwInDockerTaskName() {
-    if (hasCliArg(CLI_ARGS.PW_DOCKER_UPDATE_SNAPSHOTS)) {
+    if (UUI_TEST_PARAM_UPDATE_SNAPSHOTS) {
         return YARN_TASKS.DOCKER_TEST_E2E_UPDATE;
-    } else if (hasCliArg(CLI_ARGS.PW_DOCKER_PROJECT_CHROMIUM)) {
-        return YARN_TASKS.DOCKER_TEST_E2E_CHROMIUM;
     }
     return YARN_TASKS.DOCKER_TEST_E2E;
 }
 
+function getEnvParamsForDocker(): string[] {
+    const env = [`-e ${ENV_UUI_PARAMS.UUI_DOCKER_HOST_MACHINE_IP}=${currentMachineIpv4}`];
+    if (UUI_TEST_PARAM_PROJECT) {
+        env.push(`-e ${ENV_UUI_PARAMS.UUI_TEST_PARAM_PROJECT}=${UUI_TEST_PARAM_PROJECT}`);
+    }
+    if (UUI_TEST_PARAM_ONLY_FAILED) {
+        env.push(`-e ${ENV_UUI_PARAMS.UUI_TEST_PARAM_ONLY_FAILED}=true`);
+    }
+    return env;
+}
+
 function getVolumesMapArgs() {
     // files/folders to mount volumes
     return [

diff --git a/uui-e2e-tests/scripts/constants.ts b/uui-e2e-tests/scripts/constants.ts
@@ -1,10 +1,15 @@
-export const CLI_ARGS = {
-    PW_DOCKER_UPDATE_SNAPSHOTS: '--updateSnapshots',
-    PW_DOCKER_PROJECT_CHROMIUM: '--projectChromium',
-};
+/**
+ * UUI-specific ENV param names
+ */
+export enum ENV_UUI_PARAMS {
+    UUI_TEST_PARAM_UPDATE_SNAPSHOTS= 'UUI_TEST_PARAM_UPDATE_SNAPSHOTS',
+    UUI_TEST_PARAM_PROJECT = 'UUI_TEST_PARAM_PROJECT',
+    UUI_TEST_PARAM_CHECK_ISSUES = 'UUI_TEST_PARAM_CHECK_ISSUES',
+    UUI_TEST_PARAM_ONLY_FAILED = 'UUI_TEST_PARAM_ONLY_FAILED',
+    UUI_DOCKER_HOST_MACHINE_IP = 'UUI_DOCKER_HOST_MACHINE_IP'
+}
 export const YARN_TASKS = {
     DOCKER_TEST_E2E: 'docker-test-e2e',
-    DOCKER_TEST_E2E_CHROMIUM: 'docker-test-e2e-chromium',
     DOCKER_TEST_E2E_UPDATE: 'docker-test-e2e-update',
 };
 export const DOCKER_IMAGE_TAGS = {

diff --git a/uui-e2e-tests/scripts/envFileUtils.ts b/uui-e2e-tests/scripts/envFileUtils.ts
@@ -1,16 +1,15 @@
 import dotenv from 'dotenv';
 import path from 'node:path';
-import { readEnvParams } from './cliUtils';
 import { ENV_FILES, HOST_IP_PH } from './constants';
+import { readUuiSpecificEnvVariables } from './envParamUtils';
 
-const { UUI_DOCKER_HOST_MACHINE_IP = 'localhost', isDocker } = readEnvParams();
+const { UUI_DOCKER_HOST_MACHINE_IP = 'localhost', isDocker } = readUuiSpecificEnvVariables();
 
 const envFileName = isDocker ? ENV_FILES.DOCKER : ENV_FILES.LOCAL;
 
 type TEnvParams = {
     UUI_APP_BASE_URL: string,
     UUI_DOCKER_CONTAINER_ENGINE: string,
-    UUI_REPORT_OBSOLETE_SCREENSHOTS: string
 };
 
 export function readEnvFile(): TEnvParams {

diff --git a/uui-e2e-tests/scripts/envParamUtils.ts b/uui-e2e-tests/scripts/envParamUtils.ts
@@ -0,0 +1,31 @@
+type TEnvParams = {
+    isCi: boolean;
+    isDocker: boolean;
+    UUI_DOCKER_HOST_MACHINE_IP?: string;
+    UUI_TEST_PARAM_PROJECT?: string,
+    UUI_TEST_PARAM_ONLY_FAILED?: boolean,
+    UUI_TEST_PARAM_CHECK_ISSUES?: boolean,
+    UUI_TEST_PARAM_UPDATE_SNAPSHOTS?: boolean,
+};
+export function readUuiSpecificEnvVariables(): TEnvParams {
+    const {
+        UUI_IS_DOCKER,
+        CI,
+        UUI_DOCKER_HOST_MACHINE_IP,
+        //
+        UUI_TEST_PARAM_PROJECT,
+        UUI_TEST_PARAM_ONLY_FAILED,
+        UUI_TEST_PARAM_CHECK_ISSUES,
+        UUI_TEST_PARAM_UPDATE_SNAPSHOTS,
+        //
+    } = process.env;
+    return {
+        isDocker: UUI_IS_DOCKER === 'true',
+        isCi: !!CI,
+        UUI_DOCKER_HOST_MACHINE_IP,
+        UUI_TEST_PARAM_PROJECT,
+        UUI_TEST_PARAM_ONLY_FAILED: UUI_TEST_PARAM_ONLY_FAILED === 'true',
+        UUI_TEST_PARAM_CHECK_ISSUES: UUI_TEST_PARAM_CHECK_ISSUES === 'true',
+        UUI_TEST_PARAM_UPDATE_SNAPSHOTS: UUI_TEST_PARAM_UPDATE_SNAPSHOTS === 'true',
+    };
+}
diff --git a/uui-e2e-tests/src/constants.ts b/uui-e2e-tests/src/constants.ts
@@ -1,5 +1,6 @@
 import { PlaywrightTestOptions } from '@playwright/test';
 
+export const PLATFORM = 'linux';
 export const PREVIEW_URL = '/preview';
 export const PlayWrightInterfaceName = '_uui_playwright_interface';
 

diff --git a/uui-e2e-tests/src/types.ts b/uui-e2e-tests/src/types.ts
@@ -1,5 +1,5 @@
 import { TComponentId, TPreviewIdByComponentId } from './data/testData';
-import { PreviewPage } from './pages/previewPage';
+import type { PreviewPage } from './pages/previewPage';
 
 export type TClip = { x: number, y: number, width: number, height: number };
 

diff --git a/uui-e2e-tests/src/utils/ctx.ts b/uui-e2e-tests/src/utils/ctx.ts
diff --git a/uui-e2e-tests/src/utils/failedTestsUtils.ts b/uui-e2e-tests/src/utils/failedTestsUtils.ts
@@ -0,0 +1,23 @@
+import { outputJsonFile } from '../../playwright.config';
+import fs from 'node:fs';
+
+type TJsonReport = {
+    suites: {
+        specs: { ok: boolean, title: string }[]
+    }[]
+};
+
+export function getFailedTestNamesFromLastRun(): Set<string> {
+    const set = new Set<string>();
+    if (fs.existsSync(outputJsonFile)) {
+        const parsed: TJsonReport = JSON.parse(fs.readFileSync(outputJsonFile).toString());
+        parsed.suites.forEach(({ specs }) => {
+            specs.forEach((sp) => {
+                if (!sp.ok) {
+                    set.add(sp.title);
+                }
+            });
+        });
+    }
+    return set;
+}