From 273c3b37daf24c5cca09da096980535bf1a834fe Mon Sep 17 00:00:00 2001 From: Nicholas Sakaimbo Date: Mon, 24 Feb 2020 10:50:40 -0500 Subject: [PATCH] [tests] Add end-to-end testing utility (#778) To-date, executing the end-to-end ("e2e") test suite locally (for debugging or augmentation purposes) has been cumbersome as it involved manually invoking the application, Chromedriver and mocha processes for each test run (as well as shutting down and re-starting each process for subsequent test runs). In addition, developers had to cross-reference the Circle CI script and dashboard to verify the environment variables necessary to invoke the test-suite. This commit adds a utility script that: 1. makes the required environment variables explicit 2. automatically invokes all requisite processes, and 3. automatically kills invoked processes upon test completion or error. In addition, this logic ensures a clean exit if the developer interrupts a test run via `SIGINT` ( `CTRL` + `C`) These changes combined make running (and re-running) the tests much more streamlined, thereby significantly improving the developer workflow. --- .circleci/config.yml | 13 +----- Makefile | 4 ++ README.md | 5 +++ package.json | 2 +- test/run/run-e2e-tests.js | 94 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 13 deletions(-) create mode 100644 test/run/run-e2e-tests.js diff --git a/.circleci/config.yml b/.circleci/config.yml index f2da88d41..5b2aa86bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -396,18 +396,7 @@ jobs: killall touristd killall UserNotificationCenter - LOGDIR=$(echo "$(pwd)/test/logs") - mkdir -p "$LOGDIR" - APPLOG=$(echo "$LOGDIR/app.log") - DRIVERLOG=$(echo "$LOGDIR/chromedriver.log") - - pushd "release/mac/WordPress.com.app/Contents/MacOS" - eval './WordPress.com --disable-renderer-backgrounding --disable-http-cache --start-maximized --remote-debugging-port=9222 > "$APPLOG" 2>&1 &' && sleep 5 - popd - - ./node_modules/.bin/chromedriver --port=9515 --verbose > "$DRIVERLOG" 2>&1 & - ./node_modules/.bin/mocha test/tests/e2e.js - osascript -e 'quit app "WordPress.com"' + make e2e - run: name: Release cleanup command: | diff --git a/Makefile b/Makefile index d8641a75e..dddbd86fe 100644 --- a/Makefile +++ b/Makefile @@ -191,3 +191,7 @@ clean: @rm -rf .$/build .PHONY: test build-source + +.PHONY: +e2e: + @npm run e2e diff --git a/README.md b/README.md index e730cc0ab..0958575e0 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,11 @@ Need more detailed instructions? [We have them.](docs/install.md) The app is split between Electron code and Calypso code, and so the [development guide](docs/development.md) may help you find where to change stuff. +# Running The End-To-End Test Suite + +1. Set the environment variables `E2EUSERNAME`, `E2EPASSWORD` and `E2E_MAILOSAUR_INBOX`. +2. Use `npm run e2e` or `make e2e` to invoke the test suite. + # Building & Packaging a Release While running the app locally in a development environment is great, you will eventually need to [build a release version](docs/release.md) you can share. diff --git a/package.json b/package.json index 4824aec9f..196ebe4e0 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "build:app": "make build CONFIG_ENV=release NODE_ENV=production CALYPSO_ENV=desktop", "install-if-deps-outdated": "node calypso/bin/install-if-deps-outdated.js", "dev": "make start CONFIG_ENV=development NODE_ENV=development CALYPSO_ENV=desktop-development", - "e2e": "mocha test/tests/e2e.js --timeout 20000" + "e2e": "node test/run/run-e2e-tests.js" }, "keywords": [ "desktop", diff --git a/test/run/run-e2e-tests.js b/test/run/run-e2e-tests.js new file mode 100644 index 000000000..e11de2697 --- /dev/null +++ b/test/run/run-e2e-tests.js @@ -0,0 +1,94 @@ +#!/usr/bin/env node + +const path = require( 'path' ); +const { promisify } = require( 'util' ); +const { openSync, mkdirSync } = require( 'fs' ); +const { execSync, spawn } = require( 'child_process' ); + +const PROJECT_DIR = path.join( __dirname, '../../' ); +const BUILT_APP_DIR = path.join( PROJECT_DIR, 'release', 'mac', 'WordPress.com.app', 'Contents', 'MacOS' ); + +function spawnDetached( cwd, command, args, output ) { + const app = spawn( command, args, { stdio: [ 'ignore', output, output ], detached: true, cwd } ); + app.on( 'error', err => { + throw `failed to initialize command "${ command }": "${ err }"`; + } ); + return app; +} + +function initLogs( timestamp ) { + const dir = path.join( PROJECT_DIR, 'test', 'logs', `${ timestamp }` ); + + mkdirSync( dir, { recursive: true } ); + + const appLog = openSync( path.join( dir, `app-${ timestamp }.log` ), 'a' ); + const driverLog = openSync( path.join( dir, `chromedriver-${ timestamp }.log` ), 'a' ); + + if ( !appLog || !driverLog ) { + throw 'failed to initialize logs'; + } + + return { appLog, driverLog }; +} + +const delay = promisify( setTimeout ); + +let app; +let driver; + +function handleExit() { + if ( driver ) { + driver.kill(); + } + if ( app ) { + app.kill(); + } +} + +// Handle both user-initiated (SIGINT) and normal termination. +process.on( 'SIGINT', function() { + handleExit(); + process.exit(); +} ); + +process.on( 'exit', handleExit ); + +async function run() { + try { + const requiredENVs = [ 'E2EUSERNAME', 'E2EPASSWORD', 'E2E_MAILOSAUR_INBOX' ]; + const missingENVs = requiredENVs.filter( name => ! process.env[name] || process.env[name] === '' ); + if ( missingENVs.length ) { + throw `Missing non-empty ENV for: ${ missingENVs.join( ', ' ) }`; + } + + // Replace `:` with `-` to format timestamp as YYYY-MM-DDTHH-MM-SS.mmmZ + const timestamp = ( new Date() ).toJSON().replace( /:/g, '-' ); + const { appLog, driverLog } = initLogs( timestamp ); + + app = spawnDetached( BUILT_APP_DIR, './WordPress.com', [ + '--disable-renderer-backgrounding', + '--disable-http-cache', + '--start-maximized', + '--remote-debugging-port=9222', + ], appLog ); + await delay( 5000 ); + + driver = spawnDetached( PROJECT_DIR, 'npx', [ + 'chromedriver', + '--port=9515', + '--verbose', + ], driverLog ); + + const tests = path.join( PROJECT_DIR, 'test', 'tests', 'e2e.js' ); + execSync( `npx mocha ${ tests } --timeout 20000`, { stdio: 'inherit' } ); + } + catch ( err ) { + console.error( err ); + } + finally { + // Explicitly call process.exit to ensure that spawned processes are killed. + process.exit(); + } +} + +run(); \ No newline at end of file