[tests] Add end-to-end testing utility (#778)

To-date, executing the end-to-end ("e2e") test suite locally (for debugging or augmentation purposes) has been cumbersome as it involved manually invoking the application, Chromedriver and mocha processes for each test run (as well as shutting down and re-starting each process for subsequent test runs). In addition, developers had to cross-reference the Circle CI script and dashboard to verify the environment variables necessary to invoke the test-suite.

This commit adds a utility script that: 

1. makes the required environment variables explicit 
2. automatically invokes all requisite processes, and 
3. automatically kills invoked processes upon test completion or error. In addition, this logic ensures a clean exit if the developer interrupts a test run via `SIGINT` ( `CTRL` + `C`) 

These changes combined make running (and re-running) the tests much more streamlined, thereby significantly improving the developer workflow.

.circleci/config.yml

@@ -396,18 +396,7 @@ jobs:
                   killall touristd
                   killall UserNotificationCenter
 
-                  LOGDIR=$(echo "$(pwd)/test/logs")
-                  mkdir -p "$LOGDIR"
-                  APPLOG=$(echo "$LOGDIR/app.log")
-                  DRIVERLOG=$(echo "$LOGDIR/chromedriver.log")
-
-                  pushd "release/mac/WordPress.com.app/Contents/MacOS" 
-                  eval './WordPress.com --disable-renderer-backgrounding --disable-http-cache --start-maximized --remote-debugging-port=9222 > "$APPLOG" 2>&1 &' && sleep 5
-                  popd
-                  
-                  ./node_modules/.bin/chromedriver --port=9515 --verbose > "$DRIVERLOG" 2>&1 &
-                  ./node_modules/.bin/mocha test/tests/e2e.js
-                  osascript -e 'quit app "WordPress.com"'
+                  make e2e
       - run:
           name: Release cleanup
           command: |

Makefile

@@ -191,3 +191,7 @@ clean:
 	@rm -rf .$/build
 
 .PHONY: test build-source
+
+.PHONY:
+e2e:
+	@npm run e2e

README.md

@@ -21,6 +21,11 @@ Need more detailed instructions? [We have them.](docs/install.md)
 
 The app is split between Electron code and Calypso code, and so the [development guide](docs/development.md) may help you find where to change stuff.
 
+# Running The End-To-End Test Suite
+
+1. Set the environment variables `E2EUSERNAME`, `E2EPASSWORD` and `E2E_MAILOSAUR_INBOX`.
+2. Use `npm run e2e` or `make e2e` to invoke the test suite.
+
 # Building & Packaging a Release
 
 While running the app locally in a development environment is great, you will eventually need to [build a release version](docs/release.md) you can share.

package.json

@@ -21,7 +21,7 @@
     "build:app": "make build CONFIG_ENV=release NODE_ENV=production CALYPSO_ENV=desktop",
     "install-if-deps-outdated": "node calypso/bin/install-if-deps-outdated.js",
     "dev": "make start CONFIG_ENV=development NODE_ENV=development CALYPSO_ENV=desktop-development",
-    "e2e": "mocha test/tests/e2e.js --timeout 20000"
+    "e2e": "node test/run/run-e2e-tests.js"
   },
   "keywords": [
     "desktop",

test/run/run-e2e-tests.js

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+
+const path = require( 'path' );
+const { promisify } = require( 'util' );
+const { openSync, mkdirSync } = require( 'fs' );
+const { execSync, spawn } = require( 'child_process' );
+
+const PROJECT_DIR = path.join( __dirname, '../../' );
+const BUILT_APP_DIR = path.join( PROJECT_DIR, 'release', 'mac', 'WordPress.com.app', 'Contents', 'MacOS' );
+
+function spawnDetached( cwd, command, args, output ) {
+    const app = spawn( command, args, { stdio: [ 'ignore', output, output ], detached: true, cwd } );
+    app.on( 'error', err => {
+        throw `failed to initialize command "${ command }": "${ err }"`;
+    } );
+    return app;
+}
+
+function initLogs( timestamp ) {
+    const dir = path.join( PROJECT_DIR, 'test', 'logs', `${ timestamp }` );
+
+    mkdirSync( dir, { recursive: true } );
+
+    const appLog = openSync( path.join( dir, `app-${ timestamp }.log` ), 'a' );
+    const driverLog = openSync( path.join( dir, `chromedriver-${ timestamp }.log` ), 'a' );
+
+    if ( !appLog || !driverLog ) {
+        throw 'failed to initialize logs';
+    }
+
+    return { appLog, driverLog };
+}
+
+const delay = promisify( setTimeout );
+
+let app;
+let driver;
+
+function handleExit() {
+    if ( driver ) {
+        driver.kill();
+    }
+    if ( app ) {
+        app.kill();
+    }
+}
+
+// Handle both user-initiated (SIGINT) and normal termination.
+process.on( 'SIGINT', function() {
+    handleExit();
+    process.exit();
+} );
+
+process.on( 'exit', handleExit );
+
+async function run() {
+    try {
+        const requiredENVs = [ 'E2EUSERNAME', 'E2EPASSWORD', 'E2E_MAILOSAUR_INBOX' ];
+        const missingENVs = requiredENVs.filter( name => ! process.env[name] || process.env[name] === '' );
+        if ( missingENVs.length ) {
+            throw `Missing non-empty ENV for: ${ missingENVs.join( ', ' ) }`;
+        }
+
+        // Replace `:` with `-` to format timestamp as YYYY-MM-DDTHH-MM-SS.mmmZ
+        const timestamp = ( new Date() ).toJSON().replace( /:/g, '-' );
+        const { appLog, driverLog } = initLogs( timestamp );
+
+        app = spawnDetached( BUILT_APP_DIR, './WordPress.com', [
+            '--disable-renderer-backgrounding',
+            '--disable-http-cache',
+            '--start-maximized',
+            '--remote-debugging-port=9222',
+        ], appLog );
+        await delay( 5000 );
+
+        driver = spawnDetached( PROJECT_DIR, 'npx', [
+            'chromedriver',
+            '--port=9515',
+            '--verbose',
+        ], driverLog );
+
+        const tests = path.join( PROJECT_DIR, 'test', 'tests', 'e2e.js' );
+        execSync( `npx mocha ${ tests } --timeout 20000`, { stdio: 'inherit' } );
+    }
+    catch ( err ) {
+        console.error( err );
+    }
+    finally {
+        // Explicitly call process.exit to ensure that spawned processes are killed.
+        process.exit();
+    }
+}
+
+run();