From 1205bce9ec3c8d5a20ef2692327bfb5e438d7755 Mon Sep 17 00:00:00 2001 From: Yaroslav Admin Date: Thu, 23 May 2019 02:11:41 +0200 Subject: [PATCH] chore(test): fix flaky test cases (#3314) * fix(test): fixed flaky reconnecting test https://github.com/karma-runner/karma/commit/cc2eff27deb680f789afb34577fd337d2ad5dcac introduced different behavior depending on whether socket reconnected or browser reconnected after reload. This lead to the reconnecting test being flaky as, depending on the timing, it will trigger second test run and result in unexpected output. * fix(test): remove broken monitor action for Cucumber The issue is that it resulted in the following sequence of actions: - run karma start - run karma run - and then instantly kill karma start command This resulted in flaky tests because the kill could race, resulting in incomplete output. In fact test was unnecessary complicated as it was enough to use runOut, which asserts output of the karma run command. * fix(test): guard from repeated executions of the karma run 'data' event handler may be called multiple times, which will result in multiple calls to karma run. To take it even further, its execution is delayed by setTimeout, which means that subsequent executions will run while next test scenario is in progress and may mess it up. To prevent this make sure that karma run is executed only once independently of how many time 'data' event is fired. --- test/e2e/error.feature | 6 +-- test/e2e/step_definitions/core_steps.js | 50 +++++++++++-------------- test/e2e/stop.feature | 2 +- test/e2e/support/reconnecting/test.js | 3 +- 4 files changed, 27 insertions(+), 34 deletions(-) diff --git a/test/e2e/error.feature b/test/e2e/error.feature index 9d0f3d98d..df9c2ed94 100644 --- a/test/e2e/error.feature +++ b/test/e2e/error.feature @@ -18,7 +18,7 @@ Feature: Error Display """ SyntaxError: Unexpected token } """ - Scenario: Single-run Syntax Error in a test file + Scenario: Not single-run Syntax Error in a test file Given a configuration with: """ files = ['error/test.js', 'error/under-test.js']; @@ -29,10 +29,8 @@ Feature: Error Display ]; singleRun = false; """ - When I monitor Karma - And I stop when the log contains 'SyntaxError' + When I runOut Karma Then it fails with like: """ SyntaxError: Unexpected token } """ - And I stop a server programmatically diff --git a/test/e2e/step_definitions/core_steps.js b/test/e2e/step_definitions/core_steps.js index 3d7581424..fac3852f7 100644 --- a/test/e2e/step_definitions/core_steps.js +++ b/test/e2e/step_definitions/core_steps.js @@ -64,7 +64,8 @@ cucumber.defineSupportCode((a) => { } const runOut = command === 'runOut' - if (command === 'run' || command === 'runOut' || command === 'monitor') { + if (command === 'run' || command === 'runOut') { + let isRun = false this.child = spawn('' + runtimePath, ['start', '--log-level', 'warn', configFile]) const done = () => { cleansingNeeded = true @@ -84,22 +85,24 @@ cucumber.defineSupportCode((a) => { this.child.stdout.on('data', (chunk) => { this.lastRun.stdout += chunk.toString() const cmd = runtimePath + ' run ' + configFile + ' ' + additionalArgs - setTimeout(() => { - exec(cmd, { - cwd: baseDir - }, (error, stdout) => { - if (error) { - this.lastRun.error = error - } - if (runOut) { - this.lastRun.stdout = stdout - } - done() - }) - if (command === 'monitor') { - done() - } - }, 1000) + if (!isRun) { + isRun = true + + setTimeout(() => { + exec(cmd, { + cwd: baseDir + }, (error, stdout, stderr) => { + if (error) { + this.lastRun.error = error + } + if (runOut) { + this.lastRun.stdout = stdout + this.lastRun.stderr = stderr + } + done() + }) + }, 1000) + } }) } else { executor((error, stdout, stderr) => { @@ -132,8 +135,8 @@ cucumber.defineSupportCode((a) => { setTimeout(function () { stopper.stop(_this.configFile, function (exitCode) { _this.stopperExitCode = exitCode + callback() }) - callback() }, 1000) }) @@ -160,7 +163,7 @@ cucumber.defineSupportCode((a) => { defineParameterType({ name: 'command', - regexp: /run|runOut|start|init|stop|monitor/ + regexp: /run|runOut|start|init|stop/ }) defineParameterType({ @@ -180,15 +183,6 @@ cucumber.defineSupportCode((a) => { execKarma.apply(this, [command, undefined, proxyPort, proxyPath, callback]) }) - When('I stop when the log contains {string}', function (message, callback) { - setInterval(() => { - if (this.lastRun.stdout.includes(message)) { - this.child && this.child.kill() - callback() - } - }, 100) - }) - defineParameterType({ name: 'exact', regexp: /no\sdebug|like/ diff --git a/test/e2e/stop.feature b/test/e2e/stop.feature index ce356c2cb..889569e12 100644 --- a/test/e2e/stop.feature +++ b/test/e2e/stop.feature @@ -44,7 +44,7 @@ Feature: Stop karma """ - Scenario: A server can be stopped programically + Scenario: A server can be stopped programmatically Given a configuration with: """ files = ['basic/plus.js', 'basic/test.js']; diff --git a/test/e2e/support/reconnecting/test.js b/test/e2e/support/reconnecting/test.js index 238a51aaa..a3b682e1b 100644 --- a/test/e2e/support/reconnecting/test.js +++ b/test/e2e/support/reconnecting/test.js @@ -22,8 +22,9 @@ describe('plus', function () { it('should re-connect', function (done) { expect(4).toBe(4) + // Emit reconnect, so Karma will not start new test run after reconnecting. + socket().emit('reconnect') socket().connect() - // window.parent.socket.socket.connect() done() })