[ftr] take screenshots on failure (elastic#11709)

* [tests/functional] move screenshots to their own service

* [ftr] add testFailure and testHookFailure lifecycle hooks

* [tests/functional/screenshots] cleanup old screenshots at startup

* [test/functional/screenshots] take screenshots when tests fail

* [cli_plugin/install] fix test

* [ui/scanner] fix test

package.json

@@ -258,7 +258,7 @@
     "load-grunt-config": "0.19.2",
     "makelogs": "4.0.1",
     "marked-text-renderer": "0.1.0",
-    "mocha": "2.5.3",
+    "mocha": "3.3.0",
     "mock-fs": "4.2.0",
     "murmurhash3js": "3.0.1",
     "ncp": "2.0.0",
@@ -272,6 +272,7 @@
     "sinon": "1.17.2",
     "source-map": "0.5.6",
     "source-map-support": "0.2.10",
+    "strip-ansi": "^3.0.1",
     "supertest": "1.2.0",
     "supertest-as-promised": "2.0.2",
     "tree-kill": "1.1.0",

src/cli_plugin/install/__tests__/zip.js

@@ -54,12 +54,12 @@ describe('kibana cli', function () {
       });
     });
 
-    it('handles a corrupt zip archive', async (done) => {
+    it('handles a corrupt zip archive', async () => {
       try {
         await extractArchive(path.resolve(repliesPath, 'corrupt.zip'));
-        done(false);
+        throw new Error('This should have failed');
       } catch(e) {
-        done();
+        return;
       }
     });
   });

src/functional_test_runner/__tests__/fixtures/failure_hooks/config.js

@@ -0,0 +1,28 @@
+import { delay } from 'bluebird';
+
+export default function () {
+  return {
+    testFiles: [
+      require.resolve('./tests/before_hook'),
+      require.resolve('./tests/it'),
+      require.resolve('./tests/after_hook')
+    ],
+    services: {
+      hookIntoLIfecycle({ getService }) {
+        const log = getService('log');
+
+        getService('lifecycle')
+          .on('testFailure', async (err, test) => {
+            log.info('testFailure %s %s', err.message, test.fullTitle());
+            await delay(10);
+            log.info('testFailureAfterDelay %s %s', err.message, test.fullTitle());
+          })
+          .on('testHookFailure', async (err, test) => {
+            log.info('testHookFailure %s %s', err.message, test.fullTitle());
+            await delay(10);
+            log.info('testHookFailureAfterDelay %s %s', err.message, test.fullTitle());
+          });
+      }
+    }
+  };
+}

src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/after_hook.js

@@ -0,0 +1,8 @@
+export default function () {
+  describe('failing after hook', () => {
+    it('stub test', () => {});
+    after('$FAILING_AFTER_HOOK$', () => {
+      throw new Error('$FAILING_AFTER_ERROR$');
+    });
+  });
+}

src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/before_hook.js

@@ -0,0 +1,9 @@
+export default function () {
+  describe('failing before hook', () => {
+    before('$FAILING_BEFORE_HOOK$', () => {
+      throw new Error('$FAILING_BEFORE_ERROR$');
+    });
+
+    it('stub test', () => {});
+  });
+}

src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/it.js

@@ -0,0 +1,7 @@
+export default function () {
+  describe('failing test', () => {
+    it('$FAILING_TEST$', () => {
+      throw new Error('$FAILING_TEST_ERROR$');
+    });
+  });
+}

src/functional_test_runner/__tests__/integration/failure_hooks.js

@@ -0,0 +1,51 @@
+import { spawnSync } from 'child_process';
+import { resolve } from 'path';
+
+import stripAnsi from 'strip-ansi';
+import expect from 'expect.js';
+
+const SCRIPT = resolve(__dirname, '../../../../scripts/functional_test_runner.js');
+const FAILURE_HOOKS_CONFIG = resolve(__dirname, '../fixtures/failure_hooks/config.js');
+
+describe('failure hooks', function () {
+  this.timeout(60 * 1000);
+
+  it('runs and prints expected output', () => {
+    const proc = spawnSync(process.execPath, [SCRIPT, '--config', FAILURE_HOOKS_CONFIG]);
+    const lines = stripAnsi(proc.stdout.toString('utf8')).split(/\r?\n/);
+    const tests = [
+      {
+        flag: '$FAILING_BEFORE_HOOK$',
+        assert(lines) {
+          expect(lines.shift()).to.match(/info\s+testHookFailure\s+\$FAILING_BEFORE_ERROR\$/);
+          expect(lines.shift()).to.match(/info\s+testHookFailureAfterDelay\s+\$FAILING_BEFORE_ERROR\$/);
+        }
+      },
+      {
+        flag: '$FAILING_TEST$',
+        assert(lines) {
+          expect(lines.shift()).to.match(/global before each/);
+          expect(lines.shift()).to.match(/info\s+testFailure\s+\$FAILING_TEST_ERROR\$/);
+          expect(lines.shift()).to.match(/info\s+testFailureAfterDelay\s+\$FAILING_TEST_ERROR\$/);
+        }
+      },
+      {
+        flag: '$FAILING_AFTER_HOOK$',
+        assert(lines) {
+          expect(lines.shift()).to.match(/info\s+testHookFailure\s+\$FAILING_AFTER_ERROR\$/);
+          expect(lines.shift()).to.match(/info\s+testHookFailureAfterDelay\s+\$FAILING_AFTER_ERROR\$/);
+        }
+      },
+    ];
+
+    while (lines.length && tests.length) {
+      const line = lines.shift();
+      if (line.includes(tests[0].flag)) {
+        const test = tests.shift();
+        test.assert(lines);
+      }
+    }
+
+    expect(tests).to.have.length(0);
+  });
+});

src/functional_test_runner/lib/index.js

@@ -1,5 +1,4 @@
 export { createLifecycle } from './lifecycle';
 export { readConfigFile } from './config';
 export { createProviderCollection } from './create_provider_collection';
-export { setupMocha } from './setup_mocha';
-export { runTests } from './run_tests';
+export { setupMocha, runTests } from './mocha';

src/functional_test_runner/lib/lifecycle.js

@@ -3,6 +3,8 @@ export function createLifecycle() {
     beforeLoadTests: [],
     beforeTests: [],
     beforeEachTest: [],
+    testFailure: [],
+    testHookFailure: [],
     cleanup: [],
     phaseStart: [],
     phaseEnd: [],
@@ -15,6 +17,7 @@ export function createLifecycle() {
       }
 
       listeners[name].push(fn);
+      return this;
     }
 
     async trigger(name, ...args) {

src/functional_test_runner/lib/mocha/assignment_proxy.js

@@ -0,0 +1,7 @@
+export function createAssignmentProxy(object, interceptor) {
+  return new Proxy(object, {
+    set(target, property, value) {
+      return Reflect.set(target, property, interceptor(property, value));
+    }
+  });
+}

src/functional_test_runner/lib/mocha/decorate_mocha_ui.js

@@ -0,0 +1,134 @@
+import { createAssignmentProxy } from './assignment_proxy';
+import { wrapFunction } from './wrap_function';
+import { wrapRunnableArgsWithErrorHandler } from './wrap_runnable_args';
+
+export function decorateMochaUi(lifecycle, context) {
+  // incremented at the start of each suite, decremented after
+  // so that in each non-suite call we can know if we are within
+  // a suite, or that when a suite is defined it is within a suite
+  let suiteLevel = 0;
+
+  // incremented at the start of each suite, used to know when a
+  // suite is not the first suite
+  let suiteCount = 0;
+
+  /**
+   *  Wrap the describe() function in the mocha UI to ensure
+   *  that the first call made when defining a test file is a
+   *  "describe()", and that there is only one describe call at
+   *  the top level of that file.
+   *
+   *  @param  {String} name
+   *  @param  {Function} fn
+   *  @return {Function}
+   */
+  function wrapSuiteFunction(name, fn) {
+    return wrapFunction(fn, {
+      before() {
+        if (suiteCount > 0 && suiteLevel === 0) {
+          throw new Error(`
+            Test files must only define a single top-level suite. Please ensure that
+            all calls to \`describe()\` are within a single \`describe()\` call in this file.
+          `);
+        }
+
+        suiteCount += 1;
+        suiteLevel += 1;
+      },
+      after() {
+        suiteLevel -= 1;
+      }
+    });
+  }
+
+  /**
+   *  Wrap test functions to emit "testFailure" lifecycle hooks
+   *  when they fail and throw when they are called outside of
+   *  a describe
+   *
+   *  @param  {String} name
+   *  @param  {Function} fn
+   *  @return {Function}
+   */
+  function wrapTestFunction(name, fn) {
+    return wrapNonSuiteFunction(name, wrapRunnableArgsWithErrorHandler(fn, async (err, test) => {
+      await lifecycle.trigger('testFailure', err, test);
+    }));
+  }
+
+  /**
+   *  Wrap test hook functions to emit "testHookFailure" lifecycle
+   *  hooks when they fail and throw when they are called outside
+   *  of a describe
+   *
+   *  @param  {String} name
+   *  @param  {Function} fn
+   *  @return {Function}
+   */
+  function wrapTestHookFunction(name, fn) {
+    return wrapNonSuiteFunction(name, wrapRunnableArgsWithErrorHandler(fn, async (err, test) => {
+      await lifecycle.trigger('testHookFailure', err, test);
+    }));
+  }
+
+  /**
+   *  Wrap all non describe() mocha ui functions to ensure
+   *  that they are not called outside of a describe block
+   *
+   *  @param  {String} name
+   *  @param  {Function} fn
+   *  @return {Function}
+   */
+  function wrapNonSuiteFunction(name, fn) {
+    return wrapFunction(fn, {
+      before() {
+        if (suiteLevel === 0) {
+          throw new Error(`
+            All ${name}() calls in test files must be within a describe() call.
+          `);
+        }
+      }
+    });
+  }
+
+  /**
+   *  called for every assignment while defining the mocha ui
+   *  and can return an alternate value that will be used for that
+   *  assignment
+   *
+   *  @param  {String} property
+   *  @param  {Any} value
+   *  @return {Any} replacement function
+   */
+  function assignmentInterceptor(property, value) {
+    if (typeof value !== 'function') {
+      return value;
+    }
+
+    switch (property) {
+      case 'describe':
+      case 'xdescribe':
+      case 'context':
+      case 'xcontext':
+        return wrapSuiteFunction(property, value);
+
+      case 'it':
+      case 'xit':
+      case 'specify':
+      case 'xspecify':
+        return wrapTestFunction(property, value);
+
+      case 'before':
+      case 'beforeEach':
+      case 'after':
+      case 'afterEach':
+      case 'run':
+        return wrapTestHookFunction(property, value);
+
+      default:
+        return wrapNonSuiteFunction(property, value);
+    }
+  }
+
+  return createAssignmentProxy(context, assignmentInterceptor);
+}

src/functional_test_runner/lib/mocha/index.js

@@ -0,0 +1,2 @@
+export { setupMocha } from './setup_mocha';
+export { runTests } from './run_tests';