diff --git a/package.json b/package.json index 2fa392874c43e..e940a376ea0ae 100644 --- a/package.json +++ b/package.json @@ -258,7 +258,7 @@ "load-grunt-config": "0.19.2", "makelogs": "4.0.1", "marked-text-renderer": "0.1.0", - "mocha": "2.5.3", + "mocha": "3.3.0", "mock-fs": "4.2.0", "murmurhash3js": "3.0.1", "ncp": "2.0.0", @@ -272,6 +272,7 @@ "sinon": "1.17.2", "source-map": "0.5.6", "source-map-support": "0.2.10", + "strip-ansi": "^3.0.1", "supertest": "1.2.0", "supertest-as-promised": "2.0.2", "tree-kill": "1.1.0", diff --git a/src/cli_plugin/install/__tests__/zip.js b/src/cli_plugin/install/__tests__/zip.js index 28477703729e5..3b5e48027fa59 100644 --- a/src/cli_plugin/install/__tests__/zip.js +++ b/src/cli_plugin/install/__tests__/zip.js @@ -54,12 +54,12 @@ describe('kibana cli', function () { }); }); - it('handles a corrupt zip archive', async (done) => { + it('handles a corrupt zip archive', async () => { try { await extractArchive(path.resolve(repliesPath, 'corrupt.zip')); - done(false); + throw new Error('This should have failed'); } catch(e) { - done(); + return; } }); }); diff --git a/src/functional_test_runner/__tests__/fixtures/failure_hooks/config.js b/src/functional_test_runner/__tests__/fixtures/failure_hooks/config.js new file mode 100644 index 0000000000000..b7d2fee094f44 --- /dev/null +++ b/src/functional_test_runner/__tests__/fixtures/failure_hooks/config.js @@ -0,0 +1,28 @@ +import { delay } from 'bluebird'; + +export default function () { + return { + testFiles: [ + require.resolve('./tests/before_hook'), + require.resolve('./tests/it'), + require.resolve('./tests/after_hook') + ], + services: { + hookIntoLIfecycle({ getService }) { + const log = getService('log'); + + getService('lifecycle') + .on('testFailure', async (err, test) => { + log.info('testFailure %s %s', err.message, test.fullTitle()); + await delay(10); + log.info('testFailureAfterDelay %s %s', err.message, test.fullTitle()); + }) + .on('testHookFailure', async (err, test) => { + log.info('testHookFailure %s %s', err.message, test.fullTitle()); + await delay(10); + log.info('testHookFailureAfterDelay %s %s', err.message, test.fullTitle()); + }); + } + } + }; +} diff --git a/src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/after_hook.js b/src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/after_hook.js new file mode 100644 index 0000000000000..b569d871e9810 --- /dev/null +++ b/src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/after_hook.js @@ -0,0 +1,8 @@ +export default function () { + describe('failing after hook', () => { + it('stub test', () => {}); + after('$FAILING_AFTER_HOOK$', () => { + throw new Error('$FAILING_AFTER_ERROR$'); + }); + }); +} diff --git a/src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/before_hook.js b/src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/before_hook.js new file mode 100644 index 0000000000000..9b683d9552067 --- /dev/null +++ b/src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/before_hook.js @@ -0,0 +1,9 @@ +export default function () { + describe('failing before hook', () => { + before('$FAILING_BEFORE_HOOK$', () => { + throw new Error('$FAILING_BEFORE_ERROR$'); + }); + + it('stub test', () => {}); + }); +} diff --git a/src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/it.js b/src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/it.js new file mode 100644 index 0000000000000..1eb80814ca9e4 --- /dev/null +++ b/src/functional_test_runner/__tests__/fixtures/failure_hooks/tests/it.js @@ -0,0 +1,7 @@ +export default function () { + describe('failing test', () => { + it('$FAILING_TEST$', () => { + throw new Error('$FAILING_TEST_ERROR$'); + }); + }); +} diff --git a/src/functional_test_runner/__tests__/integration/failure_hooks.js b/src/functional_test_runner/__tests__/integration/failure_hooks.js new file mode 100644 index 0000000000000..becf4d178e043 --- /dev/null +++ b/src/functional_test_runner/__tests__/integration/failure_hooks.js @@ -0,0 +1,51 @@ +import { spawnSync } from 'child_process'; +import { resolve } from 'path'; + +import stripAnsi from 'strip-ansi'; +import expect from 'expect.js'; + +const SCRIPT = resolve(__dirname, '../../../../scripts/functional_test_runner.js'); +const FAILURE_HOOKS_CONFIG = resolve(__dirname, '../fixtures/failure_hooks/config.js'); + +describe('failure hooks', function () { + this.timeout(60 * 1000); + + it('runs and prints expected output', () => { + const proc = spawnSync(process.execPath, [SCRIPT, '--config', FAILURE_HOOKS_CONFIG]); + const lines = stripAnsi(proc.stdout.toString('utf8')).split(/\r?\n/); + const tests = [ + { + flag: '$FAILING_BEFORE_HOOK$', + assert(lines) { + expect(lines.shift()).to.match(/info\s+testHookFailure\s+\$FAILING_BEFORE_ERROR\$/); + expect(lines.shift()).to.match(/info\s+testHookFailureAfterDelay\s+\$FAILING_BEFORE_ERROR\$/); + } + }, + { + flag: '$FAILING_TEST$', + assert(lines) { + expect(lines.shift()).to.match(/global before each/); + expect(lines.shift()).to.match(/info\s+testFailure\s+\$FAILING_TEST_ERROR\$/); + expect(lines.shift()).to.match(/info\s+testFailureAfterDelay\s+\$FAILING_TEST_ERROR\$/); + } + }, + { + flag: '$FAILING_AFTER_HOOK$', + assert(lines) { + expect(lines.shift()).to.match(/info\s+testHookFailure\s+\$FAILING_AFTER_ERROR\$/); + expect(lines.shift()).to.match(/info\s+testHookFailureAfterDelay\s+\$FAILING_AFTER_ERROR\$/); + } + }, + ]; + + while (lines.length && tests.length) { + const line = lines.shift(); + if (line.includes(tests[0].flag)) { + const test = tests.shift(); + test.assert(lines); + } + } + + expect(tests).to.have.length(0); + }); +}); diff --git a/src/functional_test_runner/lib/describe_nesting_validator.js b/src/functional_test_runner/lib/describe_nesting_validator.js deleted file mode 100644 index d09e1e86c6e5a..0000000000000 --- a/src/functional_test_runner/lib/describe_nesting_validator.js +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Creates an object that enables us to intercept all calls to mocha - * interface functions `describe()`, `before()`, etc. and ensure that: - * - * - all calls are made within a `describe()` - * - there is only one top-level `describe()` - * - * To do this we create a proxy to another object, `context`. Mocha - * interfaces will assign all of their exposed methods on this Proxy - * which will wrap all functions with checks for the above rules. - * - * @return {any} the context that mocha-ui interfaces will assign to - */ -export function createDescribeNestingValidator(context) { - let describeCount = 0; - let describeLevel = 0; - - function createContextProxy() { - return new Proxy(context, { - set(target, property, value) { - return Reflect.set(target, property, wrapContextAssignmentValue(property, value)); - } - }); - } - - function wrapContextAssignmentValue(name, value) { - if (typeof value !== 'function') { - return value; - } - - if (name === 'describe') { - return createDescribeProxy(value); - } - - return createNonDescribeProxy(name, value); - } - - function createDescribeProxy(describe) { - return new Proxy(describe, { - apply(target, thisArg, args) { - try { - if (describeCount > 0 && describeLevel === 0) { - throw new Error(` - Test files must only define a single top-level suite. Please ensure that - all calls to \`describe()\` are within a single \`describe()\` call in this file. - `); - } - - describeCount += 1; - describeLevel += 1; - return Reflect.apply(describe, thisArg, args); - } finally { - describeLevel -= 1; - } - } - }); - } - - function createNonDescribeProxy(name, nonDescribe) { - return new Proxy(nonDescribe, { - apply(target, thisArg, args) { - if (describeCount === 0) { - throw new Error(` - All ${name}() calls in test files must be within a describe() call. - `); - } - - return Reflect.apply(nonDescribe, thisArg, args); - } - }); - } - - return createContextProxy(); -} diff --git a/src/functional_test_runner/lib/index.js b/src/functional_test_runner/lib/index.js index 5452afe8c4414..97e034acd14f5 100644 --- a/src/functional_test_runner/lib/index.js +++ b/src/functional_test_runner/lib/index.js @@ -1,5 +1,4 @@ export { createLifecycle } from './lifecycle'; export { readConfigFile } from './config'; export { createProviderCollection } from './create_provider_collection'; -export { setupMocha } from './setup_mocha'; -export { runTests } from './run_tests'; +export { setupMocha, runTests } from './mocha'; diff --git a/src/functional_test_runner/lib/lifecycle.js b/src/functional_test_runner/lib/lifecycle.js index 7483c967dfc13..4259a4d72187c 100644 --- a/src/functional_test_runner/lib/lifecycle.js +++ b/src/functional_test_runner/lib/lifecycle.js @@ -3,6 +3,8 @@ export function createLifecycle() { beforeLoadTests: [], beforeTests: [], beforeEachTest: [], + testFailure: [], + testHookFailure: [], cleanup: [], phaseStart: [], phaseEnd: [], @@ -15,6 +17,7 @@ export function createLifecycle() { } listeners[name].push(fn); + return this; } async trigger(name, ...args) { diff --git a/src/functional_test_runner/lib/mocha/assignment_proxy.js b/src/functional_test_runner/lib/mocha/assignment_proxy.js new file mode 100644 index 0000000000000..e0c1dcd168b21 --- /dev/null +++ b/src/functional_test_runner/lib/mocha/assignment_proxy.js @@ -0,0 +1,7 @@ +export function createAssignmentProxy(object, interceptor) { + return new Proxy(object, { + set(target, property, value) { + return Reflect.set(target, property, interceptor(property, value)); + } + }); +} diff --git a/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js b/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js new file mode 100644 index 0000000000000..80b84d319db6b --- /dev/null +++ b/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js @@ -0,0 +1,134 @@ +import { createAssignmentProxy } from './assignment_proxy'; +import { wrapFunction } from './wrap_function'; +import { wrapRunnableArgsWithErrorHandler } from './wrap_runnable_args'; + +export function decorateMochaUi(lifecycle, context) { + // incremented at the start of each suite, decremented after + // so that in each non-suite call we can know if we are within + // a suite, or that when a suite is defined it is within a suite + let suiteLevel = 0; + + // incremented at the start of each suite, used to know when a + // suite is not the first suite + let suiteCount = 0; + + /** + * Wrap the describe() function in the mocha UI to ensure + * that the first call made when defining a test file is a + * "describe()", and that there is only one describe call at + * the top level of that file. + * + * @param {String} name + * @param {Function} fn + * @return {Function} + */ + function wrapSuiteFunction(name, fn) { + return wrapFunction(fn, { + before() { + if (suiteCount > 0 && suiteLevel === 0) { + throw new Error(` + Test files must only define a single top-level suite. Please ensure that + all calls to \`describe()\` are within a single \`describe()\` call in this file. + `); + } + + suiteCount += 1; + suiteLevel += 1; + }, + after() { + suiteLevel -= 1; + } + }); + } + + /** + * Wrap test functions to emit "testFailure" lifecycle hooks + * when they fail and throw when they are called outside of + * a describe + * + * @param {String} name + * @param {Function} fn + * @return {Function} + */ + function wrapTestFunction(name, fn) { + return wrapNonSuiteFunction(name, wrapRunnableArgsWithErrorHandler(fn, async (err, test) => { + await lifecycle.trigger('testFailure', err, test); + })); + } + + /** + * Wrap test hook functions to emit "testHookFailure" lifecycle + * hooks when they fail and throw when they are called outside + * of a describe + * + * @param {String} name + * @param {Function} fn + * @return {Function} + */ + function wrapTestHookFunction(name, fn) { + return wrapNonSuiteFunction(name, wrapRunnableArgsWithErrorHandler(fn, async (err, test) => { + await lifecycle.trigger('testHookFailure', err, test); + })); + } + + /** + * Wrap all non describe() mocha ui functions to ensure + * that they are not called outside of a describe block + * + * @param {String} name + * @param {Function} fn + * @return {Function} + */ + function wrapNonSuiteFunction(name, fn) { + return wrapFunction(fn, { + before() { + if (suiteLevel === 0) { + throw new Error(` + All ${name}() calls in test files must be within a describe() call. + `); + } + } + }); + } + + /** + * called for every assignment while defining the mocha ui + * and can return an alternate value that will be used for that + * assignment + * + * @param {String} property + * @param {Any} value + * @return {Any} replacement function + */ + function assignmentInterceptor(property, value) { + if (typeof value !== 'function') { + return value; + } + + switch (property) { + case 'describe': + case 'xdescribe': + case 'context': + case 'xcontext': + return wrapSuiteFunction(property, value); + + case 'it': + case 'xit': + case 'specify': + case 'xspecify': + return wrapTestFunction(property, value); + + case 'before': + case 'beforeEach': + case 'after': + case 'afterEach': + case 'run': + return wrapTestHookFunction(property, value); + + default: + return wrapNonSuiteFunction(property, value); + } + } + + return createAssignmentProxy(context, assignmentInterceptor); +} diff --git a/src/functional_test_runner/lib/mocha/index.js b/src/functional_test_runner/lib/mocha/index.js new file mode 100644 index 0000000000000..672165b567523 --- /dev/null +++ b/src/functional_test_runner/lib/mocha/index.js @@ -0,0 +1,2 @@ +export { setupMocha } from './setup_mocha'; +export { runTests } from './run_tests'; diff --git a/src/functional_test_runner/lib/load_test_files.js b/src/functional_test_runner/lib/mocha/load_test_files.js similarity index 85% rename from src/functional_test_runner/lib/load_test_files.js rename to src/functional_test_runner/lib/mocha/load_test_files.js index f98c7b56941e8..e167a5d3ef980 100644 --- a/src/functional_test_runner/lib/load_test_files.js +++ b/src/functional_test_runner/lib/mocha/load_test_files.js @@ -1,7 +1,7 @@ import { isAbsolute } from 'path'; -import { loadTracer } from './load_tracer'; -import { createDescribeNestingValidator } from './describe_nesting_validator'; +import { loadTracer } from '../load_tracer'; +import { decorateMochaUi } from './decorate_mocha_ui'; /** * Load an array of test files into a mocha instance @@ -12,7 +12,7 @@ import { createDescribeNestingValidator } from './describe_nesting_validator'; * @param {String} path * @return {undefined} - mutates mocha, no return value */ -export const loadTestFiles = (mocha, log, providers, paths) => { +export const loadTestFiles = (mocha, log, lifecycle, providers, paths) => { const innerLoadTestFile = (path) => { if (typeof path !== 'string' || !isAbsolute(path)) { throw new TypeError('loadTestFile() only accepts absolute paths'); @@ -38,7 +38,7 @@ export const loadTestFiles = (mocha, log, providers, paths) => { loadTracer(provider, `testProvider[${path}]`, () => { // mocha.suite hocus-pocus comes from: https://git.io/vDnXO - mocha.suite.emit('pre-require', createDescribeNestingValidator(global), path, mocha); + mocha.suite.emit('pre-require', decorateMochaUi(lifecycle, global), path, mocha); const returnVal = provider({ loadTestFile: innerLoadTestFile, diff --git a/src/functional_test_runner/lib/run_tests.js b/src/functional_test_runner/lib/mocha/run_tests.js similarity index 100% rename from src/functional_test_runner/lib/run_tests.js rename to src/functional_test_runner/lib/mocha/run_tests.js diff --git a/src/functional_test_runner/lib/setup_mocha.js b/src/functional_test_runner/lib/mocha/setup_mocha.js similarity index 91% rename from src/functional_test_runner/lib/setup_mocha.js rename to src/functional_test_runner/lib/mocha/setup_mocha.js index 32130e24af8f7..189d77f876bd5 100644 --- a/src/functional_test_runner/lib/setup_mocha.js +++ b/src/functional_test_runner/lib/mocha/setup_mocha.js @@ -27,6 +27,6 @@ export async function setupMocha(lifecycle, log, config, providers) { await lifecycle.trigger('beforeEachTest'); }); - loadTestFiles(mocha, log, providers, config.get('testFiles')); + loadTestFiles(mocha, log, lifecycle, providers, config.get('testFiles')); return mocha; } diff --git a/src/functional_test_runner/lib/mocha/wrap_function.js b/src/functional_test_runner/lib/mocha/wrap_function.js new file mode 100644 index 0000000000000..1635800cba037 --- /dev/null +++ b/src/functional_test_runner/lib/mocha/wrap_function.js @@ -0,0 +1,80 @@ + +/** + * Get handler that will intercept calls to `toString` + * on the function, since Function.prototype.toString() + * does not like being called on Proxy objects + * + * @param {[type]} target [description] + * @param {[type]} property [description] + * @param {[type]} receiver [description] + * @return {[type]} [description] + */ +function commonGetHandler(target, property, receiver) { + if (property === 'toString') { + return (...args) => target.toString(...args); + } + + return Reflect.get(target, property, receiver); +} + +/** + * Wrap the execution of a function with a series of Hooks + * + * @param {Function} fn + * @param {Object} [hooks={}] + * @property {Function} hooks.before + * @property {Function} hooks.after + * @return {Any} + */ +export function wrapFunction(fn, hooks = {}) { + return new Proxy(fn, { + get: commonGetHandler, + apply(target, thisArg, argumentsList) { + try { + if (hooks.before) { + hooks.before(target, thisArg, argumentsList); + } + return Reflect.apply(target, thisArg, argumentsList); + } finally { + if (hooks.after) { + hooks.after(target, thisArg, argumentsList); + } + } + } + }); +} + +/** + * Wrap the execution of an async function with a series of Hooks + * + * @param {AsyncFunction} fn + * @param {Object} [hooks={}] + * @property {AsyncFunction} hooks.before + * @property {AsyncFunction} hooks.handleError + * @property {AsyncFunction} hooks.after + * @return {Any} + */ +export function wrapAsyncFunction(fn, hooks = {}) { + return new Proxy(fn, { + get: commonGetHandler, + async apply(target, thisArg, argumentsList) { + try { + if (hooks.before) { + await hooks.before(target, thisArg, argumentsList); + } + + return await Reflect.apply(target, thisArg, argumentsList); + } catch (err) { + if (hooks.handleError) { + return await hooks.handleError(target, thisArg, argumentsList, err); + } + + throw err; + } finally { + if (hooks.after) { + await hooks.after(target, thisArg, argumentsList); + } + } + } + }); +} diff --git a/src/functional_test_runner/lib/mocha/wrap_runnable_args.js b/src/functional_test_runner/lib/mocha/wrap_runnable_args.js new file mode 100644 index 0000000000000..4a730c953140a --- /dev/null +++ b/src/functional_test_runner/lib/mocha/wrap_runnable_args.js @@ -0,0 +1,31 @@ +import { wrapFunction, wrapAsyncFunction } from './wrap_function'; + +/** + * Wraps a "runnable" defining function (it(), beforeEach(), etc.) + * so that any "runnable" arguments passed to it are wrapped and will + * trigger a lifecycle event if they throw an error. + * + * @param {Function} fn + * @param {String} eventName + * @return {Function} + */ +export function wrapRunnableArgsWithErrorHandler(fn, handler) { + return wrapFunction(fn, { + before(target, thisArg, argumentsList) { + for (let i = 0; i < argumentsList.length; i++) { + if (typeof argumentsList[i] === 'function') { + argumentsList[i] = wrapRunnableError(argumentsList[i], handler); + } + } + } + }); +} + +function wrapRunnableError(runnable, handler) { + return wrapAsyncFunction(runnable, { + async handleError(target, thisArg, argumentsList, err) { + await handler(err, thisArg.test); + throw err; + } + }); +} diff --git a/src/ui/public/utils/__tests__/scanner.js b/src/ui/public/utils/__tests__/scanner.js index 0fb7fafa06612..9e317f1510ed6 100644 --- a/src/ui/public/utils/__tests__/scanner.js +++ b/src/ui/public/utils/__tests__/scanner.js @@ -53,16 +53,15 @@ describe('Scanner', function () { scroll = sinon.stub(scanner.client, 'scroll', (req, cb) => cb(null, mockScroll)); }); - it('should reject when an error occurs', function (done) { + it('should reject when an error occurs', function () { search.restore(); search = sinon.stub(scanner.client, 'search', (req, cb) => cb(new Error('fail.'))); return scanner.scanAndMap('') .then(function () { - done(new Error('should reject')); + throw new Error('should reject'); }) .catch(function (error) { expect(error.message).to.be('fail.'); - done(); }); }); diff --git a/tasks/config/clean.js b/tasks/config/clean.js index 199f3fe381996..19482a9545bd1 100644 --- a/tasks/config/clean.js +++ b/tasks/config/clean.js @@ -2,7 +2,6 @@ module.exports = function () { return { build: 'build', target: 'target', - screenshots: 'test/screenshots/session', testsFromModules: 'build/kibana/node_modules/**/{test,tests}/**', devSourceForTestbed: 'build/kibana/src/core_plugins/testbed/', }; diff --git a/tasks/test.js b/tasks/test.js index f9c51defef693..733c7cc8881aa 100644 --- a/tasks/test.js +++ b/tasks/test.js @@ -58,7 +58,6 @@ module.exports = function (grunt) { 'checkPlugins', 'esvm:ui', 'run:testUIServer', - 'clean:screenshots', 'functionalTestRunner', 'esvm_shutdown:ui', 'stop:testUIServer' @@ -68,7 +67,6 @@ module.exports = function (grunt) { 'checkPlugins', 'esvm:ui', 'run:testUIReleaseServer', - 'clean:screenshots', 'functionalTestRunner', 'esvm_shutdown:ui', 'stop:testUIReleaseServer' diff --git a/test/functional/apps/console/_console.js b/test/functional/apps/console/_console.js index 9f40cedf08a0a..64d7190f57709 100644 --- a/test/functional/apps/console/_console.js +++ b/test/functional/apps/console/_console.js @@ -14,6 +14,7 @@ GET _search export default function ({ getService, getPageObjects }) { const retry = getService('retry'); const log = getService('log'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'console']); describe('console app', function describeIndexTests() { @@ -23,11 +24,11 @@ export default function ({ getService, getPageObjects }) { }); it('should show the default request', function () { - PageObjects.common.saveScreenshot('Console-help-expanded'); + screenshots.take('Console-help-expanded'); // collapse the help pane because we only get the VISIBLE TEXT, not the part that is scrolled return PageObjects.console.collapseHelp() .then(function () { - PageObjects.common.saveScreenshot('Console-help-collapsed'); + screenshots.take('Console-help-collapsed'); return retry.try(function () { return PageObjects.console.getRequest() .then(function (actualRequest) { @@ -42,7 +43,7 @@ export default function ({ getService, getPageObjects }) { return PageObjects.console.clickPlay() .then(function () { - PageObjects.common.saveScreenshot('Console-default-request'); + screenshots.take('Console-default-request'); return retry.try(function () { return PageObjects.console.getResponse() .then(function (actualResponse) { diff --git a/test/functional/apps/dashboard/_dashboard.js b/test/functional/apps/dashboard/_dashboard.js index 3c243003fd613..99bd7b03418d4 100644 --- a/test/functional/apps/dashboard/_dashboard.js +++ b/test/functional/apps/dashboard/_dashboard.js @@ -8,6 +8,7 @@ import { export default function ({ getService, getPageObjects }) { const retry = getService('retry'); const log = getService('log'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'dashboard', 'header', 'visualize']); describe('dashboard tab', function describeIndexTests() { @@ -16,7 +17,7 @@ export default function ({ getService, getPageObjects }) { }); it('should be able to add visualizations to dashboard', async function addVisualizations() { - await PageObjects.common.saveScreenshot('Dashboard-no-visualizations'); + await screenshots.take('Dashboard-no-visualizations'); // This flip between apps fixes the url so state is preserved when switching apps in test mode. // Without this flip the url in test mode looks something like @@ -29,7 +30,7 @@ export default function ({ getService, getPageObjects }) { await PageObjects.dashboard.addVisualizations(PageObjects.dashboard.getTestVisualizationNames()); log.debug('done adding visualizations'); - await PageObjects.common.saveScreenshot('Dashboard-add-visualizations'); + await screenshots.take('Dashboard-add-visualizations'); }); it('set the timepicker time to that which contains our test data', async function setTimepicker() { @@ -46,7 +47,7 @@ export default function ({ getService, getPageObjects }) { log.debug('now re-load previously saved dashboard'); return PageObjects.dashboard.loadSavedDashboard(dashboardName); }); - await PageObjects.common.saveScreenshot('Dashboard-load-saved'); + await screenshots.take('Dashboard-load-saved'); }); it('should have all the expected visualizations', function checkVisualizations() { @@ -58,7 +59,7 @@ export default function ({ getService, getPageObjects }) { }); }) .then(function () { - PageObjects.common.saveScreenshot('Dashboard-has-visualizations'); + screenshots.take('Dashboard-has-visualizations'); }); }); @@ -79,7 +80,7 @@ export default function ({ getService, getPageObjects }) { return PageObjects.dashboard.getPanelSizeData() .then(function (panelTitles) { log.info('visualization titles = ' + panelTitles); - PageObjects.common.saveScreenshot('Dashboard-visualization-sizes'); + screenshots.take('Dashboard-visualization-sizes'); expect(panelTitles).to.eql(visObjects); }); }); @@ -139,7 +140,7 @@ export default function ({ getService, getPageObjects }) { return retry.tryForTime(10000, async function () { const panelTitles = await PageObjects.dashboard.getPanelSizeData(); log.info('visualization titles = ' + panelTitles.map(item => item.title)); - PageObjects.common.saveScreenshot('Dashboard-visualization-sizes'); + screenshots.take('Dashboard-visualization-sizes'); expect(panelTitles.length).to.eql(visualizations.length + 1); }); }); diff --git a/test/functional/apps/discover/_collapse_expand.js b/test/functional/apps/discover/_collapse_expand.js index 6569467b206ba..0fb0a11cc2d58 100644 --- a/test/functional/apps/discover/_collapse_expand.js +++ b/test/functional/apps/discover/_collapse_expand.js @@ -4,6 +4,7 @@ export default function ({ getService, getPageObjects }) { const log = getService('log'); const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'discover', 'header']); describe('discover tab', function describeIndexTests() { @@ -36,7 +37,7 @@ export default function ({ getService, getPageObjects }) { describe('field data', function () { it('should initially be expanded', function () { - PageObjects.common.saveScreenshot('Discover-sidebar-expanded'); + screenshots.take('Discover-sidebar-expanded'); return PageObjects.discover.getSidebarWidth() .then(function (width) { log.debug('expanded sidebar width = ' + width); @@ -47,7 +48,7 @@ export default function ({ getService, getPageObjects }) { it('should collapse when clicked', function () { return PageObjects.discover.toggleSidebarCollapse() .then(function () { - PageObjects.common.saveScreenshot('Discover-sidebar-collapsed'); + screenshots.take('Discover-sidebar-collapsed'); log.debug('PageObjects.discover.getSidebarWidth()'); return PageObjects.discover.getSidebarWidth(); }) diff --git a/test/functional/apps/discover/_discover.js b/test/functional/apps/discover/_discover.js index 431d44155c729..66b9d295ca1e7 100644 --- a/test/functional/apps/discover/_discover.js +++ b/test/functional/apps/discover/_discover.js @@ -6,6 +6,7 @@ export default function ({ getService, getPageObjects }) { const esArchiver = getService('esArchiver'); const remote = getService('remote'); const kibanaServer = getService('kibanaServer'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'discover', 'header']); describe('discover app', function describeIndexTests() { @@ -48,7 +49,7 @@ export default function ({ getService, getPageObjects }) { const expectedToastMessage = `Discover: Saved Data Source "${queryName1}"`; expect(toastMessage).to.be(expectedToastMessage); - await PageObjects.common.saveScreenshot('Discover-save-query-toast'); + await screenshots.take('Discover-save-query-toast'); await PageObjects.header.waitForToastMessageGone(); const actualQueryNameString = await PageObjects.discover.getCurrentQueryName(); @@ -62,7 +63,7 @@ export default function ({ getService, getPageObjects }) { await retry.try(async function() { expect(await PageObjects.discover.getCurrentQueryName()).to.be(queryName1); }); - await PageObjects.common.saveScreenshot('Discover-load-query'); + await screenshots.take('Discover-load-query'); }); it('should show the correct hit count', async function () { @@ -208,7 +209,7 @@ export default function ({ getService, getPageObjects }) { it('should show "no results"', async () => { const isVisible = await PageObjects.discover.hasNoResults(); expect(isVisible).to.be(true); - await PageObjects.common.saveScreenshot('Discover-no-results'); + await screenshots.take('Discover-no-results'); }); it('should suggest a new time range is picked', async () => { diff --git a/test/functional/apps/discover/_field_data.js b/test/functional/apps/discover/_field_data.js index 47df8e6b86e17..421ff77775977 100644 --- a/test/functional/apps/discover/_field_data.js +++ b/test/functional/apps/discover/_field_data.js @@ -5,6 +5,7 @@ export default function ({ getService, getPageObjects }) { const retry = getService('retry'); const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'header', 'discover']); describe('discover app', function describeIndexTests() { @@ -44,7 +45,7 @@ export default function ({ getService, getPageObjects }) { return retry.try(function tryingForTime() { return PageObjects.discover.getHitCount() .then(function compareData(hitCount) { - PageObjects.common.saveScreenshot('Discover-field-data'); + screenshots.take('Discover-field-data'); expect(hitCount).to.be(expectedHitCount); }); }); @@ -217,7 +218,7 @@ export default function ({ getService, getPageObjects }) { return retry.try(function tryingForTime() { return PageObjects.discover.getDocTableIndex(1) .then(function (rowData) { - PageObjects.common.saveScreenshot('Discover-sort-down'); + screenshots.take('Discover-sort-down'); expect(rowData).to.be(ExpectedDoc); }); }); @@ -232,7 +233,7 @@ export default function ({ getService, getPageObjects }) { return PageObjects.header.getToastMessage(); }) .then(function (toastMessage) { - PageObjects.common.saveScreenshot('Discover-syntax-error-toast'); + screenshots.take('Discover-syntax-error-toast'); expect(toastMessage).to.be(expectedError); }) .then(function () { diff --git a/test/functional/apps/discover/_shared_links.js b/test/functional/apps/discover/_shared_links.js index 40841514f900e..028146e860aff 100644 --- a/test/functional/apps/discover/_shared_links.js +++ b/test/functional/apps/discover/_shared_links.js @@ -5,6 +5,7 @@ export default function ({ getService, getPageObjects }) { const log = getService('log'); const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'discover', 'header']); describe('shared links', function describeIndexTests() { @@ -60,7 +61,7 @@ export default function ({ getService, getPageObjects }) { const expectedCaption = 'Share saved'; return PageObjects.discover.clickShare() .then(function () { - PageObjects.common.saveScreenshot('Discover-share-link'); + screenshots.take('Discover-share-link'); return PageObjects.discover.getShareCaption(); }) .then(function (actualCaption) { @@ -90,7 +91,7 @@ export default function ({ getService, getPageObjects }) { return PageObjects.header.getToastMessage(); }) .then(function (toastMessage) { - PageObjects.common.saveScreenshot('Discover-copy-to-clipboard-toast'); + screenshots.take('Discover-copy-to-clipboard-toast'); expect(toastMessage).to.match(expectedToastMessage); }) .then(function () { @@ -104,7 +105,7 @@ export default function ({ getService, getPageObjects }) { return PageObjects.discover.clickShortenUrl() .then(function () { return retry.try(function tryingForTime() { - PageObjects.common.saveScreenshot('Discover-shorten-url-button'); + screenshots.take('Discover-shorten-url-button'); return PageObjects.discover.getSharedUrl() .then(function (actualUrl) { expect(actualUrl).to.match(re); diff --git a/test/functional/apps/management/_creation_form_changes.js b/test/functional/apps/management/_creation_form_changes.js index f6f011b718bff..66bafe4bebd26 100644 --- a/test/functional/apps/management/_creation_form_changes.js +++ b/test/functional/apps/management/_creation_form_changes.js @@ -2,6 +2,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const kibanaServer = getService('kibanaServer'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['settings', 'common']); describe('user input reactions', function () { @@ -22,7 +23,7 @@ export default function ({ getService, getPageObjects }) { .then(function () { return PageObjects.settings.getCreateButton().isEnabled() .then(function (enabled) { - PageObjects.common.saveScreenshot('Settings-indices-enable-creation'); + screenshots.take('Settings-indices-enable-creation'); expect(enabled).to.be.ok(); }); }); diff --git a/test/functional/apps/management/_index_pattern_create_delete.js b/test/functional/apps/management/_index_pattern_create_delete.js index cd68c650e5fa1..fb12f5f4fadd8 100644 --- a/test/functional/apps/management/_index_pattern_create_delete.js +++ b/test/functional/apps/management/_index_pattern_create_delete.js @@ -5,6 +5,7 @@ export default function ({ getService, getPageObjects }) { const remote = getService('remote'); const log = getService('log'); const retry = getService('retry'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['settings', 'common']); describe('creating and deleting default index', function describeIndexTests() { @@ -27,7 +28,7 @@ export default function ({ getService, getPageObjects }) { it('should have index pattern in page header', function () { return PageObjects.settings.getIndexPageHeading().getVisibleText() .then(function (patternName) { - PageObjects.common.saveScreenshot('Settings-indices-new-index-pattern'); + screenshots.take('Settings-indices-new-index-pattern'); expect(patternName).to.be('logstash-*'); }); }); @@ -75,7 +76,7 @@ export default function ({ getService, getPageObjects }) { const expectedAlertText = 'Are you sure you want to remove this index pattern?'; return PageObjects.settings.removeIndexPattern() .then(function (alertText) { - PageObjects.common.saveScreenshot('Settings-indices-confirm-remove-index-pattern'); + screenshots.take('Settings-indices-confirm-remove-index-pattern'); expect(alertText).to.be(expectedAlertText); }); }); diff --git a/test/functional/apps/management/_index_pattern_popularity.js b/test/functional/apps/management/_index_pattern_popularity.js index 5a8c26c56addc..0aa2ddb3da1a7 100644 --- a/test/functional/apps/management/_index_pattern_popularity.js +++ b/test/functional/apps/management/_index_pattern_popularity.js @@ -3,6 +3,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const kibanaServer = getService('kibanaServer'); const log = getService('log'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['settings', 'common']); describe('index result popularity', function describeIndexTests() { @@ -51,7 +52,7 @@ export default function ({ getService, getPageObjects }) { const popularity = await PageObjects.settings.getPopularity(); log.debug('popularity = ' + popularity); expect(popularity).to.be('1'); - PageObjects.common.saveScreenshot('Settings-indices-result-popularity-updated'); + screenshots.take('Settings-indices-result-popularity-updated'); }); it('should be reset on cancel', async function () { @@ -73,7 +74,7 @@ export default function ({ getService, getPageObjects }) { const popularity = await PageObjects.settings.getPopularity(); log.debug('popularity = ' + popularity); expect(popularity).to.be('1'); - PageObjects.common.saveScreenshot('Settings-indices-result-popularity-saved'); + screenshots.take('Settings-indices-result-popularity-saved'); }); }); // end 'change popularity' }); // end index result popularity diff --git a/test/functional/apps/management/_index_pattern_results_sort.js b/test/functional/apps/management/_index_pattern_results_sort.js index f1b850a38d6c5..9f9f2a70aeaa1 100644 --- a/test/functional/apps/management/_index_pattern_results_sort.js +++ b/test/functional/apps/management/_index_pattern_results_sort.js @@ -3,6 +3,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const kibanaServer = getService('kibanaServer'); const retry = getService('retry'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['settings', 'common']); describe('index result field sort', function describeIndexTests() { @@ -50,7 +51,7 @@ export default function ({ getService, getPageObjects }) { return col.selector(); }) .then(function (rowText) { - PageObjects.common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-ascending`); + screenshots.take(`Settings-indices-column-${col.heading}-sort-ascending`); expect(rowText).to.be(col.first); }); }); @@ -64,7 +65,7 @@ export default function ({ getService, getPageObjects }) { return col.selector(); }) .then(function (rowText) { - PageObjects.common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-descending`); + screenshots.take(`Settings-indices-column-${col.heading}-sort-descending`); expect(rowText).to.be(col.last); }); }); @@ -108,7 +109,7 @@ export default function ({ getService, getPageObjects }) { for (let pageNum = 1; pageNum <= LAST_PAGE_NUMBER; pageNum += 1) { await PageObjects.settings.goToPage(pageNum); const pageFieldNames = await retry.tryMethod(PageObjects.settings, 'getFieldNames'); - await PageObjects.common.saveScreenshot(`Settings-indexed-fields-page-${pageNum}`); + await screenshots.take(`Settings-indexed-fields-page-${pageNum}`); if (pageNum === LAST_PAGE_NUMBER) { expect(pageFieldNames).to.have.length(EXPECTED_LAST_PAGE_COUNT); diff --git a/test/functional/apps/management/_kibana_settings.js b/test/functional/apps/management/_kibana_settings.js index 1d8cb08a7e561..51bb18c8a7dd5 100644 --- a/test/functional/apps/management/_kibana_settings.js +++ b/test/functional/apps/management/_kibana_settings.js @@ -3,6 +3,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const kibanaServer = getService('kibanaServer'); const log = getService('log'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['settings', 'common']); describe('creating and deleting default index', function describeIndexTests() { @@ -32,12 +33,12 @@ export default function ({ getService, getPageObjects }) { it('should allow setting advanced settings', function () { return PageObjects.settings.clickKibanaSettings() .then(function TestCallSetAdvancedSettingsForTimezone() { - PageObjects.common.saveScreenshot('Settings-advanced-tab'); + screenshots.take('Settings-advanced-tab'); log.debug('calling setAdvancedSetting'); return PageObjects.settings.setAdvancedSettings('dateFormat:tz', 'America/Phoenix'); }) .then(function GetAdvancedSetting() { - PageObjects.common.saveScreenshot('Settings-set-timezone'); + screenshots.take('Settings-set-timezone'); return PageObjects.settings.getAdvancedSettings('dateFormat:tz'); }) .then(function (advancedSetting) { @@ -48,7 +49,7 @@ export default function ({ getService, getPageObjects }) { after(function () { return PageObjects.settings.clickKibanaSettings() .then(function TestCallSetAdvancedSettingsForTimezone() { - PageObjects.common.saveScreenshot('Settings-advanced-tab'); + screenshots.take('Settings-advanced-tab'); log.debug('calling setAdvancedSetting'); return PageObjects.settings.setAdvancedSettings('dateFormat:tz', 'UTC'); }); diff --git a/test/functional/apps/management/_scripted_fields.js b/test/functional/apps/management/_scripted_fields.js index a949796c70e5f..32a11b3450be8 100644 --- a/test/functional/apps/management/_scripted_fields.js +++ b/test/functional/apps/management/_scripted_fields.js @@ -18,6 +18,7 @@ export default function ({ getService, getPageObjects }) { const log = getService('log'); const remote = getService('remote'); const retry = getService('retry'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'header', 'settings', 'visualize', 'discover']); describe('scripted fields', () => { @@ -102,7 +103,7 @@ export default function ({ getService, getPageObjects }) { await log.debug('getDataTableData = ' + data.split('\n')); await log.debug('data=' + data); await log.debug('data.length=' + data.length); - await PageObjects.common.saveScreenshot('Visualize-vertical-bar-chart'); + await screenshots.take('Visualize-vertical-bar-chart'); expect(data.trim().split('\n')).to.eql(expectedChartValues); }); }); @@ -169,7 +170,7 @@ export default function ({ getService, getPageObjects }) { await log.debug('getDataTableData = ' + data.split('\n')); await log.debug('data=' + data); await log.debug('data.length=' + data.length); - await PageObjects.common.saveScreenshot('Visualize-vertical-bar-chart'); + await screenshots.take('Visualize-vertical-bar-chart'); expect(data.trim().split('\n')).to.eql(expectedChartValues); }); }); diff --git a/test/functional/apps/status_page/index.js b/test/functional/apps/status_page/index.js index c3ac3b2059132..53c78ebda6c34 100644 --- a/test/functional/apps/status_page/index.js +++ b/test/functional/apps/status_page/index.js @@ -2,6 +2,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const retry = getService('retry'); + const screenshots = getService('screenshots'); const testSubjects = getService('testSubjects'); const PageObjects = getPageObjects(['common']); @@ -15,7 +16,7 @@ export default function ({ getService, getPageObjects }) { return testSubjects.find('statusBreakdown') .getVisibleText() .then(function (text) { - PageObjects.common.saveScreenshot('Status'); + screenshots.take('Status'); expect(text.indexOf('plugin:kibana')).to.be.above(-1); }); }); diff --git a/test/functional/apps/visualize/_area_chart.js b/test/functional/apps/visualize/_area_chart.js index d3638a60c29b2..4e6eb25274bf5 100644 --- a/test/functional/apps/visualize/_area_chart.js +++ b/test/functional/apps/visualize/_area_chart.js @@ -3,6 +3,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const log = getService('log'); const retry = getService('retry'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'visualize', 'header', 'settings']); describe('visualize app', function describeIndexTests() { @@ -85,7 +86,7 @@ export default function ({ getService, getPageObjects }) { return PageObjects.visualize.saveVisualization(vizName1) .then(function (message) { log.debug('Saved viz message = ' + message); - PageObjects.common.saveScreenshot('Visualize-area-chart-save-toast'); + screenshots.take('Visualize-area-chart-save-toast'); expect(message).to.be('Visualization Editor: Saved Visualization \"' + vizName1 + '\"'); }) .then(function testVisualizeWaitForToastMessageGone() { @@ -134,7 +135,7 @@ export default function ({ getService, getPageObjects }) { .then(function (paths) { log.debug('expectedAreaChartData = ' + expectedAreaChartData); log.debug('actual chart data = ' + paths); - PageObjects.common.saveScreenshot('Visualize-area-chart'); + screenshots.take('Visualize-area-chart'); expect(paths).to.eql(expectedAreaChartData); }); }); diff --git a/test/functional/apps/visualize/_chart_types.js b/test/functional/apps/visualize/_chart_types.js index c265f3435b582..30644869b70b5 100644 --- a/test/functional/apps/visualize/_chart_types.js +++ b/test/functional/apps/visualize/_chart_types.js @@ -2,6 +2,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const log = getService('log'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'visualize']); describe('visualize app', function describeIndexTests() { @@ -34,7 +35,7 @@ export default function ({ getService, getPageObjects }) { .then(function testChartTypes(chartTypes) { log.debug('returned chart types = ' + chartTypes); log.debug('expected chart types = ' + expectedChartTypes); - PageObjects.common.saveScreenshot('Visualize-chart-types'); + screenshots.take('Visualize-chart-types'); expect(chartTypes).to.eql(expectedChartTypes); }); }); diff --git a/test/functional/apps/visualize/_data_table.js b/test/functional/apps/visualize/_data_table.js index d77cfc8ffeaac..06d1c4ff18546 100644 --- a/test/functional/apps/visualize/_data_table.js +++ b/test/functional/apps/visualize/_data_table.js @@ -3,6 +3,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const log = getService('log'); const retry = getService('retry'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'visualize', 'header']); describe('visualize app', function describeIndexTests() { @@ -78,7 +79,7 @@ export default function ({ getService, getPageObjects }) { return PageObjects.visualize.getDataTableData() .then(function showData(data) { log.debug(data.split('\n')); - PageObjects.common.saveScreenshot('Visualize-data-table'); + screenshots.take('Visualize-data-table'); expect(data.split('\n')).to.eql(expectedChartData); }); }); diff --git a/test/functional/apps/visualize/_heatmap_chart.js b/test/functional/apps/visualize/_heatmap_chart.js index 17ac350031d51..93508932319bd 100644 --- a/test/functional/apps/visualize/_heatmap_chart.js +++ b/test/functional/apps/visualize/_heatmap_chart.js @@ -2,6 +2,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const log = getService('log'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'visualize', 'header']); describe('visualize app', function describeIndexTests() { @@ -85,7 +86,7 @@ export default function ({ getService, getPageObjects }) { .then(function showData(data) { log.debug('data=' + data); log.debug('data.length=' + data.length); - PageObjects.common.saveScreenshot('Visualize-heatmap-chart'); + screenshots.take('Visualize-heatmap-chart'); expect(data).to.eql(expectedChartValues); }); }); diff --git a/test/functional/apps/visualize/_line_chart.js b/test/functional/apps/visualize/_line_chart.js index 85021aa740208..d98c57ba467d2 100644 --- a/test/functional/apps/visualize/_line_chart.js +++ b/test/functional/apps/visualize/_line_chart.js @@ -3,6 +3,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const log = getService('log'); const retry = getService('retry'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'visualize', 'header']); describe('visualize app', function describeIndexTests() { @@ -62,7 +63,7 @@ export default function ({ getService, getPageObjects }) { return PageObjects.visualize.getLineChartData('fill="#6eadc1"') .then(function showData(data) { log.debug('data=' + data); - PageObjects.common.saveScreenshot('Visualize-line-chart'); + screenshots.take('Visualize-line-chart'); const tolerance = 10; // the y-axis scale is 10000 so 10 is 0.1% for (let x = 0; x < data.length; x++) { log.debug('x=' + x + ' expectedChartData[x].split(\' \')[1] = ' + @@ -92,7 +93,7 @@ export default function ({ getService, getPageObjects }) { return PageObjects.visualize.getLineChartData('fill="#6eadc1"') .then(function showData(data) { log.debug('data=' + data); - PageObjects.common.saveScreenshot('Visualize-line-chart'); + screenshots.take('Visualize-line-chart'); const tolerance = 10; // the y-axis scale is 10000 so 10 is 0.1% for (let x = 0; x < data.length; x++) { log.debug('x=' + x + ' expectedChartData[x].split(\' \')[1] = ' + diff --git a/test/functional/apps/visualize/_metric_chart.js b/test/functional/apps/visualize/_metric_chart.js index a6840035bedd6..401d69eb51932 100644 --- a/test/functional/apps/visualize/_metric_chart.js +++ b/test/functional/apps/visualize/_metric_chart.js @@ -3,6 +3,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const log = getService('log'); const retry = getService('retry'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'visualize', 'header']); describe('visualize app', function describeIndexTests() { @@ -34,7 +35,7 @@ export default function ({ getService, getPageObjects }) { return retry.try(function tryingForTime() { return PageObjects.visualize.getMetric() .then(function (metricValue) { - PageObjects.common.saveScreenshot('Visualize-metric-chart'); + screenshots.take('Visualize-metric-chart'); expect(expectedCount).to.eql(metricValue.split('\n')); }); }); diff --git a/test/functional/apps/visualize/_pie_chart.js b/test/functional/apps/visualize/_pie_chart.js index b787f2c51be1f..cd0ceffb9458f 100644 --- a/test/functional/apps/visualize/_pie_chart.js +++ b/test/functional/apps/visualize/_pie_chart.js @@ -2,6 +2,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const log = getService('log'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'visualize', 'header', 'settings']); describe('visualize app', function describeIndexTests() { @@ -84,7 +85,7 @@ export default function ({ getService, getPageObjects }) { return PageObjects.visualize.getPieChartData() .then(function (pieData) { log.debug('pieData.length = ' + pieData.length); - PageObjects.common.saveScreenshot('Visualize-pie-chart'); + screenshots.take('Visualize-pie-chart'); expect(pieData.length).to.be(expectedPieChartSliceCount); }); }); diff --git a/test/functional/apps/visualize/_point_series_options.js b/test/functional/apps/visualize/_point_series_options.js index 11b47465737ef..71e4642b01299 100644 --- a/test/functional/apps/visualize/_point_series_options.js +++ b/test/functional/apps/visualize/_point_series_options.js @@ -2,6 +2,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const log = getService('log'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'visualize', 'header', 'pointSeries']); const pointSeriesVis = PageObjects.pointSeries; @@ -101,7 +102,7 @@ export default function ({ getService, getPageObjects }) { .then(function showData(data) { log.debug('count data=' + data); log.debug('data.length=' + data.length); - PageObjects.common.saveScreenshot('Visualize-secondary-value-axis'); + screenshots.take('Visualize-secondary-value-axis'); expect(data).to.eql(expectedChartValues[0]); }) .then(function () { diff --git a/test/functional/apps/visualize/_tile_map.js b/test/functional/apps/visualize/_tile_map.js index 3035d49844fae..e3d17177b84a1 100644 --- a/test/functional/apps/visualize/_tile_map.js +++ b/test/functional/apps/visualize/_tile_map.js @@ -3,6 +3,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const log = getService('log'); const retry = getService('retry'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'visualize', 'header', 'settings']); describe('visualize app', function describeIndexTests() { @@ -112,7 +113,7 @@ export default function ({ getService, getPageObjects }) { }) .then(function takeScreenshot() { log.debug('Take screenshot (success)'); - PageObjects.common.saveScreenshot('map-at-zoom-0'); + screenshots.take('map-at-zoom-0'); }); }); @@ -267,7 +268,7 @@ export default function ({ getService, getPageObjects }) { }) .then(function takeScreenshot() { log.debug('Take screenshot'); - PageObjects.common.saveScreenshot('Visualize-site-map'); + screenshots.take('Visualize-site-map'); }); }); diff --git a/test/functional/apps/visualize/_vertical_bar_chart.js b/test/functional/apps/visualize/_vertical_bar_chart.js index ea1cfea3d04d7..218f352ca1652 100644 --- a/test/functional/apps/visualize/_vertical_bar_chart.js +++ b/test/functional/apps/visualize/_vertical_bar_chart.js @@ -2,6 +2,7 @@ import expect from 'expect.js'; export default function ({ getService, getPageObjects }) { const log = getService('log'); + const screenshots = getService('screenshots'); const PageObjects = getPageObjects(['common', 'visualize', 'header']); describe('visualize app', function describeIndexTests() { @@ -84,7 +85,7 @@ export default function ({ getService, getPageObjects }) { .then(function showData(data) { log.debug('data=' + data); log.debug('data.length=' + data.length); - PageObjects.common.saveScreenshot('Visualize-vertical-bar-chart'); + screenshots.take('Visualize-vertical-bar-chart'); expect(data).to.eql(expectedChartValues); }); }); diff --git a/test/functional/config.js b/test/functional/config.js index 2bda3bc756460..050a8d4d0c3ce 100644 --- a/test/functional/config.js +++ b/test/functional/config.js @@ -11,7 +11,7 @@ import { VisualizePageProvider, SettingsPageProvider, MonitoringPageProvider, - PointSeriesPageProvider + PointSeriesPageProvider, } from './page_objects'; import { @@ -22,7 +22,8 @@ import { KibanaServerProvider, EsProvider, EsArchiverProvider, - DocTableProvider + DocTableProvider, + ScreenshotsProvider, } from './services'; import { servers, apps } from '../server_config'; @@ -50,7 +51,7 @@ export default function () { visualize: VisualizePageProvider, settings: SettingsPageProvider, monitoring: MonitoringPageProvider, - pointSeries: PointSeriesPageProvider + pointSeries: PointSeriesPageProvider, }, services: { kibanaServer: KibanaServerProvider, @@ -60,15 +61,16 @@ export default function () { testSubjects: TestSubjectsProvider, es: EsProvider, esArchiver: EsArchiverProvider, - docTable: DocTableProvider + docTable: DocTableProvider, + screenshots: ScreenshotsProvider, }, servers, apps, esArchiver: { - directory: resolve(__dirname, '../../src/fixtures/es_archives') + directory: resolve(__dirname, '../../src/fixtures/es_archives'), }, screenshots: { - directory: resolve(__dirname, '../screenshots/session') + directory: resolve(__dirname, '../screenshots'), } }; } diff --git a/test/functional/page_objects/common_page.js b/test/functional/page_objects/common_page.js index 7463afd73679c..54632e272ae77 100644 --- a/test/functional/page_objects/common_page.js +++ b/test/functional/page_objects/common_page.js @@ -1,13 +1,7 @@ -import { delay, promisify } from 'bluebird'; -import fs from 'fs'; -import mkdirp from 'mkdirp'; -import { resolve } from 'path'; +import { delay } from 'bluebird'; import getUrl from '../../utils/get_url'; -const mkdirpAsync = promisify(mkdirp); -const writeFileAsync = promisify(fs.writeFile); - export function CommonPageProvider({ getService, getPageObjects }) { const log = getService('log'); const config = getService('config'); @@ -17,7 +11,6 @@ export function CommonPageProvider({ getService, getPageObjects }) { const kibanaServer = getService('kibanaServer'); const PageObjects = getPageObjects(['shield']); - const screenshotDirectory = config.get('screenshots.directory'); const defaultTryTimeout = config.get('timeouts.try'); const defaultFindTimeout = config.get('timeouts.find'); @@ -194,21 +187,6 @@ export function CommonPageProvider({ getService, getPageObjects }) { }; } - async saveScreenshot(fileName, isFailure = false) { - try { - const type = isFailure ? 'failure' : 'session'; - const directory = resolve(screenshotDirectory, type); - const path = resolve(directory, `${fileName}.png`); - log.debug(`Taking screenshot "${path}"`); - - const screenshot = await remote.takeScreenshot(); - await mkdirpAsync(directory); - await writeFileAsync(path, screenshot); - } catch (err) { - log.warning(`SCREENSHOT FAILED: ${err}`); - } - } - async waitUntilUrlIncludes(path) { await retry.try(async () => { const url = await remote.getCurrentUrl(); diff --git a/test/functional/services/index.js b/test/functional/services/index.js index 4082e3a0edb49..3164ae26ec683 100644 --- a/test/functional/services/index.js +++ b/test/functional/services/index.js @@ -6,3 +6,4 @@ export { KibanaServerProvider } from './kibana_server'; export { EsProvider } from './es'; export { EsArchiverProvider } from './es_archiver'; export { DocTableProvider } from './doc_table'; +export { ScreenshotsProvider } from './screenshots'; diff --git a/test/functional/services/screenshots.js b/test/functional/services/screenshots.js new file mode 100644 index 0000000000000..3c90995f0616e --- /dev/null +++ b/test/functional/services/screenshots.js @@ -0,0 +1,49 @@ +import { resolve, dirname } from 'path'; +import { writeFile } from 'fs'; + +import { fromNode as fcb } from 'bluebird'; +import mkdirp from 'mkdirp'; +import del from 'del'; + +export async function ScreenshotsProvider({ getService }) { + const log = getService('log'); + const config = getService('config'); + const remote = getService('remote'); + const lifecycle = getService('lifecycle'); + + const SESSION_DIRECTORY = resolve(config.get('screenshots.directory'), 'session'); + const FAILURE_DIRECTORY = resolve(config.get('screenshots.directory'), 'failure'); + await del([SESSION_DIRECTORY, FAILURE_DIRECTORY]); + + class Screenshots { + async take(name) { + return await this._take(resolve(SESSION_DIRECTORY, `${name}.png`)); + } + + async takeForFailure(name) { + return await this._take(resolve(FAILURE_DIRECTORY, `${name}.png`)); + } + + async _take(path) { + try { + log.debug(`Taking screenshot "${path}"`); + const [screenshot] = await Promise.all([ + remote.takeScreenshot(), + fcb(cb => mkdirp(dirname(path), cb)), + ]); + await fcb(cb => writeFile(path, screenshot, cb)); + } catch (err) { + log.error('SCREENSHOT FAILED'); + log.error(err); + } + } + } + + const screenshots = new Screenshots(); + + lifecycle + .on('testFailure', (err, test) => screenshots.takeForFailure(test.fullTitle())) + .on('testHookFailure', (err, test) => screenshots.takeForFailure(test.fullTitle())); + + return screenshots; +}