Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.x] adding runPipeline tests (#27015) #33637

Merged
merged 1 commit into from
Mar 21, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@
"packages/*",
"x-pack",
"x-pack/plugins/*",
"test/plugin_functional/plugins/*"
"test/plugin_functional/plugins/*",
"test/interpreter_functional/plugins/*"
],
"nohoist": [
"**/@types/*",
Expand Down Expand Up @@ -386,6 +387,7 @@
"normalize-path": "^3.0.0",
"pixelmatch": "4.0.2",
"pkg-up": "^2.0.0",
"pngjs": "^3.4.0",
"postcss": "^7.0.5",
"postcss-url": "^8.0.0",
"prettier": "^1.14.3",
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-pm/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ export function getProjectPaths(rootPath: string, options: IProjectPathOptions)
// In anyway, have a plugin declaring their own dependencies is the
// correct and the expect behavior.
projectPaths.push(resolve(rootPath, 'test/plugin_functional/plugins/*'));
projectPaths.push(resolve(rootPath, 'test/interpreter_functional/plugins/*'));

if (!ossOnly) {
projectPaths.push(resolve(rootPath, 'x-pack'));
Expand Down
1 change: 1 addition & 0 deletions scripts/functional_tests.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,5 @@ require('@kbn/test').runTestsCli([
require.resolve('../test/functional/config.js'),
require.resolve('../test/api_integration/config.js'),
require.resolve('../test/plugin_functional/config.js'),
require.resolve('../test/interpreter_functional/config.js'),
]);
7 changes: 7 additions & 0 deletions src/functional_test_runner/lib/config/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,13 @@ export const schema = Joi.object()
})
.default(),

// settings for the snapshots module
snapshots: Joi.object()
.keys({
directory: Joi.string().default(defaultRelativeToConfigPath('snapshots')),
})
.default(),

// settings for the failureDebugging module
failureDebugging: Joi.object()
.keys({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ export const visualization = () => ({

handlers.onDestroy(() => visualizationLoader.destroy());

await visualizationLoader.render(domNode, handlers.vis, visData, visConfig || handlers.vis.params, uiState, params).then(() => {
await visualizationLoader.render(domNode, handlers.vis, visData, handlers.vis.params, uiState, params).then(() => {
if (handlers.done) handlers.done();
});
},
Expand Down
11 changes: 11 additions & 0 deletions tasks/config/run.js
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,17 @@ module.exports = function (grunt) {
],
},

interpreterFunctionalTestsRelease: {
cmd: process.execPath,
args: [
'scripts/functional_tests',
'--config', 'test/interpreter_functional/config.js',
'--bail',
'--debug',
'--kibana-install-dir', KIBANA_INSTALL_DIR,
],
},

pluginFunctionalTestsRelease: {
cmd: process.execPath,
args: [
Expand Down
33 changes: 33 additions & 0 deletions test/functional/services/browser.js
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,39 @@ export async function BrowserProvider({ getService }) {
}
}));
}

async executeAsync(fn, ...args) {
return await driver.executeAsyncScript(fn, ...cloneDeep(args, arg => {
if (arg instanceof WebElementWrapper) {
return arg._webElement;
}
}));
}

getScrollTop() {
return driver
.executeScript('return document.body.scrollTop')
.then(scrollSize => parseInt(scrollSize, 10));
}

getScrollLeft() {
return driver
.executeScript('return document.body.scrollLeft')
.then(scrollSize => parseInt(scrollSize, 10));
}

// return promise with REAL scroll position
setScrollTop(scrollSize) {
return driver
.executeScript('document.body.scrollTop = ' + scrollSize)
.then(() => this.getScrollTop(driver));
}

setScrollLeft(scrollSize) {
return driver
.executeScript('document.body.scrollLeft = ' + scrollSize)
.then(() => this.getScrollLeft(driver));
}
}

return new BrowserService();
Expand Down
3 changes: 3 additions & 0 deletions test/functional/services/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ import { RenderableProvider } from './renderable';
// @ts-ignore not TS yet
import { ScreenshotsProvider } from './screenshots';
// @ts-ignore not TS yet
import { SnapshotsProvider } from './snapshots';
// @ts-ignore not TS yet
import { TableProvider } from './table';
// @ts-ignore not TS yet
import { TestSubjectsProvider } from './test_subjects';
Expand All @@ -70,6 +72,7 @@ export const services = {
testSubjects: TestSubjectsProvider,
docTable: DocTableProvider,
screenshots: ScreenshotsProvider,
snapshots: SnapshotsProvider,
dashboardVisualizations: DashboardVisualizationProvider,
dashboardExpect: DashboardExpectProvider,
failureDebugging: FailureDebuggingProvider,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import { scrollIntoViewIfNecessary } from './scroll_into_view_if_necessary';
import { delay } from 'bluebird';
import { PNG } from 'pngjs';
import cheerio from 'cheerio';
import testSubjSelector from '@kbn/test-subj-selector';

Expand Down Expand Up @@ -443,4 +444,19 @@ export class WebElementWrapper {

return $;
}

/**
* Creates the screenshot of the element
*
* @returns {Promise<void>}
*/
async takeScreenshot() {
const screenshot = await this._driver.takeScreenshot();
const buffer = Buffer.from(screenshot.toString(), 'base64');
const { width, height, x, y } = await this.getPosition();
const src = PNG.sync.read(buffer);
const dst = new PNG({ width, height });
PNG.bitblt(src, dst, x, y, width, height, 0, 0);
return PNG.sync.write(dst);
}
}
20 changes: 9 additions & 11 deletions test/functional/services/screenshots.js
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ export async function ScreenshotsProvider({ getService }) {
* @param updateBaselines {boolean} optional, pass true to update the baseline snapshot.
* @return {Promise.<number>} Percentage difference between the baseline and the current snapshot.
*/
async compareAgainstBaseline(name, updateBaselines) {
async compareAgainstBaseline(name, updateBaselines, el) {
log.debug('compareAgainstBaseline');
const sessionPath = resolve(SESSION_DIRECTORY, `${name}.png`);
await this._take(sessionPath);
await this._take(sessionPath, el);

const baselinePath = resolve(BASELINE_DIRECTORY, `${name}.png`);
const failurePath = resolve(FAILURE_DIRECTORY, `${name}.png`);
Expand All @@ -63,21 +63,19 @@ export async function ScreenshotsProvider({ getService }) {
}
}

async take(name) {
return await this._take(resolve(SESSION_DIRECTORY, `${name}.png`));
async take(name, el) {
return await this._take(resolve(SESSION_DIRECTORY, `${name}.png`), el);
}

async takeForFailure(name) {
await this._take(resolve(FAILURE_DIRECTORY, `${name}.png`));
async takeForFailure(name, el) {
await this._take(resolve(FAILURE_DIRECTORY, `${name}.png`), el);
}

async _take(path) {
async _take(path, el) {
try {
log.info(`Taking screenshot "${path}"`);
const [screenshot] = await Promise.all([
browser.takeScreenshot(),
fcb(cb => mkdirp(dirname(path), cb)),
]);
const screenshot = await (el ? el.takeScreenshot() : browser.takeScreenshot());
await fcb(cb => mkdirp(dirname(path), cb));
await fcb(cb => writeFile(path, screenshot, 'base64', cb));
} catch (err) {
log.error('SCREENSHOT FAILED');
Expand Down
85 changes: 85 additions & 0 deletions test/functional/services/snapshots.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import expect from 'expect.js';
import { dirname, resolve } from 'path';
import { writeFile, readFileSync } from 'fs';
import { fromNode as fcb, promisify } from 'bluebird';
import mkdirp from 'mkdirp';
import del from 'del';

const writeFileAsync = promisify(writeFile);

export async function SnapshotsProvider({ getService }) {
const log = getService('log');
const config = getService('config');

const SESSION_DIRECTORY = resolve(config.get('snapshots.directory'), 'session');
const BASELINE_DIRECTORY = resolve(config.get('snapshots.directory'), 'baseline');
await del([SESSION_DIRECTORY]);

class Snapshots {

/**
*
* @param name {string} name of the file to use for comparison
* @param value {object} value to compare to baseline.
* @param updateBaselines {boolean} optional, pass true to update the baseline snapshot.
* @return {Promise.<number>} returns 0 if successful.
*/
async compareAgainstBaseline(name, value, updateBaselines) {
log.debug('compareAgainstBaseline');
const sessionPath = resolve(SESSION_DIRECTORY, `${name}.json`);
await this._take(sessionPath, value);

const baselinePath = resolve(BASELINE_DIRECTORY, `${name}.json`);

if (updateBaselines) {
await writeFileAsync(baselinePath, readFileSync(sessionPath));
return 0;
} else {
log.debug('comparing');
return this._compare(sessionPath, baselinePath);
}
}

_compare(sessionPath, baselinePath) {
const currentObject = readFileSync(sessionPath, { encoding: 'utf8' });
const baselineObject = readFileSync(baselinePath, { encoding: 'utf8' });
expect(currentObject).to.eql(baselineObject);
return 0;
}

async take(name) {
return await this._take(resolve(SESSION_DIRECTORY, `${name}.json`));
}

async _take(path, snapshot) {
try {
await fcb(cb => mkdirp(dirname(path), cb));
await fcb(cb => writeFile(path, JSON.stringify(snapshot), 'utf8', cb));
} catch (err) {
log.error('SNAPSHOT FAILED');
log.error(err);
}
}
}

return new Snapshots();
}
21 changes: 21 additions & 0 deletions test/interpreter_functional/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Interpreter Functional Tests

This folder contains interpreter functional tests.

Add new test suites into the `test_suites` folder and reference them from the
`config.js` file. These test suites work the same as regular functional test.

## Run the test

To run these tests during development you can use the following commands:

```
# Start the test server (can continue running)
node scripts/functional_tests_server.js --config test/interpreter_functional/config.js
# Start a test run
node scripts/functional_test_runner.js --config test/interpreter_functional/config.js
```

# Writing tests

Look into test_suites/run_pipeline/basic.js for examples
57 changes: 57 additions & 0 deletions test/interpreter_functional/config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import path from 'path';
import fs from 'fs';

export default async function ({ readConfigFile }) {
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));

// Find all folders in ./plugins since we treat all them as plugin folder
const allFiles = fs.readdirSync(path.resolve(__dirname, 'plugins'));
const plugins = allFiles.filter(file => fs.statSync(path.resolve(__dirname, 'plugins', file)).isDirectory());

return {
testFiles: [
require.resolve('./test_suites/run_pipeline'),
],
services: functionalConfig.get('services'),
pageObjects: functionalConfig.get('pageObjects'),
servers: functionalConfig.get('servers'),
esTestCluster: functionalConfig.get('esTestCluster'),
apps: functionalConfig.get('apps'),
esArchiver: {
directory: path.resolve(__dirname, '../es_archives')
},
screenshots: functionalConfig.get('screenshots'),
snapshots: {
directory: path.resolve(__dirname, 'snapshots'),
},
junit: {
reportName: 'Interpreter Functional Tests',
},
kbnTestServer: {
...functionalConfig.get('kbnTestServer'),
serverArgs: [
...functionalConfig.get('kbnTestServer.serverArgs'),
...plugins.map(pluginDir => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}`),
],
},
};
}
Loading