adding runPipeline tests (#27015) (#33637)

package.json

@@ -81,7 +81,8 @@
       "packages/*",
       "x-pack",
       "x-pack/plugins/*",
-      "test/plugin_functional/plugins/*"
+      "test/plugin_functional/plugins/*",
+      "test/interpreter_functional/plugins/*"
     ],
     "nohoist": [
       "**/@types/*",
@@ -386,6 +387,7 @@
     "normalize-path": "^3.0.0",
     "pixelmatch": "4.0.2",
     "pkg-up": "^2.0.0",
+    "pngjs": "^3.4.0",
     "postcss": "^7.0.5",
     "postcss-url": "^8.0.0",
     "prettier": "^1.14.3",

packages/kbn-pm/src/config.ts

@@ -43,6 +43,7 @@ export function getProjectPaths(rootPath: string, options: IProjectPathOptions)
   // In anyway, have a plugin declaring their own dependencies is the
   // correct and the expect behavior.
   projectPaths.push(resolve(rootPath, 'test/plugin_functional/plugins/*'));
+  projectPaths.push(resolve(rootPath, 'test/interpreter_functional/plugins/*'));
 
   if (!ossOnly) {
     projectPaths.push(resolve(rootPath, 'x-pack'));

scripts/functional_tests.js

@@ -22,4 +22,5 @@ require('@kbn/test').runTestsCli([
   require.resolve('../test/functional/config.js'),
   require.resolve('../test/api_integration/config.js'),
   require.resolve('../test/plugin_functional/config.js'),
+  require.resolve('../test/interpreter_functional/config.js'),
 ]);

src/functional_test_runner/lib/config/schema.ts

@@ -220,6 +220,13 @@ export const schema = Joi.object()
       })
       .default(),
 
+    // settings for the snapshots module
+    snapshots: Joi.object()
+      .keys({
+        directory: Joi.string().default(defaultRelativeToConfigPath('snapshots')),
+      })
+      .default(),
+
     // settings for the failureDebugging module
     failureDebugging: Joi.object()
       .keys({

src/legacy/core_plugins/interpreter/public/renderers/visualization.js

@@ -49,7 +49,7 @@ export const visualization = () => ({
 
     handlers.onDestroy(() => visualizationLoader.destroy());
 
-    await visualizationLoader.render(domNode, handlers.vis, visData, visConfig || handlers.vis.params, uiState, params).then(() => {
+    await visualizationLoader.render(domNode, handlers.vis, visData, handlers.vis.params, uiState, params).then(() => {
       if (handlers.done) handlers.done();
     });
   },

tasks/config/run.js

@@ -208,6 +208,17 @@ module.exports = function (grunt) {
       ],
     },
 
+    interpreterFunctionalTestsRelease: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests',
+        '--config', 'test/interpreter_functional/config.js',
+        '--bail',
+        '--debug',
+        '--kibana-install-dir', KIBANA_INSTALL_DIR,
+      ],
+    },
+
     pluginFunctionalTestsRelease: {
       cmd: process.execPath,
       args: [

test/functional/services/browser.js

@@ -323,6 +323,39 @@ export async function BrowserProvider({ getService }) {
         }
       }));
     }
+
+    async executeAsync(fn, ...args) {
+      return await driver.executeAsyncScript(fn, ...cloneDeep(args, arg => {
+        if (arg instanceof WebElementWrapper) {
+          return arg._webElement;
+        }
+      }));
+    }
+
+    getScrollTop() {
+      return driver
+        .executeScript('return document.body.scrollTop')
+        .then(scrollSize => parseInt(scrollSize, 10));
+    }
+
+    getScrollLeft() {
+      return driver
+        .executeScript('return document.body.scrollLeft')
+        .then(scrollSize => parseInt(scrollSize, 10));
+    }
+
+    // return promise with REAL scroll position
+    setScrollTop(scrollSize) {
+      return driver
+        .executeScript('document.body.scrollTop = ' + scrollSize)
+        .then(() => this.getScrollTop(driver));
+    }
+
+    setScrollLeft(scrollSize) {
+      return driver
+        .executeScript('document.body.scrollLeft = ' + scrollSize)
+        .then(() => this.getScrollLeft(driver));
+    }
   }
 
   return  new BrowserService();

test/functional/services/index.ts

@@ -54,6 +54,8 @@ import { RenderableProvider } from './renderable';
 // @ts-ignore not TS yet
 import { ScreenshotsProvider } from './screenshots';
 // @ts-ignore not TS yet
+import { SnapshotsProvider } from './snapshots';
+// @ts-ignore not TS yet
 import { TableProvider } from './table';
 // @ts-ignore not TS yet
 import { TestSubjectsProvider } from './test_subjects';
@@ -70,6 +72,7 @@ export const services = {
   testSubjects: TestSubjectsProvider,
   docTable: DocTableProvider,
   screenshots: ScreenshotsProvider,
+  snapshots: SnapshotsProvider,
   dashboardVisualizations: DashboardVisualizationProvider,
   dashboardExpect: DashboardExpectProvider,
   failureDebugging: FailureDebuggingProvider,

test/functional/services/lib/web_element_wrapper/web_element_wrapper.js

@@ -19,6 +19,7 @@
 
 import { scrollIntoViewIfNecessary } from './scroll_into_view_if_necessary';
 import { delay } from 'bluebird';
+import { PNG } from 'pngjs';
 import cheerio from 'cheerio';
 import testSubjSelector from '@kbn/test-subj-selector';
 
@@ -443,4 +444,19 @@ export class WebElementWrapper {
 
     return $;
   }
+
+  /**
+   * Creates the screenshot of the element
+   *
+   * @returns {Promise<void>}
+   */
+  async takeScreenshot() {
+    const screenshot = await this._driver.takeScreenshot();
+    const buffer = Buffer.from(screenshot.toString(), 'base64');
+    const { width, height, x, y } = await this.getPosition();
+    const src = PNG.sync.read(buffer);
+    const dst = new PNG({ width, height });
+    PNG.bitblt(src, dst, x, y, width, height, 0, 0);
+    return PNG.sync.write(dst);
+  }
 }

test/functional/services/screenshots.js

@@ -45,10 +45,10 @@ export async function ScreenshotsProvider({ getService }) {
      * @param updateBaselines {boolean} optional, pass true to update the baseline snapshot.
      * @return {Promise.<number>} Percentage difference between the baseline and the current snapshot.
      */
-    async compareAgainstBaseline(name, updateBaselines) {
+    async compareAgainstBaseline(name, updateBaselines, el) {
       log.debug('compareAgainstBaseline');
       const sessionPath = resolve(SESSION_DIRECTORY, `${name}.png`);
-      await this._take(sessionPath);
+      await this._take(sessionPath, el);
 
       const baselinePath = resolve(BASELINE_DIRECTORY, `${name}.png`);
       const failurePath = resolve(FAILURE_DIRECTORY, `${name}.png`);
@@ -63,21 +63,19 @@ export async function ScreenshotsProvider({ getService }) {
       }
     }
 
-    async take(name) {
-      return await this._take(resolve(SESSION_DIRECTORY, `${name}.png`));
+    async take(name, el) {
+      return await this._take(resolve(SESSION_DIRECTORY, `${name}.png`), el);
     }
 
-    async takeForFailure(name) {
-      await this._take(resolve(FAILURE_DIRECTORY, `${name}.png`));
+    async takeForFailure(name, el) {
+      await this._take(resolve(FAILURE_DIRECTORY, `${name}.png`), el);
     }
 
-    async _take(path) {
+    async _take(path, el) {
       try {
         log.info(`Taking screenshot "${path}"`);
-        const [screenshot] = await Promise.all([
-          browser.takeScreenshot(),
-          fcb(cb => mkdirp(dirname(path), cb)),
-        ]);
+        const screenshot = await (el ? el.takeScreenshot() : browser.takeScreenshot());
+        await fcb(cb => mkdirp(dirname(path), cb));
         await fcb(cb => writeFile(path, screenshot, 'base64', cb));
       } catch (err) {
         log.error('SCREENSHOT FAILED');

test/functional/services/snapshots.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from 'expect.js';
+import { dirname, resolve } from 'path';
+import { writeFile, readFileSync } from 'fs';
+import { fromNode as fcb, promisify } from 'bluebird';
+import mkdirp from 'mkdirp';
+import del from 'del';
+
+const writeFileAsync = promisify(writeFile);
+
+export async function SnapshotsProvider({ getService }) {
+  const log = getService('log');
+  const config = getService('config');
+
+  const SESSION_DIRECTORY = resolve(config.get('snapshots.directory'), 'session');
+  const BASELINE_DIRECTORY = resolve(config.get('snapshots.directory'), 'baseline');
+  await del([SESSION_DIRECTORY]);
+
+  class Snapshots {
+
+    /**
+     *
+     * @param name {string} name of the file to use for comparison
+     * @param value {object} value to compare to baseline.
+     * @param updateBaselines {boolean} optional, pass true to update the baseline snapshot.
+     * @return {Promise.<number>} returns 0 if successful.
+     */
+    async compareAgainstBaseline(name, value, updateBaselines) {
+      log.debug('compareAgainstBaseline');
+      const sessionPath = resolve(SESSION_DIRECTORY, `${name}.json`);
+      await this._take(sessionPath, value);
+
+      const baselinePath = resolve(BASELINE_DIRECTORY, `${name}.json`);
+
+      if (updateBaselines) {
+        await writeFileAsync(baselinePath, readFileSync(sessionPath));
+        return 0;
+      } else {
+        log.debug('comparing');
+        return this._compare(sessionPath, baselinePath);
+      }
+    }
+
+    _compare(sessionPath, baselinePath) {
+      const currentObject = readFileSync(sessionPath, { encoding: 'utf8' });
+      const baselineObject = readFileSync(baselinePath, { encoding: 'utf8' });
+      expect(currentObject).to.eql(baselineObject);
+      return 0;
+    }
+
+    async take(name) {
+      return await this._take(resolve(SESSION_DIRECTORY, `${name}.json`));
+    }
+
+    async _take(path, snapshot) {
+      try {
+        await fcb(cb => mkdirp(dirname(path), cb));
+        await fcb(cb => writeFile(path, JSON.stringify(snapshot), 'utf8', cb));
+      } catch (err) {
+        log.error('SNAPSHOT FAILED');
+        log.error(err);
+      }
+    }
+  }
+
+  return new Snapshots();
+}

test/interpreter_functional/README.md

@@ -0,0 +1,21 @@
+# Interpreter Functional Tests
+
+This folder contains interpreter functional tests.
+
+Add new test suites into the `test_suites` folder and reference them from the
+`config.js` file. These test suites work the same as regular functional test.
+
+## Run the test
+
+To run these tests during development you can use the following commands:
+
+```
+# Start the test server (can continue running)
+node scripts/functional_tests_server.js --config test/interpreter_functional/config.js
+# Start a test run
+node scripts/functional_test_runner.js --config test/interpreter_functional/config.js
+```
+
+# Writing tests
+
+Look into test_suites/run_pipeline/basic.js for examples

test/interpreter_functional/config.js

@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import path from 'path';
+import fs from 'fs';
+
+export default async function ({ readConfigFile }) {
+  const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
+
+  // Find all folders in ./plugins since we treat all them as plugin folder
+  const allFiles = fs.readdirSync(path.resolve(__dirname, 'plugins'));
+  const plugins = allFiles.filter(file => fs.statSync(path.resolve(__dirname, 'plugins', file)).isDirectory());
+
+  return {
+    testFiles: [
+      require.resolve('./test_suites/run_pipeline'),
+    ],
+    services: functionalConfig.get('services'),
+    pageObjects: functionalConfig.get('pageObjects'),
+    servers: functionalConfig.get('servers'),
+    esTestCluster: functionalConfig.get('esTestCluster'),
+    apps: functionalConfig.get('apps'),
+    esArchiver: {
+      directory: path.resolve(__dirname, '../es_archives')
+    },
+    screenshots: functionalConfig.get('screenshots'),
+    snapshots: {
+      directory: path.resolve(__dirname, 'snapshots'),
+    },
+    junit: {
+      reportName: 'Interpreter Functional Tests',
+    },
+    kbnTestServer: {
+      ...functionalConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...functionalConfig.get('kbnTestServer.serverArgs'),
+        ...plugins.map(pluginDir => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}`),
+      ],
+    },
+  };
+}