Merge pull request elastic#7442 from LeeDr/runTestsOnChrome

Re-implement most of PR elastic#7024 switch to Chrome browser

Former-commit-id: e2ff6bb

CONTRIBUTING.md

@@ -146,10 +146,7 @@ Run the tests for just your particular plugin. Assuming you plugin lives outside
 
 #### Running browser automation tests:
 
-*The Selenium server that is started currently only runs the tests in a recent version of Firefox.*
-*You can use the `PATH` environment variable to specify which version of Firefox to use.*
-
-The following will start Kibana, Elasticsearch and Selenium for you. To run the functional UI tests use the following commands
+The following will start Kibana, Elasticsearch and the chromedriver for you. To run the functional UI tests use the following commands
 
 `npm run test:ui`
 Run the functional UI tests one time and exit. This is used by the CI systems and is great for quickly checking that things pass. It is essentially a combination of the next two tasks.
@@ -158,18 +155,7 @@ Run the functional UI tests one time and exit. This is used by the CI systems an
 Start the server required for the `test:ui:runner` tasks. Once the server is started `test:ui:runner` can be run multiple times without waiting for the server to start.
 
 `npm run test:ui:runner`
-Execute the front-end selenium tests. This requires the server started by the `test:ui:server` task.
-
-##### If you already have ElasticSearch, Kibana, and Selenium Server running:
-
-Set your es and kibana ports in `test/intern.js` to 9220 and 5620, respectively. You can configure your Selenium server to run the tests on Chrome,IE, or other browsers here.
-
-Once you've got the services running, execute the following:
-
-```
-sh
-npm run test:ui:runner
-```
+Execute the front-end browser tests. This requires the server started by the `test:ui:server` task.
 
 #### Browser automation notes:
 

package.json

@@ -30,6 +30,7 @@
     "Jon Budzenski <[email protected]>",
     "Juan Thomassie <[email protected]>",
     "Khalah Jones-Golden <[email protected]>",
+    "Lee Drengenberg <[email protected]>",
     "Lukas Olson <[email protected]>",
     "Matt Bargar <[email protected]>",
     "Nicolás Bevacqua <[email protected]>",
@@ -154,6 +155,7 @@
     "auto-release-sinon": "1.0.3",
     "babel-eslint": "4.1.8",
     "chokidar": "1.4.3",
+    "chromedriver": "2.21.2",
     "dot": "1.0.3",
     "elasticdump": "2.1.1",
     "eslint": "1.10.3",
@@ -175,7 +177,7 @@
     "html-entities": "1.1.3",
     "husky": "0.8.1",
     "image-diff": "1.6.0",
-    "intern": "3.0.1",
+    "intern": "3.2.3",
     "istanbul-instrumenter-loader": "0.1.3",
     "karma": "0.13.9",
     "karma-chrome-launcher": "0.2.0",

tasks/config/intern.js

@@ -6,6 +6,7 @@ module.exports = function (grunt) {
     options: {
       runType: 'runner',
       config: 'test/intern',
+      bail: true,
       reporters: ['Console']
     },
     dev: {},

tasks/config/run.js

@@ -6,6 +6,7 @@ module.exports = function (grunt) {
   let binScript =  /^win/.test(platform) ? '.\\bin\\kibana.bat' : './bin/kibana';
   let buildScript =  /^win/.test(platform) ? '.\\build\\kibana\\bin\\kibana.bat' : './build/kibana/bin/kibana';
   let uiConfig = require(root('test/server_config'));
+  let chromedriver = require('chromedriver');
 
   const stdDevArgs = [
     '--env.name=development',
@@ -113,35 +114,31 @@ module.exports = function (grunt) {
       ]
     },
 
-    seleniumServer: {
+    chromeDriver: {
       options: {
         wait: false,
-        ready: /Selenium Server is up and running/,
-        quiet: true,
+        ready: /Starting ChromeDriver/,
+        quiet: false,
         failOnError: false
       },
-      cmd: 'java',
+      cmd: chromedriver.path,
       args: [
-        '-jar',
-        '<%= downloadSelenium.options.selenium.path %>',
-        '-port',
-        uiConfig.servers.webdriver.port,
+        `--port=${uiConfig.servers.webdriver.port}`,
+        '--url-base=wd/hub',
       ]
     },
 
-    devSeleniumServer: {
+    devChromeDriver: {
       options: {
         wait: false,
-        ready: /Selenium Server is up and running/,
+        ready: /Starting ChromeDriver/,
         quiet: false,
         failOnError: false
       },
-      cmd: 'java',
+      cmd: chromedriver.path,
       args: [
-        '-jar',
-        '<%= downloadSelenium.options.selenium.path %>',
-        '-port',
-        uiConfig.servers.webdriver.port,
+        `--port=${uiConfig.servers.webdriver.port}`,
+        '--url-base=wd/hub',
       ]
     },
 

tasks/test.js

@@ -30,20 +30,18 @@ module.exports = function (grunt) {
   grunt.registerTask('test:ui', [
     'esvm:ui',
     'run:testUIServer',
-    'downloadSelenium',
-    'run:seleniumServer',
+    'run:chromeDriver',
     'clean:screenshots',
     'intern:dev',
     'esvm_shutdown:ui',
-    'stop:seleniumServer',
+    'stop:chromeDriver',
     'stop:testUIServer'
   ]);
 
   grunt.registerTask('test:ui:server', [
     'esvm:ui',
     'run:testUIServer',
-    'downloadSelenium',
-    'run:devSeleniumServer:keepalive'
+    'run:devChromeDriver:keepalive'
   ]);
 
   grunt.registerTask('test:ui:runner', [

test/functional/apps/dashboard/_dashboard.js

@@ -67,8 +67,8 @@ bdd.describe('dashboard tab', function describeIndexTests() {
       // .then(function () {
       common.debug('Set absolute time range from \"' + fromTime + '\" to \"' + toTime + '\"');
       return headerPage.setAbsoluteRange(fromTime, toTime)
-      .then(function sleep() {
-        return common.sleep(4000);
+      .then(function () {
+        return headerPage.getSpinnerDone();
       })
       .then(function takeScreenshot() {
         common.saveScreenshot('Dashboard-set-timepicker');
@@ -81,10 +81,16 @@ bdd.describe('dashboard tab', function describeIndexTests() {
       return dashboardPage.saveDashboard(dashboardName)
       // click New Dashboard just to clear the one we just created
       .then(function () {
-        return dashboardPage.clickNewDashboard();
+        return common.try(function () {
+          common.debug('saved Dashboard, now click New Dashboard');
+          return dashboardPage.clickNewDashboard();
+        });
       })
       .then(function () {
-        return dashboardPage.loadSavedDashboard(dashboardName);
+        return common.try(function () {
+          common.debug('now re-load previously saved dashboard');
+          return dashboardPage.loadSavedDashboard(dashboardName);
+        });
       })
       .then(function () {
         common.saveScreenshot('Dashboard-load-saved');

test/functional/apps/management/_kibana_settings.js

@@ -12,33 +12,28 @@ bdd.describe('creating and deleting default index', function describeIndexTests(
   bdd.before(function () {
     // delete .kibana index and then wait for Kibana to re-create it
     return esClient.deleteAndUpdateConfigDoc()
+    .then(function () {
+      return settingsPage.createIndexPattern();
+    })
     .then(function () {
       return settingsPage.navigateTo();
     });
   });
 
-  bdd.describe('index pattern creation', function indexPatternCreation() {
-    bdd.before(function () {
-      return settingsPage.createIndexPattern()
-      .then(function () {
-        return settingsPage.navigateTo();
-      });
-    });
 
-    bdd.it('should allow setting advanced settings', function () {
-      return settingsPage.clickKibanaSettings()
-      .then(function TestCallSetAdvancedSettingsForTimezone() {
-        common.saveScreenshot('Settings-advanced-tab');
-        common.debug('calling setAdvancedSetting');
-        return settingsPage.setAdvancedSettings('dateFormat:tz', 'America/Phoenix');
-      })
-      .then(function GetAdvancedSetting() {
-        common.saveScreenshot('Settings-set-timezone');
-        return settingsPage.getAdvancedSettings('dateFormat:tz');
-      })
-      .then(function (advancedSetting) {
-        expect(advancedSetting).to.be('America/Phoenix');
-      });
+  bdd.it('should allow setting advanced settings', function () {
+    return settingsPage.clickKibanaSettings()
+    .then(function TestCallSetAdvancedSettingsForTimezone() {
+      common.saveScreenshot('Settings-advanced-tab');
+      common.debug('calling setAdvancedSetting');
+      return settingsPage.setAdvancedSettings('dateFormat:tz', 'America/Phoenix');
+    })
+    .then(function GetAdvancedSetting() {
+      common.saveScreenshot('Settings-set-timezone');
+      return settingsPage.getAdvancedSettings('dateFormat:tz');
+    })
+    .then(function (advancedSetting) {
+      expect(advancedSetting).to.be('America/Phoenix');
     });
   });
 });

test/intern.js

@@ -10,7 +10,7 @@ define(function (require) {
       'idle-timeout': 99
     },
     environments: [{
-      browserName: 'firefox'
+      browserName: 'chrome'
     }],
     tunnelOptions: serverConfig.servers.webdriver,
     functionalSuites: [

test/support/pages/common.js

@@ -64,17 +64,17 @@ export default (function () {
 
     navigateToApp: function (appName, testStatusPage) {
       var self = this;
-      // navUrl includes user:password@ for use with Shield
-      // appUrl excludes user:password@ to match what getCurrentUrl returns
-      var navUrl = getUrl(config.servers.kibana, config.apps[appName]);
       var appUrl = getUrl.noAuth(config.servers.kibana, config.apps[appName]);
-      self.debug('navigating to ' + appName + ' url: ' + navUrl);
+      self.debug('navigating to ' + appName + ' url: ' + appUrl);
 
       var doNavigation = function (url) {
         return self.try(function () {
           // since we're using hash URLs, always reload first to force re-render
           self.debug('navigate to: ' + url);
           return self.remote.get(url)
+          .then(function () {
+            return self.sleep(700);
+          })
           .then(function () {
             self.debug('returned from get, calling refresh');
             return self.remote.refresh();
@@ -113,7 +113,20 @@ export default (function () {
           })
           .then(function (currentUrl) {
             currentUrl = currentUrl.replace(/\/\/\w+:\w+@/, '//');
-            var navSuccessful = new RegExp(appUrl).test(currentUrl);
+            var maxAdditionalLengthOnNavUrl = 230;
+            // On several test failures at the end of the TileMap test we try to navigate back to
+            // Visualize so we can create the next Vertical Bar Chart, but we can see from the
+            // logging and the screenshot that it's still on the TileMap page. Why didn't the "get"
+            // with a new timestamped URL go? I thought that sleep(700) between the get and the
+            // refresh would solve the problem but didn't seem to always work.
+            // So this hack fails the navSuccessful check if the currentUrl doesn't match the
+            // appUrl plus up to 230 other chars.
+            // Navigating to Settings when there is a default index pattern has a URL length of 196
+            // (from debug output). Some other tabs may also be long. But a rather simple configured
+            // visualization is about 1000 chars long. So at least we catch that case.
+            var navSuccessful = new RegExp(appUrl + '.{0,' + maxAdditionalLengthOnNavUrl + '}$')
+            .test(currentUrl);
+
             if (!navSuccessful) {
               var msg = 'App failed to load: ' + appName +
               ' in ' + defaultFindTimeout + 'ms' +
@@ -128,7 +141,7 @@ export default (function () {
         });
       };
 
-      return doNavigation(navUrl)
+      return doNavigation(appUrl)
       .then(function (currentUrl) {
         var lastUrl = currentUrl;
         return self.try(function () {