Merge pull request elastic#7410 from cjcenizal/improvement/ui-screens…

…hot-coverage Take screenshots of UI throughout functional tests.
airow · Jun 14, 2016 · aa47599 · aa47599
2 parents d8512a8 + 9e353fd
commit aa47599
Show file tree

Hide file tree

Showing 79 changed files with 509 additions and 126 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,6 +15,7 @@ target
 /test/screenshots/diff
 /test/screenshots/failure
 /test/screenshots/session
+/test/screenshots/visual_regression_gallery.html
 /esvm
 .htpasswd
 .eslintcache

diff --git a/package.json b/package.json
@@ -38,7 +38,7 @@
     "Tim Sullivan <[email protected]>"
   ],
   "scripts": {
-    "test": "grunt test",
+    "test": "grunt test; grunt test:visualRegression",
     "test:dev": "grunt test:dev",
     "test:quick": "grunt test:quick",
     "test:browser": "grunt test:browser",
@@ -59,8 +59,7 @@
     "makelogs": "makelogs",
     "mocha": "mocha",
     "mocha:debug": "mocha --debug-brk",
-    "sterilize": "grunt sterilize",
-    "compareScreenshots": "node utilities/compareScreenshots"
+    "sterilize": "grunt sterilize"
   },
   "repository": {
     "type": "git",
@@ -153,6 +152,7 @@
     "auto-release-sinon": "1.0.3",
     "babel-eslint": "4.1.8",
     "chokidar": "1.4.3",
+    "dot": "1.0.3",
     "elasticdump": "2.1.1",
     "eslint": "1.10.3",
     "eslint-plugin-mocha": "1.1.0",
@@ -191,7 +191,7 @@
     "nock": "2.10.0",
     "npm": "2.11.0",
     "portscanner": "1.0.0",
-    "simple-git": "1.8.0",
+    "simple-git": "1.37.0",
     "sinon": "1.17.2",
     "source-map": "0.4.4",
     "source-map-support": "0.4.0",

diff --git a/tasks/config/clean.js b/tasks/config/clean.js
@@ -3,6 +3,7 @@ module.exports = function (grunt) {
   return {
     build: 'build',
     target: 'target',
+    screenshots: 'test/screenshots/session',
     testsFromModules: 'build/kibana/node_modules/**/{test,tests}/**',
     deepModuleBins: 'build/kibana/node_modules/*/node_modules/**/.bin/{' + modules.join(',') + '}',
     deepModules: 'build/kibana/node_modules/*/node_modules/**/{' + modules.join(',') + '}/',

diff --git a/tasks/test.js b/tasks/test.js
@@ -1,5 +1,16 @@
-var _ = require('lodash');
+const _ = require('lodash');
+const visualRegression = require('../utilities/visual_regression');
+
 module.exports = function (grunt) {
+  grunt.registerTask(
+    'test:visualRegression',
+    'Compare screenshots and generate diff images.',
+    function () {
+      const done = this.async();
+      visualRegression.run(done);
+    }
+  );
+
   grunt.registerTask('test:server', [ 'esvm:test', 'simplemocha:all', 'esvm_shutdown:test' ]);
   grunt.registerTask('test:browser', [ 'run:testServer', 'karma:unit' ]);
   grunt.registerTask('test:coverage', [ 'run:testCoverageServer', 'karma:coverage' ]);
@@ -21,6 +32,7 @@ module.exports = function (grunt) {
     'run:testUIServer',
     'downloadSelenium',
     'run:seleniumServer',
+    'clean:screenshots',
     'intern:dev',
     'esvm_shutdown:ui',
     'stop:seleniumServer',
@@ -35,6 +47,7 @@ module.exports = function (grunt) {
   ]);
 
   grunt.registerTask('test:ui:runner', [
+    'clean:screenshots',
     'intern:dev'
   ]);
 
@@ -55,7 +68,7 @@ module.exports = function (grunt) {
     'intern:api'
   ]);
 
-  grunt.registerTask('test', function (subTask) {
+  grunt.registerTask('test', subTask => {
     if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`);
 
     grunt.task.run(_.compact([

diff --git a/test/functional/apps/console/_console.js b/test/functional/apps/console/_console.js
@@ -13,7 +13,6 @@ bdd.describe('console app', function describeIndexTests() {
     return common.navigateToApp('console', false);
   });
 
-
   bdd.it('should show the default request', function () {
     var expectedRequest = [
       'GET _search',
@@ -24,9 +23,11 @@ bdd.describe('console app', function describeIndexTests() {
       '}',
       ''
     ];
+    common.saveScreenshot('Console-help-expanded');
     // collapse the help pane because we only get the VISIBLE TEXT, not the part that is scrolled
     return consolePage.collapseHelp()
     .then(function () {
+      common.saveScreenshot('Console-help-collapsed');
       return common.try(function () {
         return consolePage.getRequest()
         .then(function (actualRequest) {
@@ -40,6 +41,7 @@ bdd.describe('console app', function describeIndexTests() {
     var expectedResponseContains = '"_index": ".kibana",';
     return consolePage.clickPlay()
     .then(function () {
+      common.saveScreenshot('Console-default-request');
       return common.try(function () {
         return consolePage.getResponse()
         .then(function (actualResponse) {

diff --git a/test/functional/apps/dashboard/_dashboard.js b/test/functional/apps/dashboard/_dashboard.js
@@ -42,6 +42,8 @@ bdd.describe('dashboard tab', function describeIndexTests() {
     ];
 
     bdd.it('should be able to add visualizations to dashboard', function addVisualizations() {
+      common.saveScreenshot('Dashboard-no-visualizations');
+
       function addVisualizations(arr) {
         return arr.reduce(function (promise, vizName) {
           return promise
@@ -54,13 +56,13 @@ bdd.describe('dashboard tab', function describeIndexTests() {
       return addVisualizations(visualizations)
       .then(function () {
         common.debug('done adding visualizations');
+        common.saveScreenshot('Dashboard-add-visualizations');
       });
     });
 
     bdd.it('set the timepicker time to that which contains our test data', function setTimepicker() {
       var fromTime = '2015-09-19 06:31:44.000';
       var toTime = '2015-09-23 18:31:44.000';
-      var testSubName = 'Dashboard Test 1';
 
       // .then(function () {
       common.debug('Set absolute time range from \"' + fromTime + '\" to \"' + toTime + '\"');
@@ -69,21 +71,23 @@ bdd.describe('dashboard tab', function describeIndexTests() {
         return common.sleep(4000);
       })
       .then(function takeScreenshot() {
-        common.debug('Take screenshot');
-        common.saveScreenshot('./screenshot-' + testSubName + '.png');
+        common.saveScreenshot('Dashboard-set-timepicker');
       });
     });
 
     bdd.it('should save and load dashboard', function saveAndLoadDashboard() {
-      var testSubName = 'Dashboard Test 1';
+      const dashboardName = 'Dashboard Test 1';
       // TODO: save time on the dashboard and test it
-      return dashboardPage.saveDashboard(testSubName)
+      return dashboardPage.saveDashboard(dashboardName)
       // click New Dashboard just to clear the one we just created
       .then(function () {
         return dashboardPage.clickNewDashboard();
       })
       .then(function () {
-        return dashboardPage.loadSavedDashboard(testSubName);
+        return dashboardPage.loadSavedDashboard(dashboardName);
+      })
+      .then(function () {
+        common.saveScreenshot('Dashboard-load-saved');
       });
     });
 
@@ -94,6 +98,9 @@ bdd.describe('dashboard tab', function describeIndexTests() {
           common.log('visualization titles = ' + panelTitles);
           expect(panelTitles).to.eql(visualizations);
         });
+      })
+      .then(function () {
+        common.saveScreenshot('Dashboard-has-visualizations');
       });
     });
 
@@ -110,6 +117,7 @@ bdd.describe('dashboard tab', function describeIndexTests() {
         return dashboardPage.getPanelData()
         .then(function (panelTitles) {
           common.log('visualization titles = ' + panelTitles);
+          common.saveScreenshot('Dashboard-visualization-sizes');
           expect(panelTitles).to.eql(visObjects);
         });
       });

diff --git a/test/functional/apps/discover/_collapse_expand.js b/test/functional/apps/discover/_collapse_expand.js
@@ -38,6 +38,7 @@ bdd.describe('discover tab', function describeIndexTests() {
 
   bdd.describe('field data', function () {
     bdd.it('should initially be expanded', function () {
+      common.saveScreenshot('Discover-sidebar-expanded');
       return discoverPage.getSidebarWidth()
         .then(function (width) {
           common.debug('expanded sidebar width = ' + width);
@@ -48,6 +49,7 @@ bdd.describe('discover tab', function describeIndexTests() {
     bdd.it('should collapse when clicked', function () {
       return discoverPage.toggleSidebarCollapse()
         .then(function () {
+          common.saveScreenshot('Discover-sidebar-collapsed');
           common.debug('discoverPage.getSidebarWidth()');
           return discoverPage.getSidebarWidth();
         })

diff --git a/test/functional/apps/discover/_discover.js b/test/functional/apps/discover/_discover.js
@@ -58,6 +58,7 @@ bdd.describe('discover app', function describeIndexTests() {
         return headerPage.getToastMessage();
       })
       .then(function (toastMessage) {
+        common.saveScreenshot('Discover-save-query-toast');
         expect(toastMessage).to.be(expectedSavedQueryMessage);
       })
       .then(function () {
@@ -80,6 +81,7 @@ bdd.describe('discover app', function describeIndexTests() {
         return discoverPage.getCurrentQueryName();
       })
       .then(function (actualQueryNameString) {
+        common.saveScreenshot('Discover-load-query');
         expect(actualQueryNameString).to.be(queryName1);
       });
     });
@@ -289,6 +291,7 @@ bdd.describe('discover app', function describeIndexTests() {
 
     bdd.it('should show "no results"', () => {
       return discoverPage.hasNoResults().then(visible => {
+        common.saveScreenshot('Discover-no-results');
         expect(visible).to.be(true);
       });
     });

diff --git a/test/functional/apps/discover/_field_data.js b/test/functional/apps/discover/_field_data.js
@@ -49,6 +49,7 @@ bdd.describe('discover app', function describeIndexTests() {
         return common.try(function tryingForTime() {
           return discoverPage.getHitCount()
           .then(function compareData(hitCount) {
+            common.saveScreenshot('Discover-field-data');
             expect(hitCount).to.be(expectedHitCount);
           });
         });
@@ -64,7 +65,6 @@ bdd.describe('discover app', function describeIndexTests() {
       });
     });
 
-
     bdd.it('search _type:apache should show the correct hit count', function () {
       var expectedHitCount = '11,156';
       return discoverPage.query('_type:apache')
@@ -222,6 +222,7 @@ bdd.describe('discover app', function describeIndexTests() {
         return common.try(function tryingForTime() {
           return discoverPage.getDocTableIndex(1)
           .then(function (rowData) {
+            common.saveScreenshot('Discover-sort-down');
             expect(rowData).to.be(ExpectedDoc);
           });
         });
@@ -237,6 +238,7 @@ bdd.describe('discover app', function describeIndexTests() {
         return headerPage.getToastMessage();
       })
       .then(function (toastMessage) {
+        common.saveScreenshot('Discover-syntax-error-toast');
         expect(toastMessage).to.be(expectedError);
       })
       .then(function () {

diff --git a/test/functional/apps/discover/_shared_links.js b/test/functional/apps/discover/_shared_links.js
@@ -56,6 +56,7 @@ bdd.describe('shared links', function describeIndexTests() {
       var expectedCaption = 'Share a link';
       return discoverPage.clickShare()
       .then(function () {
+        common.saveScreenshot('Discover-share-link');
         return discoverPage.getShareCaption();
       })
       .then(function (actualCaption) {
@@ -85,6 +86,7 @@ bdd.describe('shared links', function describeIndexTests() {
         return headerPage.getToastMessage();
       })
       .then(function (toastMessage) {
+        common.saveScreenshot('Discover-copy-to-clipboard-toast');
         expect(toastMessage).to.match(expectedToastMessage);
       })
       .then(function () {
@@ -98,6 +100,7 @@ bdd.describe('shared links', function describeIndexTests() {
       return discoverPage.clickShortenUrl()
       .then(function () {
         return common.try(function tryingForTime() {
+          common.saveScreenshot('Discover-shorten-url-button');
           return discoverPage.getShortenedUrl()
           .then(function (actualUrl) {
             expect(actualUrl).to.match(re);

diff --git a/test/functional/apps/settings/_advanced_settings.js b/test/functional/apps/settings/_advanced_settings.js
@@ -25,10 +25,12 @@ bdd.describe('creating and deleting default index', function describeIndexTests(
     bdd.it('should allow setting advanced settings', function () {
       return settingsPage.clickAdvancedTab()
       .then(function TestCallSetAdvancedSettingsForTimezone() {
+        common.saveScreenshot('Settings-advanced-tab');
         common.debug('calling setAdvancedSetting');
         return settingsPage.setAdvancedSettings('dateFormat:tz', 'America/Phoenix');
       })
       .then(function GetAdvancedSetting() {
+        common.saveScreenshot('Settings-set-timezone');
         return settingsPage.getAdvancedSettings('dateFormat:tz');
       })
       .then(function (advancedSetting) {

diff --git a/test/functional/apps/settings/_creation_form_changes.js b/test/functional/apps/settings/_creation_form_changes.js
@@ -30,6 +30,7 @@ bdd.describe('user input reactions', function () {
       return settingsPage.getTimeBasedIndexPatternCheckbox(waitTime);
     })
     .then(function () {
+      common.saveScreenshot('Settings-indices-hide-time-based-index-pattern');
       // we expect the promise above to fail
       var handler = common.handleError(self);
       var msg = 'Found time based index pattern checkbox';
@@ -47,6 +48,7 @@ bdd.describe('user input reactions', function () {
     .then(function () {
       return settingsPage.getCreateButton().isEnabled()
       .then(function (enabled) {
+        common.saveScreenshot('Settings-indices-enable-creation');
         expect(enabled).to.be.ok();
       });
     });

diff --git a/test/functional/apps/settings/_index_pattern_create_delete.js b/test/functional/apps/settings/_index_pattern_create_delete.js
@@ -26,6 +26,7 @@ bdd.describe('creating and deleting default index', function describeIndexTests(
     bdd.it('should have index pattern in page header', function pageHeader() {
       return settingsPage.getIndexPageHeading().getVisibleText()
       .then(function (patternName) {
+        common.saveScreenshot('Settings-indices-new-index-pattern');
         expect(patternName).to.be('logstash-*');
       });
     });
@@ -72,6 +73,7 @@ bdd.describe('creating and deleting default index', function describeIndexTests(
       var expectedAlertText = 'Are you sure you want to remove this index pattern?';
       return settingsPage.removeIndexPattern()
       .then(function (alertText) {
+        common.saveScreenshot('Settings-indices-confirm-remove-index-pattern');
         expect(alertText).to.be(expectedAlertText);
       });
     });

diff --git a/test/functional/apps/settings/_index_pattern_popularity.js b/test/functional/apps/settings/_index_pattern_popularity.js
@@ -63,6 +63,7 @@ bdd.describe('index result popularity', function describeIndexTests() {
       .then(function (popularity) {
         common.debug('popularity = ' + popularity);
         expect(popularity).to.be('1');
+        common.saveScreenshot('Settings-indices-result-popularity-updated');
       });
     });
 
@@ -101,6 +102,7 @@ bdd.describe('index result popularity', function describeIndexTests() {
       .then(function (popularity) {
         common.debug('popularity = ' + popularity);
         expect(popularity).to.be('1');
+        common.saveScreenshot('Settings-indices-result-popularity-saved');
       });
     });
   }); // end 'change popularity'

diff --git a/test/functional/apps/settings/_index_pattern_results_sort.js b/test/functional/apps/settings/_index_pattern_results_sort.js
@@ -51,6 +51,7 @@ bdd.describe('index result field sort', function describeIndexTests() {
           return col.selector();
         })
         .then(function (rowText) {
+          common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-ascending`);
           expect(rowText).to.be(col.first);
         });
       });
@@ -64,6 +65,7 @@ bdd.describe('index result field sort', function describeIndexTests() {
           return col.selector();
         })
         .then(function (rowText) {
+          common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-descending`);
           expect(rowText).to.be(col.last);
         });
       });
@@ -115,6 +117,7 @@ bdd.describe('index result field sort', function describeIndexTests() {
             return settingsPage.getPageFieldCount();
           })
           .then(function (pageCount) {
+            common.saveScreenshot('Settings-indices-paged');
             var expectedSize = (val < 4) ? expectedDefaultPageSize : expectedLastPageCount;
             expect(pageCount.length).to.be(expectedSize);
           });

diff --git a/test/functional/apps/settings/_initial_state.js b/test/functional/apps/settings/_initial_state.js
@@ -20,6 +20,7 @@ bdd.describe('initial state', function () {
   bdd.it('should load with time pattern checked', function () {
     return settingsPage.getTimeBasedEventsCheckbox().isSelected()
     .then(function (selected) {
+      common.saveScreenshot('Settings-initial-state');
       expect(selected).to.be.ok();
     });
   });