Add visual regression screenshot gallery.

- Add 'test:visualRegression' grunt task.
- Run 'test:visualRegression' as part of npm script 'test'.
- Add 'clean:screenshots task'.
- Clean screenshots/session when funtional tests are run.

.gitignore

@@ -15,6 +15,7 @@ target
 /test/screenshots/diff
 /test/screenshots/failure
 /test/screenshots/session
+/test/screenshots/visual_regression_gallery.html
 /esvm
 .htpasswd
 .eslintcache

package.json

@@ -38,7 +38,7 @@
     "Tim Sullivan <[email protected]>"
   ],
   "scripts": {
-    "test": "grunt test",
+    "test": "grunt test; grunt test:visualRegression",
     "test:dev": "grunt test:dev",
     "test:quick": "grunt test:quick",
     "test:browser": "grunt test:browser",
@@ -59,8 +59,7 @@
     "makelogs": "makelogs",
     "mocha": "mocha",
     "mocha:debug": "mocha --debug-brk",
-    "sterilize": "grunt sterilize",
-    "compareScreenshots": "node utilities/compareScreenshots"
+    "sterilize": "grunt sterilize"
   },
   "repository": {
     "type": "git",
@@ -153,6 +152,7 @@
     "auto-release-sinon": "1.0.3",
     "babel-eslint": "4.1.8",
     "chokidar": "1.4.3",
+    "dot": "1.0.3",
     "elasticdump": "2.1.1",
     "eslint": "1.10.3",
     "eslint-plugin-mocha": "1.1.0",
@@ -191,7 +191,7 @@
     "nock": "2.10.0",
     "npm": "2.11.0",
     "portscanner": "1.0.0",
-    "simple-git": "1.8.0",
+    "simple-git": "1.37.0",
     "sinon": "1.17.2",
     "source-map": "0.4.4",
     "source-map-support": "0.4.0",

tasks/config/clean.js

@@ -3,6 +3,7 @@ module.exports = function (grunt) {
   return {
     build: 'build',
     target: 'target',
+    screenshots: 'test/screenshots/session',
     testsFromModules: 'build/kibana/node_modules/**/{test,tests}/**',
     deepModuleBins: 'build/kibana/node_modules/*/node_modules/**/.bin/{' + modules.join(',') + '}',
     deepModules: 'build/kibana/node_modules/*/node_modules/**/{' + modules.join(',') + '}/',

tasks/test.js

@@ -1,5 +1,16 @@
-var _ = require('lodash');
+const _ = require('lodash');
+const visualRegression = require('../utilities/visual_regression');
+
 module.exports = function (grunt) {
+  grunt.registerTask(
+    'test:visualRegression',
+    'Compare screenshots and generate diff images.',
+    function () {
+      const done = this.async();
+      visualRegression.run(done);
+    }
+  );
+
   grunt.registerTask('test:server', [ 'esvm:test', 'simplemocha:all', 'esvm_shutdown:test' ]);
   grunt.registerTask('test:browser', [ 'run:testServer', 'karma:unit' ]);
   grunt.registerTask('test:coverage', [ 'run:testCoverageServer', 'karma:coverage' ]);
@@ -21,6 +32,7 @@ module.exports = function (grunt) {
     'run:testUIServer',
     'downloadSelenium',
     'run:seleniumServer',
+    'clean:screenshots',
     'intern:dev',
     'esvm_shutdown:ui',
     'stop:seleniumServer',
@@ -35,6 +47,7 @@ module.exports = function (grunt) {
   ]);
 
   grunt.registerTask('test:ui:runner', [
+    'clean:screenshots',
     'intern:dev'
   ]);
 
@@ -55,7 +68,7 @@ module.exports = function (grunt) {
     'intern:api'
   ]);
 
-  grunt.registerTask('test', function (subTask) {
+  grunt.registerTask('test', subTask => {
     if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`);
 
     grunt.task.run(_.compact([

test/functional/apps/settings/_index_pattern_results_sort.js

@@ -51,7 +51,7 @@ bdd.describe('index result field sort', function describeIndexTests() {
           return col.selector();
         })
         .then(function (rowText) {
-          common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-ascending.png`);
+          common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-ascending`);
           expect(rowText).to.be(col.first);
         });
       });
@@ -65,7 +65,7 @@ bdd.describe('index result field sort', function describeIndexTests() {
           return col.selector();
         })
         .then(function (rowText) {
-          common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-descending.png`);
+          common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-descending`);
           expect(rowText).to.be(col.last);
         });
       });

test/functional/apps/visualize/_metric_chart.js

@@ -14,7 +14,6 @@ bdd.describe('visualize app', function describeIndexTests() {
   var toTime = '2015-09-23 18:31:44.000';
 
   bdd.before(function () {
-    var vizName1 = 'Visualization MetricChart';
     common.debug('navigateToApp visualize');
     return common.navigateToApp('visualize')
     .then(function () {