Merge pull request #898 from rashidkpc/10mappings

Mapping updates for Elasticsearch 1.0 release

README.md

@@ -14,6 +14,7 @@ setup and start using (seriously). Kibana strives to be easy to get started with
 flexible and powerful
 
 ### Requirements
+* Elasticsearch 0.90.9 or above
 * A modern web browser. The latest version of Chrome, Safari and Firefox have all been tested to
 work. IE9 and greater should work. IE8 does not.
 * A webserver. No extensions are required, as long as it can serve plain html it will work

src/app/app.js

@@ -30,6 +30,9 @@ function (angular, $, _, appLevelRequire) {
   // This stores the Kibana revision number, @REV@ is replaced by grunt.
   app.constant('kbnVersion',"@REV@");
 
+  // The minimum version that must be in the cluster
+  app.constant('esMinVersion','0.90.9');
+
   // Use this for cache busting partials
   app.constant('cacheBust',"cache-bust="+Date.now());
 

src/app/controllers/dash.js

@@ -31,8 +31,6 @@ function (angular, config, _) {
   module.controller('DashCtrl', function(
     $scope, $route, ejsResource, fields, dashboard, alertSrv, panelMove, esVersion, kbnVersion) {
 
-    $scope.requiredElasticSearchVersion = ">=0.90.3";
-
     $scope.editor = {
       index: 0
     };

src/app/dashboards/default.json

@@ -52,7 +52,7 @@
           ],
           "type": "text",
           "mode": "markdown",
-          "content": "### Welcome to Kibana. \nGlad you could make it. Happy to have you here! Lets get started, shall we?\n##### Requirements\n* **A good browser.**  \n    The latest version of Chrome or Firefox is recommended. Safari (latest version) and Internet Explorer 9 and above are also supported.\n* **A webserver.**  \n    Just somewhere to host the HTML and Javascript. Basically any webserver will work.\n* **Elasticsearch**  \n   0.20.5 or above. Kibana will soon move to requiring Elasticsearch 0.90 or above, so upgrading is recommended.\n\n##### Configuration\nIf Kibana and Elasticsearch are on the same host, and you're using the default Elasticsearch port, then you're all set. Kibana is configured to use that setup by default!  \n\nIf not, you need to edit *config.js* and set the *elasticsearch* parameter with the URL (including port, probably 9200) of your Elasticsearch server. The host part should be the entire, fully qualified domain name, or IP, **not localhost**.\n#### Are you a Logstash User?\n+ **YES** - Great! We have a prebuilt dashboard: [(Logstash Dashboard)](index.html#/dashboard/file/logstash.json). See the note to the right about making it your global default  \n\n+ **NO** - Hey, no problem, you just have a bit of setup to do. You have a few choices:  \n\n    1. [Sample Dashboard](index.html#/dashboard/file/guided.json) *I don't have much data yet, please extract some basics for me*  \n    2. [Unconfigured Dashboard](index.html#/dashboard/file/noted.json) *I have a lot of data and I don't want Kibana to query it at once*\n    3. [Blank Dashboard](index.html#/dashboard/file/blank.json) *I'm comfortable figuring it out on my own*",
+          "content": "### Welcome to Kibana. \nGlad you could make it. Happy to have you here! Lets get started, shall we?\n##### Requirements\n* **A good browser.**  \n    The latest version of Chrome or Firefox is recommended. Safari (latest version) and Internet Explorer 9 and above are also supported.\n* **A webserver.**  \n    Just somewhere to host the HTML and Javascript. Basically any webserver will work.\n* **Elasticsearch**  \n   0.90.9 or above. \n\n##### Configuration\nIf Kibana and Elasticsearch are on the same host, and you're using the default Elasticsearch port, then you're all set. Kibana is configured to use that setup by default!  \n\nIf not, you need to edit *config.js* and set the *elasticsearch* parameter with the URL (including port, probably 9200) of your Elasticsearch server. The host part should be the entire, fully qualified domain name, or IP, **not localhost**.\n#### Are you a Logstash User?\n+ **YES** - Great! We have a prebuilt dashboard: [(Logstash Dashboard)](index.html#/dashboard/file/logstash.json). See the note to the right about making it your global default  \n\n+ **NO** - Hey, no problem, you just have a bit of setup to do. You have a few choices:  \n\n    1. [Sample Dashboard](index.html#/dashboard/file/guided.json) *I don't have much data yet, please extract some basics for me*  \n    2. [Unconfigured Dashboard](index.html#/dashboard/file/noted.json) *I have a lot of data and I don't want Kibana to query it at once*\n    3. [Blank Dashboard](index.html#/dashboard/file/blank.json) *I'm comfortable figuring it out on my own*",
           "style": {},
           "status": "Stable"
         }

src/app/directives/kibanaPanel.js

@@ -74,8 +74,7 @@ function (angular,$) {
 
           $scope.kbnJqUiDraggableOptions = {
             revert: 'invalid',
-            helper: function(event) {
-              console.log(event);
+            helper: function() {
               return $('<div style="width:200px;height:100px;background: rgba(100,100,100,0.50);"/>');
             },
             placeholder: 'keep'

src/app/filters/all.js

@@ -29,18 +29,6 @@ define([
     };
   });
 
-  /*
-    Filter an array of objects by elasticsearch version requirements
-  */
-  module.filter('esVersion', function(esVersion) {
-    return function(items, require) {
-      var ret = _.filter(items,function(qt) {
-        return esVersion.is(qt[require]) ? true : false;
-      });
-      return ret;
-    };
-  });
-
   module.filter('slice', function() {
     return function(arr, start, end) {
       if(!_.isUndefined(arr)) {

src/app/panels/query/meta.html

@@ -8,7 +8,7 @@
 
   <fieldset>
     <select class="input-small" ng-model="querySrv.list[id].type" ng-change="typeChange(querySrv.list[id])">
-      <option ng-repeat="type in queryTypes|esVersion:'require'">{{type.name}}</option>
+      <option ng-repeat="type in queryTypes">{{type}}</option>
     </select> &nbsp<a href="" class="small" ng-click="queryHelp(querySrv.list[id].type)"> About the {{querySrv.list[id].type}} query</a>
 
     <hr class="small">

src/app/panels/query/module.js

@@ -38,12 +38,7 @@ define([
     $scope.querySrv = querySrv;
 
     // A list of query types for the query config popover
-    $scope.queryTypes = _.map(querySrv.queryTypes, function(v,k) {
-      return {
-        name:k,
-        require:v.require
-      };
-    });
+    $scope.queryTypes = querySrv.types;
 
     var queryHelpModal = $modal({
       template: './app/panels/query/helpModal.html',

src/app/services/dashboard.js

@@ -15,7 +15,7 @@ function (angular, $, kbn, _, config, moment, Modernizr) {
 
   module.service('dashboard', function(
     $routeParams, $http, $rootScope, $injector, $location, $timeout,
-    ejsResource, timer, kbnIndex, alertSrv
+    ejsResource, timer, kbnIndex, alertSrv, esVersion, esMinVersion
   ) {
     // A hash of defaults to use when loading a dashboard
 
@@ -79,7 +79,14 @@ function (angular, $, kbn, _, config, moment, Modernizr) {
       // Clear the current dashboard to prevent reloading
       self.current = {};
       self.indices = [];
-      route();
+      esVersion.isMinimum().then(function(isMinimum) {
+        if(isMinimum) {
+          route();
+        } else {
+          alertSrv.set('Upgrade Required',"Your version of Elasticsearch is too old. Kibana requires" +
+            " Elasticsearch " + esMinVersion + " or above.", "error");
+        }
+      });
     });
 
     var route = function() {

src/app/services/esVersion.js

@@ -8,47 +8,55 @@ function (angular, _, config) {
 
   var module = angular.module('kibana.services');
 
-  module.service('esVersion', function($http, alertSrv) {
+  module.service('esVersion', function($http, alertSrv, esMinVersion, $q) {
 
     this.versions = [];
 
     // save a reference to this
-    var self = this;
+    var self = this,
+      defer = $q.defer();
 
     this.init = function() {
       getVersions();
     };
 
     var getVersions = function() {
-      var nodeInfo = $http({
-        url: config.elasticsearch + '/_nodes',
-        method: "GET"
-      }).error(function(data, status) {
-        if(status === 0) {
-          alertSrv.set('Error',"Could not contact Elasticsearch at "+config.elasticsearch+
-            ". Please ensure that Elasticsearch is reachable from your system." ,'error');
-        } else {
-          alertSrv.set('Error',"Could not reach "+config.elasticsearch+"/_nodes. If you"+
-          " are using a proxy, ensure it is configured correctly",'error');
-        }
-      });
+      if(self.versions.length !== 0) {
+        defer.resolve(self.versions);
+        return defer.promise;
+      } else {
+        var nodeInfo = $http({
+          url: config.elasticsearch + '/_nodes',
+          method: "GET",
+        }).error(function(data, status) {
+          if(status === 0) {
+            alertSrv.set('Error',"Could not contact Elasticsearch at "+config.elasticsearch+
+              ". Please ensure that Elasticsearch is reachable from your system." ,'error');
+          } else {
+            alertSrv.set('Error',"Could not reach "+config.elasticsearch+"/_nodes. If you"+
+            " are using a proxy, ensure it is configured correctly",'error');
+          }
+        });
 
-      return nodeInfo.then(function(p) {
-        _.each(p.data.nodes, function(v) {
-          self.versions.push(v.version.split('-')[0]);
+        return nodeInfo.then(function(p) {
+          _.each(p.data.nodes, function(v) {
+            self.versions.push(v.version.split('-')[0]);
+          });
+          self.versions = sortVersions(_.uniq(self.versions));
+          return self.versions;
         });
-        self.versions = sortVersions(_.uniq(self.versions));
-      });
+      }
+
     };
 
     // Get the max version in this cluster
-    this.max = function() {
-      return _.last(self.versions);
+    this.max = function(versions) {
+      return _.last(versions);
     };
 
     // Return the lowest version in the cluster
-    this.min = function() {
-      return _.first(self.versions);
+    this.min = function(versions) {
+      return _.first(versions);
     };
 
     // Sort versions from lowest to highest
@@ -90,29 +98,47 @@ function (angular, _, config) {
       return _cf;
     };
 
+    this.isMinimum = function() {
+      return self.gte(esMinVersion);
+    };
+
     // check if lowest version in cluster = `version`
     this.eq = function(version) {
-      return version === self.min() ? true : false;
+      return getVersions().then(function(v) {
+        return version === self.min(v) ? true : false;
+      });
+
     };
 
     // version > lowest version in cluster?
     this.gt = function(version) {
-      return version === self.min() ? false : self.gte(version);
+      return getVersions().then(function(v) {
+        return version === self.min(v) ? false : self.gte(version);
+      });
+
     };
 
     // version < highest version in cluster?
     this.lt = function(version) {
-      return version === self.max() ? false : self.lte(version);
+      return getVersions().then(function(v) {
+        return version === self.max(v) ? false : self.lte(version);
+      });
+
     };
 
     // Check if the lowest version in the cluster is >= to `version`
     this.gte = function(version) {
-      return self.compare(version,self.min());
+      return getVersions().then(function(v) {
+        return self.compare(version,self.min(v));
+      });
+
     };
 
     // Check if the highest version in the cluster is <= to `version`
     this.lte = function(version) {
-      return self.compare(self.max(),version);
+      return getVersions().then(function(v) {
+        return self.compare(self.max(v),version);
+      });
     };
 
     // Determine if a specific version is greater than or equal to another

src/app/services/fields.js

@@ -8,7 +8,8 @@ function (angular, _, config) {
 
   var module = angular.module('kibana.services');
 
-  module.service('fields', function(dashboard, $rootScope, $http, alertSrv) {
+  module.service('fields', function(dashboard, $rootScope, $http, esVersion, alertSrv) {
+
     // Save a reference to this
     var self = this;
 
@@ -36,15 +37,16 @@ function (angular, _, config) {
       var fields = [];
       _.each(m, function(types) {
         _.each(types, function(type) {
-          fields = _.without(_.union(fields,_.keys(type)),'_all','_source');
+          fields = _.difference(_.union(fields,_.keys(type)),
+            ['_parent','_routing','_size','_ttl','_all','_uid','_version','_boost','_source']);
         });
       });
       return fields;
     };
 
     this.map = function(indices) {
       var request = $http({
-        url: config.elasticsearch + "/" + indices.join(',') + "/_mapping",
+        url: config.elasticsearch + "/" + indices.join(',') + "/_mapping/field/*",
         method: "GET"
       }).error(function(data, status) {
         if(status === 0) {
@@ -60,43 +62,18 @@ function (angular, _, config) {
       // Flatten the mapping of each index into dot notated keys.
       return request.then(function(p) {
         var mapping = {};
-        _.each(p.data, function(type,index) {
-          mapping[index] = {};
-          _.each(type, function (fields,typename) {
-            mapping[index][typename] = flatten(fields);
+        return esVersion.gte('1.0.0.RC1').then(function(version) {
+          _.each(p.data, function(indexMap,index) {
+            mapping[index] = {};
+            _.each((version ? indexMap.mappings : indexMap), function (typeMap,type) {
+              mapping[index][type] = typeMap;
+            });
           });
+          return mapping;
         });
-        return mapping;
       });
     };
 
-    var flatten = function(obj,prefix) {
-      var propName = (prefix) ? prefix :  '',
-        dot = (prefix) ? '.':'',
-        ret = {};
-      for(var attr in obj){
-        if(attr === 'dynamic_templates' || attr === '_default_') {
-          continue;
-        }
-        // For now only support multi field on the top level
-        // and if there is a default field set.
-        if(obj[attr]['type'] === 'multi_field') {
-          ret[attr] = obj[attr]['fields'][attr] || obj[attr];
-          var keys = _.without(_.keys(obj[attr]['fields']),attr);
-          for(var key in keys) {
-            ret[attr+'.'+keys[key]] = obj[attr]['fields'][keys[key]];
-          }
-        } else if (attr === 'properties') {
-          _.extend(ret,flatten(obj[attr], propName));
-        } else if(typeof obj[attr] === 'object'){
-          _.extend(ret,flatten(obj[attr], propName + dot + attr));
-        } else {
-          ret[propName] = obj;
-        }
-      }
-      return ret;
-    };
-
   });
 
 });

src/app/services/querySrv.js

@@ -9,7 +9,10 @@ function (angular, _, config, kbn) {
 
   var module = angular.module('kibana.services');
 
-  module.service('querySrv', function(dashboard, ejsResource, filterSrv, $q) {
+  module.service('querySrv', function(dashboard, ejsResource, filterSrv, esVersion, $q) {
+
+    // Save a reference to this
+    var self = this;
 
     // Create an object to hold our service state on the dashboard
     dashboard.current.services.query = dashboard.current.services.query || {};
@@ -71,7 +74,7 @@ function (angular, _, config, kbn) {
         }
       },
       regex: {
-        require:">=0.90.3",
+        require:">=0.90.12",
         icon: "icon-circle",
         resolve: function(query) {
           // Simply returns itself
@@ -123,8 +126,14 @@ function (angular, _, config, kbn) {
       }
     };
 
-    // Save a reference to this
-    var self = this;
+    self.types = [];
+    _.each(self.queryTypes,function(type,name){
+      esVersion.is(type.require).then(function(is) {
+        if(is) {
+          self.types.push(name);
+        }
+      });
+    });
 
     this.init = function() {
       self.list = dashboard.current.services.query.list;