Merge pull request #5656 from spalger/upgrade/elasticsearch/master

[es] upgrade to master

README.md

@@ -4,7 +4,7 @@ Kibana is an open source ([Apache Licensed](https://github.com/elastic/kibana/bl
 
 ## Requirements
 
-- Elasticsearch version 2.2.0 or later
+- Elasticsearch master
 - Kibana binary package
 
 ## Installation

docs/setup.asciidoc

@@ -3,7 +3,7 @@
 You can set up Kibana and start exploring your Elasticsearch indices in minutes.
 All you need is:
 
-* Elasticsearch 2.1 or later
+* Elasticsearch master
 * A modern web browser - http://www.elastic.co/subscriptions/matrix#matrix_browsers[Supported Browsers].
 * Information about your Elasticsearch installation:
 ** URL of the Elasticsearch instance you want to connect to.
@@ -82,8 +82,8 @@ simply be the name of a single index.
 reads the index mapping to list all of the fields that contain a timestamp. If your index doesn't have time-based data,
 disable the *Index contains time-based events* option.
 +
-WARNING: Using event times to create index names is *deprecated* in this release of Kibana. Support for this functionality 
-will be removed entirely in the next major Kibana release. Elasticsearch 2.1 includes sophisticated date parsing APIs that 
+WARNING: Using event times to create index names is *deprecated* in this release of Kibana. Support for this functionality
+will be removed entirely in the next major Kibana release. Elasticsearch 2.1 includes sophisticated date parsing APIs that
 Kibana uses to determine date information, removing the need to specify dates in the index pattern name.
 +
 . Click *Create* to add the index pattern. This first pattern is automatically configured as the default.

package.json

@@ -92,8 +92,8 @@
     "commander": "2.8.1",
     "css-loader": "0.17.0",
     "d3": "3.5.6",
-    "elasticsearch": "8.0.1",
-    "elasticsearch-browser": "8.0.1",
+    "elasticsearch": "10.1.2",
+    "elasticsearch-browser": "10.1.2",
     "expiry-js": "0.1.7",
     "exports-loader": "0.6.2",
     "expose-loader": "0.7.0",
@@ -154,7 +154,7 @@
     "grunt-cli": "0.1.13",
     "grunt-contrib-clean": "0.6.0",
     "grunt-contrib-copy": "0.8.1",
-    "grunt-esvm": "2.0.0",
+    "grunt-esvm": "2.1.1",
     "grunt-karma": "0.12.0",
     "grunt-run": "0.5.0",
     "grunt-s3": "0.2.0-alpha.3",

src/plugins/devMode/public/ngMock.js

@@ -1,5 +1,7 @@
 import angular from 'angular';
 import 'angular-mocks';
+import 'mocha';
+
 if (angular.mocks) {
   throw new Error(
     'Don\'t require angular-mocks directly or the tests ' +

src/plugins/elasticsearch/index.js

@@ -28,8 +28,8 @@ module.exports = function ({ Plugin }) {
           cert: string(),
           key: string()
         }).default(),
-        apiVersion: string().default('2.0'),
-        engineVersion: string().valid('^2.1.0').default('^2.1.0')
+        apiVersion: Joi.string().default('master'),
+        engineVersion: Joi.string().valid('^3.0.0').default('^3.0.0')
       }).default();
     },
 

src/plugins/elasticsearch/lib/__tests__/routes.js

@@ -99,7 +99,7 @@ describe('plugins/elasticsearch', function () {
     testRoute({
       method: 'POST',
       url: '/elasticsearch/_msearch?timeout=0&ignore_unavailable=true&preference=1429577952339',
-      payload: '{"index":"logstash-2015.04.21","ignore_unavailable":true}\n{"size":500,"sort":{"@timestamp":"desc"},"query":{"filtered":{"query":{"query_string":{"analyze_wildcard":true,"query":"*"}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":1429577068175,"lte":1429577968175}}}],"must_not":[]}}}},"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}}},"aggs":{"2":{"date_histogram":{"field":"@timestamp","interval":"30s","pre_zone":"-07:00","pre_zone_adjust_large_interval":true,"min_doc_count":0,"extended_bounds":{"min":1429577068175,"max":1429577968175}}}},"fields":["*","_source"],"script_fields":{},"fielddata_fields":["timestamp_offset","@timestamp","utc_time"]}\n' // eslint-disable-line max-len
+      payload: '{"index":"logstash-2015.04.21","ignore_unavailable":true}\n{"size":500,"sort":{"@timestamp":"desc"},"query":{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"*"}},{"bool":{"must":[{"range":{"@timestamp":{"gte":1429577068175,"lte":1429577968175}}}],"must_not":[]}}],"must_not":[]}},"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}}},"aggs":{"2":{"date_histogram":{"field":"@timestamp","interval":"30s","pre_zone":"-07:00","pre_zone_adjust_large_interval":true,"min_doc_count":0,"extended_bounds":{"min":1429577068175,"max":1429577968175}}}},"fields":["*","_source"],"script_fields":{},"fielddata_fields":["timestamp_offset","@timestamp","utc_time"]}\n' // eslint-disable-line max-len
     });
 
   });

src/plugins/elasticsearch/lib/expose_client.js

@@ -1,5 +1,6 @@
 import elasticsearch from 'elasticsearch';
 import _ from 'lodash';
+import Bluebird from 'bluebird';
 const readFile = (file) => require('fs').readFileSync(file, 'utf8');
 import util from 'util';
 import url from 'url';
@@ -44,6 +45,9 @@ module.exports = function (server) {
       plugins: options.plugins,
       apiVersion: options.apiVersion,
       keepAlive: options.keepAlive,
+      defer: function () {
+        return Bluebird.defer();
+      },
       log: function () {
         this.error = function (err) {
           server.log(['error', 'elasticsearch'], err);

src/ui/public/agg_types/param_types/field.js

@@ -56,8 +56,10 @@ export default function FieldAggParamFactory(Private) {
     var field = aggConfig.params.field;
 
     if (field.scripted) {
-      output.params.script = field.script;
-      output.params.lang = field.lang;
+      output.params.script = {
+        script: field.script,
+        lang: field.lang,
+      };
     } else {
       output.params.field = field.name;
     }

src/ui/public/courier/data_source/_abstract.js

@@ -279,9 +279,7 @@ export default function SourceAbstractFactory(Private, Promise, PromiseEmitter)
           };
         }
 
-        if (flatState.body.size === 0) {
-          flatState.search_type = 'count';
-        } else {
+        if (flatState.body.size > 0) {
           var computedFields = flatState.index.getComputedFields();
           flatState.body.fields = computedFields.fields;
           flatState.body.script_fields = flatState.body.script_fields || {};
@@ -307,6 +305,21 @@ export default function SourceAbstractFactory(Private, Promise, PromiseEmitter)
           };
         };
 
+        /**
+        * Translate a filter into a query to support es 3+
+        * @param  {Object} filter - The fitler to translate
+        * @return {Object} the query version of that filter
+        */
+        var translateToQuery = function (filter) {
+          if (!filter) return;
+
+          if (filter.query) {
+            return filter.query;
+          }
+
+          return filter;
+        };
+
         /**
          * Clean out any invalid attributes from the filters
          * @param {object} filter The filter to clean
@@ -326,19 +339,45 @@ export default function SourceAbstractFactory(Private, Promise, PromiseEmitter)
             });
 
             flatState.body.query = {
-              filtered: {
-                query: flatState.body.query,
-                filter: {
-                  bool: {
-                    must: _(flatState.filters).filter(filterNegate(false)).map(cleanFilter).value(),
-                    must_not: _(flatState.filters).filter(filterNegate(true)).map(cleanFilter).value()
-                  }
-                }
+              bool: {
+                must: (
+                  [flatState.body.query].concat(
+                    (flatState.filters || [])
+                    .filter(filterNegate(false))
+                    .map(translateToQuery)
+                    .map(cleanFilter)
+                  )
+                ),
+                must_not: (
+                  (flatState.filters || [])
+                  .filter(filterNegate(true))
+                  .map(translateToQuery)
+                  .map(cleanFilter)
+                )
               }
             };
           }
           delete flatState.filters;
         }
+
+        // re-write filters within filter aggregations
+        (function recurse(aggBranch) {
+          if (!aggBranch) return;
+          Object.keys(aggBranch).forEach(function (id) {
+            const agg = aggBranch[id];
+
+            if (agg.filters) {
+              // translate filters aggregations
+              const filters = agg.filters.filters;
+
+              Object.keys(filters).forEach(function (filterId) {
+                filters[filterId] = translateToQuery(filters[filterId]);
+              });
+            }
+
+            recurse(agg.aggs || agg.aggregations);
+          });
+        }(flatState.body.aggs || flatState.body.aggregations));
       }
 
       return flatState;

src/ui/public/courier/data_source/_decorate_query.js

@@ -15,4 +15,3 @@ export default function DecorateQuery(config) {
     return query;
   };
 };
-

src/ui/public/courier/fetch/request/segmented.js

@@ -93,8 +93,6 @@ export default function CourierSegmentedReqProvider(es, Private, Promise, Notifi
         params.body.size = self._pickSizeForIndices(indices);
       }
 
-      if (params.body.size === 0) params.search_type = 'count';
-
       return params;
     });
   };
@@ -126,7 +124,6 @@ export default function CourierSegmentedReqProvider(es, Private, Promise, Notifi
     return SearchReq.prototype.complete.call(this);
   };
 
-
   /*********
    ** SegmentedReq specific methods
    *********/

src/ui/public/directives/__tests__/timepicker.js

@@ -152,7 +152,7 @@ describe('timepicker directive', function () {
 
     it('should default the interval on the courier with incorrect values', function (done) {
       // Change refresh interval and digest
-      $scope.setRefreshInterval('undefined');
+      $scope.setRefreshInterval();
       $elem.scope().$digest();
       expect($courier.searchLooper.loopInterval()).to.be(0);
       done();

src/ui/public/es.js

@@ -1,4 +1,4 @@
-import 'elasticsearch-browser/elasticsearch.angular.js';
+import 'elasticsearch-browser';
 import _ from 'lodash';
 import uiModules from 'ui/modules';
 

src/ui/public/index_patterns/_get_computed_fields.js

@@ -9,7 +9,12 @@ export default function () {
   fielddataFields = _.pluck(self.fields.byType.date, 'name');
 
   _.each(self.getScriptedFields(), function (field) {
-    scriptFields[field.name] = { script: field.script, lang: field.lang };
+    scriptFields[field.name] = {
+      script: {
+        script: field.script,
+        lang: field.lang
+      }
+    };
   });
 
   return {

src/ui/public/index_patterns/_mapper.js

@@ -61,7 +61,7 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) {
       return promise.then(function (indexList) {
         return es.indices.getFieldMapping({
           index: indexList,
-          field: '*',
+          fields: '*',
           ignoreUnavailable: _.isArray(indexList),
           allowNoIndices: false,
           includeDefaults: true

src/ui/public/timepicker/timepicker.js

@@ -145,7 +145,7 @@ module.directive('kbnTimepicker', function (quickRanges, timeUnits, refreshInter
       };
 
       $scope.setRefreshInterval = function (interval) {
-        interval = _.clone(interval);
+        interval = _.clone(interval || {});
         notify.log('before: ' + interval.pause);
         interval.pause = (interval.pause == null || interval.pause === false) ? false : true;
 
@@ -158,4 +158,3 @@ module.directive('kbnTimepicker', function (quickRanges, timeUnits, refreshInter
     }
   };
 });
-

src/ui/public/utils/__tests__/scanner.js

@@ -1,14 +1,28 @@
 import Scanner from 'ui/utils/scanner';
 import expect from 'expect.js';
-import elasticsearch from 'elasticsearch-browser';
+import Bluebird from 'bluebird';
+import 'elasticsearch-browser';
+import ngMock from 'ngMock';
 import sinon from 'sinon';
 
-var es = new elasticsearch.Client({
-  host: 'http://localhost:9210',
-});
+describe('Scanner', function () {
+  let es;
+
+  beforeEach(ngMock.module('kibana'));
+  beforeEach(ngMock.inject(function (esFactory) {
+    es = esFactory({
+      host: 'http://localhost:9210',
+      defer: function () {
+        return Bluebird.defer();
+      }
+    });
+  }));
 
+  afterEach(function () {
+    es.close();
+    es = null;
+  });
 
-describe('Scanner', function () {
   describe('initialization', function () {
     it('should throw errors if missing arguments on initialization', function () {
       expect(() => new Scanner()).to.throwError();

src/ui/public/utils/mapping_setup.js

@@ -30,7 +30,7 @@ define(function () {
         // check all types
         type: '*',
         // limit the response to just the _source field for each index
-        field: '_source'
+        fields: '_source'
       }).then(function (resp) {
         return _.keys(resp[indexName].mappings);
       });

tasks/config/esvm.js

@@ -6,7 +6,7 @@ module.exports = function (grunt) {
 
   return {
     options: {
-      branch: '2.1',
+      branch: 'master',
       fresh: !grunt.option('esvm-no-fresh'),
       config: {
         network: {

tasks/jenkins.js

@@ -1,8 +1,9 @@
 module.exports = function (grunt) {
   let { compact } = require('lodash');
-
   grunt.registerTask('jenkins', 'Jenkins build script', function () {
+    process.env.JAVA_HOME = '/usr/lib/jvm/jdk8';
     grunt.option('os-packages', true);
+
     grunt.task.run(compact([
       'test',
       process.env.JOB_NAME === 'kibana_core' ? 'build' : null

test/fixtures/scenario_manager.js

@@ -9,7 +9,10 @@ function ScenarioManager(server) {
   // NOTE: some large sets of test data can take several minutes to load
   this.client = new elasticsearch.Client({
     host: server,
-    requestTimeout: 300000
+    requestTimeout: 300000,
+    defer: function () {
+      return Promise.defer();
+    }
   });
 }
 

test/fixtures/scenarios/logstashFunctional/makelogsIndexDefinition.js

@@ -19,14 +19,10 @@ module.exports = {
       'dynamic_templates': [{
         'string_fields': {
           'mapping': {
-            'type': 'multi_field',
-            'doc_values': true,
+            'type': 'string',
+            'index': 'analyzed',
+            'omit_norms': true,
             'fields': {
-              '{name}': {
-                'index': 'analyzed',
-                'omit_norms': true,
-                'type': 'string'
-              },
               'raw': {
                 'index': 'not_analyzed',
                 'type': 'string',
@@ -47,7 +43,7 @@ module.exports = {
         },
         'id': {
           'type': 'integer',
-          'index': 'not_analyzed',
+          'index': true,
           'include_in_all': false
         },
         'clientip': {
@@ -98,7 +94,7 @@ module.exports = {
                 },
                 'lastname': {
                   'type': 'integer',
-                  'index': 'not_analyzed'
+                  'index': true
                 }
               }
             }