Close elastic#198 - Optimized extractShards. Went from approx. 450ms …

…to 3ms

- Fixing tests
- Adding fakeState() method
- Fixing a bug with replicas>1
- Closes elastic#225

kibana/lib/extractShards.js

@@ -20,13 +20,17 @@ define(function (require) {
       return shard;
     }
 
-    var data  = _.chain(state.routing_nodes.nodes)
-      .values()
-      .flatten()
-      .union(state.routing_nodes.unassigned)
-      .map(setNodeName)
-      .value(); 
+    var pushShardToData = function (shard) {
+      data.push(setNodeName(shard)); 
+    };
 
+    var data = [];
+    _.each(state.routing_nodes.nodes, function (node) {
+      _.each(node, pushShardToData);
+    });
+
+    _.each(state.routing_nodes.unassigned, pushShardToData);
+
     return data;
   };
 

kibana/lib/fakeState.js

@@ -0,0 +1,86 @@
+define(function (require) {
+	'use strict';
+	var _ = require('lodash');
+	return function (options) {
+
+		options = _.defaults(options || {}, {
+			indices: 20,
+			nodes: 1,
+			shards: 5,
+			replicas: 1
+		});
+
+		var state = {
+      _id: new Date().getTime()+Math.random(),
+			cluster_name: 'elasticsearch',
+			version: 1,
+			nodes: {},
+			routing_nodes: {
+				unassigned: [],
+				nodes: {}
+			}
+		};
+
+		// Generate Nodes
+		_.times(options.nodes, function (n) {
+			var id = 'node_'+(n+1);
+			state.nodes[id] = {
+				name: 'Node '+(n+1),
+				transport_address: 'inet[localhost/127.0.0.1:'+(9300+n)+']',
+				attributes: {}
+			};
+			state.routing_nodes.nodes[id] = [];
+		});
+
+		var nodeIds       = _.keys(state.nodes);
+		state.master_node = nodeIds[0];
+		var currentNode   = 0;
+
+		var getNode = function () {
+			var node = nodeIds[currentNode];
+			if (options.nodes > 1) {
+				if (currentNode < (options.nodes-1)) {
+					currentNode++;
+				} else {
+					currentNode = 0;
+				}
+			}
+			return node;
+		};
+
+		// Generate Indices
+		_.times(options.indices, function (n) {
+			var index = moment().subtract('days', n).format('[logstash-]YYYY.MM.DD');
+			_.times(options.shards, function (shard) {
+
+				// Generate Primary Shards   
+				var node = getNode();
+				state.routing_nodes.nodes[node].push({
+					state           : 'STARTED',
+					primary         : true,
+					node            : node,
+					relocating_node : null,
+					shard           : shard,
+					index           : index
+				});
+
+				// Generate Replica Shards
+				_.times(options.replicas, function (replica) {
+					var shardState      = (options.nodes < 2) ? 'UNASSIGNED' : 'STARTED';
+					var replicaNode     = (options.nodes < 2) ? null : getNode();
+					var shardCollection = (options.nodes < 2) ? state.routing_nodes.unassigned : state.routing_nodes.nodes[replicaNode];
+					shardCollection.push({
+						state           : shardState,
+						primary         : false,
+						node            : replicaNode,
+						relocating_node : null,
+						shard           : shard,
+						index           : index
+					});
+				});
+
+			});
+		});
+		return state;
+	};
+});

kibana/panels/shard_allocation/components/shards.js

@@ -71,11 +71,7 @@ define(function (require) {
   return React.createClass({
     displayName: 'Shards',
     createShard: function (shard) {
-      var key = [];
-      key.push('shard.'+shard.shard);
-      key.push(shard.state);
-      key.push(shard.primary ? 'primary' : 'replica');
-      return Shard({ shard: shard, key: key.join('.') });
+      return Shard({ shard: shard });
     },
     render: function () {
       return D.div({ className: 'shards' }, 

package.json

@@ -39,7 +39,8 @@
     "grunt-contrib-jade": "~0.10.0",
     "lodash": "~2.4.1",
     "grunt-contrib-watch": "~0.5.3",
-    "grunt-blanket-mocha": "^0.4.0"
+    "grunt-blanket-mocha": "^0.4.0",
+    "moment": "~2.6.0"
   },
   "dependencies": {}
 }

test/fixtures/greenState.js

@@ -2,7 +2,7 @@ define(function () {
   'use strict';
   return function () {
     return {
-      _id: new Date().getTime(), 
+      _id: new Date().getTime()+Math.random(), 
       status: 'green',
       nodes: {
         'node-1':  {

test/fixtures/redStateOneIndex.js

@@ -2,7 +2,7 @@ define(function () {
   'use strict';
   return function () {
     return {
-      _id: new Date().getTime(), 
+      _id: new Date().getTime()+Math.random(), 
       status: 'red',
       nodes: {
         'node-1':  {

test/fixtures/redStateOneIndexInitializingPrimary.js

@@ -2,7 +2,7 @@ define(function () {
   'use strict';
   return function () {
     return {
-      _id: new Date().getTime(), 
+      _id: new Date().getTime()+Math.random(), 
       status: 'red',
       nodes: {
         'node-1':  {

test/fixtures/redStateTwoIndices.js

@@ -2,7 +2,7 @@ define(function () {
   'use strict';
   return function () {
     return {
-      _id: new Date().getTime(), 
+      _id: new Date().getTime()+Math.random(), 
       status: 'red',
       nodes: {
         'node-1':  {

test/fixtures/redStateTwoIndicesInitializingPrimaries.js

@@ -2,7 +2,7 @@ define(function () {
   'use strict';
   return function () {
     return {
-      _id: new Date().getTime(), 
+      _id: new Date().getTime()+Math.random(), 
       status: 'red',
       nodes: {
         'node-1':  {

test/fixtures/relocatingState.js

@@ -2,7 +2,7 @@ define(function () {
   'use strict';
   return function () {
     return {
-      _id: new Date().getTime(), 
+      _id: new Date().getTime()+Math.random(), 
       status: 'green',
       nodes: {
         'node-1':  {

test/fixtures/yellowStateOneIndex.js

@@ -2,7 +2,7 @@ define(function () {
   'use strict';
   return function () {
     return {
-      _id: new Date().getTime(), 
+      _id: new Date().getTime()+Math.random(), 
       status: 'yellow',
       nodes: {
         'node-1':  {

test/fixtures/yellowStateOneIndexInitializing.js

@@ -2,7 +2,7 @@ define(function () {
   'use strict';
   return function () {
     return {
-      _id: new Date().getTime(), 
+      _id: new Date().getTime()+Math.random(), 
       status: 'yellow',
       nodes: {
         'node-1':  {

test/fixtures/yellowStateTwoIndices.js

@@ -2,7 +2,7 @@ define(function () {
   'use strict';
   return function () {
     return {
-      _id: new Date().getTime(), 
+      _id: new Date().getTime()+Math.random(), 
       status: 'yellow',
       nodes: {
         'node-1':  {

test/unit/shard_allocation/extractShards.js

@@ -6,10 +6,23 @@ define(function (require) {
   var _ = require('lodash');
   var state = relocatingState();
   var shards = extractShards(state);
+  var fakeState = require('lib/fakeState');
 
   describe('shard_allocation', function () {
     describe('transforms/extractShards.js', function () {
 
+      it('should test performance', function() {
+        var _state = fakeState({
+          indices: 2000,
+          replicas: 2,
+          nodes: 100
+        });
+        var _shards = extractShards(_state); 
+        expect(_shards)
+          .to.be.instanceOf(Array)
+          .to.have.length(30000);
+      });
+
       it('should return an array', function () {
         expect(shards)
           .to.be.instanceOf(Array)