feat: add dropBufferSupport option to improve the performance (#293)

redis · May 7, 2016 · 1a8700c · 1a8700c
1 parent bffd4b9
commit 1a8700c
Show file tree

Hide file tree

Showing 7 changed files with 181 additions and 177 deletions.
diff --git a/API.md b/API.md
@@ -59,6 +59,8 @@ Creates a Redis instance
 | [options.connectionName] | <code>string</code> | <code>null</code> | Connection name. |
 | [options.db] | <code>number</code> | <code>0</code> | Database index to use. |
 | [options.password] | <code>string</code> | <code>null</code> | If set, client will send AUTH command with the value of this option when connected. |
+| [options.parser] | <code>string</code> | <code>null</code> | Either "hiredis" or "javascript". If not set, "hiredis" parser will be used if it's installed (`npm install hiredis`), otherwise "javascript" parser will be used. |
+| [options.dropBufferSupport] | <code>boolean</code> | <code>false</code> | Drop the buffer support for better performance. This option is recommanded to be enabled when "hiredis" parser is used. Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details. |
 | [options.enableReadyCheck] | <code>boolean</code> | <code>true</code> | When a connection is established to the Redis server, the server might still be loading the database from disk. While loading, the server not respond to any commands. To work around this, when this option is `true`, ioredis will check the status of the Redis server, and when the Redis server is able to process commands, a `ready` event will be emitted. |
 | [options.enableOfflineQueue] | <code>boolean</code> | <code>true</code> | By default, if there is no active connection to the Redis server, commands are added to a queue and are executed once the connection is "ready" (when `enableReadyCheck` is `true`, "ready" means the Redis server has loaded the database from disk, otherwise means the connection to the Redis server has been established). If this option is false, when execute the command when the connection isn't ready, an error will be returned. |
 | [options.connectTimeout] | <code>number</code> | <code>10000</code> | The milliseconds before a timeout occurs during the initial connection to the Redis server. |

diff --git a/README.md b/README.md
@@ -807,10 +807,9 @@ var cluster = new Redis.Cluster([
 });
 ```
 
-## Native Parser
-If [hiredis](https://github.com/redis/hiredis-node) is installed (by `npm install hiredis`),
-ioredis will use it by default. Otherwise, a pure JavaScript parser will be used.
-Typically, there's not much difference between them in terms of performance.
+## Improve Performance
+ioredis supports two parsers, "hiredis" and "javascript". Refer to https://github.com/luin/ioredis/wiki/Improve-Performance
+for details about the differences between them in terms of performance.
 
 <hr>
 

diff --git a/benchmarks/single_node.js b/benchmarks/single_node.js
@@ -1,221 +1,98 @@
 'use strict';
 
 var childProcess = require('child_process');
-var nodeRedis = require('redis');
-var IORedis = require('../');
-var ndredis, ioredis;
+var Redis = require('../');
 
 console.log('==========================');
-console.log('ioredis: ' + require('../package.json').version);
-console.log('node_redis: ' + require('redis/package.json').version);
+console.log('redis: ' + require('../package.json').version);
 var os = require('os');
 console.log('CPU: ' + os.cpus().length);
 console.log('OS: ' + os.platform() + ' ' + os.arch());
 console.log('node version: ' + process.version);
 console.log('current commit: ' + childProcess.execSync('git rev-parse --short HEAD'));
 console.log('==========================');
 
+var redisJD, redisJ, redisBD, redisB;
 var waitReady = function (next) {
-  var pending = 2;
-  ndredis.on('ready', function () {
+  var pending = 4;
+  function check() {
     if (!--pending) {
       next();
     }
-  });
+  }
+  redisJD = new Redis({ parser: 'javascript', dropBufferSupport: true });
+  redisJ = new Redis({ parser: 'javascript', dropBufferSupport: false });
+  redisBD = new Redis({ parser: 'hiredis', dropBufferSupport: true });
+  redisB = new Redis({ parser: 'hiredis', dropBufferSupport: false });
+  redisJD.on('ready', check);
+  redisJ.on('ready', check);
+  redisBD.on('ready', check);
+  redisB.on('ready', check);
+};
 
-  ioredis.on('ready', function () {
-    if (!--pending) {
-      next();
-    }
-  });
+var quit = function () {
+  redisJD.quit();
+  redisJ.quit();
+  redisBD.quit();
+  redisB.quit();
 };
 
-suite('simple set', function () {
+suite('SET foo bar', function () {
   set('mintime', 5000);
   set('concurrency', 300);
   before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
     waitReady(start);
   });
 
-  bench('ioredis', function (next) {
-    ioredis.set('foo', 'bar', next);
-  });
-
-  bench('node_redis', function (next) {
-    ndredis.set('foo', 'bar', next);
+  bench('javascript parser + dropBufferSupport: true', function (next) {
+    redisJD.set('foo', 'bar', next);
   });
 
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
-});
-
-suite('simple get', function () {
-  set('mintime', 5000);
-  set('concurrency', 300);
-  before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
-    waitReady(function () {
-      ndredis.set('foo', 'bar', start);
-    });
+  bench('javascript parser', function (next) {
+    redisJ.setBuffer('foo', 'bar', next);
   });
 
-  bench('ioredis', function (next) {
-    ioredis.get('foo', next);
+  bench('hiredis parser + dropBufferSupport: true', function (next) {
+    redisBD.set('foo', 'bar', next);
   });
 
-  bench('node_redis', function (next) {
-    ndredis.get('foo', next);
+  bench('hiredis parser', function (next) {
+    redisB.setBuffer('foo', 'bar', next);
   });
 
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
+  after(quit);
 });
 
-suite('simple get with pipeline', function () {
+suite('LRANGE foo 0 99', function () {
   set('mintime', 5000);
   set('concurrency', 300);
   before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
-    waitReady(function () {
-      ndredis.set('foo', 'bar', start);
-    });
-  });
-
-  bench('ioredis', function (next) {
-    var pipeline = ioredis.pipeline();
-    for (var i = 0; i < 10; ++i) {
-      pipeline.get('foo');
-    }
-    pipeline.exec(next);
-  });
-
-  bench('node_redis', function (next) {
-    var pending = 0;
-    for (var i = 0; i < 10; ++i) {
-      pending += 1;
-      ndredis.get('foo', check);
+    var redis = new Redis();
+    var item = [];
+    for (var i = 0; i < 100; ++i) {
+      item.push((Math.random() * 100000 | 0) + 'str');
     }
-    function check() {
-      if (!--pending) {
-        next();
-      }
-    }
-  });
-
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
-});
-
-suite('lrange 100', function () {
-  set('mintime', 5000);
-  set('concurrency', 300);
-  before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
-    waitReady(function () {
-      var item = [];
-      for (var i = 0; i < 100; ++i) {
-        item.push((Math.random() * 100000 | 0) + 'str');
-      }
-      ndredis.del('foo');
-      ndredis.lpush('foo', item, start);
-    });
-  });
-
-  bench('ioredis', function (next) {
-    ioredis.lrange('foo', 0, 99, next);
-  });
-
-  bench('node_redis', function (next) {
-    ndredis.lrange('foo', 0, 99, next);
-  });
-
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
-});
-
-suite('publish', function () {
-  set('mintime', 5000);
-  set('concurrency', 300);
-
-  before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
-    waitReady(function () {
-      start();
+    redis.del('foo');
+    redis.lpush('foo', item, function () {
+      waitReady(start);
     });
   });
 
-  bench('ioredis', function (next) {
-    ioredis.publish('foo', 'bar', next);
-  });
-
-  bench('node_redis', function (next) {
-    ndredis.publish('foo', 'bar', next);
+  bench('javascript parser + dropBufferSupport: true', function (next) {
+    redisJD.lrange('foo', 0, 99, next);
   });
 
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
-});
-
-suite('subscribe', function () {
-  set('mintime', 5000);
-  set('concurrency', 300);
-
-  var ndpublisher = null;
-  var iopublisher = null;
-  var ndsubscriber = null;
-  var iosubscriber = null;
-
-  before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
-    waitReady(function () {
-      ndsubscriber = ndredis;
-      ndsubscriber.subscribe('foo');
-      iosubscriber = ioredis;
-      iosubscriber.subscribe('foo');
-
-      ndredis = nodeRedis.createClient();
-      ioredis = new IORedis();
-      waitReady(function () {
-        ndpublisher = ndredis;
-        iopublisher = ioredis;
-        start();
-      });
-    });
+  bench('javascript parser', function (next) {
+    redisJ.lrangeBuffer('foo', 0, 99, next);
   });
 
-  bench('ioredis', function (next) {
-    iosubscriber.removeAllListeners('message');
-    ndsubscriber.removeAllListeners('message');
-    iosubscriber.on('message', next);
-    iopublisher.publish('foo', 'bar');
+  bench('hiredis parser + dropBufferSupport: true', function (next) {
+    redisBD.lrange('foo', 0, 99, next);
   });
 
-  bench('node_redis', function (next) {
-    iosubscriber.removeAllListeners('message');
-    ndsubscriber.removeAllListeners('message');
-    ndsubscriber.on('message', next);
-    ndpublisher.publish('foo', 'bar');
+  bench('hiredis parser', function (next) {
+    redisB.lrangeBuffer('foo', 0, 99, next);
   });
 
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
+  after(quit);
 });
diff --git a/lib/commander.js b/lib/commander.js
@@ -3,6 +3,11 @@
 var _ = require('lodash');
 var Command = require('./command');
 var Script = require('./script');
+var Promise = require('bluebird');
+
+var DROP_BUFFER_SUPPORT_ERROR = '*Buffer methods are not available ' +
+  'because "dropBufferSupport" option is enabled.' +
+  'Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details.';
 
 /**
  * Commander
@@ -106,7 +111,16 @@ function generateFunction(_commandName, _encoding) {
       args[i - firstArgIndex] = arguments[i];
     }
 
-    var options = { replyEncoding: _encoding };
+    var options;
+    if (this.options.dropBufferSupport) {
+      if (!_encoding) {
+        return Promise.reject(new Error(DROP_BUFFER_SUPPORT_ERROR)).nodeify(callback);
+      }
+      options = { replyEncoding: null };
+    } else {
+      options = { replyEncoding: _encoding };
+    }
+
     if (this.options.showFriendlyErrorStack) {
       options.errorStack = new Error().stack;
     }
@@ -133,7 +147,16 @@ function generateScriptingFunction(_script, _encoding) {
       args[i] = arguments[i];
     }
 
-    var options = { replyEncoding: _encoding };
+    var options;
+    if (this.options.dropBufferSupport) {
+      if (!_encoding) {
+        return Promise.reject(new Error(DROP_BUFFER_SUPPORT_ERROR)).nodeify(callback);
+      }
+      options = { replyEncoding: null };
+    } else {
+      options = { replyEncoding: _encoding };
+    }
+
     if (this.options.showFriendlyErrorStack) {
       options.errorStack = new Error().stack;
     }

diff --git a/lib/redis.js b/lib/redis.js
@@ -36,6 +36,11 @@ var ScanStream = require('./scan_stream');
  * @param {number} [options.db=0] - Database index to use.
  * @param {string} [options.password=null] - If set, client will send AUTH command
  * with the value of this option when connected.
+ * @param {string} [options.parser=null] - Either "hiredis" or "javascript". If not set, "hiredis" parser
+ * will be used if it's installed (`npm install hiredis`), otherwise "javascript" parser will be used.
+ * @param {boolean} [options.dropBufferSupport=false] - Drop the buffer support for better performance.
+ * This option is recommanded to be enabled when "hiredis" parser is used.
+ * Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details.
  * @param {boolean} [options.enableReadyCheck=true] - When a connection is established to
  * the Redis server, the server might still be loading the database from disk.
  * While loading, the server not respond to any commands.
@@ -166,6 +171,7 @@ Redis.defaultOptions = {
   db: 0,
   // Others
   parser: null,
+  dropBufferSupport: false,
   enableOfflineQueue: true,
   enableReadyCheck: true,
   autoResubscribe: true,

diff --git a/lib/redis/parser.js b/lib/redis/parser.js
@@ -20,7 +20,7 @@ exports.initParser = function () {
   this.replyParser = new Parser({
     name: this.options.parser,
     stringNumbers: this.options.stringNumbers,
-    returnBuffers: true,
+    returnBuffers: !this.options.dropBufferSupport,
     returnError: function (err) {
       _this.returnError(new ReplyError(err.message));
     },
@@ -33,6 +33,12 @@ exports.initParser = function () {
       _this.disconnect(true);
     }
   });
+
+  if (this.replyParser.name === 'hiredis' && !this.options.dropBufferSupport) {
+    console.warn('[WARN] ioredis is using hiredis parser, however "dropBufferSupport" is disabled. ' +
+     'It\'s highly recommanded to enable this option. ' +
+     'Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details.');
+  }
 };
 
 exports.returnError = function (err) {