Lazy load file hash

- Load file hash only if the file is new or the modified time is changed
- Lock the file when resolving the cache

lib/box/file.js

@@ -103,4 +103,9 @@ File.prototype.statSync = function(options) {
   return fs.statSync(this.source);
 };
 
+File.TYPE_CREATE = 'create';
+File.TYPE_UPDATE = 'update';
+File.TYPE_SKIP = 'skip';
+File.TYPE_DELETE = 'delete';
+
 module.exports = File;

lib/box/index.js

@@ -94,11 +94,9 @@ Box.prototype.process = function(callback) {
     if (!exist) return;
     return self._loadFiles();
   }).then(function(files) {
-    if (!files || !files.length) return;
-
-    return self._process(files).finally(function() {
-      files.length = 0;
-    });
+    if (files && files.length) {
+      return self._process(files);
+    }
   }).asCallback(callback);
 };
 
@@ -107,7 +105,7 @@ Box.prototype.load = Box.prototype.process;
 function listDir(path) {
   return fs.listDir(path).catch(function(err) {
     // Return an empty array if path does not exist
-    if (err.cause.code === 'ENOENT') return [];
+    if (err.cause && err.cause.code === 'ENOENT') return [];
     throw err;
   }).map(escapeBackslash);
 }
@@ -141,7 +139,7 @@ Box.prototype._loadFiles = function() {
 
       result.push({
         path: item,
-        type: 'create'
+        type: File.TYPE_CREATE
       });
     }
 
@@ -151,26 +149,24 @@ Box.prototype._loadFiles = function() {
       if (~files.indexOf(item)) {
         result.push({
           path: item,
-          type: 'update'
+          type: File.TYPE_UPDATE
         });
       } else {
         result.push({
           path: item,
-          type: 'delete'
+          type: File.TYPE_DELETE
         });
       }
     }
 
     return result;
   }).map(function(item) {
-    existed.length = 0;
-
     switch (item.type){
-      case 'create':
-      case 'update':
+      case File.TYPE_CREATE:
+      case File.TYPE_UPDATE:
         return self._handleUpdatedFile(item.path);
 
-      case 'delete':
+      case File.TYPE_DELETE:
         return self._handleDeletedFile(item.path);
     }
   });
@@ -190,65 +186,64 @@ function getHash(path) {
 }
 
 Box.prototype._handleUpdatedFile = function(path) {
+  // Skip the file if it's processing
+  if (this.processingFiles[path]) return Promise.resolve();
+
+  this.processingFiles[path] = true;
+
   var Cache = this.Cache;
   var ctx = this.context;
   var fullPath = join(this.base, path);
+  var id = escapeBackslash(fullPath.substring(ctx.base_dir.length));
+  var self = this;
 
-  return Promise.all([
-    getHash(fullPath),
-    fs.stat(fullPath)
-  ]).spread(function(hash, stats) {
-    var id = escapeBackslash(fullPath.substring(ctx.base_dir.length));
-    var cache = Cache.findById(id);
-
-    if (!cache) {
-      ctx.log.debug('Added: %s', chalk.magenta(id));
-
-      return Cache.insert({
-        _id: id,
-        hash: hash,
-        modified: stats.mtime
-      }).thenReturn({
-        type: 'create',
-        path: path
-      });
-    } else if (cache.hash === hash) {
-      ctx.log.debug('Unchanged: %s', chalk.magenta(id));
+  return Cache.compareFile(
+    id,
+    function() {
+      return getHash(fullPath);
+    },
 
-      return {
-        type: 'skip',
-        path: path
-      };
+    function() {
+      return fs.stat(fullPath);
     }
-
-    ctx.log.debug('Updated: %s', chalk.magenta(id));
-
-    cache.hash = hash;
-    cache.modified = stats.mtime;
-
-    return cache.save().thenReturn({
-      type: 'update',
-      path: path
-    });
+  ).then(function(result) {
+    result.path = path;
+    return result;
+  }).finally(function() {
+    // Unlock the file
+    self.processingFiles[path] = false;
   });
 };
 
 Box.prototype._handleDeletedFile = function(path) {
+  // Skip the file if it's processing
+  if (this.processingFiles[path]) return Promise.resolve();
+
+  this.processingFiles[path] = true;
+
   var fullPath = join(this.base, path);
   var ctx = this.context;
   var Cache = this.Cache;
+  var id = escapeBackslash(fullPath.substring(ctx.base_dir.length));
+  var cache = Cache.findById(id);
+  var result = {
+    type: File.TYPE_DELETE,
+    path: path
+  };
+  var self = this;
 
-  return new Promise(function(resolve, reject) {
-    var id = escapeBackslash(fullPath.substring(ctx.base_dir.length));
-    var cache = Cache.findById(id);
-    if (!cache) return resolve();
-
-    ctx.log.debug('Deleted: %s', chalk.magenta(id));
-    cache.remove().then(resolve, reject);
-  }).thenReturn({
-    type: 'delete',
+  function unlock() {
+    self.processingFiles[path] = false;
+  }
+
+  if (!cache) {
+    return Promise.resolve(result).finally(unlock);
+  }
+
+  return cache.remove().thenReturn({
+    type: File.TYPE_DELETE,
     path: path
-  });
+  }).finally(unlock);
 };
 
 Box.prototype._process = function(files) {
@@ -308,6 +303,10 @@ Box.prototype._dispatch = function(item) {
 };
 
 Box.prototype.watch = function(callback) {
+  if (this.isWatching()) {
+    return Promise.reject(new Error('Watcher has already started.'));
+  }
+
   var base = this.base;
   var baseLength = base.length;
   var self = this;
@@ -323,7 +322,7 @@ Box.prototype.watch = function(callback) {
     if (timer) clearTimeout(timer);
 
     // Add data to the queue
-    queue.push(data);
+    if (data) queue.push(data);
 
     // Start the timer
     setTimeout(generate, 100);
@@ -338,10 +337,7 @@ Box.prototype.watch = function(callback) {
     });
   }
 
-  return new Promise(function(resolve, reject) {
-    if (self.isWatching()) return reject(new Error('Watcher has already started.'));
-    self.process().then(resolve, reject);
-  }).then(function() {
+  return this.process().then(function() {
     return fs.watch(base, self.options);
   }).then(function(watcher) {
     self.watcher = watcher;
@@ -364,10 +360,10 @@ Box.prototype.watch = function(callback) {
 };
 
 Box.prototype.unwatch = function() {
-  if (!this.isWatching()) throw new Error('Watcher hasn\'t started yet.');
-
-  this.watcher.close();
-  this.watcher = null;
+  if (this.isWatching()) {
+    this.watcher.close();
+    this.watcher = null;
+  }
 };
 
 Box.prototype.isWatching = function() {

lib/models/cache.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var Schema = require('warehouse').Schema;
+var Promise = require('bluebird');
 
 module.exports = function(ctx) {
   var Cache = new Schema({
@@ -9,5 +10,61 @@ module.exports = function(ctx) {
     modified: {type: Number, default: Date.now}
   });
 
+  Cache.static('compareFile', function(id, hashFn, statFn) {
+    var cache = this.findById(id);
+    var self = this;
+    var mtime;
+
+    // If cache does not exist, then it must be a new file. We have to get both
+    // file hash and stats.
+    if (!cache) {
+      return Promise.all([hashFn(id), statFn(id)]).spread(function(hash, stats) {
+        return self.insert({
+          _id: id,
+          hash: hash,
+          modified: stats.mtime
+        });
+      }).thenReturn({
+        type: 'create'
+      });
+    }
+
+    // Get file stats
+    return statFn(id).then(function(stats) {
+      mtime = stats.mtime;
+
+      // Skip the file if the modified time is unchanged
+      if (cache.modified === mtime) {
+        return {
+          type: 'skip'
+        };
+      }
+
+      // Get file hash
+      return hashFn(id);
+    }).then(function(result) {
+      // If the result is an object, skip the following steps because it's an
+      // unchanged file
+      if (typeof result === 'object') return result;
+
+      var hash = result;
+
+      // Skip the file if the hash is unchanged
+      if (cache.hash === hash) {
+        return {
+          type: 'skip'
+        };
+      }
+
+      // Update cache info
+      cache.hash = hash;
+      cache.modified = mtime;
+
+      return cache.save().thenReturn({
+        type: 'update'
+      });
+    });
+  });
+
   return Cache;
 };