diff --git a/lib/crawler.js b/lib/crawler.js index 0619f68..6d01718 100644 --- a/lib/crawler.js +++ b/lib/crawler.js @@ -262,22 +262,19 @@ Crawler.prototype._makeCrawlerRequest = function _makeCrawlerRequest (options) { Crawler.prototype._executeCrawlerRequest = function _executeCrawlerRequest (options) { var self = this; + var cacheData = self.cache[options.uri]; - if (useCache(options)) { + //If a query has already been made to self URL, don't callback again + if (useCache(options) && cacheData) { - var cacheData = self.cache[options.uri]; - - //If a query has already been made to self URL, don't callback again - if (cacheData) { - - // Make sure we actually have cached data, and not just a note - // that the page was already crawled - if (_.isArray(cacheData)) { - self._onContent(null, options, cacheData[0], true); - } else { - self.emit('pool:release', options); - } + // Make sure we actually have cached data, and not just a note + // that the page was already crawled + if (_.isArray(cacheData)) { + self._onContent(null, options, cacheData[0], true); + } else { + self.emit('pool:release', options); } + } else { self._buildHttpRequest(options); } diff --git a/tests/cacheOption.test.js b/tests/cacheOption.test.js index 178b681..d28e3d4 100644 --- a/tests/cacheOption.test.js +++ b/tests/cacheOption.test.js @@ -31,12 +31,26 @@ describe('Cache features tests', function() { }); }); - //describe('Skip Duplicate', function() { - // afterEach(function () { - // c = {}; - // }); - // it('should skip previous crawled urls', function (done) {}); - // it('should not skip one single url', function (done) {}); - //}); + describe('Skip Duplicate active', function() { + afterEach(function () { + c = {}; + }); + + it('should not skip one single url', function (done) { + c = new Crawler({ + jquery: false, + skipDuplicates: true, + callback: function (error, result) { + expect(error).to.be.null; + expect(result.statusCode).to.equal(200); + done(); + }, + }); + + c.queue('http://' + httpbinHost + '/status/200'); + }); + + //it('should skip previous crawled urls', function (done) {}); + }); });