diff --git a/api/lib/models/extension.js b/api/lib/models/extension.js index 9a756bc..3128937 100644 --- a/api/lib/models/extension.js +++ b/api/lib/models/extension.js @@ -30,6 +30,20 @@ exports.stream = function(query){ return Extension.find(query).stream(); } +exports.updateGitHub = function(data){ + Extension.findById(data.id, function(err, ext){ + if (err || !ext){ + return; + } + + ext.stars = data.stars; + ext.forks = data.forks; + ext.githubTimestamp = new Date(); + + ext.save(); + }); +} + exports.process = function(extension, callback){ var id = extension.metadata.name; Download = Download || mongoose.model('Download'); diff --git a/api/lib/services/github.js b/api/lib/services/github.js new file mode 100644 index 0000000..efff8ea --- /dev/null +++ b/api/lib/services/github.js @@ -0,0 +1,129 @@ +var request = require('request'), + Q = require('q'), + _ = require('lodash'), + bus = require('./bus'), + cache = require('./cache'), + userAgent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36', + mongoose = require('mongoose'), + Extension = require('../models/extension'); + +var routes = { + limit: 'https://api.github.com/rate_limit' +} + +function init(app){ + bus.on(bus.list.REGISTRY.CACHED, onRegistryCached); +} + +function onRegistryCached(registry){ + knowYourLimits().then(function(data){ + if (data.resources && data.resources.core){ + var remaining = data.resources.core.remaining, + reset = data.resources.core.reset; + + console.log('GitHub requests remaining: ' + remaining + ', reset at ' + new Date(reset * 1000)); + onRemaining(remaining, registry); + } else { + onRemaining(0, registry); + } + }, function(err){ + onRemaining(0, registry); + }); +} + +function onRemaining(remaining, registry){ + var extensions = _.shuffle(registry); + if (remaining <= 0) return; + + _.each(extensions, function(extension){ + try{ + if (typeof extension.homepage !== 'string'){ return true; } + + var id = extension._id, + repository = extension.homepage.replace('github.com', 'api.github.com/repos'), + today = new Date().getDate(); + + if (remaining > 0){ + remaining --; + getGitHubInfo(repository, extension).then(function(data){ + extension.stars = data.stargazers_count || 0; + extension.forks = data.forks || 0; + extension.githubTimestamp = new Date().toISOString(); + + Extension.updateGitHub({ + id: id, + stars: extension.stars, + forks: extension.forks + }); + + console.log(id); + }); + } + + if (remaining <= 0){ + return false; + } + } catch(e){ + console.log(e); + } + }); +} + +function getGitHubInfo(repoUrl, extension){ + var defer = Q.defer(); + + if (typeof repoUrl !== 'string' || repoUrl.indexOf('github.com') === -1){ + defer.reject(); + return defer.promise; + } + + if (repoUrl.indexOf('http') === -1){ + repoUrl = 'http://' + repoUrl; + } + + request({ + url: repoUrl, + headers:{ + 'User-Agent': userAgent + } + }, function(err, response, body){ + if (err){ + console.log(err); + defer.reject(err); + } else { + if (typeof body === 'string'){ + try{ + body = JSON.parse(body); + } catch (e){ + defer.reject(e); + } + } + defer.resolve(body); + } + }); + + return defer.promise; +} + +function knowYourLimits(){ + var defer = Q.defer(); + request({ + url: routes.limit, + headers: { + 'User-Agent': userAgent + } + }, function(err, response, body){ + if (err){ + defer.reject(err); + } else { + if (typeof body === 'string'){ + body = JSON.parse(body); + } + defer.resolve(body); + } + }); + + return defer.promise; +} + +module.exports = init; diff --git a/api/server.js b/api/server.js index 6d57c56..07d5b61 100644 --- a/api/server.js +++ b/api/server.js @@ -23,6 +23,7 @@ fs.readdirSync(modelsPath).forEach(function (file) { var app = express(); require('./lib/config/express')(app); require('./lib/routes')(app); +require('./lib/services/github')(app); // Start server app.listen(config.port, config.ip, function () {