From 84f2f9459ff518b319239d6b6f7cba9ba3c2b19f Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Tue, 4 Mar 2014 19:59:55 +0800 Subject: [PATCH] Add missing properies and sync missing star users. fixed #235 * add readmeFilename and missing created, modified time. * Sync star users on pkg sync flow * Show star users on registry.show * Show star users on web package show * Need to create new table module_star --- common/mysql.js | 7 +++ controllers/registry/module.js | 51 ++++++++++++++-- controllers/web/package.js | 6 +- docs/db.sql | 10 +++ package.json | 1 + proxy/module_star.js | 47 ++++++++++++++ proxy/sync_module_worker.js | 78 ++++++++++++++++++++++-- test/controllers/registry/module.test.js | 15 +++-- test/proxy/module_star.test.js | 64 +++++++++++++++++++ test/sync.js | 6 +- view/web/package.html | 14 ++--- 11 files changed, 273 insertions(+), 26 deletions(-) create mode 100644 proxy/module_star.js create mode 100644 test/proxy/module_star.test.js diff --git a/common/mysql.js b/common/mysql.js index fcb954631..33978738a 100644 --- a/common/mysql.js +++ b/common/mysql.js @@ -15,6 +15,7 @@ * Module dependencies. */ +var thunkify = require('thunkify-wrap'); var ready = require('ready'); var mysql = require('mysql'); var config = require('../config'); @@ -35,6 +36,10 @@ var pool = mysql.createPool({ exports.pool = pool; exports.query = function (sql, values, cb) { + if (typeof values === 'function') { + cb = values; + values = null; + } pool.query(sql, values, function (err, rows) { cb(err, rows); }); @@ -59,6 +64,8 @@ exports.escape = function (val) { ready(exports); +thunkify(exports); + function init() { exports.query('show tables', function (err, rows) { if (err) { diff --git a/controllers/registry/module.js b/controllers/registry/module.js index 79249cb67..43acced68 100644 --- a/controllers/registry/module.js +++ b/controllers/registry/module.js @@ -36,6 +36,7 @@ var DownloadTotal = require('../../proxy/download'); var SyncModuleWorker = require('../../proxy/sync_module_worker'); var logger = require('../../common/logger'); var ModuleDeps = require('../../proxy/module_deps'); +var ModuleStar = require('../../proxy/module_star'); /** * show all version of a module @@ -43,9 +44,19 @@ var ModuleDeps = require('../../proxy/module_deps'); exports.show = function *(next) { var name = this.params.name; - var r = yield [Module.listTags(name), Module.listByName(name)]; + var r = yield [ + Module.listTags(name), + Module.listByName(name), + ModuleStar.listUsers(name) + ]; var tags = r[0]; var rows = r[1]; + var users = r[2]; + var userMap = {}; + for (var i = 0; i < users.length; i++) { + userMap[users[i]] = true; + } + users = userMap; debug('show module, user: %s, allowSync: %s, isAdmin: %s', this.session.name, this.session.allowSync, this.session.isAdmin); @@ -81,6 +92,8 @@ exports.show = function *(next) { var versions = {}; var times = {}; var attachments = {}; + var createdTime = null; + var modifiedTime = null; for (var i = 0; i < rows.length; i++) { var row = rows[i]; if (row.version === 'next') { @@ -91,12 +104,30 @@ exports.show = function *(next) { common.setDownloadURL(pkg, this); pkg._cnpm_publish_time = row.publish_time; versions[pkg.version] = pkg; - times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified; + var t = times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified; if ((!distTags.latest && !latestMod) || distTags.latest === row.version) { latestMod = row; readme = pkg.readme; } delete pkg.readme; + + if (!createdTime || t < createdTime) { + createdTime = t; + } + if (!modifiedTime || t > modifiedTime) { + modifiedTime = t; + } + } + + if (modifiedTime && createdTime) { + var ts = { + modified: modifiedTime, + created: createdTime, + }; + for (var t in times) { + ts[t] = times[t]; + } + times = ts; } if (!latestMod) { @@ -112,21 +143,29 @@ exports.show = function *(next) { rev = String(nextMod.id); } + var pkg = latestMod.package; + var info = { _id: name, _rev: rev, name: name, - description: latestMod.package.description, + description: pkg.description, "dist-tags": distTags, - maintainers: latestMod.package.maintainers, + maintainers: pkg.maintainers, time: times, - author: latestMod.package.author, - repository: latestMod.package.repository, + users: users, + author: pkg.author, + repository: pkg.repository, versions: versions, readme: readme, _attachments: attachments, }; + info.readmeFilename = pkg.readmeFilename; + info.homepage = pkg.homepage; + info.bugs = pkg.bugs; + info.license = pkg.license; + debug('show module %s: %s, latest: %s', name, rev, latestMod.version); this.body = info; diff --git a/controllers/web/package.js b/controllers/web/package.js index 816b211aa..da36c7404 100644 --- a/controllers/web/package.js +++ b/controllers/web/package.js @@ -28,6 +28,7 @@ var sync = require('../sync'); var Log = require('../../proxy/module_log'); var ModuleDeps = require('../../proxy/module_deps'); var setDownloadURL = require('../../lib/common').setDownloadURL; +var ModuleStar = require('../../proxy/module_star'); exports.display = function *(next) { var params = this.params; @@ -47,13 +48,15 @@ exports.display = function *(next) { var r = yield [ Module[getPackageMethod].apply(Module, getPackageArgs), down.total(name), - ModuleDeps.list(name) + ModuleDeps.list(name), + ModuleStar.listUsers(name), ]; var pkg = r[0]; var download = r[1]; var dependents = (r[2] || []).map(function (item) { return item.deps; }); + var users = r[3]; if (!pkg || !pkg.package) { return yield next; @@ -61,6 +64,7 @@ exports.display = function *(next) { pkg.package.fromNow = moment(pkg.publish_time).fromNow(); pkg = pkg.package; + pkg.users = users; pkg.readme = marked(pkg.readme || ''); if (!pkg.readme) { pkg.readme = pkg.description || ''; diff --git a/docs/db.sql b/docs/db.sql index 8a5d131f9..1604595f8 100644 --- a/docs/db.sql +++ b/docs/db.sql @@ -25,6 +25,16 @@ CREATE TABLE `module_keyword` ( KEY `name` (`name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module keyword'; +CREATE TABLE `module_star` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `gmt_create` datetime NOT NULL COMMENT 'create time', + `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name', + `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name', + PRIMARY KEY (`id`), + UNIQUE KEY `user_module_name` (`user`,`name`), + KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star'; + CREATE TABLE `module` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key', `gmt_create` datetime NOT NULL COMMENT 'create time', diff --git a/package.json b/package.json index 2c9e28ef8..7b3a7760e 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ }, "devDependencies": { "autod": ">=0.0.13", + "co": "3.0.4", "contributors": "*", "cov": "*", "istanbul": "git://github.com/gotwarlost/istanbul.git#harmony", diff --git a/proxy/module_star.js b/proxy/module_star.js new file mode 100644 index 000000000..e178deae8 --- /dev/null +++ b/proxy/module_star.js @@ -0,0 +1,47 @@ +/**! + * cnpmjs.org - proxy/module_star.js + * + * Copyright(c) cnpmjs.org and other contributors. + * MIT Licensed + * + * Authors: + * fengmk2 (http://fengmk2.github.com) + */ + +'use strict'; + +/** + * Module dependencies. + */ + +var mysql = require('../common/mysql'); + +exports.add = function *add(name, user) { + var sql = 'INSERT INTO module_star(name, user) VALUES(?, ?);'; + try { + yield mysql.query(sql, [name, user]); + } catch (err) { + if (err.code !== 'ER_DUP_ENTRY') { + throw err; + } + } +}; + +exports.remove = function *(name, user) { + var sql = 'DELETE FROM module_star WHERE name = ? AND user = ?;'; + return yield mysql.query(sql, [name, user]); +}; + +exports.listUsers = function *(name) { + var sql = 'SELECT user FROM module_star WHERE name = ?;'; + return (yield mysql.query(sql, [name])).map(function (r) { + return r.user; + }); +}; + +exports.listUserModules = function *(user) { + var sql = 'SELECT name FROM module_star WHERE user = ?;'; + return (yield mysql.query(sql, [user])).map(function (r) { + return r.name; + }); +}; diff --git a/proxy/sync_module_worker.js b/proxy/sync_module_worker.js index de050c969..14d353217 100644 --- a/proxy/sync_module_worker.js +++ b/proxy/sync_module_worker.js @@ -15,6 +15,7 @@ * Module dependencies. */ +var co = require('co'); var thunkify = require('thunkify-wrap'); var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker'); var EventEmitter = require('events').EventEmitter; @@ -33,6 +34,7 @@ var Module = require('./module'); var ModuleDeps = require('./module_deps'); var Log = require('./module_log'); var config = require('../config'); +var ModuleStar = require('./module_star'); function SyncModuleWorker(options) { EventEmitter.call(this); @@ -145,6 +147,36 @@ SyncModuleWorker.prototype.next = function (concurrencyId) { }); }; +function _listStarUsers(modName, callback) { + co(function *() { + var users; + var err; + try { + users = yield ModuleStar.listUsers(modName); + var userMap = {}; + for (var i = 0; i < users.length; i++) { + userMap[users[i]] = true; + } + users = userMap; + } catch (e) { + err = e; + } + callback(err, users); + })(); +} + +function _addStar(modName, username, callback) { + co(function *() { + var err; + try { + yield ModuleStar.add(modName, username); + } catch (e) { + err = e; + } + callback(err); + })(); +}; + SyncModuleWorker.prototype._sync = function (name, pkg, callback) { var username = this.username; var that = this; @@ -194,12 +226,24 @@ SyncModuleWorker.prototype._sync = function (name, pkg, callback) { ep.emit('existsTags', tags); })); + _listStarUsers(name, ep.done('existsStarUsers')); + var missingVersions = []; var missingTags = []; var missingDescriptions = []; var missingReadmes = []; + var missingStarUsers = []; + + ep.all('existsMap', 'existsTags', 'existsStarUsers', function (map, tags, existsStarUsers) { + var starUsers = pkg.users || {}; + for (var k in starUsers) { + if (!existsStarUsers[k]) { + missingStarUsers.push(k); + } + } + + that.log(' [%s] found %d missing star users', name, missingStarUsers.length); - ep.all('existsMap', 'existsTags', function (map, tags) { var times = pkg.time || {}; pkg.versions = pkg.versions || {}; var versionNames = Object.keys(times); @@ -300,7 +344,7 @@ SyncModuleWorker.prototype._sync = function (name, pkg, callback) { return a.publish_time - b.publish_time; }); missingVersions = versions; - that.log(' [%s] %d versions', name, versions.length); + that.log(' [%s] %d versions need to sync', name, versions.length); ep.emit('syncModule', missingVersions.shift()); }); @@ -318,13 +362,16 @@ SyncModuleWorker.prototype._sync = function (name, pkg, callback) { var nextVersion = missingVersions.shift(); if (!nextVersion) { + ep.unbind('syncModule'); return ep.emit('syncDone', result); } + + // next ep.emit('syncModule', nextVersion); }); }); - ep.on('syncDone', function () { + ep.once('syncDone', function () { if (missingDescriptions.length === 0) { return ep.emit('descriptionDone'); } @@ -346,7 +393,7 @@ SyncModuleWorker.prototype._sync = function (name, pkg, callback) { }); }); - ep.on('syncDone', function () { + ep.once('syncDone', function () { if (missingTags.length === 0) { return ep.emit('tagDone'); } @@ -387,7 +434,28 @@ SyncModuleWorker.prototype._sync = function (name, pkg, callback) { }); }); - ep.all('tagDone', 'descriptionDone', 'readmeDone', function () { + ep.once('syncDone', function () { + if (missingStarUsers.length === 0) { + return ep.emit('starUserDone'); + } + + that.log(' [%s] saving %d star users', name, missingStarUsers.length); + missingStarUsers.forEach(function (username) { + _addStar(name, username, function (err) { + if (err) { + that.log(' add star user error, %s', err); + } + ep.emitLater('addStarUser'); + }); + }); + + ep.after('addStarUser', missingStarUsers.length, function () { + ep.emit('starUserDone'); + }); + }); + + ep.all('tagDone', 'descriptionDone', 'readmeDone', 'starUserDone', + function () { // TODO: set latest version callback(null, versionNames); }); diff --git a/test/controllers/registry/module.test.js b/test/controllers/registry/module.test.js index 4871b2bb2..7fb49e465 100644 --- a/test/controllers/registry/module.test.js +++ b/test/controllers/registry/module.test.js @@ -79,9 +79,15 @@ describe('controllers/registry/module.test.js', function () { etag = res.headers.etag; res.body.should.have.keys('_id', '_rev', 'name', 'description', 'versions', 'dist-tags', 'readme', 'maintainers', - 'time', 'author', 'repository', '_attachments'); + 'time', 'author', 'repository', '_attachments', + 'users', 'readmeFilename', 'homepage', 'bugs', 'license'); res.body.name.should.equal('cnpmjs.org'); - res.body.versions[Object.keys(res.body.versions)[0]].dist.tarball.should.include('/cnpmjs.org/download'); + res.body.versions[Object.keys(res.body.versions)[0]] + .dist.tarball.should.include('/cnpmjs.org/download'); + res.body.time.should.have.property('modified'); + res.body.time.modified.should.be.a.String; + res.body.time.should.have.property('created'); + res.body.time.created.should.be.a.String; done(); }); }); @@ -99,7 +105,8 @@ describe('controllers/registry/module.test.js', function () { etag = res.headers.etag; res.body.should.have.keys('_id', '_rev', 'name', 'description', 'versions', 'dist-tags', 'readme', 'maintainers', - 'time', 'author', 'repository', '_attachments'); + 'time', 'author', 'repository', '_attachments', + 'users', 'readmeFilename', 'homepage', 'bugs', 'license'); res.body.name.should.equal('cnpmjs.org'); res.body.versions[Object.keys(res.body.versions)[0]].dist.tarball.should.include('/cnpmjs.org/download'); done(); @@ -237,7 +244,7 @@ describe('controllers/registry/module.test.js', function () { .expect(200, function (err, res) { should.not.exist(err); res.body.should.have.keys('_id', '_rev', 'name', 'description', 'versions', 'dist-tags', - 'readme', 'maintainers', 'time', '_attachments'); + 'readme', 'maintainers', 'time', '_attachments', 'users'); res.body.versions.should.eql({}); res.body.time.should.eql({}); res.body['dist-tags'].should.eql({}); diff --git a/test/proxy/module_star.test.js b/test/proxy/module_star.test.js new file mode 100644 index 000000000..8f7061a26 --- /dev/null +++ b/test/proxy/module_star.test.js @@ -0,0 +1,64 @@ +/**! + * cnpmjs.org - test/proxy/module_star.test.js + * + * Copyright(c) cnpmjs.org and other contributors. + * MIT Licensed + * + * Authors: + * fengmk2 (http://fengmk2.github.com) + */ + +'use strict'; + +/** + * Module dependencies. + */ + +var should = require('should'); +var co = require('co'); +var Star = require('../../proxy/module_star'); + +describe('proxy/module_star.test.js', function () { + before(function (done) { + co(function *() { + yield Star.remove('testmodule', 'fengmk2'); + yield Star.remove('testmodule', 'mk1'); + yield Star.remove('testmodule', 'mk2'); + done(); + })(); + }); + + it('should add a star', function (done) { + co(function *() { + yield Star.add('testmodule', 'fengmk2'); + // again should be ok + yield Star.add('testmodule', 'fengmk2'); + yield Star.add('testmodule', 'fengmk2'); + done(); + })(); + }); + + it('should get all star users', function (done) { + co(function *() { + yield Star.add('testmodule', 'fengmk2'); + yield Star.add('testmodule', 'mk1'); + yield Star.add('testmodule', 'mk2'); + + var rows = yield Star.listUsers('testmodule'); + rows.should.containDeep(['fengmk2', 'mk1', 'mk2']) + done(); + })(); + }); + + it('should get user all star modules', function (done) { + co(function *() { + yield Star.add('testmodule', 'fengmk2'); + yield Star.add('testmodule1', 'fengmk2'); + yield Star.add('testmodule2', 'fengmk2'); + + var rows = yield Star.listUserModules('fengmk2'); + rows.should.containDeep(['testmodule', 'testmodule1', 'testmodule2']) + done(); + })(); + }); +}); diff --git a/test/sync.js b/test/sync.js index f19095b63..86f141f3a 100644 --- a/test/sync.js +++ b/test/sync.js @@ -18,7 +18,7 @@ var SyncModuleWorker = require('../proxy/sync_module_worker'); var mysql = require('../common/mysql'); var Log = require('../proxy/module_log'); -var name = process.argv[2] || 'address,pedding'; +var name = process.argv[2] || 'byte'; var names = name.split(','); Log.create({ @@ -30,8 +30,8 @@ Log.create({ name: names, username: 'fengmk2', concurrency: names.length, - noDep: true, - publish: true, + // noDep: true, + // publish: true, }); worker.start(); diff --git a/view/web/package.html b/view/web/package.html index 7a06de60e..f0b7fd455 100644 --- a/view/web/package.html +++ b/view/web/package.html @@ -188,27 +188,27 @@

- <% if (package.users) { - var starredBy = package.starredBy - var l = starredBy.length + <% if (package.users.length > 0) { + var users = package.users + var l = users.length %> - Starred by<%= l > 0 ? ' (' + l + ')' : '' %> + Starred by (<%= l %>) <% var max = 20 if (l > max) { - starredBy = starredBy.sort(function (a, b) { + users = users.sort(function (a, b) { return Math.random() * 2 - 1 }).slice(0, max) } - starredBy.forEach(function (user, i) { + users.forEach(function (user, i) { if (i > 0) { %>, <% } %><%= user %><% }) if (l > max) { %>
and - <%= (l-max) %> more<% + <%= (l - max) %> more<% } %>