/^text$/i",
"You_can_use_an_emoji_as_avatar": "You can also use an emoji as an avatar.",
"You_can_use_webhooks_to_easily_integrate_livechat_with_your_CRM": "You can use webhooks to easily integrate livechat with your CRM.",
diff --git a/packages/rocketchat-importer-hipchat-enterprise/server/importer.js b/packages/rocketchat-importer-hipchat-enterprise/server/importer.js
index e2e67a1d536b..92b7a69c8096 100644
--- a/packages/rocketchat-importer-hipchat-enterprise/server/importer.js
+++ b/packages/rocketchat-importer-hipchat-enterprise/server/importer.js
@@ -154,6 +154,7 @@ export class HipChatEnterpriseImporter extends Base {
isPrivate: r.Room.privacy === 'private',
isArchived: r.Room.is_archived,
topic: r.Room.topic,
+ members: r.Room.members,
});
count++;
@@ -189,7 +190,7 @@ export class HipChatEnterpriseImporter extends Base {
}
async storeUserTempMessages(tempMessages, roomIdentifier, index) {
- this.logger.debug('dumping messages to database');
+ this.logger.debug(`dumping ${ tempMessages.length } messages from room ${ roomIdentifier } to database`);
await this.collection.insert({
import: this.importRecord._id,
importer: this.name,
@@ -207,16 +208,30 @@ export class HipChatEnterpriseImporter extends Base {
this.logger.debug(`preparing room with ${ file.length } messages `);
for (const m of file) {
if (m.PrivateUserMessage) {
- msgs.push({
- type: 'user',
- id: `hipchatenterprise-${ m.PrivateUserMessage.id }`,
- senderId: m.PrivateUserMessage.sender.id,
- receiverId: m.PrivateUserMessage.receiver.id,
- text: m.PrivateUserMessage.message.indexOf('/me ') === -1 ? m.PrivateUserMessage.message : `${ m.PrivateUserMessage.message.replace(/\/me /, '_') }_`,
- ts: new Date(m.PrivateUserMessage.timestamp.split(' ')[0]),
- attachment: m.PrivateUserMessage.attachment,
- attachment_path: m.PrivateUserMessage.attachment_path,
- });
+ // If the message id is already on the list, skip it
+ if (this.preparedMessages[m.PrivateUserMessage.id] !== undefined) {
+ continue;
+ }
+ this.preparedMessages[m.PrivateUserMessage.id] = true;
+
+ const newId = `hipchatenterprise-private-${ m.PrivateUserMessage.id }`;
+ const skipMessage = this._checkIfMessageExists(newId);
+ const skipAttachment = skipMessage && (m.PrivateUserMessage.attachment_path ? this._checkIfMessageExists(`${ newId }-attachment`) : true);
+
+ if (!skipMessage || !skipAttachment) {
+ msgs.push({
+ type: 'user',
+ id: newId,
+ senderId: m.PrivateUserMessage.sender.id,
+ receiverId: m.PrivateUserMessage.receiver.id,
+ text: m.PrivateUserMessage.message.indexOf('/me ') === -1 ? m.PrivateUserMessage.message : `${ m.PrivateUserMessage.message.replace(/\/me /, '_') }_`,
+ ts: new Date(m.PrivateUserMessage.timestamp.split(' ')[0]),
+ attachment: m.PrivateUserMessage.attachment,
+ attachment_path: m.PrivateUserMessage.attachment_path,
+ skip: skipMessage,
+ skipAttachment,
+ });
+ }
}
if (msgs.length >= 500) {
@@ -232,6 +247,14 @@ export class HipChatEnterpriseImporter extends Base {
return msgs.length;
}
+ _checkIfMessageExists(messageId) {
+ if (this._hasAnyImportedMessage === false) {
+ return false;
+ }
+
+ return Boolean(RocketChat.models.Messages.findOne({ _id: messageId }, { fields: { _id: 1 }, limit: 1 }));
+ }
+
async prepareRoomMessagesFile(file, roomIdentifier, id, index) {
let roomMsgs = [];
this.logger.debug(`preparing room with ${ file.length } messages `);
@@ -239,36 +262,56 @@ export class HipChatEnterpriseImporter extends Base {
for (const m of file) {
if (m.UserMessage) {
- roomMsgs.push({
- type: 'user',
- id: `hipchatenterprise-${ id }-${ m.UserMessage.id }`,
- userId: m.UserMessage.sender.id,
- text: m.UserMessage.message.indexOf('/me ') === -1 ? m.UserMessage.message : `${ m.UserMessage.message.replace(/\/me /, '_') }_`,
- ts: new Date(m.UserMessage.timestamp.split(' ')[0]),
- attachment: m.UserMessage.attachment,
- attachment_path: m.UserMessage.attachment_path,
- });
+ const newId = `hipchatenterprise-${ id }-user-${ m.UserMessage.id }`;
+ const skipMessage = this._checkIfMessageExists(newId);
+ const skipAttachment = (skipMessage && m.UserMessage.attachment_path ? this._checkIfMessageExists(`${ newId }-attachment`) : true);
+
+ if (!skipMessage || !skipAttachment) {
+ roomMsgs.push({
+ type: 'user',
+ id: newId,
+ userId: m.UserMessage.sender.id,
+ text: m.UserMessage.message.indexOf('/me ') === -1 ? m.UserMessage.message : `${ m.UserMessage.message.replace(/\/me /, '_') }_`,
+ ts: new Date(m.UserMessage.timestamp.split(' ')[0]),
+ attachment: m.UserMessage.attachment,
+ attachment_path: m.UserMessage.attachment_path,
+ skip: skipMessage,
+ skipAttachment,
+ });
+ }
} else if (m.NotificationMessage) {
const text = m.NotificationMessage.message.indexOf('/me ') === -1 ? m.NotificationMessage.message : `${ m.NotificationMessage.message.replace(/\/me /, '_') }_`;
-
- roomMsgs.push({
- type: 'user',
- id: `hipchatenterprise-${ id }-${ m.NotificationMessage.id }`,
- userId: 'rocket.cat',
- alias: m.NotificationMessage.sender,
- text: m.NotificationMessage.message_format === 'html' ? turndownService.turndown(text) : text,
- ts: new Date(m.NotificationMessage.timestamp.split(' ')[0]),
- attachment: m.NotificationMessage.attachment,
- attachment_path: m.NotificationMessage.attachment_path,
- });
+ const newId = `hipchatenterprise-${ id }-notif-${ m.NotificationMessage.id }`;
+ const skipMessage = this._checkIfMessageExists(newId);
+ const skipAttachment = skipMessage && (m.NotificationMessage.attachment_path ? this._checkIfMessageExists(`${ newId }-attachment`) : true);
+
+ if (!skipMessage || !skipAttachment) {
+ roomMsgs.push({
+ type: 'user',
+ id: newId,
+ userId: 'rocket.cat',
+ alias: m.NotificationMessage.sender,
+ text: m.NotificationMessage.message_format === 'html' ? turndownService.turndown(text) : text,
+ ts: new Date(m.NotificationMessage.timestamp.split(' ')[0]),
+ attachment: m.NotificationMessage.attachment,
+ attachment_path: m.NotificationMessage.attachment_path,
+ skip: skipMessage,
+ skipAttachment,
+ });
+ }
} else if (m.TopicRoomMessage) {
- roomMsgs.push({
- type: 'topic',
- id: `hipchatenterprise-${ id }-${ m.TopicRoomMessage.id }`,
- userId: m.TopicRoomMessage.sender.id,
- ts: new Date(m.TopicRoomMessage.timestamp.split(' ')[0]),
- text: m.TopicRoomMessage.message,
- });
+ const newId = `hipchatenterprise-${ id }-topic-${ m.TopicRoomMessage.id }`;
+ const skipMessage = this._checkIfMessageExists(newId);
+ if (!skipMessage) {
+ roomMsgs.push({
+ type: 'topic',
+ id: newId,
+ userId: m.TopicRoomMessage.sender.id,
+ ts: new Date(m.TopicRoomMessage.timestamp.split(' ')[0]),
+ text: m.TopicRoomMessage.message,
+ skip: skipMessage,
+ });
+ }
} else {
this.logger.warn('HipChat Enterprise importer isn\'t configured to handle this message:', m);
}
@@ -326,6 +369,9 @@ export class HipChatEnterpriseImporter extends Base {
break;
case 'history.json':
return await this.prepareMessagesFile(file, info);
+ case 'emoticons.json':
+ this.logger.warn('HipChat Enterprise importer doesn\'t import emoticons.', info);
+ break;
default:
this.logger.warn(`HipChat Enterprise importer doesn't know what to do with the file "${ fileName }" :o`, info);
break;
@@ -339,10 +385,16 @@ export class HipChatEnterpriseImporter extends Base {
this.collection.remove({});
this.emailList = [];
+ this._hasAnyImportedMessage = Boolean(RocketChat.models.Messages.findOne({ _id: /hipchatenterprise\-.*/ }));
+
this.usersCount = 0;
this.channelsCount = 0;
this.messagesCount = 0;
+ // HipChat duplicates direct messages (one for each user)
+ // This object will keep track of messages that have already been prepared so it doesn't try to do it twice
+ this.preparedMessages = {};
+
const promise = new Promise((resolve, reject) => {
this.extract.on('entry', Meteor.bindEnvironment((header, stream, next) => {
this.logger.debug(`new entry from import file: ${ header.name }`);
@@ -382,7 +434,6 @@ export class HipChatEnterpriseImporter extends Base {
super.addCountToTotal(this.messagesCount);
// Check if any of the emails used are already taken
-
if (this.emailList.length > 0) {
const conflictingUsers = RocketChat.models.Users.find({ 'emails.address': { $in: this.emailList } });
const conflictingUserEmails = [];
@@ -401,7 +452,7 @@ export class HipChatEnterpriseImporter extends Base {
}
// Ensure we have some users, channels, and messages
- if (!this.usersCount || !this.channelsCount || this.messagesCount === 0) {
+ if (!this.usersCount && !this.channelsCount && !this.messagesCount) {
this.logger.debug(`users: ${ this.usersCount }, channels: ${ this.channelsCount }, messages = ${ this.messagesCount }`);
super.updateProgress(ProgressStep.ERROR);
reject(new Meteor.Error('error-import-file-is-empty'));
@@ -485,12 +536,10 @@ export class HipChatEnterpriseImporter extends Base {
Meteor.runAsUser(existingUserId, () => {
RocketChat.models.Users.update({ _id: existingUserId }, { $addToSet: { importIds: userToImport.id } });
- Meteor.call('setUsername', userToImport.username, { joinDefaultChannelsSilenced: true });
-
// TODO: Use moment timezone to calc the time offset - Meteor.call 'userSetUtcOffset', user.tz_offset / 3600
RocketChat.models.Users.setName(existingUserId, userToImport.name);
- // TODO: Think about using a custom field for the users "title" field
+ // TODO: Think about using a custom field for the users "title" field
if (userToImport.avatar) {
Meteor.call('setAvatarFromService', `data:image/png;base64,${ userToImport.avatar }`);
}
@@ -525,13 +574,12 @@ export class HipChatEnterpriseImporter extends Base {
this.addUserError(userToImport.id, e);
}
} else {
- const user = { email: userToImport.email, password: Random.id() };
- // if (u.is_email_taken && u.email) {
- // user.email = user.email.replace('@', `+rocket.chat_${ Math.floor(Math.random() * 10000).toString() }@`);
- // }
+ const user = { email: userToImport.email, password: Random.id(), username: userToImport.username };
if (!user.email) {
delete user.email;
- user.username = userToImport.username;
+ }
+ if (!user.username) {
+ delete user.username;
}
try {
@@ -655,25 +703,26 @@ export class HipChatEnterpriseImporter extends Base {
startImport(importSelection) {
super.startImport(importSelection);
+ this._userDataCache = {};
const started = Date.now();
this._applyUserSelections(importSelection);
const startedByUserId = Meteor.userId();
- Meteor.defer(() => {
+ Meteor.defer(async() => {
try {
- super.updateProgress(ProgressStep.IMPORTING_USERS);
- this._importUsers(startedByUserId);
+ await super.updateProgress(ProgressStep.IMPORTING_USERS);
+ await this._importUsers(startedByUserId);
- super.updateProgress(ProgressStep.IMPORTING_CHANNELS);
- this._importChannels(startedByUserId);
+ await super.updateProgress(ProgressStep.IMPORTING_CHANNELS);
+ await this._importChannels(startedByUserId);
- super.updateProgress(ProgressStep.IMPORTING_MESSAGES);
- this._importMessages(startedByUserId);
- this._importDirectMessages();
+ await super.updateProgress(ProgressStep.IMPORTING_MESSAGES);
+ await this._importMessages(startedByUserId);
+ await this._importDirectMessages();
// super.updateProgress(ProgressStep.FINISHING);
- super.updateProgress(ProgressStep.DONE);
+ await super.updateProgress(ProgressStep.DONE);
} catch (e) {
super.updateRecord({ 'error-record': JSON.stringify(e, Object.getOwnPropertyNames(e)) });
this.logger.error(e);
@@ -712,6 +761,36 @@ export class HipChatEnterpriseImporter extends Base {
});
}
+ _createSubscriptions(channelToImport, roomOrRoomId) {
+ if (!channelToImport || !channelToImport.members) {
+ return;
+ }
+
+ let room;
+ if (roomOrRoomId && typeof roomOrRoomId === 'string') {
+ room = RocketChat.models.Rooms.findOneByIdOrName(roomOrRoomId);
+ } else {
+ room = roomOrRoomId;
+ }
+
+ const extra = { open: true };
+ channelToImport.members.forEach((hipchatUserId) => {
+ if (hipchatUserId === channelToImport.creator) {
+ // Creators are subscribed automatically
+ return;
+ }
+
+ const user = this.getRocketUserFromUserId(hipchatUserId);
+ if (!user) {
+ this.logger.warn(`User ${ hipchatUserId } not found on Rocket.Chat database.`);
+ return;
+ }
+
+ this.logger.info(`Creating user's subscription to room ${ room._id }, rocket.chat user is ${ user._id }, hipchat user is ${ hipchatUserId }`);
+ RocketChat.models.Subscriptions.createWithRoomAndUser(room, user, extra);
+ });
+ }
+
_importChannel(channelToImport, startedByUserId) {
Meteor.runAsUser(startedByUserId, () => {
const existingRoom = RocketChat.models.Rooms.findOneByName(channelToImport.name);
@@ -720,6 +799,8 @@ export class HipChatEnterpriseImporter extends Base {
channelToImport.rocketId = channelToImport.name.toUpperCase() === 'GENERAL' ? 'GENERAL' : existingRoom._id;
this._saveRoomIdReference(channelToImport.id, channelToImport.rocketId);
RocketChat.models.Rooms.update({ _id: channelToImport.rocketId }, { $addToSet: { importIds: channelToImport.id } });
+
+ this._createSubscriptions(channelToImport, existingRoom || 'general');
} else {
// Find the rocketchatId of the user who created this channel
const creatorId = this._getUserRocketId(channelToImport.creator) || startedByUserId;
@@ -737,6 +818,7 @@ export class HipChatEnterpriseImporter extends Base {
if (channelToImport.rocketId) {
RocketChat.models.Rooms.update({ _id: channelToImport.rocketId }, { $set: { ts: channelToImport.created, topic: channelToImport.topic }, $addToSet: { importIds: channelToImport.id } });
+ this._createSubscriptions(channelToImport, channelToImport.rocketId);
}
}
@@ -768,7 +850,7 @@ export class HipChatEnterpriseImporter extends Base {
}
_importAttachment(msg, room, sender) {
- if (msg.attachment_path) {
+ if (msg.attachment_path && !msg.skipAttachment) {
const details = {
message_id: `${ msg.id }-attachment`,
name: msg.attachment.name,
@@ -784,7 +866,6 @@ export class HipChatEnterpriseImporter extends Base {
_importSingleMessage(msg, roomIdentifier, room) {
if (isNaN(msg.ts)) {
this.logger.warn(`Timestamp on a message in ${ roomIdentifier } is invalid`);
- super.addCountCompleted(1);
return;
}
@@ -795,17 +876,19 @@ export class HipChatEnterpriseImporter extends Base {
switch (msg.type) {
case 'user':
- RocketChat.sendMessage(creator, {
- _id: msg.id,
- ts: msg.ts,
- msg: msg.text,
- rid: room._id,
- alias: msg.alias,
- u: {
- _id: creator._id,
- username: creator.username,
- },
- }, room, true);
+ if (!msg.skip) {
+ RocketChat.insertMessage(creator, {
+ _id: msg.id,
+ ts: msg.ts,
+ msg: msg.text,
+ rid: room._id,
+ alias: msg.alias,
+ u: {
+ _id: creator._id,
+ username: creator.username,
+ },
+ }, room, false);
+ }
break;
case 'topic':
RocketChat.models.Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser('room_changed_topic', room._id, msg.text, creator, { _id: msg.id, ts: msg.ts });
@@ -816,48 +899,65 @@ export class HipChatEnterpriseImporter extends Base {
console.error(e);
this.addMessageError(e, msg);
}
-
- super.addCountCompleted(1);
}
- _importMessages(startedByUserId) {
- const messageListIds = this.collection.find({
- import: this.importRecord._id,
- importer: this.name,
- type: 'messages',
- }, { _id : true }).fetch();
+ async _importMessageList(startedByUserId, messageListId) {
+ const list = this.collection.findOneById(messageListId);
+ if (!list) {
+ return;
+ }
- messageListIds.forEach((item) => {
- const list = this.collection.findOneById(item._id);
- if (!list) {
- return;
- }
+ if (!list.messages) {
+ return;
+ }
- if (!list.messages) {
- return;
- }
+ const { roomIdentifier, hipchatRoomId, name } = list;
+ const rid = await this._getRoomRocketId(hipchatRoomId);
- const { roomIdentifier, hipchatRoomId, name } = list;
- const rid = this._getRoomRocketId(hipchatRoomId);
+ // If there's no rocketId for the channel, then it wasn't imported
+ if (!rid) {
+ this.logger.debug(`Ignoring room ${ roomIdentifier } ( ${ name } ), as there's no rid to use.`);
+ return;
+ }
- // If there's no rocketId for the channel, then it wasn't imported
- if (!rid) {
- this.logger.debug(`Ignoring room ${ roomIdentifier } ( ${ name } ), as there's no rid to use.`);
- return;
- }
+ const room = await RocketChat.models.Rooms.findOneById(rid, { fields: { usernames: 1, t: 1, name: 1 } });
+ await super.updateRecord({
+ messagesstatus: `${ roomIdentifier }.${ list.messages.length }`,
+ 'count.completed': this.progress.count.completed,
+ });
- const room = RocketChat.models.Rooms.findOneById(rid, { fields: { usernames: 1, t: 1, name: 1 } });
- super.updateRecord({
- messagesstatus: `${ roomIdentifier }.${ list.messages.length }`,
- 'count.completed': this.progress.count.completed,
- });
+ await Meteor.runAsUser(startedByUserId, async() => {
+ let msgCount = 0;
+ try {
+ for (const msg of list.messages) {
+ await this._importSingleMessage(msg, roomIdentifier, room);
+ msgCount++;
+ if (msgCount >= 50) {
+ super.addCountCompleted(msgCount);
+ msgCount = 0;
+ }
+ }
+ } catch (e) {
+ this.logger.error(e);
+ }
- Meteor.runAsUser(startedByUserId, () => {
- list.messages.forEach((msg) => {
- this._importSingleMessage(msg, roomIdentifier, room);
- });
- });
+ if (msgCount > 0) {
+ super.addCountCompleted(msgCount);
+ }
});
+
+ }
+
+ async _importMessages(startedByUserId) {
+ const messageListIds = this.collection.find({
+ import: this.importRecord._id,
+ importer: this.name,
+ type: 'messages',
+ }, { fields: { _id: true } }).fetch();
+
+ for (const item of messageListIds) {
+ await this._importMessageList(startedByUserId, item._id);
+ }
}
_importDirectMessages() {
@@ -865,15 +965,24 @@ export class HipChatEnterpriseImporter extends Base {
import: this.importRecord._id,
importer: this.name,
type: 'user-messages',
- }, { _id : true }).fetch();
+ }, { fields: { _id: true } }).fetch();
+
+ this.logger.info(`${ messageListIds.length } lists of messages to import.`);
+
+ // HipChat duplicates direct messages (one for each user)
+ // This object will keep track of messages that have already been imported so it doesn't try to insert them twice
+ const importedMessages = {};
messageListIds.forEach((item) => {
+ this.logger.debug(`New list of user messages: ${ item._id }`);
const list = this.collection.findOneById(item._id);
if (!list) {
+ this.logger.warn('Record of user-messages list not found');
return;
}
if (!list.messages) {
+ this.logger.warn('No message list found on record.');
return;
}
@@ -883,18 +992,20 @@ export class HipChatEnterpriseImporter extends Base {
return;
}
+ this.logger.debug(`${ list.messages.length } messages on this list`);
super.updateRecord({
messagesstatus: `${ list.name }.${ list.messages.length }`,
'count.completed': this.progress.count.completed,
});
+ let msgCount = 0;
const roomUsers = {};
const roomObjects = {};
list.messages.forEach((msg) => {
+ msgCount++;
if (isNaN(msg.ts)) {
this.logger.warn(`Timestamp on a message in ${ list.name } is invalid`);
- super.addCountCompleted(1);
return;
}
@@ -905,7 +1016,6 @@ export class HipChatEnterpriseImporter extends Base {
if (!roomUsers[msg.senderId]) {
this.logger.warn('Skipping message due to missing sender.');
- super.addCountCompleted(1);
return;
}
@@ -916,7 +1026,6 @@ export class HipChatEnterpriseImporter extends Base {
if (!roomUsers[msg.receiverId]) {
this.logger.warn('Skipping message due to missing receiver.');
- super.addCountCompleted(1);
return;
}
@@ -930,6 +1039,7 @@ export class HipChatEnterpriseImporter extends Base {
let room = roomObjects[roomId];
if (!room) {
+ this.logger.debug('DM room not found, creating it.');
Meteor.runAsUser(sender._id, () => {
const roomInfo = Meteor.call('createDirectMessage', receiver.username);
@@ -940,17 +1050,28 @@ export class HipChatEnterpriseImporter extends Base {
try {
Meteor.runAsUser(sender._id, () => {
+ if (importedMessages[msg.id] !== undefined) {
+ return;
+ }
+ importedMessages[msg.id] = true;
+
if (msg.attachment_path) {
- const details = {
- message_id: `${ msg.id }-attachment`,
- name: msg.attachment.name,
- size: msg.attachment.size,
- userId: sender._id,
- rid: room._id,
- };
- this.uploadFile(details, msg.attachment.url, sender, room, msg.ts);
- } else {
- RocketChat.sendMessage(sender, {
+ if (!msg.skipAttachment) {
+ this.logger.debug('Uploading DM file');
+ const details = {
+ message_id: `${ msg.id }-attachment`,
+ name: msg.attachment.name,
+ size: msg.attachment.size,
+ userId: sender._id,
+ rid: room._id,
+ };
+ this.uploadFile(details, msg.attachment.url, sender, room, msg.ts);
+ }
+ }
+
+ if (!msg.skip) {
+ this.logger.debug('Inserting DM message');
+ RocketChat.insertMessage(sender, {
_id: msg.id,
ts: msg.ts,
msg: msg.text,
@@ -959,7 +1080,7 @@ export class HipChatEnterpriseImporter extends Base {
_id: sender._id,
username: sender.username,
},
- }, room, true);
+ }, room, false);
}
});
} catch (e) {
@@ -967,8 +1088,15 @@ export class HipChatEnterpriseImporter extends Base {
this.addMessageError(e, msg);
}
- super.addCountCompleted(1);
+ if (msgCount >= 50) {
+ super.addCountCompleted(msgCount);
+ msgCount = 0;
+ }
});
+
+ if (msgCount > 0) {
+ super.addCountCompleted(msgCount);
+ }
});
}
@@ -992,14 +1120,23 @@ export class HipChatEnterpriseImporter extends Base {
return new Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
+ _getBasicUserData(userId) {
+ if (this._userDataCache[userId]) {
+ return this._userDataCache[userId];
+ }
+
+ this._userDataCache[userId] = RocketChat.models.Users.findOneById(userId, { fields: { username: 1 } });
+ return this._userDataCache[userId];
+ }
+
getRocketUserFromUserId(userId) {
if (userId === 'rocket.cat') {
- return RocketChat.models.Users.findOneById(userId, { fields: { username: 1 } });
+ return this._getBasicUserData('rocket.cat');
}
const rocketId = this._getUserRocketId(userId);
if (rocketId) {
- return RocketChat.models.Users.findOneById(rocketId, { fields: { username: 1 } });
+ return this._getBasicUserData(rocketId);
}
}
diff --git a/packages/rocketchat-importer/client/admin/adminImportProgress.html b/packages/rocketchat-importer/client/admin/adminImportProgress.html
index 6f8d95c00e66..9acfb08a8ec1 100644
--- a/packages/rocketchat-importer/client/admin/adminImportProgress.html
+++ b/packages/rocketchat-importer/client/admin/adminImportProgress.html
@@ -2,4 +2,6 @@
{{> loading}}
{{step}}
{{completed}} / {{total}}
+ +{{_ "You_can_close_this_window_now"}}
diff --git a/packages/rocketchat-importer/server/methods/getImportFileData.js b/packages/rocketchat-importer/server/methods/getImportFileData.js index 6e143aca2f8d..242954bf62d9 100644 --- a/packages/rocketchat-importer/server/methods/getImportFileData.js +++ b/packages/rocketchat-importer/server/methods/getImportFileData.js @@ -34,7 +34,11 @@ Meteor.methods({ ]; if (waitingSteps.indexOf(importer.instance.progress.step) >= 0) { - return { waiting: true }; + if (importer.instance.importRecord && importer.instance.importRecord.valid) { + return { waiting: true }; + } else { + throw new Meteor.Error('error-import-operation-invalid', 'Invalid Import Operation', { method: 'getImportFileData' }); + } } const readySteps = [ @@ -62,6 +66,7 @@ Meteor.methods({ return data; }).catch((e) => { + console.error(e); throw new Meteor.Error(e); }); diff --git a/packages/rocketchat-lib/package.js b/packages/rocketchat-lib/package.js index b7ebecbd8118..93051b49fa76 100644 --- a/packages/rocketchat-lib/package.js +++ b/packages/rocketchat-lib/package.js @@ -125,6 +125,7 @@ Package.onUse(function(api) { api.addFiles('server/functions/saveCustomFields.js', 'server'); api.addFiles('server/functions/saveCustomFieldsWithoutValidation.js', 'server'); api.addFiles('server/functions/sendMessage.js', 'server'); + api.addFiles('server/functions/insertMessage.js', 'server'); api.addFiles('server/functions/settings.js', 'server'); api.addFiles('server/functions/setUserAvatar.js', 'server'); api.addFiles('server/functions/setUsername.js', 'server'); @@ -271,6 +272,8 @@ Package.onUse(function(api) { api.addFiles('startup/defaultRoomTypes.js'); api.addFiles('startup/index.js', 'server'); + api.addFiles('server/startup/rateLimiter.js', 'server'); + // EXPORT api.export('RocketChat'); api.export('handleError', 'client'); diff --git a/packages/rocketchat-lib/server/functions/insertMessage.js b/packages/rocketchat-lib/server/functions/insertMessage.js new file mode 100644 index 000000000000..6979fd0af05c --- /dev/null +++ b/packages/rocketchat-lib/server/functions/insertMessage.js @@ -0,0 +1,149 @@ +import { Match, check } from 'meteor/check'; + +const objectMaybeIncluding = (types) => Match.Where((value) => { + Object.keys(types).forEach((field) => { + if (value[field] != null) { + try { + check(value[field], types[field]); + } catch (error) { + error.path = field; + throw error; + } + } + }); + + return true; +}); + +const validateAttachmentsFields = (attachmentField) => { + check(attachmentField, objectMaybeIncluding({ + short: Boolean, + title: String, + value: Match.OneOf(String, Match.Integer, Boolean), + })); + + if (typeof attachmentField.value !== 'undefined') { + attachmentField.value = String(attachmentField.value); + } +}; + +const validateAttachmentsActions = (attachmentActions) => { + check(attachmentActions, objectMaybeIncluding({ + type: String, + text: String, + url: String, + image_url: String, + is_webview: Boolean, + webview_height_ratio: String, + msg: String, + msg_in_chat_window: Boolean, + })); +}; + +const validateAttachment = (attachment) => { + check(attachment, objectMaybeIncluding({ + color: String, + text: String, + ts: Match.OneOf(String, Match.Integer), + thumb_url: String, + button_alignment: String, + actions: [Match.Any], + message_link: String, + collapsed: Boolean, + author_name: String, + author_link: String, + author_icon: String, + title: String, + title_link: String, + title_link_download: Boolean, + image_url: String, + audio_url: String, + video_url: String, + fields: [Match.Any], + })); + + if (attachment.fields && attachment.fields.length) { + attachment.fields.map(validateAttachmentsFields); + } + + if (attachment.actions && attachment.actions.length) { + attachment.actions.map(validateAttachmentsActions); + } +}; + +const validateBodyAttachments = (attachments) => attachments.map(validateAttachment); + +RocketChat.insertMessage = function(user, message, room, upsert = false) { + if (!user || !message || !room._id) { + return false; + } + + check(message, objectMaybeIncluding({ + _id: String, + msg: String, + text: String, + alias: String, + emoji: String, + avatar: String, + attachments: [Match.Any], + })); + + if (Array.isArray(message.attachments) && message.attachments.length) { + validateBodyAttachments(message.attachments); + } + + if (!message.ts) { + message.ts = new Date(); + } + const { _id, username } = user; + message.u = { + _id, + username, + }; + message.rid = room._id; + + if (!Match.test(message.msg, String)) { + message.msg = ''; + } + + if (message.ts == null) { + message.ts = new Date(); + } + + if (message.parseUrls !== false) { + message.html = message.msg; + message = RocketChat.Markdown.code(message); + + const urls = message.html.match(/([A-Za-z]{3,9}):\/\/([-;:&=\+\$,\w]+@{1})?([-A-Za-z0-9\.]+)+:?(\d+)?((\/[-\+=!:~%\/\.@\,\(\)\w]*)?\??([-\+=&!:;%@\/\.\,\w]+)?(?:#([^\s\)]+))?)?/g); + if (urls) { + message.urls = urls.map((url) => ({ url })); + } + + message = RocketChat.Markdown.mountTokensBack(message, false); + message.msg = message.html; + delete message.html; + delete message.tokens; + } + + // Avoid saving sandstormSessionId to the database + let sandstormSessionId = null; + if (message.sandstormSessionId) { + sandstormSessionId = message.sandstormSessionId; + delete message.sandstormSessionId; + } + + if (message._id && upsert) { + const { _id } = message; + delete message._id; + RocketChat.models.Messages.upsert({ + _id, + 'u._id': message.u._id, + }, message); + message._id = _id; + } else { + message._id = RocketChat.models.Messages.insert(message); + } + + message.sandstormSessionId = sandstormSessionId; + return message; +}; diff --git a/packages/rocketchat-lib/server/functions/sendMessage.js b/packages/rocketchat-lib/server/functions/sendMessage.js index 4a5218e7ad65..4fcda633e5c6 100644 --- a/packages/rocketchat-lib/server/functions/sendMessage.js +++ b/packages/rocketchat-lib/server/functions/sendMessage.js @@ -58,9 +58,17 @@ const validateAttachment = (attachment) => { title: String, title_link: String, title_link_download: Boolean, + image_dimensions: Object, image_url: String, + image_preview: String, + image_type: String, + image_size: Number, audio_url: String, + audio_type: String, + audio_size: Number, video_url: String, + video_type: String, + video_size: Number, fields: [Match.Any], })); diff --git a/packages/rocketchat-lib/server/startup/rateLimiter.js b/packages/rocketchat-lib/server/startup/rateLimiter.js new file mode 100644 index 000000000000..8e936f827cc6 --- /dev/null +++ b/packages/rocketchat-lib/server/startup/rateLimiter.js @@ -0,0 +1,187 @@ +import _ from 'underscore'; +import { Meteor } from 'meteor/meteor'; +import { DDPRateLimiter } from 'meteor/ddp-rate-limiter'; +import { RateLimiter } from 'meteor/rate-limit'; +import { settings } from 'meteor/rocketchat:settings'; +import { metrics } from 'meteor/rocketchat:metrics'; + +// Get initial set of names already registered for rules +const names = new Set(Object.values(DDPRateLimiter.printRules()) + .map((rule) => rule._matchers) + .filter((match) => typeof match.name === 'string') + .map((match) => match.name)); + +// Override the addRule to save new names added after this point +const { addRule } = DDPRateLimiter; +DDPRateLimiter.addRule = (matcher, calls, time, callback) => { + if (matcher && typeof matcher.name === 'string') { + names.add(matcher.name); + } + return addRule.call(DDPRateLimiter, matcher, calls, time, callback); +}; + +// Need to override the meteor's code duo to a problem with the callback reply +// being shared among all matchs +RateLimiter.prototype.check = function(input) { + const self = this; + const reply = { + allowed: true, + timeToReset: 0, + numInvocationsLeft: Infinity, + }; + + const matchedRules = self._findAllMatchingRules(input); + _.each(matchedRules, function(rule) { + // ==== BEGIN OVERRIDE ==== + const callbackReply = { + allowed: true, + timeToReset: 0, + numInvocationsLeft: Infinity, + }; + // ==== END OVERRIDE ==== + + const ruleResult = rule.apply(input); + let numInvocations = rule.counters[ruleResult.key]; + + if (ruleResult.timeToNextReset < 0) { + // Reset all the counters since the rule has reset + rule.resetCounter(); + ruleResult.timeSinceLastReset = new Date().getTime() - rule._lastResetTime; + ruleResult.timeToNextReset = rule.options.intervalTime; + numInvocations = 0; + } + + if (numInvocations > rule.options.numRequestsAllowed) { + // Only update timeToReset if the new time would be longer than the + // previously set time. This is to ensure that if this input triggers + // multiple rules, we return the longest period of time until they can + // successfully make another call + if (reply.timeToReset < ruleResult.timeToNextReset) { + reply.timeToReset = ruleResult.timeToNextReset; + } + reply.allowed = false; + reply.numInvocationsLeft = 0; + + // ==== BEGIN OVERRIDE ==== + callbackReply.timeToReset = ruleResult.timeToNextReset; + callbackReply.allowed = false; + callbackReply.numInvocationsLeft = 0; + rule._executeCallback(callbackReply, input); + // ==== END OVERRIDE ==== + } else { + // If this is an allowed attempt and we haven't failed on any of the + // other rules that match, update the reply field. + if (rule.options.numRequestsAllowed - numInvocations < reply.numInvocationsLeft && reply.allowed) { + reply.timeToReset = ruleResult.timeToNextReset; + reply.numInvocationsLeft = rule.options.numRequestsAllowed - numInvocations; + } + + // ==== BEGIN OVERRIDE ==== + callbackReply.timeToReset = ruleResult.timeToNextReset; + callbackReply.numInvocationsLeft = rule.options.numRequestsAllowed - numInvocations; + rule._executeCallback(callbackReply, input); + // ==== END OVERRIDE ==== + } + }); + return reply; +}; + +const checkNameNonStream = (name) => name && !names.has(name) && !name.startsWith('stream-'); +const checkNameForStream = (name) => name && !names.has(name) && name.startsWith('stream-'); + +const ruleIds = {}; + +const callback = (message, name) => (reply, input) => { + if (reply.allowed === false) { + console.warn('DDP RATE LIMIT:', message); + console.warn(JSON.stringify({ ...reply, ...input }, null, 2)); + metrics.ddpRateLimitExceeded.inc({ + limit_name: name, + user_id: input.userId, + client_address: input.clientAddress, + type: input.type, + name: input.name, + connection_id: input.connectionId, + }); + // } else { + // console.log('DDP RATE LIMIT:', message); + // console.log(JSON.stringify({ ...reply, ...input }, null, 2)); + } +}; + +const messages = { + IP: 'address', + User: 'userId', + Connection: 'connectionId', + User_By_Method: 'userId per method', + Connection_By_Method: 'connectionId per method', +}; + +const reconfigureLimit = Meteor.bindEnvironment((name, rules, factor = 1) => { + if (ruleIds[name + factor]) { + DDPRateLimiter.removeRule(ruleIds[name + factor]); + } + + if (!settings.get(`DDP_Rate_Limit_${ name }_Enabled`)) { + return; + } + + ruleIds[name + factor] = addRule( + rules, + settings.get(`DDP_Rate_Limit_${ name }_Requests_Allowed`) * factor, + settings.get(`DDP_Rate_Limit_${ name }_Interval_Time`) * factor, + callback(`limit by ${ messages[name] }`, name) + ); +}); + +const configIP = _.debounce(() => { + reconfigureLimit('IP', { + clientAddress: (clientAddress) => clientAddress !== '127.0.0.1', + }); +}, 1000); + +const configUser = _.debounce(() => { + reconfigureLimit('User', { + userId: (userId) => userId != null, + }); +}, 1000); + +const configConnection = _.debounce(() => { + reconfigureLimit('Connection', { + connectionId: () => true, + }); +}, 1000); + +const configUserByMethod = _.debounce(() => { + reconfigureLimit('User_By_Method', { + type: () => true, + name: checkNameNonStream, + userId: (userId) => userId != null, + }); + reconfigureLimit('User_By_Method', { + type: () => true, + name: checkNameForStream, + userId: (userId) => userId != null, + }, 4); +}, 1000); + +const configConnectionByMethod = _.debounce(() => { + reconfigureLimit('Connection_By_Method', { + type: () => true, + name: checkNameNonStream, + connectionId: () => true, + }); + reconfigureLimit('Connection_By_Method', { + type: () => true, + name: checkNameForStream, + connectionId: () => true, + }, 4); +}, 1000); + +if (!process.env.TEST_MODE) { + settings.get(/^DDP_Rate_Limit_IP_.+/, configIP); + settings.get(/^DDP_Rate_Limit_User_[^B].+/, configUser); + settings.get(/^DDP_Rate_Limit_Connection_[^B].+/, configConnection); + settings.get(/^DDP_Rate_Limit_User_By_Method_.+/, configUserByMethod); + settings.get(/^DDP_Rate_Limit_Connection_By_Method_.+/, configConnectionByMethod); +} diff --git a/packages/rocketchat-lib/server/startup/settings.js b/packages/rocketchat-lib/server/startup/settings.js index 6204b2c91593..6d08ae5ffa68 100644 --- a/packages/rocketchat-lib/server/startup/settings.js +++ b/packages/rocketchat-lib/server/startup/settings.js @@ -2666,4 +2666,34 @@ RocketChat.settings.addGroup('Setup_Wizard', function() { }); }); +RocketChat.settings.addGroup('Rate Limiter', function() { + this.section('DDP Rate Limiter', function() { + this.add('DDP_Rate_Limit_IP_Enabled', true, { type: 'boolean' }); + this.add('DDP_Rate_Limit_IP_Requests_Allowed', 120000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_IP_Enabled', value: true } }); + this.add('DDP_Rate_Limit_IP_Interval_Time', 60000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_IP_Enabled', value: true } }); + + this.add('DDP_Rate_Limit_User_Enabled', true, { type: 'boolean' }); + this.add('DDP_Rate_Limit_User_Requests_Allowed', 1200, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_User_Enabled', value: true } }); + this.add('DDP_Rate_Limit_User_Interval_Time', 60000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_User_Enabled', value: true } }); + + this.add('DDP_Rate_Limit_Connection_Enabled', true, { type: 'boolean' }); + this.add('DDP_Rate_Limit_Connection_Requests_Allowed', 600, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_Connection_Enabled', value: true } }); + this.add('DDP_Rate_Limit_Connection_Interval_Time', 60000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_Connection_Enabled', value: true } }); + + this.add('DDP_Rate_Limit_User_By_Method_Enabled', true, { type: 'boolean' }); + this.add('DDP_Rate_Limit_User_By_Method_Requests_Allowed', 20, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_User_By_Method_Enabled', value: true } }); + this.add('DDP_Rate_Limit_User_By_Method_Interval_Time', 10000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_User_By_Method_Enabled', value: true } }); + + this.add('DDP_Rate_Limit_Connection_By_Method_Enabled', true, { type: 'boolean' }); + this.add('DDP_Rate_Limit_Connection_By_Method_Requests_Allowed', 10, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_Connection_By_Method_Enabled', value: true } }); + this.add('DDP_Rate_Limit_Connection_By_Method_Interval_Time', 10000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_Connection_By_Method_Enabled', value: true } }); + }); + + this.section('API Rate Limiter', function() { + this.add('API_Enable_Rate_Limiter_Dev', true, { type: 'boolean' }); + this.add('API_Enable_Rate_Limiter_Limit_Calls_Default', 10, { type: 'int' }); + this.add('API_Enable_Rate_Limiter_Limit_Time_Default', 60000, { type: 'int' }); + }); +}); + RocketChat.settings.init(); diff --git a/packages/rocketchat-livechat/server/api/v1/room.js b/packages/rocketchat-livechat/server/api/v1/room.js index e2e544cfec18..246f9451af11 100644 --- a/packages/rocketchat-livechat/server/api/v1/room.js +++ b/packages/rocketchat-livechat/server/api/v1/room.js @@ -153,3 +153,9 @@ RocketChat.API.v1.addRoute('livechat/room.survey', { } }, }); + +RocketChat.API.v1.addRoute('livechat/room.forward', { authRequired: true }, { + post() { + RocketChat.API.v1.success(Meteor.runAsUser(this.userId, () => Meteor.call('livechat:transfer', this.bodyParams))); + }, +}); diff --git a/packages/rocketchat-livechat/server/methods/transfer.js b/packages/rocketchat-livechat/server/methods/transfer.js index eabe665348da..d6a59b3ac243 100644 --- a/packages/rocketchat-livechat/server/methods/transfer.js +++ b/packages/rocketchat-livechat/server/methods/transfer.js @@ -16,14 +16,17 @@ Meteor.methods({ }); const room = RocketChat.models.Rooms.findOneById(transferData.roomId); - - const guest = LivechatVisitors.findOneById(room.v._id); + if (!room) { + throw new Meteor.Error('error-invalid-room', 'Invalid room', { method: 'livechat:transfer' }); + } const subscription = RocketChat.models.Subscriptions.findOneByRoomIdAndUserId(room._id, Meteor.userId(), { fields: { _id: 1 } }); if (!subscription && !RocketChat.authz.hasRole(Meteor.userId(), 'livechat-manager')) { throw new Meteor.Error('error-not-authorized', 'Not authorized', { method: 'livechat:transfer' }); } + const guest = LivechatVisitors.findOneById(room.v && room.v._id); + return RocketChat.Livechat.transfer(room, guest, transferData); }, }); diff --git a/packages/rocketchat-metrics/server/lib/metrics.js b/packages/rocketchat-metrics/server/lib/metrics.js index 7ea170de960d..95b44d5130be 100644 --- a/packages/rocketchat-metrics/server/lib/metrics.js +++ b/packages/rocketchat-metrics/server/lib/metrics.js @@ -48,6 +48,7 @@ metrics.notificationsSent = new client.Counter({ name: 'rocketchat_notification_ metrics.ddpSessions = new client.Gauge({ name: 'rocketchat_ddp_sessions_count', help: 'number of open ddp sessions' }); metrics.ddpAthenticatedSessions = new client.Gauge({ name: 'rocketchat_ddp_sessions_auth', help: 'number of authenticated open ddp sessions' }); metrics.ddpConnectedUsers = new client.Gauge({ name: 'rocketchat_ddp_connected_users', help: 'number of unique connected users' }); +metrics.ddpRateLimitExceeded = new client.Counter({ name: 'rocketchat_ddp_rate_limit_exceeded', labelNames: ['limit_name', 'user_id', 'client_address', 'type', 'name', 'connection_id'], help: 'number of times a ddp rate limiter was exceeded' }); metrics.version = new client.Gauge({ name: 'rocketchat_version', labelNames: ['version'], help: 'Rocket.Chat version' }); metrics.migration = new client.Gauge({ name: 'rocketchat_migration', help: 'migration versoin' }); diff --git a/packages/rocketchat-models/server/index.js b/packages/rocketchat-models/server/index.js index d504e4720ca2..3dcffefc6df4 100644 --- a/packages/rocketchat-models/server/index.js +++ b/packages/rocketchat-models/server/index.js @@ -10,6 +10,7 @@ import Subscriptions from './models/Subscriptions'; import Uploads from './models/Uploads'; import UserDataFiles from './models/UserDataFiles'; import Users from './models/Users'; +import Sessions from './models/Sessions'; import Statistics from './models/Statistics'; import Permissions from './models/Permissions'; import Roles from './models/Roles'; @@ -28,6 +29,7 @@ export { Uploads, UserDataFiles, Users, + Sessions, Statistics, Permissions, Roles, diff --git a/packages/rocketchat-models/server/models/Sessions.js b/packages/rocketchat-models/server/models/Sessions.js new file mode 100644 index 000000000000..cb7fb3151b19 --- /dev/null +++ b/packages/rocketchat-models/server/models/Sessions.js @@ -0,0 +1,280 @@ +import { Base } from './_Base'; + +export class Sessions extends Base { + constructor(...args) { + super(...args); + + this.tryEnsureIndex({ instanceId: 1, sessionId: 1, year: 1, month: 1, day: 1 }); + this.tryEnsureIndex({ instanceId: 1, sessionId: 1, userId: 1 }); + this.tryEnsureIndex({ instanceId: 1, sessionId: 1 }); + this.tryEnsureIndex({ year: 1, month: 1, day: 1, type: 1 }); + this.tryEnsureIndex({ _computedAt: 1 }, { expireAfterSeconds: 60 * 60 * 24 * 45 }); + } + + getUniqueUsersOfYesterday() { + const date = new Date(); + date.setDate(date.getDate() - 1); + + const year = date.getFullYear(); + const month = date.getMonth() + 1; + const day = date.getDate(); + + return { + year, + month, + day, + data: Promise.await(this.model.rawCollection().aggregate([{ + $match: { + year, + month, + day, + type: 'user_daily', + }, + }, { + $group: { + _id: { + day: '$day', + month: '$month', + year: '$year', + }, + count: { + $sum: '$count', + }, + time: { + $sum: '$time', + }, + }, + }, { + $project: { + _id: 0, + count: 1, + time: 1, + }, + }]).toArray()), + }; + } + + getUniqueUsersOfLastMonth() { + const date = new Date(); + date.setMonth(date.getMonth() - 1); + + const year = date.getFullYear(); + const month = date.getMonth() + 1; + + return { + year, + month, + data: Promise.await(this.model.rawCollection().aggregate([{ + $match: { + year, + month, + type: 'user_daily', + }, + }, { + $group: { + _id: { + month: '$month', + year: '$year', + }, + count: { + $sum: '$count', + }, + time: { + $sum: '$time', + }, + }, + }, { + $project: { + _id: 0, + count: 1, + time: 1, + }, + }]).toArray()), + }; + } + + getUniqueDevicesOfYesterday() { + const date = new Date(); + date.setDate(date.getDate() - 1); + + const year = date.getFullYear(); + const month = date.getMonth() + 1; + const day = date.getDate(); + + return { + year, + month, + day, + data: Promise.await(this.model.rawCollection().aggregate([{ + $match: { + year, + month, + day, + type: 'user_daily', + }, + }, { + $unwind: '$devices', + }, { + $group: { + _id: { + type : '$devices.type', + name : '$devices.name', + version : '$devices.version', + }, + count: { + $sum: '$count', + }, + }, + }, { + $project: { + _id: 0, + type: '$_id.type', + name: '$_id.name', + version: '$_id.version', + count: 1, + }, + }]).toArray()), + }; + } + + getUniqueOSOfYesterday() { + const date = new Date(); + date.setDate(date.getDate() - 1); + + const year = date.getFullYear(); + const month = date.getMonth() + 1; + const day = date.getDate(); + + return { + year, + month, + day, + data: Promise.await(this.model.rawCollection().aggregate([{ + $match: { + year, + month, + day, + type: 'user_daily', + 'devices.os.name': { + $exists: true, + }, + }, + }, { + $unwind: '$devices', + }, { + $group: { + _id: { + name : '$devices.os.name', + version : '$devices.os.version', + }, + count: { + $sum: '$count', + }, + }, + }, { + $project: { + _id: 0, + name: '$_id.name', + version: '$_id.version', + count: 1, + }, + }]).toArray()), + }; + } + + createOrUpdate(data = {}) { + const { year, month, day, sessionId, instanceId } = data; + + if (!year || !month || !day || !sessionId || !instanceId) { + return; + } + + const now = new Date; + + return this.upsert({ instanceId, sessionId, year, month, day }, { + $set: data, + $setOnInsert: { + createdAt: now, + }, + }); + } + + closeByInstanceIdAndSessionId(instanceId, sessionId) { + const query = { + instanceId, + sessionId, + closedAt: { $exists: 0 }, + }; + + const closeTime = new Date(); + const update = { + $set: { + closedAt: closeTime, + lastActivityAt: closeTime, + }, + }; + + return this.update(query, update); + } + + updateActiveSessionsByDateAndInstanceIdAndIds({ year, month, day } = {}, instanceId, sessions, data = {}) { + const query = { + instanceId, + year, + month, + day, + sessionId: { $in: sessions }, + closedAt: { $exists: 0 }, + }; + + const update = { + $set: data, + }; + + return this.update(query, update, { multi: true }); + } + + logoutByInstanceIdAndSessionIdAndUserId(instanceId, sessionId, userId) { + const query = { + instanceId, + sessionId, + userId, + logoutAt: { $exists: 0 }, + }; + + const logoutAt = new Date(); + const update = { + $set: { + logoutAt, + }, + }; + + return this.update(query, update, { multi: true }); + } + + createBatch(sessions) { + if (!sessions || sessions.length === 0) { + return; + } + + const ops = []; + sessions.forEach((doc) => { + const { year, month, day, sessionId, instanceId } = doc; + delete doc._id; + + ops.push({ + updateOne: { + filter: { year, month, day, sessionId, instanceId }, + update: { + $set: doc, + }, + upsert: true, + }, + }); + }); + + return this.model.rawCollection().bulkWrite(ops, { ordered: false }); + } +} + +export default new Sessions('sessions'); diff --git a/packages/rocketchat-statistics/server/functions/get.js b/packages/rocketchat-statistics/server/functions/get.js index 2b0cd965c7cf..5c70902beee4 100644 --- a/packages/rocketchat-statistics/server/functions/get.js +++ b/packages/rocketchat-statistics/server/functions/get.js @@ -5,6 +5,7 @@ import os from 'os'; import LivechatVisitors from 'meteor/rocketchat:livechat/server/models/LivechatVisitors'; import { RocketChat } from 'meteor/rocketchat:lib'; import { InstanceStatus } from 'meteor/konecty:multiple-instances-status'; +import { Sessions } from 'meteor/rocketchat:models'; const wizardFields = [ 'Organization_Type', @@ -130,5 +131,10 @@ RocketChat.statistics.get = function _getStatistics() { console.error('Error getting MongoDB version'); } + statistics.uniqueUsersOfYesterday = Sessions.getUniqueUsersOfYesterday(); + statistics.uniqueUsersOfLastMonth = Sessions.getUniqueUsersOfLastMonth(); + statistics.uniqueDevicesOfYesterday = Sessions.getUniqueDevicesOfYesterday(); + statistics.uniqueOSOfYesterday = Sessions.getUniqueOSOfYesterday(); + return statistics; }; diff --git a/packages/rocketchat-statistics/server/index.js b/packages/rocketchat-statistics/server/index.js index b6c264b84f2b..150f49e1179f 100644 --- a/packages/rocketchat-statistics/server/index.js +++ b/packages/rocketchat-statistics/server/index.js @@ -1,5 +1,6 @@ import '../lib/rocketchat'; -import './models/Statistics_import'; +import './models/import'; import './functions/get'; import './functions/save'; import './methods/getStatistics'; +import './startup/monitor'; diff --git a/packages/rocketchat-statistics/server/lib/SAUMonitor.js b/packages/rocketchat-statistics/server/lib/SAUMonitor.js new file mode 100644 index 000000000000..dc66c06d4a16 --- /dev/null +++ b/packages/rocketchat-statistics/server/lib/SAUMonitor.js @@ -0,0 +1,382 @@ +import { Meteor } from 'meteor/meteor'; +import { Accounts } from 'meteor/accounts-base'; +import UAParser from 'ua-parser-js'; +import { UAParserMobile } from './UAParserMobile'; +import { Sessions } from 'meteor/rocketchat:models'; +import { Logger } from 'meteor/rocketchat:logger'; +import { SyncedCron } from 'meteor/littledata:synced-cron'; + +const getDateObj = (dateTime = new Date()) => ({ + day: dateTime.getDate(), + month: dateTime.getMonth() + 1, + year: dateTime.getFullYear(), +}); + +const isSameDateObj = (oldest, newest) => oldest.year === newest.year && oldest.month === newest.month && oldest.day === newest.day; + +const logger = new Logger('SAUMonitor'); + +/** + * Server Session Monitor for SAU(Simultaneously Active Users) based on Meteor server sessions + */ +export class SAUMonitorClass { + constructor() { + this._started = false; + this._monitorTime = 60000; + this._timer = null; + this._today = getDateObj(); + this._instanceId = null; + } + + start(instanceId) { + if (this.isRunning()) { + return; + } + + this._instanceId = instanceId; + + if (!this._instanceId) { + logger.debug('[start] - InstanceId is not defined.'); + return; + } + + this._startMonitoring(() => { + this._started = true; + logger.debug(`[start] - InstanceId: ${ this._instanceId }`); + }); + } + + stop() { + if (!this.isRunning()) { + return; + } + + this._started = false; + + if (this._timer) { + Meteor.clearInterval(this._timer); + } + + logger.debug(`[stop] - InstanceId: ${ this._instanceId }`); + } + + isRunning() { + return this._started === true; + } + + _startMonitoring(callback) { + try { + this._handleAccountEvents(); + this._handleOnConnection(); + this._startSessionControl(); + this._initActiveServerSessions(); + this._startAggregation(); + if (callback) { + callback(); + } + } catch (err) { + throw new Meteor.Error(err); + } + } + + _startSessionControl() { + if (this.isRunning()) { + return; + } + + if (this._monitorTime < 0) { + return; + } + + this._timer = Meteor.setInterval(() => { + this._updateActiveSessions(); + }, this._monitorTime); + } + + _handleOnConnection() { + if (this.isRunning()) { + return; + } + + Meteor.onConnection((connection) => { + // this._handleSession(connection, getDateObj()); + + connection.onClose(() => { + Sessions.closeByInstanceIdAndSessionId(this._instanceId, connection.id); + }); + }); + } + + _handleAccountEvents() { + if (this.isRunning()) { + return; + } + + Accounts.onLogin((info) => { + const userId = info.user._id; + const loginAt = new Date(); + const params = { userId, loginAt, ...getDateObj() }; + this._handleSession(info.connection, params); + this._updateConnectionInfo(info.connection.id, { loginAt }); + }); + + Accounts.onLogout((info) => { + const sessionId = info.connection.id; + const userId = info.user._id; + Sessions.logoutByInstanceIdAndSessionIdAndUserId(this._instanceId, sessionId, userId); + }); + } + + _handleSession(connection, params) { + const data = this._getConnectionInfo(connection, params); + Sessions.createOrUpdate(data); + } + + _updateActiveSessions() { + if (!this.isRunning()) { + return; + } + + const { year, month, day } = this._today; + const currentDateTime = new Date(); + const currentDay = getDateObj(currentDateTime); + + if (!isSameDateObj(this._today, currentDay)) { + const beforeDateTime = new Date(this._today.year, this._today.month - 1, this._today.day, 23, 59, 59, 999); + const nextDateTime = new Date(currentDay.year, currentDay.month - 1, currentDay.day); + + const createSessions = ((objects, ids) => { + Sessions.createBatch(objects); + + Meteor.defer(() => { + Sessions.updateActiveSessionsByDateAndInstanceIdAndIds({ year, month, day }, this._instanceId, ids, { lastActivityAt: beforeDateTime }); + }); + }); + this._applyAllServerSessionsBatch(createSessions, { createdAt: nextDateTime, lastActivityAt: nextDateTime, ...currentDay }); + this._today = currentDay; + return; + } + + // Otherwise, just update the lastActivityAt field + this._applyAllServerSessionsIds((sessions) => { + Sessions.updateActiveSessionsByDateAndInstanceIdAndIds({ year, month, day }, this._instanceId, sessions, { lastActivityAt: currentDateTime }); + }); + } + + _getConnectionInfo(connection, params = {}) { + if (!connection) { + return; + } + + const ip = connection.httpHeaders ? connection.httpHeaders['x-real-ip'] || connection.httpHeaders['x-forwarded-for'] : connection.clientAddress; + const host = connection.httpHeaders && connection.httpHeaders.host; + const info = { + type: 'session', + sessionId: connection.id, + instanceId: this._instanceId, + ip, + host, + ...this._getUserAgentInfo(connection), + ...params, + }; + + if (connection.loginAt) { + info.loginAt = connection.loginAt; + } + + return info; + } + + _getUserAgentInfo(connection) { + if (!(connection && connection.httpHeaders && connection.httpHeaders['user-agent'])) { + return; + } + + const uaString = connection.httpHeaders['user-agent']; + let result; + + if (UAParserMobile.isMobileApp(uaString)) { + result = UAParserMobile.uaObject(uaString); + } else { + const ua = new UAParser(uaString); + result = ua.getResult(); + } + + const info = { + type: 'other', + }; + + const removeEmptyProps = (obj) => { + Object.keys(obj).forEach((p) => (!obj[p] || obj[p] === undefined) && delete obj[p]); + return obj; + }; + + if (result.browser && result.browser.name) { + info.type = 'browser'; + info.name = result.browser.name; + info.longVersion = result.browser.version; + } + + if (result.os && result.os.name) { + info.os = removeEmptyProps(result.os); + } + + if (result.device && (result.device.type || result.device.model)) { + info.type = 'mobile-app'; + + if (result.app && result.app.name) { + info.name = result.app.name; + info.longVersion = result.app.version; + if (result.app.bundle) { + info.longVersion += ` ${ result.app.bundle }`; + } + } + } + + if (typeof info.longVersion === 'string') { + info.version = info.longVersion.match(/(\d+\.){0,2}\d+/)[0]; + } + + return { + device: info, + }; + } + + _initActiveServerSessions() { + this._applyAllServerSessions((connectionHandle) => { + this._handleSession(connectionHandle, getDateObj()); + }); + } + + _applyAllServerSessions(callback) { + if (!callback || typeof callback !== 'function') { + return; + } + + const sessions = Object.values(Meteor.server.sessions).filter((session) => session.userId); + sessions.forEach((session) => { + callback(session.connectionHandle); + }); + } + + _applyAllServerSessionsIds(callback) { + if (!callback || typeof callback !== 'function') { + return; + } + + const sessionIds = Object.values(Meteor.server.sessions).filter((session) => session.userId).map((s) => s.id); + while (sessionIds.length) { + callback(sessionIds.splice(0, 500)); + } + } + + _updateConnectionInfo(sessionId, data = {}) { + if (!sessionId) { + return; + } + if (Meteor.server.sessions[sessionId]) { + Object.keys(data).forEach((p) => { + Object.defineProperty(Meteor.server.sessions[sessionId].connectionHandle, p, { + value: data[p], + }); + }); + } + } + + _applyAllServerSessionsBatch(callback, params) { + const batch = (arr, limit) => { + if (!arr.length) { + return Promise.resolve(); + } + const ids = []; + return Promise.all(arr.splice(0, limit).map((item) => { + ids.push(item.id); + return this._getConnectionInfo(item.connectionHandle, params); + })).then((data) => { + callback(data, ids); + return batch(arr, limit); + }).catch((e) => { + logger.debug(`Error: ${ e.message }`); + }); + }; + + const sessions = Object.values(Meteor.server.sessions).filter((session) => session.userId); + batch(sessions, 500); + } + + _startAggregation() { + logger.info('[aggregate] - Start Cron.'); + SyncedCron.add({ + name: 'aggregate-sessions', + schedule: (parser) => parser.text('at 2:00 am'), + job: () => { + this.aggregate(); + }, + }); + + SyncedCron.start(); + } + + aggregate() { + logger.info('[aggregate] - Aggregatting data.'); + + const date = new Date(); + date.setDate(date.getDate() - 0); // yesterday + const yesterday = getDateObj(date); + + const match = { + type: 'session', + year: { $lte: yesterday.year }, + month: { $lte: yesterday.month }, + day: { $lte: yesterday.day }, + }; + + Sessions.model.rawCollection().aggregate([{ + $match: { + userId: { $exists: true }, + lastActivityAt: { $exists: true }, + device: { $exists: true }, + ...match, + }, + }, { + $group: { + _id: { + userId: '$userId', + day: '$day', + month: '$month', + year: '$year', + }, + times: { $push: { $trunc: { $divide: [{ $subtract: ['$lastActivityAt', '$loginAt'] }, 1000] } } }, + devices: { $addToSet: '$device' }, + }, + }, { + $project: { + _id: '$_id', + times: { $filter: { input: '$times', as: 'item', cond: { $gt: ['$$item', 0] } } }, + devices: '$devices', + }, + }, { + $project: { + type: 'user_daily', + _computedAt: new Date(), + day: '$_id.day', + month: '$_id.month', + year: '$_id.year', + userId: '$_id.userId', + time: { $sum: '$times' }, + count: { $size: '$times' }, + devices: '$devices', + }, + }]).forEach(Meteor.bindEnvironment((record) => { + record._id = `${ record.userId }-${ record.year }-${ record.month }-${ record.day }`; + Sessions.upsert({ _id: record._id }, record); + })); + + Sessions.update(match, { + $set: { + type: 'computed-session', + _computedAt: new Date(), + }, + }, { multi: true }); + } +} diff --git a/packages/rocketchat-statistics/server/lib/UAParserMobile.js b/packages/rocketchat-statistics/server/lib/UAParserMobile.js new file mode 100644 index 000000000000..7f1a48ab38af --- /dev/null +++ b/packages/rocketchat-statistics/server/lib/UAParserMobile.js @@ -0,0 +1,106 @@ +const mergeDeep = ((target, source) => { + if (!(typeof target === 'object' && typeof source === 'object')) { + return target; + } + + for (const key in source) { + if (source[key] === null && (target[key] === undefined || target[key] === null)) { + target[key] = null; + } else if (source[key] instanceof Array) { + if (!target[key]) { target[key] = []; } + target[key] = target[key].concat(source[key]); + } else if (typeof source[key] === 'object') { + if (!target[key]) { target[key] = {}; } + mergeDeep(target[key], source[key]); + } else { + target[key] = source[key]; + } + } + + return target; +}); + +const UAParserMobile = { + appName: 'RC Mobile', + device: 'mobile', + uaSeparator: ';', + props: { + os: { + list: ['name', 'version'], + }, + app: { + list: ['version', 'bundle'], + get: (prop, value) => { + if (prop === 'bundle') { + return value.replace(/([()])/g, ''); + } + + if (prop === 'version') { + return value.replace(/^v/g, ''); + } + + return value; + }, + }, + }, + + isMobileApp(uaString) { + if (!uaString || typeof uaString !== 'string') { + return false; + } + + const splitUA = uaString.split(this.uaSeparator); + return splitUA && splitUA[0] && splitUA[0].trim() === this.appName; + }, + + uaObject(uaString) { + if (!this.isMobileApp(uaString)) { + return {}; + } + + const splitUA = uaString.split(this.uaSeparator); + + let obj = { + device: { + type: this.device, + }, + app: { + name: splitUA[0], + }, + }; + + splitUA.shift(); // remove first element + if (splitUA.length === 0) { + return obj; + } + + splitUA.forEach((element, index) => { + const splitProps = element.trim().split(' '); + const key = Object.keys(this.props)[index]; + if (!key) { + return; + } + + const props = this.props[key]; + if (!props.list || !Array.isArray(props.list) || props.list.length === 0) { + return; + } + + const subProps = {}; + splitProps.forEach((value, idx) => { + if (props.list.length > idx) { + const propName = props.list[idx]; + subProps[propName] = props.get ? props.get(propName, value) : value; + } + }); + + const prop = {}; + prop[key] = subProps; + obj = mergeDeep(obj, prop); + }); + + return obj; + }, +}; + +export { UAParserMobile }; diff --git a/packages/rocketchat-statistics/server/models/Statistics_import.js b/packages/rocketchat-statistics/server/models/import.js similarity index 100% rename from packages/rocketchat-statistics/server/models/Statistics_import.js rename to packages/rocketchat-statistics/server/models/import.js diff --git a/packages/rocketchat-statistics/server/startup/monitor.js b/packages/rocketchat-statistics/server/startup/monitor.js new file mode 100644 index 000000000000..f86bc7d099c3 --- /dev/null +++ b/packages/rocketchat-statistics/server/startup/monitor.js @@ -0,0 +1,10 @@ +import { Meteor } from 'meteor/meteor'; +import { InstanceStatus } from 'meteor/konecty:multiple-instances-status'; + +import { SAUMonitorClass } from '../lib/SAUMonitor'; + +const SAUMonitor = new SAUMonitorClass(); + +Meteor.startup(() => { + SAUMonitor.start(InstanceStatus.id()); +}); diff --git a/packages/rocketchat-ui-admin/client/rooms/adminRoomInfo.html b/packages/rocketchat-ui-admin/client/rooms/adminRoomInfo.html index 8edb049bfa7a..162c85960d93 100644 --- a/packages/rocketchat-ui-admin/client/rooms/adminRoomInfo.html +++ b/packages/rocketchat-ui-admin/client/rooms/adminRoomInfo.html @@ -21,6 +21,14 @@