RSS... nicer (#317)

package.json

@@ -45,7 +45,6 @@
     "discord-giveaways": "^3.2.3",
     "discord.js": "^12.0.2",
     "erlpack": "github:discordapp/erlpack",
-    "greenkeeper-lockfile": "^1.14.0",
     "json-query": "^2.2.2",
     "moment": "^2.24.0",
     "ms": "^2.1.2",

src/CommandManager.js

@@ -46,7 +46,7 @@ class CommandManager {
 
   async loadCommand(manifest) {
     if (this.commandCache[manifest.id]) {
-      this.logger.debug(`Found ${manifest.id} in cache`);
+      this.logger.silly(`Found ${manifest.id} in cache`);
       return this.commandCache[manifest.id];
     }
 
@@ -57,7 +57,7 @@ class CommandManager {
     // eslint-disable-next-line global-require, import/no-dynamic-require
     const Command = require(path.join(commandDir, manifest.path));
     const command = new Command(this.bot);
-    this.logger.debug(`Loading ${command.id}`);
+    this.logger.silly(`Loading ${command.id}`);
     this.commandCache[manifest.id] = command;
     return command;
   }
@@ -78,7 +78,7 @@ class CommandManager {
       });
 
       if (this.commands.length !== 0) {
-        this.logger.debug('Decaching commands');
+        this.logger.silly('Decaching commands');
         files.forEach((f) => {
           decache(path.join(commandDir, f));
         });

src/CommonFunctions.js

@@ -356,7 +356,7 @@ const embedDefaults = {
  * @returns {Array.<string>}                  Array of string chunks
  */
 const chunkify = ({
-  string, newStrings = [], breakChar = '; ', maxLength = 1000,
+  string, newStrings = [], breakChar = '; ', maxLength = 1000, checkTitle = false,
 }) => {
   let breakIndex;
   let chunk;
@@ -366,6 +366,16 @@ const chunkify = ({
       // Split message at last break character, if it exists
       chunk = string.substring(0, maxLength);
       breakIndex = chunk.lastIndexOf(breakChar) !== -1 ? chunk.lastIndexOf(breakChar) : maxLength;
+
+      if (checkTitle) {
+        // strip the last title if it starts with a title
+        if (string.endsWith('**')) {
+          const endTitle = string.matches(/\*\*(.*)\*\*\s*$/g)[1];
+          string = string.replace(/\*\*(.*)\*\*\s*$/g, ''); // eslint-disable-line no-param-reassign
+          breakIndex -= endTitle.length;
+        }
+      }
+
       newStrings.push(string.substring(0, breakIndex));
       // Skip char if split on line break
       if (breakIndex !== maxLength) {
@@ -379,6 +389,30 @@ const chunkify = ({
   return newStrings;
 };
 
+/**
+ * Convert html string content into semi-similar discord-flavored markdown
+ * @param  {string} htmlString html string to convert
+ * @returns {string}            markdinated string
+ */
+const markdinate = htmlString => htmlString
+  .split('\n').map(l => l.trim()).join('\n') // trim lines
+  .replace(/\r\n/gm, '\n') // replace CRLF with LF
+  .replace(/<\/?strong>/gm, '**') // swap <strong> tags for their md equivalent
+  .replace(/<br\s*\/?>/g, '\n') // replace manual breaks with break character
+  .replace(/<\/li>\s*<li>/gm, '</li>\n<li>') // clean up breaks between list items
+  .replace(/<li\s?(?:class=".*")?\s?(?:dir=".*")?>\n/gm, '- ') // strip list items to bullets, replace later with emoji
+  .replace(/ipsnoembed="false" /gm, '') // manually replace ipsnoembed, it causes issues given location
+  .replace(/<a href="(.*)" rel="external nofollow(?: noopener)?"\s?(?:target="_blank")?>(.*)<\/a>/gm, '[$2]($1)')
+  .replace(/&amp;/gm, '&') // replace ampersand entity... it looks weird with some titles
+  .replace(/<\/li>/gm, '') // strip li end tags
+  .replace(/<(?:.|\n)*?>/gm, '') // replace all other tags, like images and paragraphs... cause they uuugly
+  .replace(/-\s+\n/g, '- ')
+  .replace(/\n\s*\n+\s*/gm, '\n\n') // strip 2+ line endings to max 2
+  .replace(/\*\*\n\n/gm, '**\n') // strip any newlines between headers and content to collapse content
+  .replace(/^\s*-\s*\n\s*\[/g, '- [')
+  .replace(/^- /gm, ':white_small_square:') // swap bullets for emoji
+  .trim();
+
 /**
  * Check that embeds are valid, and merge values into array
  * @param  {Array.<any>} original  Original array
@@ -1073,4 +1107,5 @@ module.exports = {
   groupBy,
   games,
   giveawayDefaults,
+  markdinate,
 };

src/Logger.js

@@ -1,5 +1,6 @@
 'use strict';
 
+/* eslint-disable no-console */
 const Sentry = require('@sentry/node');
 require('colors');
 
@@ -44,21 +45,17 @@ Object.keys(levels).forEach((level) => {
     const isActive = Object.keys(levels).indexOf(level) >= Object.keys(levels).indexOf(l.logLevel);
     const nonError = Object.keys(levels).indexOf(level) < Object.keys(levels).indexOf('ERROR');
     if (isActive && nonError) {
-      // eslint-disable-next-line no-console
       console.log(simple);
     }
 
-    if (level.toLowerCase() === 'fatal' && Sentry) {
-      Sentry.captureMessage(message, {
-        level: 'fatal',
-      });
+    if (level.toLowerCase() === 'fatal') {
+      if (Sentry) Sentry.captureMessage(message, { level: 'fatal' });
+      console.error(simple);
       process.exit(4);
     }
     if (level.toLowerCase() === 'error') {
-      // eslint-disable-next-line no-console
       console.error(simple);
       if (typeof message === 'object') {
-        // eslint-disable-next-line no-console
         console.error(message);
       }
       if (Sentry) {

src/bot.js

@@ -203,7 +203,10 @@ class Genesis {
     this.client.on('error', this.logger.error);
     this.client.on('warn', this.logger.warn);
     this.client.on('debug', (message) => {
-      if (/(Sending a heartbeat|Latency of|voice)/i.test(message)) return;
+      if (/(Sending a heartbeat|Latency of|voice)/i.test(message)) {
+        this.logger.silly(message);
+        return;
+      }
       this.logger.debug(message);
     });
   }

src/commands/Ondemand/GetRSSUpdates.js

@@ -0,0 +1,39 @@
+'use strict';
+
+const Command = require('../../models/Command.js');
+const RSSEmbed = require('../../embeds/RSSEmbed.js');
+const feeds = require('../../resources/rssFeeds');
+const { setupPages } = require('../../CommonFunctions');
+
+/**
+ * Displays the most recent RSS Update for the current platform
+ */
+class GetRSSUpdates extends Command {
+  constructor(bot) {
+    super(bot, 'rss.update', 'fupdates', 'Display themost recent RSS Update entries', 'WARFRAME');
+  }
+
+  async run(message, ctx) {
+    const feedUrl = feeds.filter(f => f.key === `forum.updates.${ctx.platform}`)[0].url;
+    const matchingFeeds = this.bot.feedNotifier.feeder.list.filter(f => f.url === feedUrl);
+    const updates = matchingFeeds && matchingFeeds[0]
+        && matchingFeeds[0].items && matchingFeeds[0].items[0]
+      ? [...matchingFeeds[0].items]
+      : [];
+
+    if (!updates.length) {
+      this.messageManager.reply(message, ctx.i18n`No RSS Updates`, true, true);
+      return this.messageManager.statuses.FAILURE;
+    }
+
+    const pages = [];
+    updates.reverse().forEach((item) => {
+      pages.push(new RSSEmbed(item, feeds[0]));
+    });
+    await setupPages(pages, { message, settings: this.settings, mm: this.messageManager });
+
+    return this.messageManager.statuses.SUCCESS;
+  }
+}
+
+module.exports = GetRSSUpdates;

src/embeds/RSSEmbed.js

@@ -2,7 +2,7 @@
 
 const BaseEmbed = require('./BaseEmbed.js');
 
-const { chunkify } = require('../CommonFunctions');
+const { chunkify, markdinate } = require('../CommonFunctions');
 
 /**
  * Generates daily deal embeds
@@ -15,9 +15,32 @@ class RSSEmbed extends BaseEmbed {
    */
   constructor(feedItem, feed) {
     super();
-    [this.description] = chunkify({
-      string: (feedItem.description || '\u200B').replace(/<(?:.|\n)*?>/gm, '').replace(/\n\n+\s*/gm, '\n\n'),
+    // clean up description, falling back to an empty string
+    let strippedDesc = markdinate((feedItem.description || '\u200B')
+      .replace(new RegExp(`<strong>${feedItem.title}</strong>`, 'gm'), ''));
+    const firstLine = strippedDesc.split('\n')[0].replace(/\*\*/g, '');
+
+    if (feedItem.title.includes(firstLine)) {
+      const tokens = strippedDesc.split('\n');
+      tokens.shift();
+      strippedDesc = tokens.join('\n');
+    }
+
+
+    [strippedDesc] = chunkify({
+      string: strippedDesc,
+      maxLength: 1000,
+      breakChar: '\n',
+      checkTitle: true,
     });
+
+    // strip the last title if it starts with a title
+    if (strippedDesc.endsWith('**')) {
+      strippedDesc = strippedDesc.replace(/\*\*(.*)\*\*$/g, '');
+    }
+
+    // this.description = `\`\`\`\n${strippedDesc}\n\`\`\``;
+    this.description = strippedDesc;
     this.url = feedItem.link;
     this.timestamp = feedItem.pubdate;
     this.title = feedItem.title;
@@ -42,7 +65,6 @@ class RSSEmbed extends BaseEmbed {
       if (first) {
         if (first.startsWith('//')) {
           first = first.replace('//', 'https://');
-
           this.image = {
             url: first,
           };

src/eventHandlers/CommandHandler.js

@@ -93,7 +93,7 @@ class CommandHandler extends Handler {
     const botPingId = `<@${this.bot.client.user.id}>`;
     const botNickPing = `<@!${this.bot.client.user.id}>`;
     const ctx = await this.bot.settings.getCommandContext(message.channel, message.author);
-    this.logger.debug(`${message.author.id} is ${ctx.isBlacklisted ? '' : 'not '}blacklisted`);
+    this.logger.silly(`${message.author.id} is ${ctx.isBlacklisted ? '' : 'not '}blacklisted`);
     if (ctx.isBlacklisted) {
       return;
     }
@@ -128,7 +128,7 @@ class CommandHandler extends Handler {
     // set new context objects
     ctx.message = strippedMessage;
     ctx.i18n = I18n.use(ctx.language);
-    this.logger.debug(`Handling \`${content}\``);
+    this.logger.silly(`Handling \`${content}\``);
 
     let done = false;
     commands.forEach(async (command) => {
@@ -137,7 +137,7 @@ class CommandHandler extends Handler {
         // check if it's runnable for the user
         const canAct = await this.checkCanAct(command, strippedMessage);
         if (checkInlineCustom(canAct, ctx.allowCustom, ctx.allowInline, command)) {
-          this.logger.debug(`Matched ${command.id}`);
+          this.logger.silly(`Matched ${command.id}`);
 
           // load command
           const cmd = await this.bot.commandManager.loadCommand(command);

src/eventHandlers/ReadyRatioCheck.js

@@ -91,7 +91,7 @@ class OnReadyHandle extends Handler {
    * Run the ready handle
    */
   async execute() {
-    this.logger.debug(`Running ${this.id} for ${this.event}`);
+    this.logger.silly(`Running ${this.id} for ${this.event}`);
     setTimeout(guildRatioCheck, 4000, this);
   }
 }

src/settings/DatabaseQueries/DBMQueries.js

@@ -14,7 +14,12 @@ class DBMQueries {
    * @returns {Promise}
    */
   createSchema() {
-    return Promise.mapSeries(schema, q => this.db.query(q));
+    try {
+      return Promise.mapSeries(schema, q => this.db.query(q));
+    } catch (e) {
+      this.logger.fatal(e);
+      return undefined;
+    }
   }
 
   /**