feat: Only get script log once from Sense server

Implements #741
ptarmiganlabs · Aug 17, 2023 · f193538 · f193538
1 parent 6e7814a
commit f193538
Show file tree

Hide file tree

Showing 9 changed files with 128 additions and 34 deletions.
diff --git a/src/globals.js b/src/globals.js
@@ -1,10 +1,10 @@
+const os = require('os');
+const crypto = require('crypto');
 const fs = require('fs-extra');
 const upath = require('upath');
 const Influx = require('influx');
 const { IncomingWebhook } = require('ms-teams-webhook');
 const si = require('systeminformation');
-const os = require('os');
-const crypto = require('crypto');
 const isUncPath = require('is-unc-path');
 const winston = require('winston');
 
@@ -538,8 +538,6 @@ async function loadApprovedDirectories() {
                 fileDeleteDirectories.push(deleteDir);
             });
         }
-
-        return;
     } catch (err) {
         logger.error(`CONFIG: Getting approved directories: ${err}`);
     }

diff --git a/src/lib/incident_mgmt/new_relic.js b/src/lib/incident_mgmt/new_relic.js
@@ -3,7 +3,6 @@ const QrsInteract = require('qrs-interact');
 const { RateLimiterMemory } = require('rate-limiter-flexible');
 
 const globals = require('../../globals');
-const scriptLog = require('../scriptlog');
 
 function getQRSConfig() {
     const cfg = {
@@ -412,11 +411,19 @@ async function sendNewRelicLog(incidentConfig, reloadParams, destNewRelicAccount
             payload[0].common.attributes = Object.assign(incidentConfig.attributes, reloadParams);
 
             // Get script logs
-            const scriptLogData = await scriptLog.getScriptLog(
-                reloadParams.qs_taskId,
-                0,
-                globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.destination.log.tailScriptLogLines')
+            const scriptLogData = reloadParams.scriptLog;
+
+            // Reduce script log lines to only the ones we want to send to New Relic
+            scriptLogData.scriptLogHeadCount = 0;
+            scriptLogData.scriptLogTailCount = globals.config.get(
+                'Butler.incidentTool.newRelic.reloadTaskFailure.destination.log.tailScriptLogLines'
             );
+
+            scriptLogData.scriptLogHead = '';
+            scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
+                .slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
+                .join('\r\n');
+
             globals.logger.debug(`NEW RELIC TASK FAILED LOG: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);
 
             // Use script log data to enrich log entry sent to New Relic

diff --git a/src/lib/scriptlog.js b/src/lib/scriptlog.js
@@ -267,7 +267,7 @@ async function failedTaskStoreLogOnDisk(reloadParams) {
         const reloadLogDirRoot = globals.config.get('Butler.scriptLog.storeOnDisk.reloadTaskFailure.logDirectory');
 
         // Get misc script log info
-        const scriptLog = await getScriptLog(reloadParams.taskId, 1, 1);
+        const { scriptLog } = reloadParams;
 
         // Create directory for script log, if needed
         const logDate = reloadParams.logTimeStamp.slice(0, 10);

diff --git a/src/lib/slack_api.js b/src/lib/slack_api.js
@@ -18,6 +18,7 @@ async function slackSend(slackConfig, logger) {
         if (slackConfig.messageType === 'basic') {
             Object.assign(body, slackConfig.text);
         } else if (slackConfig.messageType === 'formatted') {
+            // Parse the JSON string into an object
             Object.assign(body, JSON.parse(slackConfig.text));
         } else if (slackConfig.messageType === 'restmsg') {
             Object.assign(body, slackConfig.text);

diff --git a/src/lib/slack_notification.js b/src/lib/slack_notification.js
@@ -5,7 +5,6 @@ const handlebars = require('handlebars');
 const { RateLimiterMemory } = require('rate-limiter-flexible');
 
 const globals = require('../globals');
-const scriptLog = require('./scriptlog');
 const slackApi = require('./slack_api');
 
 let rateLimiterMemoryFailedReloads;
@@ -374,6 +373,7 @@ async function sendSlack(slackConfig, templateContext, msgType) {
         const msg = slackConfig;
 
         if (slackConfig.messageType === 'basic') {
+            // Compile template
             compiledTemplate = handlebars.compile(slackConfig.basicMsgTemplate);
             renderedText = compiledTemplate(templateContext);
 
@@ -419,7 +419,10 @@ async function sendSlack(slackConfig, templateContext, msgType) {
         } else if (slackConfig.messageType === 'formatted') {
             try {
                 if (fs.existsSync(slackConfig.templateFile) === true) {
+                    // Read template file
                     const template = fs.readFileSync(slackConfig.templateFile, 'utf8');
+
+                    // Compile template
                     compiledTemplate = handlebars.compile(template);
 
                     if (msgType === 'reload') {
@@ -432,6 +435,7 @@ async function sendSlack(slackConfig, templateContext, msgType) {
                         templateContext.scriptLogTail = templateContext.scriptLogTail.replace(regExpText, '\\\\');
                     }
 
+                    // Render Slack message using template. Do not convert to &lt; and &gt; as Slack will not render the message correctly
                     slackMsg = compiledTemplate(templateContext);
 
                     globals.logger.debug(`SLACKNOTIF: Rendered message:\n${slackMsg}`);
@@ -472,17 +476,25 @@ function sendReloadTaskFailureNotificationSlack(reloadParams) {
                 }
 
                 // Get script logs, if enabled in the config file
-                const scriptLogData = await scriptLog.getScriptLog(
-                    reloadParams.taskId,
-                    globals.config.get('Butler.slackNotification.reloadTaskFailure.headScriptLogLines'),
-                    globals.config.get('Butler.slackNotification.reloadTaskFailure.tailScriptLogLines')
-                );
+                const scriptLogData = reloadParams.scriptLog;
+
+                // Reduce script log lines to only the ones we want to send to Slack
+                scriptLogData.scriptLogHeadCount = globals.config.get('Butler.slackNotification.reloadTaskFailure.headScriptLogLines');
+                scriptLogData.scriptLogTailCount = globals.config.get('Butler.slackNotification.reloadTaskFailure.tailScriptLogLines');
+
+                scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');
+                scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
+                    .slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
+                    .join('\r\n');
+
                 globals.logger.debug(`TASK FAILED ALERT SLACK: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);
 
                 // Get Sense URLs from config file. Can be used as template fields.
                 const senseUrls = getQlikSenseUrls();
 
                 // These are the template fields that can be used in Slack body
+                // Regular expression for converting escapöing single quote
+
                 const templateContext = {
                     hostName: reloadParams.hostName,
                     user: reloadParams.user,
@@ -519,6 +531,19 @@ function sendReloadTaskFailureNotificationSlack(reloadParams) {
                     qlikSenseHub: senseUrls.hubUrl,
                 };
 
+                // Replace all single and dpouble quotes in scriptLogHead and scriptLogTail with escaped dittos
+                // This is needed to avoid breaking the Slack message JSON
+                const regExpSingle = /'/gm;
+                const regExpDouble = /"/gm;
+                templateContext.scriptLogHead = templateContext.scriptLogHead.replace(regExpSingle, "\'").replace(regExpDouble, "\\'");
+                templateContext.scriptLogTail = templateContext.scriptLogTail.replace(regExpSingle, "\'").replace(regExpDouble, "\\'");
+
+                // Replace all single and double quotes in executionDetailsConcatenated with escaped ditto
+                // This is needed to avoid breaking the Slack message JSON
+                templateContext.executionDetailsConcatenated = templateContext.executionDetailsConcatenated
+                    .replace(regExpSingle, "\\'")
+                    .replace(regExpDouble, "\\'");
+
                 // Check if script log is longer than 3000 characters, which is max for text fields sent to Slack API
                 // https://api.slack.com/reference/block-kit/blocks#section_fields
                 if (templateContext.scriptLogHead.length >= 3000) {
@@ -600,11 +625,17 @@ function sendReloadTaskAbortedNotificationSlack(reloadParams) {
                 }
 
                 // Get script logs, if enabled in the config file
-                const scriptLogData = await scriptLog.getScriptLog(
-                    reloadParams.taskId,
-                    globals.config.get('Butler.slackNotification.reloadTaskAborted.headScriptLogLines'),
-                    globals.config.get('Butler.slackNotification.reloadTaskAborted.tailScriptLogLines')
-                );
+                const scriptLogData = reloadParams.scriptLog;
+
+                // Reduce script log lines to only the ones we want to send to Slack
+                scriptLogData.scriptLogHeadCount = globals.config.get('Butler.slackNotification.reloadTaskAborted.headScriptLogLines');
+                scriptLogData.scriptLogTailCount = globals.config.get('Butler.slackNotification.reloadTaskAborted.tailScriptLogLines');
+
+                scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');
+                scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
+                    .slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
+                    .join('\r\n');
+
                 globals.logger.debug(`TASK ABORTED ALERT SLACK: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);
 
                 // Get Sense URLs from config file. Can be used as template fields.

diff --git a/src/lib/smtp.js b/src/lib/smtp.js
@@ -7,7 +7,6 @@ const { RateLimiterMemory } = require('rate-limiter-flexible');
 const emailValidator = require('email-validator');
 
 const globals = require('../globals');
-const scriptLog = require('./scriptlog');
 const qrsUtil = require('../qrs_util');
 
 let rateLimiterMemoryFailedReloads;
@@ -491,11 +490,17 @@ async function sendReloadTaskFailureNotificationEmail(reloadParams) {
     }
 
     // Get script logs, if enabled in the config file
-    const scriptLogData = await scriptLog.getScriptLog(
-        reloadParams.taskId,
-        globals.config.get('Butler.emailNotification.reloadTaskFailure.headScriptLogLines'),
-        globals.config.get('Butler.emailNotification.reloadTaskFailure.tailScriptLogLines')
-    );
+    const scriptLogData = reloadParams.scriptLog;
+
+    // Reduce script log lines to only the ones we want to send to Slack
+    scriptLogData.scriptLogHeadCount = globals.config.get('Butler.emailNotification.reloadTaskFailure.headScriptLogLines');
+    scriptLogData.scriptLogTailCount = globals.config.get('Butler.emailNotification.reloadTaskFailure.tailScriptLogLines');
+
+    scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');;
+    scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
+        .slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
+        .join('\r\n');
+
     globals.logger.debug(`TASK FAILED ALERT EMAIL: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);
 
     // Get Sense URLs from config file. Can be used as template fields.
@@ -712,11 +717,17 @@ async function sendReloadTaskAbortedNotificationEmail(reloadParams) {
     }
 
     // Get script logs, if enabled in the config file
-    const scriptLogData = await scriptLog.getScriptLog(
-        reloadParams.taskId,
-        globals.config.get('Butler.emailNotification.reloadTaskAborted.headScriptLogLines'),
-        globals.config.get('Butler.emailNotification.reloadTaskAborted.tailScriptLogLines')
-    );
+    const scriptLogData = reloadParams.scriptLog;
+
+    // Reduce script log lines to only the ones we want to send to Slack
+    scriptLogData.scriptLogHeadCount = globals.config.get('Butler.emailNotification.reloadTaskAborted.headScriptLogLines');
+    scriptLogData.scriptLogTailCount = globals.config.get('Butler.emailNotification.reloadTaskAborted.tailScriptLogLines');
+
+    scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');;
+    scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
+        .slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
+        .join('\r\n');
+
     globals.logger.debug(`TASK ABORTED ALERT EMAIL: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);
 
     // Get Sense URLs from config file. Can be used as template fields.

diff --git a/src/routes/disk_utils.js b/src/routes/disk_utils.js
@@ -1,7 +1,7 @@
 const httpErrors = require('http-errors');
 const fs = require('fs-extra');
 const upath = require('upath');
-const { mkdirp } = require('mkdirp')
+const { mkdirp } = require('mkdirp');
 const isUncPath = require('is-unc-path');
 
 // Load global variables and functions

diff --git a/src/test/routes/disk_utils.test.js b/src/test/routes/disk_utils.test.js
@@ -1,3 +1,4 @@
+/* eslint-disable no-console */
 /* eslint-disable camelcase */
 const fs = require('fs/promises');
 const upath = require('upath');