diff --git a/src/globals.js b/src/globals.js index 9c17d8b0..ffe27d35 100644 --- a/src/globals.js +++ b/src/globals.js @@ -1,10 +1,10 @@ +const os = require('os'); +const crypto = require('crypto'); const fs = require('fs-extra'); const upath = require('upath'); const Influx = require('influx'); const { IncomingWebhook } = require('ms-teams-webhook'); const si = require('systeminformation'); -const os = require('os'); -const crypto = require('crypto'); const isUncPath = require('is-unc-path'); const winston = require('winston'); @@ -538,8 +538,6 @@ async function loadApprovedDirectories() { fileDeleteDirectories.push(deleteDir); }); } - - return; } catch (err) { logger.error(`CONFIG: Getting approved directories: ${err}`); } diff --git a/src/lib/incident_mgmt/new_relic.js b/src/lib/incident_mgmt/new_relic.js index b44b02f8..ae3dc1a5 100644 --- a/src/lib/incident_mgmt/new_relic.js +++ b/src/lib/incident_mgmt/new_relic.js @@ -3,7 +3,6 @@ const QrsInteract = require('qrs-interact'); const { RateLimiterMemory } = require('rate-limiter-flexible'); const globals = require('../../globals'); -const scriptLog = require('../scriptlog'); function getQRSConfig() { const cfg = { @@ -412,11 +411,19 @@ async function sendNewRelicLog(incidentConfig, reloadParams, destNewRelicAccount payload[0].common.attributes = Object.assign(incidentConfig.attributes, reloadParams); // Get script logs - const scriptLogData = await scriptLog.getScriptLog( - reloadParams.qs_taskId, - 0, - globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.destination.log.tailScriptLogLines') + const scriptLogData = reloadParams.scriptLog; + + // Reduce script log lines to only the ones we want to send to New Relic + scriptLogData.scriptLogHeadCount = 0; + scriptLogData.scriptLogTailCount = globals.config.get( + 'Butler.incidentTool.newRelic.reloadTaskFailure.destination.log.tailScriptLogLines' ); + + scriptLogData.scriptLogHead = ''; + scriptLogData.scriptLogTail = scriptLogData.scriptLogFull + .slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0)) + .join('\r\n'); + globals.logger.debug(`NEW RELIC TASK FAILED LOG: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`); // Use script log data to enrich log entry sent to New Relic diff --git a/src/lib/scriptlog.js b/src/lib/scriptlog.js index e6ea9f32..e6cc38e7 100644 --- a/src/lib/scriptlog.js +++ b/src/lib/scriptlog.js @@ -267,7 +267,7 @@ async function failedTaskStoreLogOnDisk(reloadParams) { const reloadLogDirRoot = globals.config.get('Butler.scriptLog.storeOnDisk.reloadTaskFailure.logDirectory'); // Get misc script log info - const scriptLog = await getScriptLog(reloadParams.taskId, 1, 1); + const { scriptLog } = reloadParams; // Create directory for script log, if needed const logDate = reloadParams.logTimeStamp.slice(0, 10); diff --git a/src/lib/slack_api.js b/src/lib/slack_api.js index 12f20f31..57bffb11 100644 --- a/src/lib/slack_api.js +++ b/src/lib/slack_api.js @@ -18,6 +18,7 @@ async function slackSend(slackConfig, logger) { if (slackConfig.messageType === 'basic') { Object.assign(body, slackConfig.text); } else if (slackConfig.messageType === 'formatted') { + // Parse the JSON string into an object Object.assign(body, JSON.parse(slackConfig.text)); } else if (slackConfig.messageType === 'restmsg') { Object.assign(body, slackConfig.text); diff --git a/src/lib/slack_notification.js b/src/lib/slack_notification.js index 11fe0b76..cc9fb2ff 100644 --- a/src/lib/slack_notification.js +++ b/src/lib/slack_notification.js @@ -5,7 +5,6 @@ const handlebars = require('handlebars'); const { RateLimiterMemory } = require('rate-limiter-flexible'); const globals = require('../globals'); -const scriptLog = require('./scriptlog'); const slackApi = require('./slack_api'); let rateLimiterMemoryFailedReloads; @@ -374,6 +373,7 @@ async function sendSlack(slackConfig, templateContext, msgType) { const msg = slackConfig; if (slackConfig.messageType === 'basic') { + // Compile template compiledTemplate = handlebars.compile(slackConfig.basicMsgTemplate); renderedText = compiledTemplate(templateContext); @@ -419,7 +419,10 @@ async function sendSlack(slackConfig, templateContext, msgType) { } else if (slackConfig.messageType === 'formatted') { try { if (fs.existsSync(slackConfig.templateFile) === true) { + // Read template file const template = fs.readFileSync(slackConfig.templateFile, 'utf8'); + + // Compile template compiledTemplate = handlebars.compile(template); if (msgType === 'reload') { @@ -432,6 +435,7 @@ async function sendSlack(slackConfig, templateContext, msgType) { templateContext.scriptLogTail = templateContext.scriptLogTail.replace(regExpText, '\\\\'); } + // Render Slack message using template. Do not convert to < and > as Slack will not render the message correctly slackMsg = compiledTemplate(templateContext); globals.logger.debug(`SLACKNOTIF: Rendered message:\n${slackMsg}`); @@ -472,17 +476,25 @@ function sendReloadTaskFailureNotificationSlack(reloadParams) { } // Get script logs, if enabled in the config file - const scriptLogData = await scriptLog.getScriptLog( - reloadParams.taskId, - globals.config.get('Butler.slackNotification.reloadTaskFailure.headScriptLogLines'), - globals.config.get('Butler.slackNotification.reloadTaskFailure.tailScriptLogLines') - ); + const scriptLogData = reloadParams.scriptLog; + + // Reduce script log lines to only the ones we want to send to Slack + scriptLogData.scriptLogHeadCount = globals.config.get('Butler.slackNotification.reloadTaskFailure.headScriptLogLines'); + scriptLogData.scriptLogTailCount = globals.config.get('Butler.slackNotification.reloadTaskFailure.tailScriptLogLines'); + + scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n'); + scriptLogData.scriptLogTail = scriptLogData.scriptLogFull + .slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0)) + .join('\r\n'); + globals.logger.debug(`TASK FAILED ALERT SLACK: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`); // Get Sense URLs from config file. Can be used as template fields. const senseUrls = getQlikSenseUrls(); // These are the template fields that can be used in Slack body + // Regular expression for converting escapöing single quote + const templateContext = { hostName: reloadParams.hostName, user: reloadParams.user, @@ -519,6 +531,19 @@ function sendReloadTaskFailureNotificationSlack(reloadParams) { qlikSenseHub: senseUrls.hubUrl, }; + // Replace all single and dpouble quotes in scriptLogHead and scriptLogTail with escaped dittos + // This is needed to avoid breaking the Slack message JSON + const regExpSingle = /'/gm; + const regExpDouble = /"/gm; + templateContext.scriptLogHead = templateContext.scriptLogHead.replace(regExpSingle, "\'").replace(regExpDouble, "\\'"); + templateContext.scriptLogTail = templateContext.scriptLogTail.replace(regExpSingle, "\'").replace(regExpDouble, "\\'"); + + // Replace all single and double quotes in executionDetailsConcatenated with escaped ditto + // This is needed to avoid breaking the Slack message JSON + templateContext.executionDetailsConcatenated = templateContext.executionDetailsConcatenated + .replace(regExpSingle, "\\'") + .replace(regExpDouble, "\\'"); + // Check if script log is longer than 3000 characters, which is max for text fields sent to Slack API // https://api.slack.com/reference/block-kit/blocks#section_fields if (templateContext.scriptLogHead.length >= 3000) { @@ -600,11 +625,17 @@ function sendReloadTaskAbortedNotificationSlack(reloadParams) { } // Get script logs, if enabled in the config file - const scriptLogData = await scriptLog.getScriptLog( - reloadParams.taskId, - globals.config.get('Butler.slackNotification.reloadTaskAborted.headScriptLogLines'), - globals.config.get('Butler.slackNotification.reloadTaskAborted.tailScriptLogLines') - ); + const scriptLogData = reloadParams.scriptLog; + + // Reduce script log lines to only the ones we want to send to Slack + scriptLogData.scriptLogHeadCount = globals.config.get('Butler.slackNotification.reloadTaskAborted.headScriptLogLines'); + scriptLogData.scriptLogTailCount = globals.config.get('Butler.slackNotification.reloadTaskAborted.tailScriptLogLines'); + + scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n'); + scriptLogData.scriptLogTail = scriptLogData.scriptLogFull + .slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0)) + .join('\r\n'); + globals.logger.debug(`TASK ABORTED ALERT SLACK: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`); // Get Sense URLs from config file. Can be used as template fields. diff --git a/src/lib/smtp.js b/src/lib/smtp.js index 2e3b8e53..3e8d6ed0 100644 --- a/src/lib/smtp.js +++ b/src/lib/smtp.js @@ -7,7 +7,6 @@ const { RateLimiterMemory } = require('rate-limiter-flexible'); const emailValidator = require('email-validator'); const globals = require('../globals'); -const scriptLog = require('./scriptlog'); const qrsUtil = require('../qrs_util'); let rateLimiterMemoryFailedReloads; @@ -491,11 +490,17 @@ async function sendReloadTaskFailureNotificationEmail(reloadParams) { } // Get script logs, if enabled in the config file - const scriptLogData = await scriptLog.getScriptLog( - reloadParams.taskId, - globals.config.get('Butler.emailNotification.reloadTaskFailure.headScriptLogLines'), - globals.config.get('Butler.emailNotification.reloadTaskFailure.tailScriptLogLines') - ); + const scriptLogData = reloadParams.scriptLog; + + // Reduce script log lines to only the ones we want to send to Slack + scriptLogData.scriptLogHeadCount = globals.config.get('Butler.emailNotification.reloadTaskFailure.headScriptLogLines'); + scriptLogData.scriptLogTailCount = globals.config.get('Butler.emailNotification.reloadTaskFailure.tailScriptLogLines'); + + scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');; + scriptLogData.scriptLogTail = scriptLogData.scriptLogFull + .slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0)) + .join('\r\n'); + globals.logger.debug(`TASK FAILED ALERT EMAIL: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`); // Get Sense URLs from config file. Can be used as template fields. @@ -712,11 +717,17 @@ async function sendReloadTaskAbortedNotificationEmail(reloadParams) { } // Get script logs, if enabled in the config file - const scriptLogData = await scriptLog.getScriptLog( - reloadParams.taskId, - globals.config.get('Butler.emailNotification.reloadTaskAborted.headScriptLogLines'), - globals.config.get('Butler.emailNotification.reloadTaskAborted.tailScriptLogLines') - ); + const scriptLogData = reloadParams.scriptLog; + + // Reduce script log lines to only the ones we want to send to Slack + scriptLogData.scriptLogHeadCount = globals.config.get('Butler.emailNotification.reloadTaskAborted.headScriptLogLines'); + scriptLogData.scriptLogTailCount = globals.config.get('Butler.emailNotification.reloadTaskAborted.tailScriptLogLines'); + + scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');; + scriptLogData.scriptLogTail = scriptLogData.scriptLogFull + .slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0)) + .join('\r\n'); + globals.logger.debug(`TASK ABORTED ALERT EMAIL: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`); // Get Sense URLs from config file. Can be used as template fields. diff --git a/src/routes/disk_utils.js b/src/routes/disk_utils.js index b98b132a..2f28d072 100644 --- a/src/routes/disk_utils.js +++ b/src/routes/disk_utils.js @@ -1,7 +1,7 @@ const httpErrors = require('http-errors'); const fs = require('fs-extra'); const upath = require('upath'); -const { mkdirp } = require('mkdirp') +const { mkdirp } = require('mkdirp'); const isUncPath = require('is-unc-path'); // Load global variables and functions diff --git a/src/test/routes/disk_utils.test.js b/src/test/routes/disk_utils.test.js index 06c96263..a033ac1c 100644 --- a/src/test/routes/disk_utils.test.js +++ b/src/test/routes/disk_utils.test.js @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ /* eslint-disable camelcase */ const fs = require('fs/promises'); const upath = require('upath'); diff --git a/src/udp/udp_handlers.js b/src/udp/udp_handlers.js index 2ae47702..b5eb0787 100644 --- a/src/udp/udp_handlers.js +++ b/src/udp/udp_handlers.js @@ -7,7 +7,7 @@ const webhookOut = require('../lib/webhook_notification'); const msteams = require('../lib/msteams_notification'); const signl4 = require('../lib/incident_mgmt/signl4'); const newRelic = require('../lib/incident_mgmt/new_relic'); -const { failedTaskStoreLogOnDisk } = require('../lib/scriptlog'); +const { failedTaskStoreLogOnDisk, getScriptLog } = require('../lib/scriptlog'); const { getTaskTags } = require('../qrs_util/task_tag_util'); const { getAppTags } = require('../qrs_util/app_tag_util'); @@ -17,6 +17,20 @@ const schedulerAborted = async (msg) => { `TASKABORTED: Received reload aborted UDP message from scheduler: UDP msg=${msg[0]}, Host=${msg[1]}, App name=${msg[3]}, Task name=${msg[2]}, Log level=${msg[8]}, Log msg=${msg[10]}` ); + // Get script log for failed reloads. + // Only done if Slack, Teams, email or New Relic alerts are enabled + let scriptLog; + if ( + (globals.config.has('Butler.incidentTool.newRelic.enable') && globals.config.get('Butler.incidentTool.newRelic.enable') === true) || + (globals.config.has('Butler.slackNotification.enable') && globals.config.get('Butler.slackNotification.enable') === true) || + (globals.config.has('Butler.teamsNotification.enable') && globals.config.get('Butler.teamsNotification.enable') === true) || + (globals.config.has('Butler.emailNotification.enable') && globals.config.get('Butler.emailNotification.enable') === true) + ) { + scriptLog = await getScriptLog(msg[5], 1, 1); + + globals.logger.verbose(`Script log for aborted reload retrieved`); + } + // First field in message (msg[0]) is message category (this is the modern/recent message format) // Check if app/task tags are used by any of the alert destinations. @@ -98,6 +112,7 @@ const schedulerAborted = async (msg) => { qs_logMessage: msg[10], qs_appTags: appTags, qs_taskTags: taskTags, + scriptLog, }); } @@ -121,6 +136,7 @@ const schedulerAborted = async (msg) => { logMessage: msg[10], qs_appTags: appTags, qs_taskTags: taskTags, + scriptLog, }); } @@ -144,6 +160,7 @@ const schedulerAborted = async (msg) => { logMessage: msg[10], qs_appTags: appTags, qs_taskTags: taskTags, + scriptLog, }); } @@ -167,6 +184,7 @@ const schedulerAborted = async (msg) => { logMessage: msg[10], qs_appTags: appTags, qs_taskTags: taskTags, + scriptLog, }); } @@ -229,6 +247,24 @@ const schedulerAborted = async (msg) => { // Handler for failed scheduler initiated reloads const schedulerFailed = async (msg, legacyFlag) => { + // Get script log for failed reloads. + // Only done if Slack, Teams, email or New Relic alerts are enabled + let scriptLog; + if ( + (globals.config.has('Butler.incidentTool.newRelic.enable') && globals.config.get('Butler.incidentTool.newRelic.enable') === true) || + (globals.config.has('Butler.slackNotification.enable') && globals.config.get('Butler.slackNotification.enable') === true) || + (globals.config.has('Butler.teamsNotification.enable') && globals.config.get('Butler.teamsNotification.enable') === true) || + (globals.config.has('Butler.emailNotification.enable') && globals.config.get('Butler.emailNotification.enable') === true) + ) { + if (legacyFlag) { + scriptLog = await getScriptLog(msg[4], 0, 0); + globals.logger.verbose(`Script log for failed reload retrieved (legacy)`); + } else { + scriptLog = await getScriptLog(msg[5], 0, 0); + globals.logger.verbose(`Script log for failed reload retrieved (new)`); + } + } + if (legacyFlag) { // First field in message (msg[0]) is host name @@ -311,6 +347,7 @@ const schedulerFailed = async (msg, legacyFlag) => { qs_logLevel: msg[7], qs_executionId: msg[8], qs_logMessage: msg[9], + scriptLog, }); } @@ -332,6 +369,7 @@ const schedulerFailed = async (msg, legacyFlag) => { logLevel: msg[7], executionId: msg[8], logMessage: msg[9], + scriptLog, }); } @@ -353,6 +391,7 @@ const schedulerFailed = async (msg, legacyFlag) => { logLevel: msg[7], executionId: msg[8], logMessage: msg[9], + scriptLog, }); } @@ -374,6 +413,7 @@ const schedulerFailed = async (msg, legacyFlag) => { logLevel: msg[7], executionId: msg[8], logMessage: msg[9], + scriptLog, }); } @@ -469,6 +509,7 @@ const schedulerFailed = async (msg, legacyFlag) => { logMessage: msg[10], qs_appTags: appTags, qs_taskTags: taskTags, + scriptLog, }); } @@ -538,6 +579,7 @@ const schedulerFailed = async (msg, legacyFlag) => { qs_logMessage: msg[10], qs_appTags: appTags, qs_taskTags: taskTags, + scriptLog, }); } @@ -561,6 +603,7 @@ const schedulerFailed = async (msg, legacyFlag) => { logMessage: msg[10], qs_appTags: appTags, qs_taskTags: taskTags, + scriptLog, }); } @@ -584,6 +627,7 @@ const schedulerFailed = async (msg, legacyFlag) => { logMessage: msg[10], qs_appTags: appTags, qs_taskTags: taskTags, + scriptLog, }); } @@ -607,6 +651,7 @@ const schedulerFailed = async (msg, legacyFlag) => { logMessage: msg[10], qs_appTags: appTags, qs_taskTags: taskTags, + scriptLog, }); }