diff --git a/src/app.js b/src/app.js index 306398a3..5ca9c221 100644 --- a/src/app.js +++ b/src/app.js @@ -1,3 +1,4 @@ +/* eslint-disable prefer-object-spread */ /* eslint-disable global-require */ // const Fastify = require('fastify'); @@ -7,6 +8,7 @@ const AutoLoad = require('fastify-autoload'); const FastifySwagger = require('fastify-swagger'); const FastifyReplyFrom = require('fastify-reply-from'); const FastifyHealthcheck = require('fastify-healthcheck'); +const FastifyRateLimit = require('fastify-rate-limit'); const globals = require('./globals'); const heartbeat = require('./lib/heartbeat'); @@ -119,6 +121,12 @@ async function build(opts = {}) { globals.logger.error(`CONFIG: Error initiating host info: ${err}`); } + // Register rate limited for API + restServer.register(FastifyRateLimit, { + max: 100, + timeWindow: '1 minute', + }); + // This loads all plugins defined in plugins. // Those should be support plugins that are reused through your application restServer.register(require('./plugins/sensible'), { options: Object.assign({}, opts) }); diff --git a/src/globals.js b/src/globals.js index 2a112f4f..797f3def 100644 --- a/src/globals.js +++ b/src/globals.js @@ -46,7 +46,9 @@ program 'debug', 'silly', ]) - ); + ) + .option('--new-relic-api-key ', 'insert API key to use with New Relic') + .option('--new-relic-account-id ', 'New Relic account ID'); // Parse command line params program.parse(process.argv); @@ -66,6 +68,7 @@ if (options.configfile && options.configfile.length > 0) { configFileBasename = path.basename(configFileExpanded, configFileExtension); if (configFileExtension.toLowerCase() !== '.yaml') { + // eslint-disable-next-line no-console console.log('Error: Config file extension must be yaml'); process.exit(1); } @@ -74,6 +77,7 @@ if (options.configfile && options.configfile.length > 0) { process.env.NODE_CONFIG_DIR = configFilePath; process.env.NODE_ENV = configFileBasename; } else { + // eslint-disable-next-line no-console console.log('Error: Specified config file does not exist'); process.exit(1); } @@ -94,6 +98,16 @@ if (options.loglevel && options.loglevel.length > 0) { config.Butler.logLevel = options.loglevel; } +// Is there a New Relic API key specified on the command line? +if (options.newRelicApiKey && options.newRelicApiKey.length > 0) { + config.Butler.thirdPartyToolsCredentials.newRelic.insertApiKey = options.newRelicApiKey; +} + +// Is there a New Relic account ID specified on the command line? +if (options.newRelicAccountId && options.newRelicAccountId.length > 0) { + config.Butler.thirdPartyToolsCredentials.newRelic.accountId = options.newRelicAccountId; +} + // Set up logger with timestamps and colors, and optional logging to disk file const logTransports = []; diff --git a/src/lib/incident_mgmt/new-relic.js b/src/lib/incident_mgmt/new-relic.js new file mode 100644 index 00000000..594a7217 --- /dev/null +++ b/src/lib/incident_mgmt/new-relic.js @@ -0,0 +1,260 @@ +const axios = require('axios'); +const { RateLimiterMemory } = require('rate-limiter-flexible'); + +const globals = require('../../globals'); + +let rateLimiterFailedReloads; +let rateLimiterAbortedReloads; + +if (globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.rateLimit')) { + rateLimiterFailedReloads = new RateLimiterMemory({ + points: 1, + duration: globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.rateLimit'), + }); +} else { + rateLimiterFailedReloads = new RateLimiterMemory({ + points: 1, + duration: 300, + }); +} + +if (globals.config.has('Butler.incidentTool.newRelic.reloadTaskAborted.rateLimit')) { + rateLimiterAbortedReloads = new RateLimiterMemory({ + points: 1, + duration: globals.config.get('Butler.incidentTool.newRelic.reloadTaskAborted.rateLimit'), + }); +} else { + rateLimiterAbortedReloads = new RateLimiterMemory({ + points: 1, + duration: 300, + }); +} + +function getReloadFailedNotificationConfigOk() { + try { + // First make sure this tool is enabled in the config file and that we have needed parameters + if ( + !globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure') || + !globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.enable') || + !globals.config.has('Butler.incidentTool.newRelic.url') + ) { + // Not enough info in config file + globals.logger.error('NEWRELIC: Reload failure New Relic config info missing in Butler config file'); + return false; + } + + // Add headers + const headers = { + 'Content-Type': 'application/json', + 'Api-Key': globals.config.get('Butler.thirdPartyToolsCredentials.newRelic.insertApiKey'), + }; + + // eslint-disable-next-line no-restricted-syntax + for (const header of globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.header')) { + headers[header.name] = header.value; + } + + // Add static attributes + const attributes = {}; + + // eslint-disable-next-line no-restricted-syntax + for (const header of globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.attribute.static')) { + attributes[header.name] = header.value; + } + + // Add dynamic attributes + attributes.version = globals.appVersion; + + const cfg = { + eventType: 'Qlik Sense reload task failed', + url: + globals.config.get('Butler.incidentTool.newRelic.url').slice(-1) === '/' + ? globals.config.get('Butler.incidentTool.newRelic.url') + : `${globals.config.get('Butler.incidentTool.newRelic.url')}/`, + rateLimit: globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.rateLimit') + ? globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.rateLimit') + : '', + headers, + attributes, + }; + + return cfg; + } catch (err) { + globals.logger.error(`NEWRELIC: ${err}`); + return false; + } +} + +function getReloadAbortedNotificationConfigOk() { + try { + // First make sure this tool is enabled in the config file and that we have needed parameters + if ( + !globals.config.has('Butler.incidentTool.newRelic.reloadTaskAborted') || + !globals.config.has('Butler.incidentTool.newRelic.reloadTaskAborted.enable') || + !globals.config.has('Butler.incidentTool.newRelic.url') + ) { + // Not enough info in config file + globals.logger.error('NEWRELIC: Reload aborted New Relic config info missing in Butler config file'); + return false; + } + + // Add headers + const headers = { + 'Content-Type': 'application/json', + 'Api-Key': globals.config.get('Butler.thirdPartyToolsCredentials.newRelic.insertApiKey'), + }; + + // eslint-disable-next-line no-restricted-syntax + for (const header of globals.config.get('Butler.incidentTool.newRelic.reloadTaskAborted.header')) { + headers[header.name] = header.value; + } + + // Add static attributes + const attributes = {}; + + // eslint-disable-next-line no-restricted-syntax + for (const header of globals.config.get('Butler.incidentTool.newRelic.reloadTaskAborted.attribute.static')) { + attributes[header.name] = header.value; + } + + // Add dynamic attributes + attributes.version = globals.appVersion; + + const cfg = { + eventType: 'Qlik Sense reload task failed', + url: + globals.config.get('Butler.incidentTool.newRelic.url').slice(-1) === '/' + ? globals.config.get('Butler.incidentTool.newRelic.url') + : `${globals.config.get('Butler.incidentTool.newRelic.url')}/`, + rateLimit: globals.config.has('Butler.incidentTool.newRelic.reloadTaskAborted.rateLimit') + ? globals.config.get('Butler.incidentTool.newRelic.reloadTaskAborted.rateLimit') + : '', + headers, + attributes, + }; + + return cfg; + } catch (err) { + globals.logger.error(`NEWRELIC: ${err}`); + return false; + } +} + +async function sendNewRelic(incidentConfig, reloadParams) { + try { + // Build final URL + const eventUrl = `${incidentConfig.url}v1/accounts/${globals.config.get( + 'Butler.thirdPartyToolsCredentials.newRelic.accountId' + )}/events`; + + // Build final payload + const payload = Object.assign(incidentConfig.attributes, reloadParams); + payload.eventType = incidentConfig.eventType; + + // Convert timestamp in log to milliseconds + let tsTmp; + if (reloadParams.qs_logTimeStamp.includes(',')) { + tsTmp = new Date(reloadParams.qs_logTimeStamp.split(',')[0]).getTime(); + tsTmp += parseInt(reloadParams.qs_logTimeStamp.split(',')[1], 10); + } else if (reloadParams.qs_logTimeStamp.includes('.')) { + tsTmp = new Date(reloadParams.qs_logTimeStamp.split('.')[0]).getTime(); + tsTmp += parseInt(reloadParams.qs_logTimeStamp.split('.')[1], 10); + } else { + tsTmp = new Date(reloadParams.qs_logTimeStamp.split(',')[0]).getTime(); + } + + payload.timestamp = tsTmp; + + // Remove log timestamp field from payload as it is no longer needed + delete payload.logTimeStamp; + + // Build body for HTTP POST + const axiosRequest = { + url: eventUrl, + method: 'post', + timeout: 10000, + data: payload, + headers: incidentConfig.headers, + }; + + const response = await axios.request(axiosRequest); + globals.logger.debug(`NEWRELIC: Response from API: ${response}`); + } catch (err) { + globals.logger.error(`NEWRELIC: ${JSON.stringify(err, null, 2)}`); + } +} + +function sendReloadTaskFailureNotification(reloadParams) { + rateLimiterFailedReloads + .consume(reloadParams.taskId, 1) + .then(async (rateLimiterRes) => { + try { + globals.logger.info( + `NEWRELIC TASK FAIL: Rate limiting ok: Sending reload failure notification to New Relic for task "${reloadParams.taskName}"` + ); + globals.logger.verbose( + `NEWRELIC TASK FAIL: Rate limiting details "${JSON.stringify(rateLimiterRes, null, 2)}"` + ); + + // Make sure Slack sending is enabled in the config file and that we have all required settings + const incidentConfig = getReloadFailedNotificationConfigOk(); + if (incidentConfig === false) { + return 1; + } + + sendNewRelic(incidentConfig, reloadParams); + return null; + } catch (err) { + globals.logger.error(`NEWRELIC TASK FAIL: ${err}`); + return null; + } + }) + .catch((rateLimiterRes) => { + globals.logger.verbose( + `NEWRELIC TASK FAIL: Rate limiting failed. Not sending reload failure notification to New Relic for task "${reloadParams.taskName}"` + ); + globals.logger.verbose( + `NEWRELIC TASK FAIL: Rate limiting details "${JSON.stringify(rateLimiterRes, null, 2)}"` + ); + }); +} + +function sendReloadTaskAbortedNotification(reloadParams) { + rateLimiterAbortedReloads + .consume(reloadParams.taskId, 1) + .then(async (rateLimiterRes) => { + try { + globals.logger.info( + `NEWRELIC TASK ABORT: Rate limiting ok: Sending reload aborted notification to New Relic for task "${reloadParams.taskName}"` + ); + globals.logger.verbose( + `NEWRELIC TASK ABORT: Rate limiting details "${JSON.stringify(rateLimiterRes, null, 2)}"` + ); + + // Make sure outgoing webhooks are enabled in the config file and that we have all required settings + const incidentConfig = getReloadAbortedNotificationConfigOk(); + if (incidentConfig === false) { + return 1; + } + + sendNewRelic(incidentConfig, reloadParams); + return null; + } catch (err) { + globals.logger.error(`NEWRELIC TASK ABORT: ${err}`); + return null; + } + }) + .catch((rateLimiterRes) => { + globals.logger.verbose( + `NEWRELIC TASK ABORT: Rate limiting failed. Not sending reload aborted notification to New Relic for task "${reloadParams.taskName}"` + ); + globals.logger.verbose( + `NEWRELIC TASK ABORT: Rate limiting details "${JSON.stringify(rateLimiterRes, null, 2)}"` + ); + }); +} + +module.exports = { + sendReloadTaskFailureNotification, + sendReloadTaskAbortedNotification, +}; diff --git a/src/lib/post-to-influxdb.js b/src/lib/post-to-influxdb.js index 244c199b..decba8b4 100755 --- a/src/lib/post-to-influxdb.js +++ b/src/lib/post-to-influxdb.js @@ -10,10 +10,10 @@ function postButlerMemoryUsageToInfluxdb(memory) { butler_instance: memory.instanceTag, }, fields: { - heap_used: memory.heapUsed, - heap_total: memory.heapTotal, - external: memory.external, - process_memory: memory.processMemory, + heap_used: memory.heapUsedMByte, + heap_total: memory.heapTotalMByte, + external: memory.externalMemoryMByte, + process_memory: memory.processMemoryMByte, }, }, ]; diff --git a/src/lib/post-to-new-relic.js b/src/lib/post-to-new-relic.js new file mode 100755 index 00000000..9aa657be --- /dev/null +++ b/src/lib/post-to-new-relic.js @@ -0,0 +1,134 @@ +/* eslint-disable guard-for-in */ +const axios = require('axios'); + +const globals = require('../globals'); + +async function postButlerUptimeToNewRelic(fields) { + try { + const payload = []; + const metrics = []; + const attributes = {}; + const ts = new Date().getTime(); // Timestamp in millisec + + // Add static fields to attributes + if (globals.config.has('Butler.uptimeMonitor.storeNewRelic.attribute.static')) { + const staticAttributes = globals.config.get('Butler.uptimeMonitor.storeNewRelic.attribute.static'); + + // eslint-disable-next-line no-restricted-syntax + for (const item of staticAttributes) { + attributes[item.name] = item.value; + } + } + + // Add version to attributes + if ( + globals.config.has('Butler.uptimeMonitor.storeNewRelic.attribute.dynamic.butlerVersion.enable') && + globals.config.get('Butler.uptimeMonitor.storeNewRelic.attribute.dynamic.butlerVersion.enable') === true + ) { + attributes.version = globals.appVersion; + } + + const common = { + timestamp: ts, + 'interval.ms': fields.intervalMillisec, + attributes, + }; + + // Add memory usage + if ( + globals.config.has('Butler.uptimeMonitor.storeNewRelic.metric.dynamic.butlerMemoryUsage.enable') && + globals.config.get('Butler.uptimeMonitor.storeNewRelic.metric.dynamic.butlerMemoryUsage.enable') === true + ) { + metrics.push({ + name: 'heapUsed', + type: 'gauge', + value: fields.heapUsed, + }); + + metrics.push({ + name: 'heapTotal', + type: 'gauge', + value: fields.heapTotal, + }); + + metrics.push({ + name: 'externalMem', + type: 'gauge', + value: fields.externalMemory, + }); + + metrics.push({ + name: 'processMem', + type: 'gauge', + value: fields.processMemory, + }); + } + + // Add uptime + if ( + globals.config.has('Butler.uptimeMonitor.storeNewRelic.metric.dynamic.butlerUptime.enable') && + globals.config.get('Butler.uptimeMonitor.storeNewRelic.metric.dynamic.butlerUptime.enable') === true + ) { + metrics.push({ + name: 'uptimeMillisec', + type: 'gauge', + value: fields.uptimeMilliSec, + }); + } + + // Build final payload + payload.push({ + common, + metrics, + }); + + globals.logger.debug(`UPTIME NEW RELIC: Payload: ${JSON.stringify(payload, null, 2)}`); + + // Preapare call to remote host + const remoteUrl = globals.config.get('Butler.uptimeMonitor.storeNewRelic.url'); + + // Add headers + const headers = { + 'Content-Type': 'application/json', + 'Api-Key': globals.config.get('Butler.thirdPartyToolsCredentials.newRelic.insertApiKey'), + }; + + // eslint-disable-next-line no-restricted-syntax + for (const header of globals.config.get('Butler.uptimeMonitor.storeNewRelic.header')) { + headers[header.name] = header.value; + } + + const res = await axios.post(remoteUrl, payload, { headers }); + globals.logger.debug( + `UPTIME NEW RELIC: Result code from posting to New Relic: ${res.status}, ${res.statusText}` + ); + globals.logger.verbose(`UPTIME NEW RELIC: Sent Butler memory usage data to New Relic`); + } catch (error) { + // handle error + globals.logger.error(`UPTIME NEW RELIC: Error sending uptime: ${error}`); + } +} + +async function postFailedReloadEventToNewRelic() { + try { + // + } catch (error) { + // handle error + globals.logger.error(`UPTIME NEW RELIC: Error posting reload failed event: ${error}`); + } +} + +async function postAbortedReloadEventToNewRelic() { + try { + // + } catch (error) { + // handle error + globals.logger.error(`UPTIME NEW RELIC: Error posting reload aborted event: ${error}`); + } +} + +module.exports = { + postButlerUptimeToNewRelic, + postFailedReloadEventToNewRelic, + postAbortedReloadEventToNewRelic, +}; diff --git a/src/lib/service_uptime.js b/src/lib/service_uptime.js index eb7f413f..f5587508 100644 --- a/src/lib/service_uptime.js +++ b/src/lib/service_uptime.js @@ -4,6 +4,7 @@ require('moment-precise-range-plugin'); const globals = require('../globals'); const postToInfluxdb = require('./post-to-influxdb'); +const postToHttp = require('./post-to-new-relic'); function serviceUptimeStart() { const uptimeLogLevel = globals.config.get('Butler.uptimeMonitor.logLevel'); @@ -34,37 +35,69 @@ function serviceUptimeStart() { const startTime = Date.now(); let startIterations = 0; + // const intervalMillisec = later.parse.text(uptimeInterval); + const sched = later.parse.text(uptimeInterval); + const nextOccurence = later.schedule(sched).next(4); + const intervalMillisec = nextOccurence[3].getTime() - nextOccurence[2].getTime(); + globals.logger.debug(`UPTIME: Interval between uptime events: ${intervalMillisec} milliseconds`); + later.setInterval(() => { startIterations += 1; const uptimeMilliSec = Date.now() - startTime; moment.duration(uptimeMilliSec); - const heapTotal = Math.round((process.memoryUsage().heapTotal / 1024 / 1024) * 100) / 100; - const heapUsed = Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100; - const processMemory = Math.round((process.memoryUsage().rss / 1024 / 1024) * 100) / 100; - const external = Math.round((process.memoryUsage().external / 1024 / 1024) * 100) / 100; + const { heapTotal } = process.memoryUsage(); + const { heapUsed } = process.memoryUsage(); + const processMemory = process.memoryUsage().rss; + const externalMemory = process.memoryUsage().external; + + const heapTotalMByte = Math.round((heapTotal / 1024 / 1024) * 100) / 100; + const heapUsedMByte = Math.round((heapUsed / 1024 / 1024) * 100) / 100; + const processMemoryMByte = Math.round((processMemory / 1024 / 1024) * 100) / 100; + const externalMemoryMByte = Math.round((externalMemory / 1024 / 1024) * 100) / 100; + + const uptimeString = moment.preciseDiff(0, uptimeMilliSec); globals.logger.log(uptimeLogLevel, '--------------------------------'); globals.logger.log( uptimeLogLevel, - `Iteration # ${formatter.format(startIterations)}, Uptime: ${moment.preciseDiff( - 0, - uptimeMilliSec - )}, Heap used ${heapUsed} MB of total heap ${heapTotal} MB. External (off-heap): ${external} MB. Memory allocated to process: ${processMemory} MB.` + `Iteration # ${formatter.format( + startIterations + )}, Uptime: ${uptimeString}, Heap used ${heapUsedMByte} MB of total heap ${heapTotalMByte} MB. External (off-heap): ${externalMemoryMByte} MB. Memory allocated to process: ${processMemoryMByte} MB.` ); - // Store to Influxdb + // Store to Influxdb if enabled const butlerMemoryInfluxTag = globals.config.has('Butler.uptimeMonitor.storeInInfluxdb.instanceTag') ? globals.config.get('Butler.uptimeMonitor.storeInInfluxdb.instanceTag') : ''; - if (globals.config.get('Butler.uptimeMonitor.storeInInfluxdb.enable') === true) { + if ( + globals.config.has('Butler.uptimeMonitor.storeInInfluxdb.enable') && + globals.config.get('Butler.uptimeMonitor.storeInInfluxdb.enable') === true + ) { postToInfluxdb.postButlerMemoryUsageToInfluxdb({ instanceTag: butlerMemoryInfluxTag, + heapUsedMByte, + heapTotalMByte, + externalMemoryMByte, + processMemoryMByte, + }); + } + + // Do generic http POST if enabled + if ( + globals.config.has('Butler.uptimeMonitor.storeNewRelic.enable') && + globals.config.get('Butler.uptimeMonitor.storeNewRelic.enable') === true + ) { + postToHttp.postButlerUptimeToNewRelic({ + intervalMillisec, heapUsed, heapTotal, - external, + externalMemory, processMemory, + startIterations, + uptimeMilliSec, + uptimeString, }); } }, later.parse.text(uptimeInterval)); diff --git a/src/lib/telemetry.js b/src/lib/telemetry.js index 4bdde75d..01dbf433 100644 --- a/src/lib/telemetry.js +++ b/src/lib/telemetry.js @@ -29,7 +29,8 @@ const callRemoteURL = async () => { heartbeat: globals.config.has('Butler.heartbeat.enable') ? globals.config.get('Butler.heartbeat.enable') : false, dockerHealthCheck: globals.config.has('Butler.dockerHealthCheck.enable') ? globals.config.get('Butler.dockerHealthCheck.enable') : false, uptimeMonitor: globals.config.has('Butler.uptimeMonitor.enable') ? globals.config.get('Butler.uptimeMonitor.enable') : false, - uptimeMonitor_storeInInfluxdb: globals.config.has('Butler.uptimeMonitor.storeInInfluxdb.butlerSOSMemoryUsage') ? globals.config.get('Butler.uptimeMonitor.storeInInfluxdb.butlerSOSMemoryUsage') : false, + uptimeMonitor_storeInInfluxdb: globals.config.has('Butler.uptimeMonitor.storeInInfluxdb.enable') ? globals.config.get('Butler.uptimeMonitor.storeInInfluxdb.enable') : false, + uptimeMonitor_storeInNewRelic: globals.config.has('Butler.uptimeMonitor.storeNewRelic.enable') ? globals.config.get('Butler.uptimeMonitor.storeNewRelic.enable') : false, teamsNotification: globals.config.has('Butler.teamsNotification.enable') ? globals.config.get('Butler.teamsNotification.enable') : false, teamsNotification_reloadTaskFailure: globals.config.has('Butler.teamsNotification.reloadTaskFailure.enable') ? globals.config.get('Butler.teamsNotification.reloadTaskFailure.enable') : false, teamsNotification_reloadTaskAborted: globals.config.has('Butler.teamsNotification.reloadTaskAborted.enable') ? globals.config.get('Butler.teamsNotification.reloadTaskAborted.enable') : false, @@ -42,6 +43,10 @@ const callRemoteURL = async () => { webhookNotification: globals.config.has('Butler.webhookNotification.enable') ? globals.config.get('Butler.webhookNotification.enable') : false, webhookNotification_reloadTaskFailure: globals.config.has('Butler.webhookNotification.reloadTaskFailure.enable') ? globals.config.get('Butler.webhookNotification.reloadTaskFailure.enable') : false, webhookNotification_reloadTaskAborted: globals.config.has('Butler.webhookNotification.reloadTaskAborted.enable') ? globals.config.get('Butler.webhookNotification.reloadTaskAborted.enable') : false, + signl4Notification_reloadTaskFailure: globals.config.has('Butler.incidentTool.signl4.reloadTaskFailure.enable') ? globals.config.get('Butler.incidentTool.signl4.reloadTaskFailure.enable') : false, + signl4Notification_reloadTaskAborted: globals.config.has('Butler.incidentTool.signl4.reloadTaskAborted.enable') ? globals.config.get('Butler.incidentTool.signl4.reloadTaskAborted.enable') : false, + newRelicNotification_reloadTaskFailure: globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.enable') ? globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.enable') : false, + newRelicNotification_reloadTaskAborted: globals.config.has('Butler.incidentTool.newRelic.reloadTaskAborted.enable') ? globals.config.get('Butler.incidentTool.newRelic.reloadTaskAborted.enable') : false, scheduler: globals.config.has('Butler.scheduler.enable') ? globals.config.get('Butler.scheduler.enable') : false, mqtt: globals.config.has('Butler.mqttConfig.enable') ? globals.config.get('Butler.mqttConfig.enable') : false, userActivityLogging: globals.config.has('Butler.userActivityLogging.enable') ? globals.config.get('Butler.userActivityLogging.enable') : false, diff --git a/src/package-lock.json b/src/package-lock.json index c9d14580..ef1ff400 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -28,6 +28,7 @@ "fastify-autoload": "^3.12.0", "fastify-healthcheck": "^3.1.0", "fastify-plugin": "^3.0.1", + "fastify-rate-limit": "^5.8.0", "fastify-reply-from": "^6.6.0", "fastify-sensible": "^3.1.2", "fastify-swagger": "^5.1.1", @@ -3268,6 +3269,21 @@ "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-3.0.1.tgz", "integrity": "sha512-qKcDXmuZadJqdTm6vlCqioEbyewF60b/0LOFCcYN1B6BIZGlYJumWWOYs70SFYLDAH4YqdE1cxH/RKMG7rFxgA==" }, + "node_modules/fastify-rate-limit": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/fastify-rate-limit/-/fastify-rate-limit-5.8.0.tgz", + "integrity": "sha512-sln2ZbEG1cb0Ok4pn+tXrZIU0zJUWEimANWB/Bq+z/Ad5fBys9YsmCySrPqhUdBcZHwk9ymX22wbgZvvNLokyQ==", + "dependencies": { + "fastify-plugin": "^3.0.1", + "ms": "^2.1.3", + "tiny-lru": "^8.0.1" + } + }, + "node_modules/fastify-rate-limit/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, "node_modules/fastify-reply-from": { "version": "6.6.0", "resolved": "https://registry.npmjs.org/fastify-reply-from/-/fastify-reply-from-6.6.0.tgz", @@ -10309,6 +10325,23 @@ "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-3.0.1.tgz", "integrity": "sha512-qKcDXmuZadJqdTm6vlCqioEbyewF60b/0LOFCcYN1B6BIZGlYJumWWOYs70SFYLDAH4YqdE1cxH/RKMG7rFxgA==" }, + "fastify-rate-limit": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/fastify-rate-limit/-/fastify-rate-limit-5.8.0.tgz", + "integrity": "sha512-sln2ZbEG1cb0Ok4pn+tXrZIU0zJUWEimANWB/Bq+z/Ad5fBys9YsmCySrPqhUdBcZHwk9ymX22wbgZvvNLokyQ==", + "requires": { + "fastify-plugin": "^3.0.1", + "ms": "^2.1.3", + "tiny-lru": "^8.0.1" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + }, "fastify-reply-from": { "version": "6.6.0", "resolved": "https://registry.npmjs.org/fastify-reply-from/-/fastify-reply-from-6.6.0.tgz", diff --git a/src/package.json b/src/package.json index d0787921..84871ee2 100644 --- a/src/package.json +++ b/src/package.json @@ -22,6 +22,7 @@ "fastify-autoload": "^3.12.0", "fastify-healthcheck": "^3.1.0", "fastify-plugin": "^3.0.1", + "fastify-rate-limit": "^5.8.0", "fastify-reply-from": "^6.6.0", "fastify-sensible": "^3.1.2", "fastify-swagger": "^5.1.1", diff --git a/src/routes/senseStartTask.js b/src/routes/senseStartTask.js index 7eec52f2..353e37a4 100644 --- a/src/routes/senseStartTask.js +++ b/src/routes/senseStartTask.js @@ -310,7 +310,7 @@ async function handlerPutStartTask(request, reply) { res.tasksId.invalid = tasksInvalid; // eslint-disable-next-line no-restricted-syntax for (const item of tasksToStartTaskId) { - res.tasksId.denied.push({taskId: item.taskId}) + res.tasksId.denied.push({ taskId: item.taskId }); } } } else { diff --git a/src/udp/udp_handlers.js b/src/udp/udp_handlers.js index 2200ac09..534fe945 100644 --- a/src/udp/udp_handlers.js +++ b/src/udp/udp_handlers.js @@ -5,6 +5,7 @@ const slack = require('../lib/slack_notification'); const webhookOut = require('../lib/webhook_notification'); const msteams = require('../lib/msteams_notification'); const signl4 = require('../lib/incident_mgmt/signl4'); +const newRelic = require('../lib/incident_mgmt/new-relic'); const { failedTaskStoreLogOnDisk } = require('../lib/scriptlog'); // Handler for failed scheduler initiated reloads @@ -204,6 +205,27 @@ const schedulerFailed = (msg, legacyFlag) => { }); } + // Post to New Relic when a task has failed, if enabled + if ( + globals.config.has('Butler.incidentTool.newRelic.enable') && + globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.enable') && + globals.config.get('Butler.incidentTool.newRelic.enable') === true && + globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.enable') === true + ) { + newRelic.sendReloadTaskFailureNotification({ + qs_hostName: msg[0], + qs_user: msg[3].replace(/\\/g, '/'), + qs_taskName: msg[1], + qs_taskId: msg[4], + qs_appName: msg[2], + qs_appId: msg[5], + qs_logTimeStamp: msg[6], + qs_logLevel: msg[7], + qs_executionId: msg[8], + qs_logMessage: msg[9], + }); + } + // Post to Slack when a task has failed, if Slack is enabled if ( globals.config.has('Butler.slackNotification.enable') && @@ -364,6 +386,27 @@ const schedulerFailed = (msg, legacyFlag) => { }); } + // Post to New Relic when a task has failed, if enabled + if ( + globals.config.has('Butler.incidentTool.newRelic.enable') && + globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.enable') && + globals.config.get('Butler.incidentTool.newRelic.enable') === true && + globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.enable') === true + ) { + newRelic.sendReloadTaskFailureNotification({ + qs_hostName: msg[1], + qs_user: msg[4].replace(/\\/g, '/'), + qs_taskName: msg[2], + qs_taskId: msg[5], + qs_appName: msg[3], + qs_appId: msg[6], + qs_logTimeStamp: msg[7], + qs_logLevel: msg[8], + qs_executionId: msg[9], + qs_logMessage: msg[10], + }); + } + // Post to Slack when a task has failed, if Slack is enabled if ( globals.config.has('Butler.slackNotification.enable') &&