Skip to content

Commit

Permalink
Showing 5 changed files with 515 additions and 109 deletions.
89 changes: 61 additions & 28 deletions src/config/production_template.yaml
Original file line number Diff line number Diff line change
@@ -265,38 +265,71 @@ Butler:
severity: 10 # Signl4 severity level for aborted reloads
newRelic:
enable: false
# New Relic uses different API URLs for different kinds of data (metrics, events, logs, ...)
# There are different URLs depending on whther you have an EU or US region New Relic account.
# The available URLs are listed here: https://docs.newrelic.com/docs/accounts/accounts-billing/account-setup/choose-your-data-center/
#
# Note that the URL path should *not* be included in the url setting below!
# As of this writing the valid options are
# https://insights-collector.eu01.nr-data.net
# https://insights-collector.newrelic.com
url: https://insights-collector.eu01.nr-data.net
url:
# As of this writing the valid options are
# https://insights-collector.eu01.nr-data.net
# https://insights-collector.newrelic.com
event: https://insights-collector.eu01.nr-data.net

# Valid options are (1) EU/rest of world and 2) US)
# https://log-api.eu.newrelic.com/log/v1
# https://log-api.newrelic.com/log/v1
log: https://log-api.eu.newrelic.com/log/v1
reloadTaskFailure:
enable: true
rateLimit: 15 # Min seconds between events sent to New Relic for a given taskID. Defaults to 5 minutes.
header: # Custom http headers
- name: X-My-Header
value: Header value
attribute:
static: # Static attributes/dimensions to attach to events sent to New Relic.
- name: service
value: butler
- name: environment
value: prod
destination:
event:
enable: true
attribute:
static: # Static attributes/dimensions to attach to events sent to New Relic.
- name: event-specific-attribute 1
value: abc 123
log:
enable: true
tailScriptLogLines: 20
attribute:
static: # Static attributes/dimensions to attach to events sent to New Relic.
- name: log-specific-attribute 1
value: def 123
sharedSettings:
rateLimit: 15 # Min seconds between events sent to New Relic for a given taskID. Defaults to 5 minutes.
header: # Custom http headers
- name: X-My-Header
value: Header value 1
attribute:
static: # Static attributes/dimensions to attach to events sent to New Relic.
- name: service
value: butler
- name: environment
value: prod
reloadTaskAborted:
enable: true
rateLimit: 15 # Min seconds between events sent to New Relic for a given taskID. Defaults to 5 minutes.
header: # Custom http headers
- name: X-My-Header
value: Header value
attribute:
static: # Static attributes/dimensions to attach to events sent to New Relic.
- name: service
value: butler
- name: environment
value: prod
destination:
event:
enable: true
attribute:
static: # Static attributes/dimensions to attach to events sent to New Relic.
- name: event-specific-attribute 2
value: abc 123
log:
enable: true
tailScriptLogLines: 20
attribute:
static: # Static attributes/dimensions to attach to events sent to New Relic.
- name: log-specific-attribute 2
value: def 123
sharedSettings:
rateLimit: 15 # Min seconds between events sent to New Relic for a given taskID. Defaults to 5 minutes.
header: # Custom http headers
- name: X-My-Header
value: Header value 2
attribute:
static: # Static attributes/dimensions to attach to events sent to New Relic.
- name: service
value: butler
- name: environment
value: prod

# Settings for notifications and messages sent using outgoing webhooks
webhookNotification:
415 changes: 352 additions & 63 deletions src/lib/incident_mgmt/new_relic.js

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/lib/post_to_new_relic.js
Original file line number Diff line number Diff line change
@@ -89,7 +89,7 @@ async function postButlerUptimeToNewRelic(fields) {

// Add headers
const headers = {
'Content-Type': 'application/json',
'Content-Type': 'application/json; charset=utf-8',
'Api-Key': globals.config.get('Butler.thirdPartyToolsCredentials.newRelic.insertApiKey'),
};

2 changes: 1 addition & 1 deletion src/routes/newrelic_metric.js
Original file line number Diff line number Diff line change
@@ -66,7 +66,7 @@ async function handlerPostNewRelicMetric(request, reply) {

// Add headers
const headers = {
'Content-Type': 'application/json',
'Content-Type': 'application/json; charset=utf-8',
'Api-Key': globals.config.get('Butler.thirdPartyToolsCredentials.newRelic.insertApiKey'),
};

116 changes: 100 additions & 16 deletions src/udp/udp_handlers.js
Original file line number Diff line number Diff line change
@@ -11,12 +11,12 @@ const { failedTaskStoreLogOnDisk } = require('../lib/scriptlog');
// Handler for failed scheduler initiated reloads
const schedulerAborted = (msg) => {
globals.logger.verbose(
`TASKFABORTED: Received reload aborted UDP message from scheduler: Host=${msg[1]}, App name=${msg[3]}, User=${msg[4]}, Log level=${msg[8]}`
`TASKABORTED: Received reload aborted UDP message from scheduler: UDP msg=${msg[0]}, Host=${msg[1]}, App name=${msg[3]}, Task name=${msg[2]}, Log level=${msg[8]}, Log msg=${msg[10]}`
);

// First field in message (msg[0]) is message category (this is the modern/recent message format)

// Post to Signl4 when a task has failed, if enabled
// Post to Signl4 when a task has been aborted
if (
globals.config.has('Butler.incidentTool.signl4.enable') &&
globals.config.has('Butler.incidentTool.signl4.reloadTaskAborted.enable') &&
@@ -37,6 +37,48 @@ const schedulerAborted = (msg) => {
});
}

// Post event to New Relic when a task has been aborted
if (
globals.config.has('Butler.incidentTool.newRelic.enable') &&
globals.config.has('Butler.incidentTool.newRelic.reloadTaskAborted.destination.event.enable') &&
globals.config.get('Butler.incidentTool.newRelic.enable') === true &&
globals.config.get('Butler.incidentTool.newRelic.reloadTaskAborted.destination.event.enable') === true
) {
newRelic.sendReloadTaskAbortedEvent({
qs_hostName: msg[1],
qs_user: msg[4].replace(/\\/g, '/'),
qs_taskName: msg[2],
qs_taskId: msg[5],
qs_appName: msg[3],
qs_appId: msg[6],
qs_logTimeStamp: msg[7],
qs_logLevel: msg[8],
qs_executionId: msg[9],
qs_logMessage: msg[10],
});
}

// Post log to New Relic when a task has been aborted
if (
globals.config.has('Butler.incidentTool.newRelic.enable') &&
globals.config.has('Butler.incidentTool.newRelic.reloadTaskAborted.destination.log.enable') &&
globals.config.get('Butler.incidentTool.newRelic.enable') === true &&
globals.config.get('Butler.incidentTool.newRelic.reloadTaskAborted.destination.log.enable') === true
) {
newRelic.sendReloadTaskAbortedLog({
qs_hostName: msg[1],
qs_user: msg[4].replace(/\\/g, '/'),
qs_taskName: msg[2],
qs_taskId: msg[5],
qs_appName: msg[3],
qs_appId: msg[6],
qs_logTimeStamp: msg[7],
qs_logLevel: msg[8],
qs_executionId: msg[9],
qs_logMessage: msg[10],
});
}

// Post to Slack when a reload task has been aborted (typically from the QMC, or via APIs), if Slack is enabled
if (
globals.config.has('Butler.slackNotification.enable') &&
@@ -155,10 +197,6 @@ const schedulerAborted = (msg) => {

// Handler for failed scheduler initiated reloads
const schedulerFailed = (msg, legacyFlag) => {
globals.logger.verbose(
`TASKFAILURE: Received reload failed UDP message from scheduler: Host=${msg[0]}, App name=${msg[2]}, Task name=${msg[1]}, Log level=${msg[7]}`
);

if (legacyFlag) {
// First field in message (msg[0]) is host name

@@ -181,7 +219,7 @@ const schedulerFailed = (msg, legacyFlag) => {
});
}

// Post to Signl4 when a task has failed, if enabled
// Post to Signl4 when a task has failed
if (
globals.config.has('Butler.incidentTool.signl4.enable') &&
globals.config.has('Butler.incidentTool.signl4.reloadTaskFailure.enable') &&
@@ -202,14 +240,35 @@ const schedulerFailed = (msg, legacyFlag) => {
});
}

// Post to New Relic when a task has failed, if enabled
// Post event to New Relic when a task has failed
if (
globals.config.has('Butler.incidentTool.newRelic.enable') &&
globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.enable') &&
globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.destination.event.enable') &&
globals.config.get('Butler.incidentTool.newRelic.enable') === true &&
globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.enable') === true
globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.destination.event.enable') === true
) {
newRelic.sendReloadTaskFailureNotification({
newRelic.sendReloadTaskFailureEvent({
qs_hostName: msg[0],
qs_user: msg[3].replace(/\\/g, '/'),
qs_taskName: msg[1],
qs_taskId: msg[4],
qs_appName: msg[2],
qs_appId: msg[5],
qs_logTimeStamp: msg[6],
qs_logLevel: msg[7],
qs_executionId: msg[8],
qs_logMessage: msg[9],
});
}

// Post log to New Relic when a task has failed
if (
globals.config.has('Butler.incidentTool.newRelic.enable') &&
globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.destination.log.enable') &&
globals.config.get('Butler.incidentTool.newRelic.enable') === true &&
globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.destination.log.enable') === true
) {
newRelic.sendReloadTaskFailureLog({
qs_hostName: msg[0],
qs_user: msg[3].replace(/\\/g, '/'),
qs_taskName: msg[1],
@@ -341,6 +400,10 @@ const schedulerFailed = (msg, legacyFlag) => {
);
}
} else {
globals.logger.verbose(
`TASKFAILURE: Received reload failed UDP message from scheduler: UDP msg=${msg[0]}, Host=${msg[1]}, App name=${msg[3]}, Task name=${msg[2]}, Log level=${msg[8]}, Log msg=${msg[10]}`
);

// First field in message (msg[0]) is message category (this is the modern/recent message format)

// Store script log to disk
@@ -362,7 +425,7 @@ const schedulerFailed = (msg, legacyFlag) => {
});
}

// Post to Signl4 when a task has failed, if enabled
// Post to Signl4 when a task has failed
if (
globals.config.has('Butler.incidentTool.signl4.enable') &&
globals.config.has('Butler.incidentTool.signl4.reloadTaskFailure.enable') &&
@@ -383,14 +446,35 @@ const schedulerFailed = (msg, legacyFlag) => {
});
}

// Post to New Relic when a task has failed, if enabled
// Post evemt to New Relic when a task has failed
if (
globals.config.has('Butler.incidentTool.newRelic.enable') &&
globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.destination.event.enable') &&
globals.config.get('Butler.incidentTool.newRelic.enable') === true &&
globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.destination.event.enable') === true
) {
newRelic.sendReloadTaskFailureEvent({
qs_hostName: msg[1],
qs_user: msg[4].replace(/\\/g, '/'),
qs_taskName: msg[2],
qs_taskId: msg[5],
qs_appName: msg[3],
qs_appId: msg[6],
qs_logTimeStamp: msg[7],
qs_logLevel: msg[8],
qs_executionId: msg[9],
qs_logMessage: msg[10],
});
}

// Post log to New Relic when a task has failed
if (
globals.config.has('Butler.incidentTool.newRelic.enable') &&
globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.enable') &&
globals.config.has('Butler.incidentTool.newRelic.reloadTaskFailure.destination.log.enable') &&
globals.config.get('Butler.incidentTool.newRelic.enable') === true &&
globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.enable') === true
globals.config.get('Butler.incidentTool.newRelic.reloadTaskFailure.destination.log.enable') === true
) {
newRelic.sendReloadTaskFailureNotification({
newRelic.sendReloadTaskFailureLog({
qs_hostName: msg[1],
qs_user: msg[4].replace(/\\/g, '/'),
qs_taskName: msg[2],

0 comments on commit d1237ab

Please sign in to comment.