Skip to content

Commit

Permalink
fix(alerts): Empty alert recipient lists no longer cause errors
Browse files Browse the repository at this point in the history 
Fixes #640
  • Loading branch information
@mountaindude
mountaindude committed Aug 22, 2023
1 parent b6312ac commit 34421ce
Show file tree
Hide file tree
Showing 4 changed files with 122 additions and 53 deletions.
19 changes: 15 additions & 4 deletions src/lib/incident_mgmt/new_relic.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,9 +430,13 @@ async function sendNewRelicLog(incidentConfig, reloadParams, destNewRelicAccount
);

scriptLogData.scriptLogHead = '';
scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
if (scriptLogData?.scriptLogFull?.length > 0) {
scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
} else {
scriptLogData.scriptLogTail = '';
}

globals.logger.debug(`NEW RELIC TASK FAILED LOG: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);

Expand Down Expand Up @@ -520,7 +524,14 @@ async function sendNewRelicLog(incidentConfig, reloadParams, destNewRelicAccount
}
}
} catch (err) {
globals.logger.error(`NEWRELIC 2: ${JSON.stringify(err, null, 2)}`);
if (err.message) {
globals.logger.error(`NEWRELIC 2 message: ${err.message}`);
}

if (err.stack) {
globals.logger.error(`NEWRELIC 2 stack: ${err.stack}`);
}

}
}

Expand Down
27 changes: 19 additions & 8 deletions src/lib/msteams_notification.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,10 +440,16 @@ function sendReloadTaskFailureNotificationTeams(reloadParams) {
scriptLogData.scriptLogHeadCount = globals.config.get('Butler.teamsNotification.reloadTaskFailure.headScriptLogLines');
scriptLogData.scriptLogTailCount = globals.config.get('Butler.teamsNotification.reloadTaskFailure.tailScriptLogLines');

scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');
scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
if (scriptLogData?.scriptLogFull?.length > 0) {
scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');

scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
} else {
scriptLogData.scriptLogHead = '';
scriptLogData.scriptLogTail = '';
}

globals.logger.debug(`TASK FAILED ALERT TEAMS: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);

Expand Down Expand Up @@ -573,10 +579,15 @@ function sendReloadTaskAbortedNotificationTeams(reloadParams) {
scriptLogData.scriptLogHeadCount = globals.config.get('Butler.teamsNotification.reloadTaskAborted.headScriptLogLines');
scriptLogData.scriptLogTailCount = globals.config.get('Butler.teamsNotification.reloadTaskAborted.tailScriptLogLines');

scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');
scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
if (scriptLogData?.scriptLogFull?.length > 0) {
scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');
scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
} else {
scriptLogData.scriptLogHead = '';
scriptLogData.scriptLogTail = '';
}

globals.logger.debug(`TASK ABORTED ALERT TEAMS: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);

Expand Down
28 changes: 20 additions & 8 deletions src/lib/slack_notification.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,10 +482,16 @@ function sendReloadTaskFailureNotificationSlack(reloadParams) {
scriptLogData.scriptLogHeadCount = globals.config.get('Butler.slackNotification.reloadTaskFailure.headScriptLogLines');
scriptLogData.scriptLogTailCount = globals.config.get('Butler.slackNotification.reloadTaskFailure.tailScriptLogLines');

scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');
scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
if (scriptLogData?.scriptLogFull?.length > 0) {
scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');

scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
} else {
scriptLogData.scriptLogHead = '';
scriptLogData.scriptLogTail = '';
}

globals.logger.debug(`TASK FAILED ALERT SLACK: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);

Expand Down Expand Up @@ -631,10 +637,16 @@ function sendReloadTaskAbortedNotificationSlack(reloadParams) {
scriptLogData.scriptLogHeadCount = globals.config.get('Butler.slackNotification.reloadTaskAborted.headScriptLogLines');
scriptLogData.scriptLogTailCount = globals.config.get('Butler.slackNotification.reloadTaskAborted.tailScriptLogLines');

scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');
scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
if (scriptLogData?.scriptLogFull?.length > 0) {
scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');

scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
} else {
scriptLogData.scriptLogHead = '';
scriptLogData.scriptLogTail = '';
}

globals.logger.debug(`TASK ABORTED ALERT SLACK: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);

Expand Down
101 changes: 68 additions & 33 deletions src/lib/smtp.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ async function sendReloadTaskFailureNotificationEmail(reloadParams) {
emailAlertCpTaskSpecificEmailAddressName
);

if (taskSpecificAlertEmailAddresses.length > 0) {
if (taskSpecificAlertEmailAddresses?.length > 0) {
globals.logger.debug(
`TASK FAILED ALERT EMAIL: Added task specific send list: ${JSON.stringify(taskSpecificAlertEmailAddresses, null, 2)}`
);
Expand All @@ -390,7 +390,7 @@ async function sendReloadTaskFailureNotificationEmail(reloadParams) {
// 2.1 Yes: Add system-wide list of recipients to send list
globals.logger.verbose(`TASK FAILED ALERT EMAIL: Send alert emails for all tasks`);

if (globalSendList.length > 0) {
if (globalSendList?.length > 0) {
globals.logger.debug(
`TASK FAILED ALERT EMAIL: Added global send list for failed task: ${JSON.stringify(globalSendList, null, 2)}`
);
Expand All @@ -412,7 +412,7 @@ async function sendReloadTaskFailureNotificationEmail(reloadParams) {
`TASK FAILED ALERT EMAIL: Added send list based on email-alert-CP: ${JSON.stringify(globalSendList, null, 2)}`
);
// 2.2.1 Yes: Add system-wide list of recipients to send list
if (globalSendList.length > 0) {
if (globalSendList?.length > 0) {
mainSendList = mainSendList.concat(globalSendList);
}
} else {
Expand Down Expand Up @@ -441,35 +441,47 @@ async function sendReloadTaskFailureNotificationEmail(reloadParams) {
// Is app owner on include list, i.e. list of app owners that should get notification emails?
// 3.1.2 No : Add list of specified app owners' email addresses to app owner send list
const includeUsers = globals.config.get('Butler.emailNotification.reloadTaskFailure.appOwnerAlert.includeOwner.user');
const matchUsers = includeUsers.filter((user) => user.directory === appOwner.directory && user.userId === appOwner.userId);
if (matchUsers.length > 0) {
const matchUsers = includeUsers?.filter((user) => user.directory === appOwner.directory && user.userId === appOwner.userId);
if (matchUsers?.length > 0) {
// App owner is in list of included users. Add to app owner send list.
appOwnerSendList.push({
email: appOwner.emails,
directory: appOwner.directory,
userId: appOwner.userId,
});
} else {
// No app owners on include list. Warn about this.
globals.logger.warn(
`TASK FAILED ALERT EMAIL: No app owners on include list for failed task. No app owners will receive notification emails.`
);
}
}

// Now evaluate the app owner exclude list
// 3.2 Is there an app owner exclude list?
const excludeUsers = globals.config.get('Butler.emailNotification.reloadTaskFailure.appOwnerAlert.excludeOwner.user');
const matchExcludedUsers = excludeUsers.filter(
const matchExcludedUsers = excludeUsers?.filter(
(user) => user.directory === appOwner.directory && user.userId === appOwner.userId
);
if (matchExcludedUsers.length > 0) {
if (matchExcludedUsers?.length > 0) {
// App owner is in list of app owners that should NOT receive notification emails
// Remove app owner email address from app owner send list (if it's already there)
// 3.2.1 Yes: Remove entries on the exclude list from app owner send list
appOwnerSendList = appOwnerSendList.filter(
appOwnerSendList = appOwnerSendList?.filter(
(user) => user.directory !== appOwner.directory && user.userId !== appOwner.userId
);
}
// 3.3 Add app owner send list to main send list
const appOwnerSendListEmails = appOwnerSendList.map((item) => item.email);
if (appOwnerSendListEmails.length > 0) {
mainSendList = mainSendList.concat(appOwnerSendListEmails[0]);
const appOwnerSendListEmails = appOwnerSendList?.map((item) => item.email);
if (appOwnerSendListEmails?.length > 0) {
mainSendList = mainSendList?.concat(appOwnerSendListEmails[0]);
}

// Does the main sendlist contain any email addresses? Warn if not
if (mainSendList?.length === 0) {
globals.logger.warn(
`TASK FAILED ALERT EMAIL: No email addresses defined for alert email to app "${reloadParams.appName}", ID=${reloadParams.appId}`
);
}
} else {
globals.logger.warn(
Expand All @@ -492,14 +504,20 @@ async function sendReloadTaskFailureNotificationEmail(reloadParams) {
// Get script logs, if enabled in the config file
const scriptLogData = reloadParams.scriptLog;

// Reduce script log lines to only the ones we want to send to Slack
// Reduce script log lines to only the ones we want to send in email
scriptLogData.scriptLogHeadCount = globals.config.get('Butler.emailNotification.reloadTaskFailure.headScriptLogLines');
scriptLogData.scriptLogTailCount = globals.config.get('Butler.emailNotification.reloadTaskFailure.tailScriptLogLines');

scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');
scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
if (scriptLogData?.scriptLogFull?.length > 0) {
scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');

scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
} else {
scriptLogData.scriptLogHead = '';
scriptLogData.scriptLogTail = '';
}

globals.logger.debug(`TASK FAILED ALERT EMAIL: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);

Expand Down Expand Up @@ -614,7 +632,7 @@ async function sendReloadTaskAbortedNotificationEmail(reloadParams) {
emailAlertCpTaskSpecificEmailAddressName
);

if (taskSpecificAlertEmailAddresses.length > 0) {
if (taskSpecificAlertEmailAddresses?.length > 0) {
mainSendList = mainSendList.concat(taskSpecificAlertEmailAddresses);
}

Expand All @@ -623,7 +641,7 @@ async function sendReloadTaskAbortedNotificationEmail(reloadParams) {
// 2.1 Yes: Add system-wide list of recipients to send list
globals.logger.verbose(`TASK ABORTED ALERT EMAIL: Send alert emails for all tasks`);

if (globalSendList.length > 0) {
if (globalSendList?.length > 0) {
mainSendList = mainSendList.concat(globalSendList);
}
} else {
Expand All @@ -639,7 +657,7 @@ async function sendReloadTaskAbortedNotificationEmail(reloadParams) {

if (sendAlert === true) {
// 2.2.1 Yes: Add system-wide list of recipients to send list
if (globalSendList.length > 0) {
if (globalSendList?.length > 0) {
mainSendList = mainSendList.concat(globalSendList);
}
} else {
Expand Down Expand Up @@ -668,8 +686,8 @@ async function sendReloadTaskAbortedNotificationEmail(reloadParams) {
// Is app owner on include list, i.e. list of app owners that should get notification emails?
// 3.1.2 No : Add list of specified app owners' email addresses to app owner send list
const includeUsers = globals.config.get('Butler.emailNotification.reloadTaskAborted.appOwnerAlert.includeOwner.user');
const matchUsers = includeUsers.filter((user) => user.directory === appOwner.directory && user.userId === appOwner.userId);
if (matchUsers.length > 0) {
const matchUsers = includeUsers?.filter((user) => user.directory === appOwner.directory && user.userId === appOwner.userId);
if (matchUsers?.length > 0) {
// App owner is in list of included users. Add to app owner send list.
appOwnerSendList.push({
email: appOwner.emails,
Expand All @@ -682,22 +700,30 @@ async function sendReloadTaskAbortedNotificationEmail(reloadParams) {
// Now evaluate the app owner exclude list
// 3.2 Is there an app owner exclude list?
const excludeUsers = globals.config.get('Butler.emailNotification.reloadTaskAborted.appOwnerAlert.excludeOwner.user');
const matchExcludedUsers = excludeUsers.filter(
const matchExcludedUsers = excludeUsers?.filter(
(user) => user.directory === appOwner.directory && user.userId === appOwner.userId
);
if (matchExcludedUsers.length > 0) {
if (matchExcludedUsers?.length > 0) {
// App owner is in list of app owners that should NOT receive notification emails
// Remove app owner email address from app owner send list (if it's already there)
// 3.2.1 Yes: Remove entries on the exclude list from app owner send list
appOwnerSendList = appOwnerSendList.filter(
appOwnerSendList = appOwnerSendList?.filter(
(user) => user.directory !== appOwner.directory && user.userId !== appOwner.userId
);
}
// 3.3 Add app owner send list to main send list
const appOwnerSendListEmails = appOwnerSendList.map((item) => item.email);
if (appOwnerSendListEmails.length > 0) {
mainSendList = mainSendList.concat(appOwnerSendListEmails[0]);
const appOwnerSendListEmails = appOwnerSendList?.map((item) => item.email);
if (appOwnerSendListEmails?.length > 0) {
mainSendList = mainSendList?.concat(appOwnerSendListEmails[0]);
}

// Does the main sendlist contain any email addresses? Warn if not
if (mainSendList?.length === 0) {
globals.logger.warn(
`TASK ABORTED ALERT EMAIL: No email addresses defined for alert email to app "${reloadParams.appName}", ID=${reloadParams.appId}`
);
}

} else {
globals.logger.warn(
`TASK ABORTED ALERT EMAIL: No email address for owner of app "${reloadParams.appName}", ID=${reloadParams.appId}`
Expand All @@ -719,14 +745,20 @@ async function sendReloadTaskAbortedNotificationEmail(reloadParams) {
// Get script logs, if enabled in the config file
const scriptLogData = reloadParams.scriptLog;

// Reduce script log lines to only the ones we want to send to Slack
// Reduce script log lines to only the ones we want to send in email
scriptLogData.scriptLogHeadCount = globals.config.get('Butler.emailNotification.reloadTaskAborted.headScriptLogLines');
scriptLogData.scriptLogTailCount = globals.config.get('Butler.emailNotification.reloadTaskAborted.tailScriptLogLines');

scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');
scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
if (scriptLogData?.scriptLogFull?.length > 0) {
scriptLogData.scriptLogHead = scriptLogData.scriptLogFull.slice(0, scriptLogData.scriptLogHeadCount).join('\r\n');

scriptLogData.scriptLogTail = scriptLogData.scriptLogFull
.slice(Math.max(scriptLogData.scriptLogFull.length - scriptLogData.scriptLogTailCount, 0))
.join('\r\n');
} else {
scriptLogData.scriptLogHead = '';
scriptLogData.scriptLogTail = '';
}

globals.logger.debug(`TASK ABORTED ALERT EMAIL: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`);

Expand Down Expand Up @@ -829,8 +861,11 @@ async function sendServiceMonitorNotificationEmail(serviceParams) {

let mainSendList = [];

if (globalSendList.length > 0) {
if (globalSendList?.length > 0) {
mainSendList = mainSendList.concat(globalSendList);
} else {
// Warn there are no recipients to send email to
globals.logger.warn(`SERVICE MONITOR EMAIL: No recipients to send alert email to.`);
}

// Make sure send list does not contain any duplicate email addresses
Expand Down

0 comments on commit 34421ce

Please sign in to comment.