Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Slack notifications
Browse files Browse the repository at this point in the history
rushi committed Sep 25, 2024
1 parent b7abd34 commit 1e5bc3d
Showing 5 changed files with 285 additions and 204 deletions.
422 changes: 231 additions & 191 deletions package-lock.json

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@
},
"dependencies": {
"@slack/bolt": "^3.3.0",
"@slack/web-api": "^7.5.0",
"axios": "^1.7.7",
"chalk": "^5.3.0",
"cheerio": "^1.0.0",
@@ -21,7 +22,7 @@
"junit-viewer": "^4.11.1",
"lodash-es": "^4.17.21",
"md5": "^2.3.0",
"node-notifier": "^10.0.1"
"slack-notify": "^2.0.7"
},
"devDependencies": {
"nodemon": "^3.1.4",
8 changes: 6 additions & 2 deletions src/monitor-elastic-agents.js
Original file line number Diff line number Diff line change
@@ -5,11 +5,12 @@ import config from "config";
import axios from "axios";
import * as cheerio from "cheerio";
import { CronJob } from "cron";
import { notify } from "./services/slack.js";

const now = () => dayjs().format("HH:mm:ss");
const url = config.get("go.url") + "/go/admin/status_reports/com.thoughtworks.gocd.elastic-agent.ecs/cluster/CI";

const MAX_PENDING_TASKS = 20;
const MAX_PENDING_TASKS = 15;
const MAX_HOST_MACHINES = 40;

async function check() {
@@ -29,6 +30,7 @@ async function check() {
console.log(`Running: ${runningTasks} Pending: ${pendingTasks}`, chalk.bold(` Total: ${totalTasks}`));
if (Number(pendingTasks) >= MAX_PENDING_TASKS) {
console.log(" ", chalk.bgRed.white.bold(`Too many pending tasks (${pendingTasks})`));
await notify(`🚨 Too many pending tasks: ${pendingTasks} Running: ${runningTasks}`);
}

process.stdout.write(chalk.bold("Instances "));
@@ -37,7 +39,8 @@ async function check() {
const total = spot + onDemand;
console.log(` Spot: ${spot} On Demand: ${onDemand}`, chalk.bold(` Total: ${total}`));
if (total >= MAX_HOST_MACHINES) {
console.log(" ", chalk.bgGreen.white.bold(`Max limit reached: ${total}`));
console.log(" ", chalk.bgGreen.white.bold(`Max limit reached: ${total}/${MAX_HOST_MACHINES}`));
await notify(`🚨 Max limit of host machines reached: ${total}/${MAX_HOST_MACHINES}`);
}

// TODO: Get errors from the page when or if they show up. I don't know the selector for that yet
@@ -48,6 +51,7 @@ async function check() {
console.log(header);
console.log(chalk.red(description));
console.log();
await notify(`🚨 ${header}\n${description}`);
}
} else {
console.log(response.status);
37 changes: 27 additions & 10 deletions src/monitor-jobs.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import chalk from "chalk";
import dayjs from "dayjs";
import { CronJob } from "cron";
import dayjs from "dayjs";
import Go from "./services/go.js";
import { notify } from "./services/slack.js";

const now = () => dayjs().format("HH:mm:ss");
const messagesToSkip = [
/Modification check failed/
];
const messagesToSkip = [/Modification check failed/];

async function check() {
const data = await Go.fetchServerHealth();
@@ -22,13 +21,21 @@ async function check() {

let counter = 0;
const pipelinesProcessed = [];

const NOTIFY_MAX_MINUTES = 5;
const pipelineNotifyMessages = [];

data.forEach((info) => {
const re = new RegExp(/\b[A-Z0-9]{2,5}-\d{1,5}\b/, "gmi");
const ticketNumber = chalk.bold(info.message.match(re)?.[0]);
const branchRe = new RegExp(/\bBranch: (.*$)\b/, "gmi");
const ticketNumber = info.message.match(re)?.[0] ?? info.message.match(branchRe)?.[0];

for (const re of messagesToSkip) {
if (re.test(info.message)) {
console.log(spacer, `${++counter} Skipped ${ticketNumber}`)
console.log(spacer, `${++counter} Skipped ${ticketNumber}`);
if (!ticketNumber) {
console.log(spacer, info.message, info.detail);
}
return;
}
}
@@ -40,18 +47,28 @@ async function check() {
return;
}

const level = /error|warn/i.test(info.level) ? "red" : "bold";
console.log(spacer, `${++counter} ${info.level}`, info.message);
if (pipelineName && duration && duration[1]) {
console.log(spacer, `${ticketNumber} Pipeline ${pipelineName[1]} waiting for ${chalk.red(duration[1])} minutes`);
const minutes = Number(duration[1]);
console.log(
spacer,
`${ticketNumber} Pipeline ${pipelineName[1]} waiting for ${chalk.red(minutes)} minutes`,
);
pipelinesProcessed.push(pipelineName);
pipelinesProcessed.push(ticketNumber);
ticketNumber && pipelinesProcessed.push(ticketNumber);
if (minutes > NOTIFY_MAX_MINUTES) {
pipelineNotifyMessages.push(`Pipeline *${pipelineName[1]}* waiting for *${minutes}* minutes`);
}
} else {
console.log(spacer, chalk.dim(info.detail));
}
})
});

console.log();

if (pipelineNotifyMessages.length > 0) {
await notify(pipelineNotifyMessages.join("\n"));
}
}

if (process.argv[2] === "start") {
19 changes: 19 additions & 0 deletions src/services/slack.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import config from "config";
import chalk from "chalk";
import { WebClient } from "@slack/web-api";

const web = new WebClient(config.get("slack.token"));

export const notify = async (text, blocks = []) => {
try {
const response = await web.chat.postMessage({
text,
blocks,
channel: config.get("slack.defaultChannel"),
});
console.log("Slack message sent", response?.ok, response?.message.text ?? response);
} catch (error) {
console.log("Error sending slack message", chalk.red(error.message));
console.log(text);
}
};

0 comments on commit 1e5bc3d

Please sign in to comment.