test: add pagerduty, check if test is running before attemmpting rerun

snyk · Mar 1, 2021 · 0576431 · 0576431
1 parent ebb8dd7
commit 0576431
Show file tree

Hide file tree

Showing 3 changed files with 81 additions and 23 deletions.
diff --git a/test/cli-alert/package-lock.json b/test/cli-alert/package-lock.json
diff --git a/test/cli-alert/package.json b/test/cli-alert/package.json
@@ -14,6 +14,7 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@octokit/rest": "^18.0.5",
+    "@pagerduty/pdjs": "^2.2.0",
     "@slack/webhook": "^5.0.3",
     "typescript": "^4.0.2"
   },

diff --git a/test/cli-alert/src/index.ts b/test/cli-alert/src/index.ts
@@ -1,14 +1,22 @@
 import { Octokit } from '@octokit/rest';
 import { IncomingWebhook } from '@slack/webhook';
 import { IncomingWebhookDefaultArguments } from '@slack/webhook';
-
-if (!process.env.USER_GITHUB_TOKEN || !process.env.SLACK_WEBHOOK_URL) {
-  console.error('Missing USER_GITHUB_TOKEN or SLACK_WEBHOOK_URL');
+import { event } from '@pagerduty/pdjs';
+
+if (
+  !process.env.USER_GITHUB_TOKEN ||
+  !process.env.SLACK_WEBHOOK_URL ||
+  !process.env.PD_ROUTING_KEY
+) {
+  console.error(
+    'Missing USER_GITHUB_TOKEN, SLACK_WEBHOOK_URL or PD_ROUTING_KEY',
+  );
   process.exit(1);
 }
 
 const GITHUB_TOKEN = process.env.USER_GITHUB_TOKEN;
 const SLACK_WEBHOOK_URL = process.env.SLACK_WEBHOOK_URL;
+const PD_ROUTING_KEY = process.env.PD_ROUTING_KEY;
 
 const octokit = new Octokit({
   auth: GITHUB_TOKEN,
@@ -54,7 +62,7 @@ async function discoverConsecutiveFailures(
       process.exit(1);
     } else if (
       'failure' === firstJob.conclusion &&
-      firstJob?.conclusion === secondJob.conclusion
+      firstJob.conclusion === secondJob.conclusion
     ) {
       console.log(`Found a job that failed 2 times in a row: ${jobName}`);
       failedJobs.push(jobName);
@@ -64,6 +72,24 @@ async function discoverConsecutiveFailures(
   return failedJobs;
 }
 
+async function sendPagerDuty() {
+  await event({
+    data: {
+      // eslint-disable-next-line @typescript-eslint/camelcase
+      routing_key: PD_ROUTING_KEY,
+      // eslint-disable-next-line @typescript-eslint/camelcase
+      event_action: 'trigger',
+      payload: {
+        summary: 'CLI Alert. Smoke tests failing',
+        source: 'Snyk CLI Smoke tests',
+        severity: 'warning',
+      },
+    },
+  })
+    .then(console.log)
+    .catch(console.error);
+}
+
 async function sendSlackAlert(failedJobs: string[]) {
   console.log('Jobs failed again. Sending Slack alert...');
   const args: IncomingWebhookDefaultArguments = {
@@ -170,21 +196,6 @@ async function run() {
 
     // Check the latest 2 smoke tests for tests that had the same job fail 2 times in a row.
     const latestWorkflowRuns = workflowRuns.workflow_runs.slice(0, 2);
-
-    console.log('Checking smoke tests jobs...');
-    const failedWorkflows = await discoverConsecutiveFailures(
-      latestWorkflowRuns[0].id,
-      latestWorkflowRuns[1].id,
-    );
-
-    if (!failedWorkflows.length || failedWorkflows.length < 1) {
-      console.log(
-        'There were no 2 consecutive fails on a job. No need to alert.',
-      );
-      return;
-    }
-
-    console.log('Trying to re-run smoke test...');
     const id = latestWorkflowRuns[0].id;
 
     // Check current status of smoke test workflow and wait if it's still running
@@ -202,6 +213,21 @@ async function run() {
       await waitForConclusion(id);
     }
 
+    console.log('Checking smoke tests jobs...');
+    const failedWorkflows = await discoverConsecutiveFailures(
+      latestWorkflowRuns[0].id,
+      latestWorkflowRuns[1].id,
+    );
+
+    if (!failedWorkflows.length || failedWorkflows.length < 1) {
+      console.log(
+        'There were no 2 consecutive fails on a job. No need to alert.',
+      );
+      return;
+    }
+
+    console.log('Trying to re-run smoke test...');
+
     // After making sure smoke test isn't currently running - try to re-run
     console.log(`Starting re-run of Smoke Test. ID number: ${id}...`);
     await octokit.actions.reRunWorkflow({
@@ -216,10 +242,13 @@ async function run() {
     const failedAgainJobs = await checkJobConclusion(id, failedWorkflows);
     console.log('Re-run completed.');
 
-    // If run failed again, send Slack alert
-    failedAgainJobs.length > 0
-      ? await sendSlackAlert(failedAgainJobs)
-      : console.log('Jobs succeeded after re-run. Do not alert.');
+    // If run failed again, send Slack alert and PagerDuty
+    if (failedAgainJobs.length > 0) {
+      await sendSlackAlert(failedAgainJobs);
+      await sendPagerDuty();
+    } else {
+      console.log('Jobs succeeded after re-run. Do not alert.');
+    }
   } catch (error) {
     console.error(error);
     process.exit(1);