Add Slack alarm for failing Route 53 health checks

infra/lib/alarms-stack.ts

@@ -0,0 +1,48 @@
+import * as cdk from "aws-cdk-lib"
+import { aws_lambda, aws_lambda_nodejs, aws_sns, StackProps } from "aws-cdk-lib"
+import { Code } from "aws-cdk-lib/aws-lambda"
+import { Secret } from "aws-cdk-lib/aws-secretsmanager"
+import {
+  LambdaSubscription,
+  UrlSubscription,
+} from "aws-cdk-lib/aws-sns-subscriptions"
+import { Construct } from "constructs"
+
+export class AlarmsStack extends cdk.Stack {
+  readonly alarmSnsTopic
+
+  constructor(scope: Construct, id: string, props: StackProps) {
+    super(scope, id, props)
+
+    this.alarmSnsTopic = new aws_sns.Topic(this, "AlarmSnsTopic")
+
+    const slackWebhookUrlSecretName = `SlackWebhookUrl`
+
+    const slackNotifierLambda = new aws_lambda_nodejs.NodejsFunction(
+      this,
+      "SlackNotifierLambda",
+      {
+        runtime: aws_lambda.Runtime.NODEJS_20_X,
+        handler: "index.handler",
+        timeout: cdk.Duration.seconds(60),
+        code: Code.fromAsset("./lambdas/slackNotifierLambda"),
+        environment: {
+          SLACK_WEBHOOK_URL_SECRET_NAME: slackWebhookUrlSecretName,
+        },
+      },
+    )
+
+    this.alarmSnsTopic.addSubscription(
+      new LambdaSubscription(slackNotifierLambda),
+    )
+    this.alarmSnsTopic.addSubscription(
+      new UrlSubscription(process.env["PAGERDUTY_ENDPOINT"]!),
+    )
+
+    Secret.fromSecretNameV2(
+      this,
+      "SlackWebhookUrlSecret",
+      slackWebhookUrlSecretName,
+    ).grantRead(slackNotifierLambda)
+  }
+}

infra/lib/lambdas/slackNotifierLambda/index.ts

@@ -0,0 +1,78 @@
+import {
+  GetSecretValueCommand,
+  SecretsManagerClient,
+} from "@aws-sdk/client-secrets-manager"
+import { Handler } from "aws-lambda"
+
+export const handler: Handler = async (event) => {
+  console.log("Received event:", JSON.stringify(event, null, 2))
+
+  const secretName = process.env.SLACK_WEBHOOK_URL_SECRET_NAME
+
+  const client = new SecretsManagerClient()
+  const response = await client.send(
+    new GetSecretValueCommand({
+      SecretId: secretName,
+    }),
+  )
+
+  const { url: slackWebhookUrl } = JSON.parse(response.SecretString!)
+
+  const message = event.Records[0].Sns.Message
+
+  const {
+    AlarmName,
+    NewStateValue,
+    NewStateReason,
+    OldStateValue,
+    StateChangeTime,
+  } = JSON.parse(message)
+
+  const date = new Date(StateChangeTime)
+
+  const formattedDate = new Intl.DateTimeFormat("fi-FI", {
+    year: "numeric",
+    month: "2-digit",
+    day: "2-digit",
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit",
+    timeZone: "Europe/Helsinki",
+    hour12: false,
+  }).format(date)
+
+  const getEmoji = () => {
+    if (OldStateValue === "ALARM" && NewStateValue === "OK") {
+      return ":sunny:"
+    } else if (NewStateValue === "ALARM") {
+      return ":thunder_cloud_and_rain:"
+    }
+    return ":question:"
+  }
+
+  const slackMessage = {
+    text: `
+    *Alarm from CloudWatch* ${getEmoji()}
+    *State changed*: \`${OldStateValue}\` :arrow_right: \`${NewStateValue}\`
+    *Alarm Name*: ${AlarmName}
+    *Reason*: ${NewStateReason}
+    *Timestamp*: ${formattedDate}
+    `,
+  }
+
+  try {
+    const fetchResponse = await fetch(slackWebhookUrl, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(slackMessage),
+    })
+
+    const responseBody = await fetchResponse.text()
+    console.log(`Response from Slack: ${responseBody}`)
+  } catch (error) {
+    console.error(`Request failed: ${error}`)
+    throw error
+  }
+}