Notify Anghammard's admins on messaging failure

When Anghammarad fails to send a message, we now notify the system's admins (DevX). The CTA for this warning notification is to review Anghammarad's logs for more information. Previously we would log information on failure but people wouldn't necessarily know to go and look for this information.
guardian · Dec 27, 2024 · e4069f0 · e4069f0
1 parent dbb4cb9
commit e4069f0
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 3 deletions.
diff --git a/anghammarad/src/main/scala/com/gu/anghammarad/Lambda.scala b/anghammarad/src/main/scala/com/gu/anghammarad/Lambda.scala
@@ -1,11 +1,12 @@
 package com.gu.anghammarad
 
+import cats.implicits._
 import com.amazonaws.services.lambda.runtime.events.SNSEvent
 import com.amazonaws.services.lambda.runtime.{Context, RequestHandler}
-import com.gu.anghammarad.models.{EmailAddress, HangoutsRoom}
+import com.gu.anghammarad.models._
 import com.gu.anghammarad.serialization.Serialization
 
-import scala.util.{Failure, Success}
+import scala.util.{Failure, Success, Try}
 
 class Lambda extends RequestHandler[SNSEvent, Unit] {
   override def handleRequest(input: SNSEvent, context: Context): Unit = {
@@ -17,7 +18,9 @@ class Lambda extends RequestHandler[SNSEvent, Unit] {
       config <- Config.loadConfig(stage)
       configuration <- Serialization.parseConfig(config)
       notification <- parseNotification
-      sent <- AnghammaradService.run(notification, configuration)
+      sent <- AnghammaradService
+        .run(notification, configuration)
+        .attemptTap(notifyOnFailure(notification, configuration))
     } yield sent
 
     // send notification if result is a failure
@@ -34,4 +37,42 @@ class Lambda extends RequestHandler[SNSEvent, Unit] {
         context.getLogger.log(s"sent ${sent.mkString(",")}")
     }
   }
+
+  private def notifyOnFailure(
+      originalNotification: Notification,
+      configuration: Configuration
+  )(
+      result: Either[Throwable, List[(Message, Contact)]]
+  ): Try[List[(Message, Contact)]] = {
+    result match {
+      case Left(err) =>
+        // if the original notification failed, alert Anghammarad's admins
+        val errorNotification = Notification(
+          subject = "Anghammarad notification failure",
+          message =
+            s"""Failed to send Anghammarad notification from ${originalNotification.sourceSystem}:
+              |
+              |Original message: ${originalNotification.subject}
+              |Targets: ${originalNotification.target.mkString(",")}
+              |Channel: ${originalNotification.channel}
+              |
+              |${err.getMessage}
+              |""".stripMargin,
+          actions = List(
+            Action(
+              cta = "Check logs",
+              url =
+                "https://logs.gutools.co.uk/s/devx/app/r/s/CPXIT" // Today's Anghammarad logs
+            )
+          ),
+          target = List(GithubTeamSlug("devx-reliability-and-ops")),
+          channel = All,
+          sourceSystem = "Anghammarad",
+          threadKey = originalNotification.threadKey
+        )
+        AnghammaradService.run(errorNotification, configuration)
+      case Right(_) =>
+        Success(Nil)
+    }
+  }
 }
diff --git a/build.sbt b/build.sbt
@@ -26,6 +26,7 @@ inThisBuild(Seq(
 ))
 
 val awsSdkVersion = "2.29.40"
+val catsVersion = "2.12.0"
 val circeVersion = "0.14.10"
 val flexmarkVersion = "0.64.8"
 val scalaTestVersion = "3.2.19"
@@ -86,6 +87,7 @@ lazy val anghammarad = project
       "org.scala-lang.modules" %% "scala-collection-compat" % "2.12.0",
       "com.amazonaws" % "aws-lambda-java-events" % "3.14.0",
       "com.amazonaws" % "aws-lambda-java-core" % "1.2.3",
+      "org.typelevel" %% "cats-core" % catsVersion,
       "software.amazon.awssdk" % "lambda" % awsSdkVersion,
       "software.amazon.awssdk" % "ses" % awsSdkVersion,
       "software.amazon.awssdk" % "s3" % awsSdkVersion,