diff --git a/Jenkinsfile b/Jenkinsfile
index 958ced8c6195a..ad5bd23b76b5d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -53,7 +53,9 @@ kibanaPipeline(timeoutMinutes: 135, checkPrChanges: true) {
       }
     }
 
-    retryable.printFlakyFailures()
-    kibanaPipeline.sendMail()
+    if (params.NOTIFY_ON_FAILURE) {
+      slackNotifications.onFailure()
+      kibanaPipeline.sendMail()
+    }
   }
 }
diff --git a/vars/slackNotifications.groovy b/vars/slackNotifications.groovy
index 8ae37d1c44637..5092f631e7b39 100644
--- a/vars/slackNotifications.groovy
+++ b/vars/slackNotifications.groovy
@@ -62,21 +62,32 @@ def getTestFailures() {
   def messages = []
   messages << "*Test Failures*"
 
-  def list = failures.collect { "• <${it.url}|${it.fullDisplayName}>" }.join("\n")
+  def list = failures.collect { "• <${it.url}|${it.fullDisplayName.split('.', 2)[-1]}>" }.join("\n")
   return "*Test Failures*\n${list}"
 }
 
-def sendFailedBuild(Map params = [:]) {
-  def displayName = "${env.JOB_NAME} ${env.BUILD_DISPLAY_NAME}"
+def getDefaultDisplayName() {
+  return "${env.JOB_NAME} ${env.BUILD_DISPLAY_NAME}"
+}
 
+def getDefaultContext() {
+  def duration = currentBuild.durationString.replace(' and counting', '')
+
+  return contextBlock([
+    "${buildUtils.getBuildStatus().toLowerCase().capitalize()} after ${duration}",
+    "<https://ci.kibana.dev/${env.JOB_BASE_NAME}/${env.BUILD_NUMBER}|ci.kibana.dev>",
+  ].join(' · '))
+}
+
+def sendFailedBuild(Map params = [:]) {
   def config = [
-    channel: '#kibana-operations',
-    title: ":broken_heart: *<${env.BUILD_URL}|${displayName}>*",
-    message: ":broken_heart: ${displayName}",
+    channel: '#kibana-operations-alerts',
+    title: ":broken_heart: *<${env.BUILD_URL}|${getDefaultDisplayName()}>*",
+    message: ":broken_heart: ${getDefaultDisplayName()}",
     color: 'danger',
     icon: ':jenkins:',
     username: 'Kibana Operations',
-    context: contextBlock("${displayName} · <https://ci.kibana.dev/${env.JOB_BASE_NAME}/${env.BUILD_NUMBER}|ci.kibana.dev>"),
+    context: getDefaultContext(),
   ] + params
 
   def blocks = [markdownBlock(config.title)]
@@ -94,18 +105,24 @@ def sendFailedBuild(Map params = [:]) {
   )
 }
 
+def onFailure(Map options = [:]) {
+  catchError {
+    def status = buildUtils.getBuildStatus()
+    if (status != "SUCCESS") {
+      catchErrors {
+        sendFailedBuild(options)
+      }
+    }
+  }
+}
+
 def onFailure(Map options = [:], Closure closure) {
   // try/finally will NOT work here, because the build status will not have been changed to ERROR when the finally{} block executes
   catchError {
     closure()
   }
 
-  def status = buildUtils.getBuildStatus()
-  if (status != "SUCCESS" && status != "UNSTABLE") {
-    catchErrors {
-      sendFailedBuild(options)
-    }
-  }
+  onFailure(options)
 }
 
 return this