From 17f52fb2891b53f71d85ac83abdda0b2ddcb3143 Mon Sep 17 00:00:00 2001
From: David Michael <dm0@redhat.com>
Date: Wed, 19 Sep 2018 12:12:49 -0400
Subject: [PATCH] jenkins: Add Slack notifications for job status changes

This wraps the body of most jobs in a try/catch block which sets
the run's status to failure if an exception or error is thrown.
At the end, that status is compared to the previous run, and a
notification is sent if they differ (treating anything other than
success as a failure).

The logic for sending notifications is stored in the utils file to
reduce code duplication, which means notifications will not be sent
for failures caused by changes to the utils file.

This expects that the Slack plugin is configured to be able to send
to the CoreOS Slack instance.  It only overrides the default Slack
channel, in case something else makes more sense for the global
configuration.
---
 Jenkinsfile.aws-test          |  7 +++++++
 Jenkinsfile.cloud             |  7 +++++++
 Jenkinsfile.rdgo              |  5 ++++-
 Jenkinsfile.treecompose       |  9 ++++++++-
 Jenkinsfile.treecompose-ootpa |  9 ++++++++-
 pipeline-utils.groovy         | 32 ++++++++++++++++++++++++++++++++
 6 files changed, 66 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile.aws-test b/Jenkinsfile.aws-test
index bb5e8572..f4975cb0 100644
--- a/Jenkinsfile.aws-test
+++ b/Jenkinsfile.aws-test
@@ -22,6 +22,7 @@ node(NODE) {
         return
     }
 
+    try {
     utils.inside_assembler_container("-v /srv:/srv") {
     stage("Sync In") {
         withCredentials([
@@ -79,4 +80,10 @@ node(NODE) {
         }
     }
     }
+    } catch (Throwable e) {
+        currentBuild.result = 'FAILURE'
+        throw e
+    } finally {
+        utils.notify_status_change currentBuild
+    }
 }
diff --git a/Jenkinsfile.cloud b/Jenkinsfile.cloud
index 72036fad..5c41e6c0 100644
--- a/Jenkinsfile.cloud
+++ b/Jenkinsfile.cloud
@@ -17,6 +17,7 @@ node(NODE) {
     utils = load("pipeline-utils.groovy")
     utils.define_properties(TIMER)
 
+    try {
     def manifest = "host-maipo.yaml"
     def manifest_data = readYaml file: "${manifest}";
     def ref = manifest_data.ref;
@@ -257,4 +258,10 @@ node(NODE) {
         string(name: 'aws_type', value: "t2.small")
     ]
     }
+    } catch (Throwable e) {
+        currentBuild.result = 'FAILURE'
+        throw e
+    } finally {
+        utils.notify_status_change currentBuild
+    }
 }
diff --git a/Jenkinsfile.rdgo b/Jenkinsfile.rdgo
index cf1fd692..3a763cb9 100644
--- a/Jenkinsfile.rdgo
+++ b/Jenkinsfile.rdgo
@@ -78,8 +78,11 @@ node(NODE) {
         build job: 'coreos-rhcos-treecompose', wait: false
     }
 
+    } catch (Throwable e) {
+        currentBuild.result = 'FAILURE'
+        throw e
     } finally {
         archiveArtifacts artifacts: "log/**", allowEmptyArchive: true
+        utils.notify_status_change currentBuild
     }
 }
-
diff --git a/Jenkinsfile.treecompose b/Jenkinsfile.treecompose
index 3009999a..5dcf7d6b 100644
--- a/Jenkinsfile.treecompose
+++ b/Jenkinsfile.treecompose
@@ -19,6 +19,7 @@ node(NODE) {
     utils = load("pipeline-utils.groovy")
     utils.define_properties(TIMER)
 
+    try {
     def manifest = "host-${OS_NAME}.yaml"
     def manifest_data = readYaml file: "${manifest}";
     // TODO - stop using the ref in favor of oscontainer:// pivot
@@ -127,5 +128,11 @@ node(NODE) {
 
         // Trigger downstream jobs
         build job: 'coreos-rhcos-cloud', wait: false
-  }
+    }
+    } catch (Throwable e) {
+        currentBuild.result = 'FAILURE'
+        throw e
+    } finally {
+        utils.notify_status_change currentBuild
+    }
 }
diff --git a/Jenkinsfile.treecompose-ootpa b/Jenkinsfile.treecompose-ootpa
index 680e507c..21a4ff9b 100644
--- a/Jenkinsfile.treecompose-ootpa
+++ b/Jenkinsfile.treecompose-ootpa
@@ -14,6 +14,7 @@ node(NODE) {
     utils = load("pipeline-utils.groovy")
     utils.define_properties(TIMER)
 
+    try {
     def manifest = "host-${OS_NAME}.yaml"
     def manifest_data = readYaml file: "${manifest}";
     // TODO - stop using the ref in favor of oscontainer:// pivot
@@ -105,5 +106,11 @@ node(NODE) {
        stage("Cleanup") { sh """
             rm ${treecompose_workdir} -rf
         """ }
-  }
+    }
+    } catch (Throwable e) {
+        currentBuild.result = 'FAILURE'
+        throw e
+    } finally {
+        utils.notify_status_change currentBuild
+    }
 }
diff --git a/pipeline-utils.groovy b/pipeline-utils.groovy
index 5b7b7586..6dc8a2c5 100644
--- a/pipeline-utils.groovy
+++ b/pipeline-utils.groovy
@@ -169,4 +169,36 @@ def inside_assembler_container(args, fn) {
     }
 }
 
+// Send a notification when a job's status changes.
+// Treat non-SUCCESS states the same, so it does not send notifications when
+// changes go between UNSTABLE and FAILURE.
+def notify_status_change(build) {
+    def color = ''
+    def message = "<${env.BUILD_URL}|Build ${env.BUILD_NUMBER} of ${env.JOB_NAME}>"
+
+    if (build.currentResult == build.previousBuild?.result)
+        return
+
+    if (build.previousBuild?.result == null) {
+        echo 'The previous build is still running; ignoring its build state.'
+        return
+    } else if (build.currentResult == 'SUCCESS') {
+        message = ":partyparrot: ${message} is working again."
+        color = 'good'
+    } else if (build.previousBuild.result == 'SUCCESS') {
+        message = ":trashfire: ${message} has started failing."
+        color = 'danger'
+    } else {
+        echo 'This and the previous build have different non-success states.'
+        return
+    }
+
+    try {
+        slackSend channel: '#jenkins-coreos', color: color, message: message
+    } catch (NoSuchMethodError err) {
+        // Log the message in the console if the Slack plugin is not installed.
+        echo message
+    }
+}
+
 return this