diff --git a/.ci/pipeline-library/src/test/slackNotifications.groovy b/.ci/pipeline-library/src/test/slackNotifications.groovy index f7e39f5fad903..33b3afed80bde 100644 --- a/.ci/pipeline-library/src/test/slackNotifications.groovy +++ b/.ci/pipeline-library/src/test/slackNotifications.groovy @@ -9,6 +9,7 @@ class SlackNotificationsTest extends KibanaBasePipelineTest { super.setUp() helper.registerAllowedMethod('slackSend', [Map.class], null) + prop('buildState', loadScript("vars/buildState.groovy")) slackNotifications = loadScript('vars/slackNotifications.groovy') } @@ -25,13 +26,49 @@ class SlackNotificationsTest extends KibanaBasePipelineTest { } @Test - void 'sendFailedBuild() should call slackSend() with message'() { + void 'sendFailedBuild() should call slackSend() with an in-progress message'() { mockFailureBuild() slackNotifications.sendFailedBuild() def args = fnMock('slackSend').args[0] + def expected = [ + channel: '#kibana-operations-alerts', + username: 'Kibana Operations', + iconEmoji: ':jenkins:', + color: 'danger', + message: ':hourglass_flowing_sand: elastic / kibana # master #1', + ] + + expected.each { + assertEquals(it.value.toString(), args[it.key].toString()) + } + + assertEquals( + ":hourglass_flowing_sand: **", + args.blocks[0].text.text.toString() + ) + + assertEquals( + "*Failed Steps*\n• ", + args.blocks[1].text.text.toString() + ) + + assertEquals( + "*Test Failures*\n• ", + args.blocks[2].text.text.toString() + ) + } + + @Test + void 'sendFailedBuild() should call slackSend() with message'() { + mockFailureBuild() + + slackNotifications.sendFailedBuild(isFinal: true) + + def args = fnMock('slackSend').args[0] + def expected = [ channel: '#kibana-operations-alerts', username: 'Kibana Operations', @@ -65,7 +102,7 @@ class SlackNotificationsTest extends KibanaBasePipelineTest { mockFailureBuild() def counter = 0 helper.registerAllowedMethod('slackSend', [Map.class], { ++counter > 1 }) - slackNotifications.sendFailedBuild() + slackNotifications.sendFailedBuild(isFinal: true) def args = fnMocks('slackSend')[1].args[0] @@ -88,6 +125,29 @@ class SlackNotificationsTest extends KibanaBasePipelineTest { ) } + @Test + void 'sendFailedBuild() should call slackSend() with a channel id and timestamp on second call'() { + mockFailureBuild() + helper.registerAllowedMethod('slackSend', [Map.class], { [ channelId: 'CHANNEL_ID', ts: 'TIMESTAMP' ] }) + slackNotifications.sendFailedBuild(isFinal: false) + slackNotifications.sendFailedBuild(isFinal: true) + + def args = fnMocks('slackSend')[1].args[0] + + def expected = [ + channel: 'CHANNEL_ID', + timestamp: 'TIMESTAMP', + username: 'Kibana Operations', + iconEmoji: ':jenkins:', + color: 'danger', + message: ':broken_heart: elastic / kibana # master #1', + ] + + expected.each { + assertEquals(it.value.toString(), args[it.key].toString()) + } + } + @Test void 'getTestFailures() should truncate list of failures to 10'() { prop('testUtils', [ diff --git a/Jenkinsfile b/Jenkinsfile index 818ba748ee165..ad1d244c78874 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,59 +4,60 @@ library 'kibana-pipeline-library' kibanaLibrary.load() kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true) { - githubPr.withDefaultPrComments { - ciStats.trackBuild { - catchError { - retryable.enable() - parallel([ - 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'), - 'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'), - 'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - 'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'), - 'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1), - 'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2), - 'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3), - 'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4), - 'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5), - 'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6), - 'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7), - 'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8), - 'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9), - 'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10), - 'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11), - 'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12), - 'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'), - // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'), - ]), - 'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - 'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'), - 'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1), - 'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2), - 'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3), - 'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4), - 'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5), - 'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6), - 'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7), - 'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8), - 'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9), - 'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10), - 'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'), - 'xpack-savedObjectsFieldMetrics': kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'), - 'xpack-securitySolutionCypress': { processNumber -> - whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/', 'x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/', 'x-pack/plugins/triggers_actions_ui/public/application/context/actions_connectors_context.tsx']) { - kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber) - } - }, + slackNotifications.onFailure(disabled: !params.NOTIFY_ON_FAILURE) { + githubPr.withDefaultPrComments { + ciStats.trackBuild { + catchError { + retryable.enable() + parallel([ + 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'), + 'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'), + 'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ + 'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'), + 'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1), + 'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2), + 'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3), + 'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4), + 'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5), + 'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6), + 'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7), + 'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8), + 'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9), + 'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10), + 'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11), + 'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12), + 'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'), + // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'), + ]), + 'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ + 'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'), + 'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1), + 'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2), + 'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3), + 'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4), + 'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5), + 'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6), + 'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7), + 'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8), + 'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9), + 'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10), + 'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'), + 'xpack-savedObjectsFieldMetrics': kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'), + 'xpack-securitySolutionCypress': { processNumber -> + whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/', 'x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/', 'x-pack/plugins/triggers_actions_ui/public/application/context/actions_connectors_context.tsx']) { + kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber) + } + }, - // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'), - ]), - ]) + // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'), + ]), + ]) + } } } } if (params.NOTIFY_ON_FAILURE) { - slackNotifications.onFailure() kibanaPipeline.sendMail() } } diff --git a/vars/githubPr.groovy b/vars/githubPr.groovy index da5348749f668..ec3dbd919fed6 100644 --- a/vars/githubPr.groovy +++ b/vars/githubPr.groovy @@ -15,7 +15,7 @@ */ def withDefaultPrComments(closure) { catchErrors { - // sendCommentOnError() needs to know if comments are enabled, so lets track it with a global + // kibanaPipeline.notifyOnError() needs to know if comments are enabled, so lets track it with a global // isPr() just ensures this functionality is skipped for non-PR builds buildState.set('PR_COMMENTS_ENABLED', isPr()) catchErrors { @@ -59,19 +59,6 @@ def sendComment(isFinal = false) { } } -def sendCommentOnError(Closure closure) { - try { - closure() - } catch (ex) { - // If this is the first failed step, it's likely that the error hasn't propagated up far enough to mark the build as a failure - currentBuild.result = 'FAILURE' - catchErrors { - sendComment(false) - } - throw ex - } -} - // Checks whether or not this currently executing build was triggered via a PR in the elastic/kibana repo def isPr() { return !!(env.ghprbPullId && env.ghprbPullLink && env.ghprbPullLink =~ /\/elastic\/kibana\//) diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index 410578886a01d..94bfe983b4653 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -16,6 +16,25 @@ def withPostBuildReporting(Closure closure) { } } +def notifyOnError(Closure closure) { + try { + closure() + } catch (ex) { + // If this is the first failed step, it's likely that the error hasn't propagated up far enough to mark the build as a failure + currentBuild.result = 'FAILURE' + catchErrors { + githubPr.sendComment(false) + } + catchErrors { + // an empty map is a valid config, but is falsey, so let's use .has() + if (buildState.has('SLACK_NOTIFICATION_CONFIG')) { + slackNotifications.sendFailedBuild(buildState.get('SLACK_NOTIFICATION_CONFIG')) + } + } + throw ex + } +} + def functionalTestProcess(String name, Closure closure) { return { processNumber -> def kibanaPort = "61${processNumber}1" @@ -35,7 +54,7 @@ def functionalTestProcess(String name, Closure closure) { "JOB=${name}", "KBN_NP_PLUGINS_BUILT=true", ]) { - githubPr.sendCommentOnError { + notifyOnError { closure() } } @@ -182,7 +201,7 @@ def bash(script, label) { } def doSetup() { - githubPr.sendCommentOnError { + notifyOnError { retryWithDelay(2, 15) { try { runbld("./test/scripts/jenkins_setup.sh", "Setup Build Environment and Dependencies") @@ -199,13 +218,13 @@ def doSetup() { } def buildOss() { - githubPr.sendCommentOnError { + notifyOnError { runbld("./test/scripts/jenkins_build_kibana.sh", "Build OSS/Default Kibana") } } def buildXpack() { - githubPr.sendCommentOnError { + notifyOnError { runbld("./test/scripts/jenkins_xpack_build_kibana.sh", "Build X-Pack Kibana") } } diff --git a/vars/slackNotifications.groovy b/vars/slackNotifications.groovy index 30f86e6d6f0ad..02aad14d8ba3f 100644 --- a/vars/slackNotifications.groovy +++ b/vars/slackNotifications.groovy @@ -105,16 +105,26 @@ def getDefaultDisplayName() { return "${env.JOB_NAME} ${env.BUILD_DISPLAY_NAME}" } -def getDefaultContext() { - def duration = currentBuild.durationString.replace(' and counting', '') +def getDefaultContext(config = [:]) { + def progressMessage = "" + if (config && !config.isFinal) { + progressMessage = "In-progress" + } else { + def duration = currentBuild.durationString.replace(' and counting', '') + progressMessage = "${buildUtils.getBuildStatus().toLowerCase().capitalize()} after ${duration}" + } return contextBlock([ - "${buildUtils.getBuildStatus().toLowerCase().capitalize()} after ${duration}", + progressMessage, "", ].join(' · ')) } -def getStatusIcon() { +def getStatusIcon(config = [:]) { + if (config && !config.isFinal) { + return ':hourglass_flowing_sand:' + } + def status = buildUtils.getBuildStatus() if (status == 'UNSTABLE') { return ':yellow_heart:' @@ -124,7 +134,7 @@ def getStatusIcon() { } def getBackupMessage(config) { - return "${getStatusIcon()} ${config.title}\n\nFirst attempt at sending this notification failed. Please check the build." + return "${getStatusIcon(config)} ${config.title}\n\nFirst attempt at sending this notification failed. Please check the build." } def sendFailedBuild(Map params = [:]) { @@ -135,19 +145,32 @@ def sendFailedBuild(Map params = [:]) { color: 'danger', icon: ':jenkins:', username: 'Kibana Operations', - context: getDefaultContext(), + isFinal: false, ] + params - def title = "${getStatusIcon()} ${config.title}" - def message = "${getStatusIcon()} ${config.message}" + config.context = config.context ?: getDefaultContext(config) + + def title = "${getStatusIcon(config)} ${config.title}" + def message = "${getStatusIcon(config)} ${config.message}" def blocks = [markdownBlock(title)] getFailedBuildBlocks().each { blocks << it } blocks << dividerBlock() blocks << config.context + def channel = config.channel + def timestamp = null + + def previousResp = buildState.get('SLACK_NOTIFICATION_RESPONSE') + if (previousResp) { + // When using `timestamp` to update a previous message, you have to use the channel ID from the previous response + channel = previousResp.channelId + timestamp = previousResp.ts + } + def resp = slackSend( - channel: config.channel, + channel: channel, + timestamp: timestamp, username: config.username, iconEmoji: config.icon, color: config.color, @@ -156,7 +179,7 @@ def sendFailedBuild(Map params = [:]) { ) if (!resp) { - slackSend( + resp = slackSend( channel: config.channel, username: config.username, iconEmoji: config.icon, @@ -165,6 +188,10 @@ def sendFailedBuild(Map params = [:]) { blocks: [markdownBlock(getBackupMessage(config))] ) } + + if (resp) { + buildState.set('SLACK_NOTIFICATION_RESPONSE', resp) + } } def onFailure(Map options = [:]) { @@ -172,6 +199,7 @@ def onFailure(Map options = [:]) { def status = buildUtils.getBuildStatus() if (status != "SUCCESS") { catchErrors { + options.isFinal = true sendFailedBuild(options) } } @@ -179,6 +207,16 @@ def onFailure(Map options = [:]) { } def onFailure(Map options = [:], Closure closure) { + if (options.disabled) { + catchError { + closure() + } + + return + } + + buildState.set('SLACK_NOTIFICATION_CONFIG', options) + // try/finally will NOT work here, because the build status will not have been changed to ERROR when the finally{} block executes catchError { closure() diff --git a/vars/workers.groovy b/vars/workers.groovy index 74ce86516e863..f5a28c97c6812 100644 --- a/vars/workers.groovy +++ b/vars/workers.groovy @@ -126,7 +126,7 @@ def intake(jobName, String script) { return { ci(name: jobName, size: 's-highmem', ramDisk: true) { withEnv(["JOB=${jobName}"]) { - githubPr.sendCommentOnError { + kibanaPipeline.notifyOnError { runbld(script, "Execute ${jobName}") } }