github: notify Slack when CI fails on merge to main/release (#24707)

Co-authored-by: James Rasell <[email protected]>

.github/workflows/build.yml

@@ -377,6 +377,25 @@ jobs:
           echo "::group::Smoke test binary"
           docker run --rm -v "$PWD:/src" registry.access.redhat.com/ubi7/ubi-minimal:7.9-1057 /src/nomad version
 
+  handle-failure:
+    needs:
+      - get-go-version
+      - get-product-version
+      - generate-metadata-file
+      - build-other
+      - build-linux
+      - build-darwin
+      - build-docker
+      - minimum-os
+    if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure')
+    uses: ./.github/workflows/test-failure-notification.yml
+    secrets: inherit
+    with:
+      actor: ${{ github.triggering_actor }}
+      git-branch: ${{ github.ref_name }}
+      workflow-run-id: ${{ github.run_id }}
+      workflow-name: ${{ github.workflow }}
+
 permissions:
   contents: read
   id-token: write

.github/workflows/security-scan.yml

@@ -69,3 +69,18 @@ jobs:
         uses: github/codeql-action/upload-sarif@8fd294e26a0e458834582b0fe4988d79966c7c0a # codeql-bundle-v2.18.4
         with:
           sarif_file: results.sarif
+
+  handle-failure:
+    permissions:
+      contents: read
+      id-token: write
+    needs:
+      - scan
+    if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure')
+    uses: ./.github/workflows/test-failure-notification.yml
+    secrets: inherit
+    with:
+      actor: ${{ github.triggering_actor }}
+      git-branch: ${{ github.ref_name }}
+      workflow-run-id: ${{ github.run_id }}
+      workflow-name: ${{ github.workflow }}

.github/workflows/test-core.yaml

@@ -121,6 +121,22 @@ jobs:
           sudo -E env "PATH=$PATH" hc-install install -version ${{env.CONSUL_VERSION}} -path '${{env.GOPATH}}/bin' consul
           sudo sed -i 's!Defaults!#Defaults!g' /etc/sudoers
           sudo -E env "PATH=$PATH" make test-nomad
+
+  handle-failure:
+    needs:
+      - checks
+      - compile
+      - tests-api
+      - tests-groups
+    if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure')
+    uses: ./.github/workflows/test-failure-notification.yml
+    secrets: inherit
+    with:
+      actor: ${{ github.triggering_actor }}
+      git-branch: ${{ github.ref_name }}
+      workflow-run-id: ${{ github.run_id }}
+      workflow-name: ${{ github.workflow }}
+
 permissions:
   contents: read
   id-token: write

.github/workflows/test-e2e.yml

@@ -82,6 +82,19 @@ jobs:
           sudo sed -i 's!Defaults!#Defaults!g' /etc/sudoers
           sudo -E env "PATH=$PATH" make integration-test-consul
 
+  handle-failure:
+    needs:
+      - test-e2e-vault
+      - test-e2e-consul
+    if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure')
+    uses: ./.github/workflows/test-failure-notification.yml
+    secrets: inherit
+    with:
+      actor: ${{ github.triggering_actor }}
+      git-branch: ${{ github.ref_name }}
+      workflow-run-id: ${{ github.run_id }}
+      workflow-name: ${{ github.workflow }}
+
 permissions:
   contents: read
   id-token: write

.github/workflows/test-failure-notification.yml

@@ -0,0 +1,75 @@
+---
+name: CI Test Failure Notification
+
+on:
+  workflow_call:
+    inputs:
+      actor:
+        description: 'Triggering actor of the failed the workflow'
+        required: true
+        type: string
+      git-branch:
+        description: 'Git branch name'
+        required: true
+        type: string
+      workflow-name:
+        description: 'Name of the GitHub Action workflow'
+        required: true
+        type: string
+      workflow-run-id:
+        description: 'GitHub Action run ID that failed'
+        required: true
+        type: string
+
+jobs:
+  send-notification:
+    runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux"]') || 'ubuntu-latest' }}
+    steps:
+      - name: Retrieve Vault-hosted Secrets
+        if: endsWith(github.repository, '-enterprise')
+        id: vault
+        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
+        with:
+          url: ${{ vars.CI_VAULT_URL }}
+          method: ${{ vars.CI_VAULT_METHOD }}
+          path: ${{ vars.CI_VAULT_PATH }}
+          jwtGithubAudience: ${{ vars.CI_VAULT_AUD }}
+          secrets: |-
+            kv/data/teams/nomad/slack-webhooks feed-nomad | SLACK_FEED_NOMAD ;
+      - name: Send Slack notification
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
+        with:
+          webhook: "${{ env.SLACK_FEED_NOMAD || secrets.SLACK_FEED_NOMAD_CI_FAILURE }}"
+          webhook-type: incoming-webhook
+          payload: |
+            {
+              "text": ":x: CI Workflow '${{ inputs.workflow-name }}' has failed",
+              "attachments": [
+                {
+                  "color": "#C41E3A",
+                  "blocks": [
+                    {
+                      "type": "section",
+                      "fields": [
+                        {
+                          "type": "mrkdwn",
+                          "text": "*Branch:* <${{ github.server_url }}/${{ github.repository }}/tree/${{ inputs.git-branch }}|${{ inputs.git-branch }}>"
+                        },
+                        {
+                          "type": "mrkdwn",
+                          "text": "*From:* @${{ inputs.actor }}"
+                        },
+                        {
+                          "type": "mrkdwn",
+                          "text": "*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ inputs.workflow-run-id }}|${{ inputs.workflow-run-id }}>"
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            }
+        
+permissions:
+  contents: read
+  id-token: write

.github/workflows/test-ui.yml

@@ -93,6 +93,21 @@ jobs:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
           PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
         run: yarn percy build:finalize
+
+  handle-failure:
+    needs:
+      - pre-test
+      - tests
+      - finalize
+    if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure')
+    uses: ./.github/workflows/test-failure-notification.yml
+    secrets: inherit
+    with:
+      actor: ${{ github.triggering_actor }}
+      git-branch: ${{ github.ref_name }}
+      workflow-run-id: ${{ github.run_id }}
+      workflow-name: ${{ github.workflow }}
+
 permissions:
   contents: read
   id-token: write

.github/workflows/test-windows.yml

@@ -98,5 +98,21 @@ jobs:
         with:
           name: results.xml
           path: results.xml
+
+  handle-failure:
+    permissions:
+      contents: read
+      id-token: write
+    needs:
+      - test-windows
+    if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure')
+    uses: ./.github/workflows/test-failure-notification.yml
+    secrets: inherit
+    with:
+      actor: ${{ github.triggering_actor }}
+      git-branch: ${{ github.ref_name }}
+      workflow-run-id: ${{ github.run_id }}
+      workflow-name: ${{ github.workflow }}
+
 permissions:
   contents: read