From e63377205c465db355559ee399bac585ca7a0e17 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Mon, 30 Sep 2024 15:12:03 -0400 Subject: [PATCH] Create a github issue when a test fails. --- .github/scripts/handle-test-failures.js | 53 ++++++++++++++++++++++ .github/workflows/prod.yml | 60 ++++++++----------------- 2 files changed, 71 insertions(+), 42 deletions(-) create mode 100644 .github/scripts/handle-test-failures.js diff --git a/.github/scripts/handle-test-failures.js b/.github/scripts/handle-test-failures.js new file mode 100644 index 000000000..1b4fad423 --- /dev/null +++ b/.github/scripts/handle-test-failures.js @@ -0,0 +1,53 @@ +module.exports = async ({ github, context, browser }) => { + const fs = require('fs'); + const path = require('path'); + + // Read the test results + const testResultsDir = './test-results'; + const files = fs.readdirSync(testResultsDir); + + for (const file of files) { + const content = fs.readFileSync(path.join(testResultsDir, file), 'utf8'); + let result; + try { + result = JSON.parse(content); + } catch (error) { + console.error(`Error parsing JSON from ${file}: ${error}`); + continue; + } + + if (result.status === 'failed') { + const testName = result.name; + const sauceLabsLink = `https://app.saucelabs.com/tests/${result.id}`; + + // Search for an existing issue + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: ['test-failure', browser] + }); + + const existingIssue = issues.data.find(issue => issue.title.includes(testName)); + + if (existingIssue) { + // Update existing issue + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existingIssue.number, + body: `Test failed again in the latest run.\nBrowser: ${browser}\nRun: ${context.runId}\n[View on SauceLabs](${sauceLabsLink})` + }); + } else { + // Create new issue + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `Test Failure: ${testName} (${browser})`, + body: `The test "${testName}" failed in the ${browser} browser.\n\nRun: ${context.runId}\n[View on SauceLabs](${sauceLabsLink})`, + labels: ['test-failure', browser] + }); + } + } + } + }; \ No newline at end of file diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index ceaca60e6..903f16e2c 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -143,22 +143,14 @@ jobs: echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV fi - - name: Notify Slack on Failure + - name: Handle Test Failures if: failure() - uses: 8398a7/action-slack@v3 + uses: actions/github-script@v7 with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); + await handleTestFailures({ github, context, browser: 'safari' }); alloy-prod-e2e-firefox: name: Firefox @@ -244,22 +236,14 @@ jobs: echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV fi - - name: Notify Slack on Failure + - name: Handle Test Failures if: failure() - uses: 8398a7/action-slack@v3 + uses: actions/github-script@v7 with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); + await handleTestFailures({ github, context, browser: 'firefox' }); alloy-prod-e2e-chrome: name: Chrome @@ -328,19 +312,11 @@ jobs: ALLOY_ENV=prod config-file: ./.sauce/prod-chrome.yml - - name: Notify Slack on Failure + - name: Handle Test Failures if: failure() - uses: 8398a7/action-slack@v3 + uses: actions/github-script@v7 with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); + await handleTestFailures({ github, context, browser: 'chrome' });