Enabling retries for Beats flaky tests (#39174)

* added retries for auditbeat flaky tests * added retries for filebeat flaky tests * added retries * test exitcode * checged exit status code for retries * set larger timeout for pytestOpts * restored timeout for pytests
elastic · May 1, 2024 · 02ea29d · 02ea29d
1 parent 5a0293e
commit 02ea29d
Show file tree

Hide file tree

Showing 16 changed files with 439 additions and 1 deletion.
diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -32,6 +32,9 @@ steps:
         command: |
           cd auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -47,6 +50,9 @@ steps:
         command: |
           cd auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -62,6 +68,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -79,6 +88,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -97,6 +109,9 @@ steps:
           make -C auditbeat crosscompile
         env:
           GOX_FLAGS: "-arch amd64"
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -115,6 +130,9 @@ steps:
           set -euo pipefail
           cd auditbeat
           mage unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -133,6 +151,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd auditbeat
           mage unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -147,6 +168,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd auditbeat
           mage unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -164,6 +188,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -180,6 +207,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -196,6 +226,9 @@ steps:
         command: |
           Set-Location -Path auditbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"

diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -45,6 +48,9 @@ steps:
         command: |
           cd filebeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -60,6 +66,9 @@ steps:
         command: |
           cd filebeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -76,6 +85,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -94,6 +106,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -118,6 +133,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -136,6 +154,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -152,6 +173,9 @@ steps:
         command: |
           cd filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -172,6 +196,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -190,6 +217,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -208,6 +238,9 @@ steps:
         command: |
           Set-Location -Path filebeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"

diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -30,6 +30,9 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -45,6 +48,9 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -61,6 +67,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -78,6 +87,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -94,6 +106,9 @@ steps:
         command:  |
           cd heartbeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -109,6 +124,9 @@ steps:
         command: |
           cd heartbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -130,6 +148,9 @@ steps:
         command: |
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -151,6 +172,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -168,6 +192,9 @@ steps:
           source .buildkite/scripts/install_macos_tools.sh
           cd heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -188,6 +215,9 @@ steps:
         command:  |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -205,6 +235,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -222,6 +255,9 @@ steps:
         command: |
           Set-Location -Path heartbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"

diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml
@@ -21,6 +21,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -38,6 +41,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage goIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -55,6 +61,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage pythonIntegTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -72,6 +81,9 @@ steps:
           set -euo pipefail
           cd libbeat
           make crosscompile
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -89,6 +101,9 @@ steps:
           set -euo pipefail
           cd libbeat
           make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS=' | go-junit-report > libbeat-stress-test.xml' stress-tests
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -108,6 +123,9 @@ steps:
           set -euo pipefail
           cd libbeat
           mage build unitTest
+        retry:
+          automatic:
+           - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"