build: retry workflow and retry test on CircleCI (#2311)

Signed-off-by: Viet Nguyen Duc <[email protected]>
SeleniumHQ · Jul 18, 2024 · 6ee6c0f · 6ee6c0f
1 parent 9bd30b0
commit 6ee6c0f
Show file tree

Hide file tree

Showing 4 changed files with 133 additions and 14 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -150,11 +150,35 @@ jobs:
       - run:
           name: "Build Docker images"
           command: |
-            PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
+            N=3
+            while [ $N -gt 0 ]; do
+              PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build || true
+              if [ $? -eq 0 ]; then
+                echo "Build images passed"
+                exit 0
+              else
+                echo "Build failed. Retrying..."
+                N=$((N-1))
+                sleep 10
+              fi
+            done
+            exit 1
       - run:
           name: "Test Docker images"
           command: |
-            USE_RANDOM_USER_ID=${USE_RANDOM_USER} PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make ${TEST_STRATEGY}
+            N=3
+            while [ $N -gt 0 ]; do
+              USE_RANDOM_USER_ID=${USE_RANDOM_USER} PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make ${TEST_STRATEGY} || true
+              if [ $? -eq 0 ]; then
+                echo "Tests passed"
+                exit 0
+              else
+                echo "Tests failed. Retrying..."
+                N=$((N-1))
+                sleep 10
+              fi
+            done
+            exit 1
 
   kubernetes-test:
     parameters:
@@ -208,7 +232,19 @@ jobs:
       - run:
           name: "Build Docker images"
           command: |
-            PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
+            N=3
+            while [ $N -gt 0 ]; do
+              PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build || true
+              if [ $? -eq 0 ]; then
+                echo "Build images passed"
+                exit 0
+              else
+                echo "Build failed. Retrying..."
+                N=$((N-1))
+                sleep 10
+              fi
+            done
+            exit 1
       - run:
           name: "Build Helm charts"
           command: |
@@ -219,12 +255,21 @@ jobs:
       - run:
           name: "Test Selenium Grid on Kubernetes"
           command: |
-            PLATFORMS=${PLATFORMS} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} \
-            TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${TEST_STRATEGY}
-      - run:
-          name: "Check video integrity"
-          command: |
-            make test_video_integrity
+            N=3
+            while [ $N -gt 0 ]; do
+              PLATFORMS=${PLATFORMS} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} \
+              TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${TEST_STRATEGY} \
+              && make test_video_integrity || true
+              if [ $? -eq 0 ]; then
+                echo "Tests passed"
+                exit 0
+              else
+                echo "Tests failed. Retrying..."
+                N=$((N-1))
+                sleep 10
+              fi
+            done
+            exit 1
       - run:
           name: "Clean-up Kubernetes environment"
           command: |

diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -9,6 +9,12 @@ on:
         type: boolean
         default: false
   workflow_dispatch:
+    inputs:
+      rerunFailedOnly:
+        description: 'Rerun only failed jobs'
+        required: false
+        type: boolean
+        default: true
   push:
     paths-ignore:
       - '**.md'
@@ -18,6 +24,12 @@ on:
 
 permissions: write-all
 
+env:
+  GH_CLI_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  RUN_ID: ${{ github.run_id }}
+  RERUN_FAILED_ONLY: ${{ github.event.inputs.rerunFailedOnly || true }}
+  RUN_ATTEMPT: ${{ github.run_attempt }}
+
 jobs:
   docker-test:
     uses: ./.github/workflows/docker-test.yml
@@ -28,3 +40,27 @@ jobs:
     uses: ./.github/workflows/helm-chart-test.yml
     with:
       release: ${{ inputs.release == 'true' }}
+
+  rerun-workflow-when-failure:
+    needs:
+      - docker-test
+      - helm-chart-test
+    if: failure() && ( github.run_attempt < 3 )
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@main
+      - name: Install GitHub CLI
+        run: |
+          sudo apt update
+          sudo apt install gh
+      - name: Authenticate GitHub CLI
+        run: |
+          echo "$GH_CLI_TOKEN" | gh auth login --with-token
+      - name: Rerun workflow when failure
+        run: |
+          echo "Rerun workflow ID $RUN_ID in attempt #$(($RUN_ATTEMPT + 1))"
+          gh workflow run rerun-failed.yml \
+            --repo $GITHUB_REPOSITORY \
+            --raw-field runId=$RUN_ID \
+            --raw-field rerunFailedOnly=$RERUN_FAILED_ONLY
diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml
@@ -157,7 +157,8 @@ jobs:
           timeout_minutes: 30
           max_attempts: 3
           command: |
-            NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }}
+            NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }} \
+            && make test_video_integrity
       - name: Test chart upgrade
         if: (matrix.test-upgrade == true)
         run: |
@@ -178,10 +179,6 @@ jobs:
           name: ${{ env.ARTIFACT_NAME }}-artifacts
           path: ./tests/tests/
           if-no-files-found: ignore
-      - name: Check video integrity
-        if: always()
-        run: |
-          make test_video_integrity
       - name: Upload test video artifacts
         if: always()
         uses: actions/upload-artifact@main

diff --git a/.github/workflows/rerun-failed.yml b/.github/workflows/rerun-failed.yml
@@ -0,0 +1,41 @@
+name: Rerun Workflows
+
+on:
+  workflow_dispatch:
+    inputs:
+      runId:
+        description: 'The ID of workflow to rerun'
+        required: true
+        type: string
+      rerunFailedOnly:
+        description: 'Rerun only failed jobs'
+        required: false
+        type: boolean
+        default: true
+
+env:
+  GH_CLI_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  RUN_ID: ${{ github.event.inputs.runId }}
+  RERUN_FAILED_ONLY: ${{ github.event.inputs.rerunFailedOnly }}
+
+jobs:
+  rerun_workflow:
+    name: Rerun ${{ github.event.inputs.runId }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@main
+      - name: Install GitHub CLI
+        run: |
+          sudo apt update
+          sudo apt install gh
+      - name: Authenticate GitHub CLI
+        run: |
+          echo "${GH_CLI_TOKEN}" | gh auth login --with-token
+      - name: "Rerun workflow ${{ env.RUN_ID }}"
+        run: |
+          if [ "${RERUN_FAILED_ONLY}" = "true" ]; then
+            gh run rerun ${RUN_ID} --failed --repo ${GITHUB_REPOSITORY}
+          else
+            gh run rerun ${RUN_ID} --repo ${GITHUB_REPOSITORY}
+          fi