diff --git a/.circleci/config.yml b/.circleci/config.yml index 2d14f7a96..9e5d3e90e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -150,11 +150,35 @@ jobs: - run: name: "Build Docker images" command: | - PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build + N=3 + while [ $N -gt 0 ]; do + PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build || true + if [ $? -eq 0 ]; then + echo "Build images passed" + exit 0 + else + echo "Build failed. Retrying..." + N=$((N-1)) + sleep 10 + fi + done + exit 1 - run: name: "Test Docker images" command: | - USE_RANDOM_USER_ID=${USE_RANDOM_USER} PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make ${TEST_STRATEGY} + N=3 + while [ $N -gt 0 ]; do + USE_RANDOM_USER_ID=${USE_RANDOM_USER} PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make ${TEST_STRATEGY} || true + if [ $? -eq 0 ]; then + echo "Tests passed" + exit 0 + else + echo "Tests failed. Retrying..." + N=$((N-1)) + sleep 10 + fi + done + exit 1 kubernetes-test: parameters: @@ -208,7 +232,19 @@ jobs: - run: name: "Build Docker images" command: | - PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build + N=3 + while [ $N -gt 0 ]; do + PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build || true + if [ $? -eq 0 ]; then + echo "Build images passed" + exit 0 + else + echo "Build failed. Retrying..." + N=$((N-1)) + sleep 10 + fi + done + exit 1 - run: name: "Build Helm charts" command: | @@ -219,12 +255,21 @@ jobs: - run: name: "Test Selenium Grid on Kubernetes" command: | - PLATFORMS=${PLATFORMS} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} \ - TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${TEST_STRATEGY} - - run: - name: "Check video integrity" - command: | - make test_video_integrity + N=3 + while [ $N -gt 0 ]; do + PLATFORMS=${PLATFORMS} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} \ + TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${TEST_STRATEGY} \ + && make test_video_integrity || true + if [ $? -eq 0 ]; then + echo "Tests passed" + exit 0 + else + echo "Tests failed. Retrying..." + N=$((N-1)) + sleep 10 + fi + done + exit 1 - run: name: "Clean-up Kubernetes environment" command: | diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 566d55dd7..73cf0685e 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -9,6 +9,12 @@ on: type: boolean default: false workflow_dispatch: + inputs: + rerunFailedOnly: + description: 'Rerun only failed jobs' + required: false + type: boolean + default: true push: paths-ignore: - '**.md' @@ -18,6 +24,12 @@ on: permissions: write-all +env: + GH_CLI_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.run_id }} + RERUN_FAILED_ONLY: ${{ github.event.inputs.rerunFailedOnly || true }} + RUN_ATTEMPT: ${{ github.run_attempt }} + jobs: docker-test: uses: ./.github/workflows/docker-test.yml @@ -28,3 +40,27 @@ jobs: uses: ./.github/workflows/helm-chart-test.yml with: release: ${{ inputs.release == 'true' }} + + rerun-workflow-when-failure: + needs: + - docker-test + - helm-chart-test + if: failure() && ( github.run_attempt < 3 ) + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@main + - name: Install GitHub CLI + run: | + sudo apt update + sudo apt install gh + - name: Authenticate GitHub CLI + run: | + echo "$GH_CLI_TOKEN" | gh auth login --with-token + - name: Rerun workflow when failure + run: | + echo "Rerun workflow ID $RUN_ID in attempt #$(($RUN_ATTEMPT + 1))" + gh workflow run rerun-failed.yml \ + --repo $GITHUB_REPOSITORY \ + --raw-field runId=$RUN_ID \ + --raw-field rerunFailedOnly=$RERUN_FAILED_ONLY diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index e0eb3959b..5d2ada608 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -157,7 +157,8 @@ jobs: timeout_minutes: 30 max_attempts: 3 command: | - NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }} + NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_EXISTING_KEDA=${TEST_EXISTING_KEDA} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }} \ + && make test_video_integrity - name: Test chart upgrade if: (matrix.test-upgrade == true) run: | @@ -178,10 +179,6 @@ jobs: name: ${{ env.ARTIFACT_NAME }}-artifacts path: ./tests/tests/ if-no-files-found: ignore - - name: Check video integrity - if: always() - run: | - make test_video_integrity - name: Upload test video artifacts if: always() uses: actions/upload-artifact@main diff --git a/.github/workflows/rerun-failed.yml b/.github/workflows/rerun-failed.yml new file mode 100644 index 000000000..c83cead31 --- /dev/null +++ b/.github/workflows/rerun-failed.yml @@ -0,0 +1,41 @@ +name: Rerun Workflows + +on: + workflow_dispatch: + inputs: + runId: + description: 'The ID of workflow to rerun' + required: true + type: string + rerunFailedOnly: + description: 'Rerun only failed jobs' + required: false + type: boolean + default: true + +env: + GH_CLI_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.event.inputs.runId }} + RERUN_FAILED_ONLY: ${{ github.event.inputs.rerunFailedOnly }} + +jobs: + rerun_workflow: + name: Rerun ${{ github.event.inputs.runId }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@main + - name: Install GitHub CLI + run: | + sudo apt update + sudo apt install gh + - name: Authenticate GitHub CLI + run: | + echo "${GH_CLI_TOKEN}" | gh auth login --with-token + - name: "Rerun workflow ${{ env.RUN_ID }}" + run: | + if [ "${RERUN_FAILED_ONLY}" = "true" ]; then + gh run rerun ${RUN_ID} --failed --repo ${GITHUB_REPOSITORY} + else + gh run rerun ${RUN_ID} --repo ${GITHUB_REPOSITORY} + fi