workflows: add wait for no operation for cleaning up GKE

When a workflow cancels, the clean-up job cannot delete clusters in the provisioning state. With this commit, the clean-up job will wait for no operation on clusters before deleting the resources. Signed-off-by: Birol Bilgin <[email protected]>
cilium · Dec 1, 2022 · cdad6d5 · cdad6d5
1 parent 1dcc5ca
commit cdad6d5
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 4 deletions.
diff --git a/.github/workflows/externalworkloads.yaml b/.github/workflows/externalworkloads.yaml
@@ -254,9 +254,12 @@ jobs:
       - name: Clean up GKE
         if: ${{ always() }}
         run: |
+          while [ "$(gcloud container operations list --filter="status=RUNNING AND targetLink~${{ env.clusterName }}" --format="value(name)")" ];do
+            echo "cluster has an ongoing operation, waiting for all operations to finish"; sleep 15
+          done
           gcloud container clusters delete ${{ env.clusterName }} --zone ${{ env.zone }} --quiet --async
           gcloud compute instances delete ${{ env.vmName }} --zone ${{ env.zone }} --quiet
-        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
+        shell: bash {0} # Disable default fail-fast behavior so that all commands run independently
 
       - name: Upload artifacts
         if: ${{ !success() }}

diff --git a/.github/workflows/gke.yaml b/.github/workflows/gke.yaml
@@ -167,8 +167,11 @@ jobs:
       - name: Clean up GKE
         if: ${{ always() }}
         run: |
+          while [ "$(gcloud container operations list --filter="status=RUNNING AND targetLink~${{ env.clusterName }}" --format="value(name)")" ];do
+            echo "cluster has an ongoing operation, waiting for all operations to finish"; sleep 15
+          done
           gcloud container clusters delete ${{ env.clusterName }} --zone ${{ env.zone }} --quiet --async
-        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
+        shell: bash {0} # Disable default fail-fast behavior so that all commands run independently
 
       - name: Upload artifacts
         if: ${{ !success() }}

diff --git a/.github/workflows/multicluster.yaml b/.github/workflows/multicluster.yaml
@@ -28,6 +28,7 @@ concurrency:
 env:
   clusterName1: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-mesh-1
   clusterName2: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-mesh-2
+  clusterNameBase: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-mesh
   zone: us-west2-a
   firewallRuleName: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-rule
   cilium_version: v1.12.2
@@ -217,10 +218,13 @@ jobs:
       - name: Clean up GKE
         if: ${{ always() }}
         run: |
-          gcloud compute firewall-rules delete ${{ env.firewallRuleName }} --quiet
+          while [ "$(gcloud container operations list --filter="status=RUNNING AND targetLink~${{ env.clusterNameBase }}" --format="value(name)")" ];do
+            echo "cluster has an ongoing operation, waiting for all operations to finish"; sleep 15
+          done
           gcloud container clusters delete ${{ env.clusterName1 }} --zone ${{ env.zone }} --quiet --async
           gcloud container clusters delete ${{ env.clusterName2 }} --zone ${{ env.zone }} --quiet --async
-        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
+          gcloud compute firewall-rules delete ${{ env.firewallRuleName }} --quiet
+        shell: bash {0} # Disable default fail-fast behavior so that all commands run independently
 
       - name: Upload artifacts
         if: ${{ !success() }}