Dedicated dra gate per snapshot/staging

elastic · Apr 30, 2024 · d6c7dc2 · d6c7dc2
1 parent 910df7e
commit d6c7dc2
Showing 1 changed file with 32 additions and 10 deletions.
diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml
@@ -12,19 +12,32 @@ env:
   PLATFORMS_ARM: "linux/arm64"
 
 steps:
-  - command: echo "--> Start of concurrency gate"
-    concurrency_group: gate
+  # we use concurrency gates (https://buildkite.com/blog/concurrency-gates)
+  # to implement two FIFO queues for DRA-snapshot and DRA-staging
+  # this prevents parallel builds and possibility of publishing out of order DRA artifacts if the first job takes longer than the second
+
+  - name: Start of concurrency group for DRA Snapshot
+    if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
+    command: echo "--> Start of concurrency gate dra-snapshot"
+    concurrency_group: "dra-gate-snapshot"
+    concurrency: 1
+    key: start-gate-snapshot
+
+  - name: Start of concurrency group for DRA Staging
+    if: build.branch =~ /^\d+\.\d+$$/
+    command: echo "--> Start of concurrency gate dra-staging"
+    concurrency_group: "dra-gate-staging"
     concurrency: 1
-    key: start-gate
+    key: start-gate-staging
 
   - wait
 
   - group: Beats dashboards
     key: dashboards
-    depends_on: start-gate
     steps:
       - label: Snapshot dashboards
         if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
+        depends_on: start-gate-snapshot
         key: dashboards-snapshot
         # TODO: container with go and make
         agents:
@@ -42,6 +55,7 @@ steps:
 
       - label: Staging dashboards
         if: build.branch =~ /^\d+\.\d+$$/
+        depends_on: start-gate-staging
         key: dashboards-staging
         # TODO: container with go and make
         agents:
@@ -60,7 +74,7 @@ steps:
   - group: Packaging snapshot
     if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
     key: packaging-snapshot
-    depends_on: start-gate
+    depends_on: start-gate-snapshot
     steps:
       - label: "SNAPSHOT: {{matrix}}"
         env:
@@ -133,7 +147,7 @@ steps:
 
   - group: Packaging Staging
     key: packaging-staging
-    depends_on: start-gate
+    depends_on: start-gate-staging
     ## Only for release
     if: build.branch =~ /^\d+\.\d+$$/
     steps:
@@ -208,7 +222,6 @@ steps:
 
   - group: DRA publish
     key: dra
-    depends_on: start-gate
     steps:
       - label: DRA Snapshot
         ## Only for release branches and main
@@ -217,6 +230,7 @@ steps:
         env:
           DRA_WORKFLOW: snapshot
         depends_on:
+          - start-gate-snapshot
           - packaging-snapshot
           - dashboards-snapshot
         command: |
@@ -235,6 +249,7 @@ steps:
         env:
           DRA_WORKFLOW: staging
         depends_on:
+          - start-gate-staging
           - packaging-staging
           - dashboards-staging
         command: |
@@ -248,7 +263,14 @@ steps:
 
   - wait
 
-  - command: echo "End of concurrency gate <--"
-    concurrency_group: gate
+  - command: echo "End of concurrency gate dra-snapshot <--"
+    if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
+    concurrency_group: "dra-gate-snapshot"
+    concurrency: 1
+    key: end-gate-snapshot
+
+  - command: echo "End of concurrency gate dra-staging <--"
+    if: build.branch =~ /^\d+\.\d+$$/
+    concurrency_group: "dra-gate-staging"
     concurrency: 1
-    key: end-gate
+    key: end-gate-staging