From b694dfad2d97871d027173bc5bdcf0a11e3e83f9 Mon Sep 17 00:00:00 2001
From: Elliot Saba <staticfloat@gmail.com>
Date: Fri, 4 Mar 2022 09:19:09 -0800
Subject: [PATCH] Try grouping packaging jobs

---
 pipelines/main/launch_unsigned_builders.yml | 12 ++-
 pipelines/main/platforms/package_linux.yml  | 91 +++++++++++----------
 pipelines/main/platforms/upload_linux.yml   |  2 +-
 3 files changed, 57 insertions(+), 48 deletions(-)

diff --git a/pipelines/main/launch_unsigned_builders.yml b/pipelines/main/launch_unsigned_builders.yml
index 321784e1..4fde9e99 100644
--- a/pipelines/main/launch_unsigned_builders.yml
+++ b/pipelines/main/launch_unsigned_builders.yml
@@ -25,9 +25,15 @@ steps:
       #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
       #buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml
 
-      # Launch all of the platform jobs.
-      bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml
-      bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/main/platforms/test_linux.arches  .buildkite/pipelines/main/platforms/test_linux.yml
+      # Launch all of the packaging jobs
+      bash .buildkite/utilities/arches_pipeline_upload.sh \
+           .buildkite/pipelines/main/platforms/package_linux.arches \
+           .buildkite/pipelines/main/platforms/package_linux.yml
+
+      # Launch all of the testing jobs
+      bash .buildkite/utilities/arches_pipeline_upload.sh \
+           .buildkite/pipelines/main/platforms/test_linux.arches \
+           .buildkite/pipelines/main/platforms/test_linux.yml
 
       # Launch the `whitespace` job last. Uploading it last actually causes it to start
       # first. We want this job to start first because we want it to finish as quickly
diff --git a/pipelines/main/platforms/package_linux.yml b/pipelines/main/platforms/package_linux.yml
index 91b25353..2d7aa8a0 100644
--- a/pipelines/main/platforms/package_linux.yml
+++ b/pipelines/main/platforms/package_linux.yml
@@ -4,53 +4,56 @@ agents:
   sandbox.jl: "true"
   os: "linux"
 steps:
-  - label: ":linux: package ${TRIPLET?}"
-    key: "package_${TRIPLET?}"
-    plugins:
-      - JuliaCI/external-buildkite#v1:
-          version: ".buildkite-external-version"
-          repo_url: "https://github.com/JuliaCI/julia-buildkite"
-      - JuliaCI/julia#v1:
-          # Drop default "registries" directory, so it is not persisted from execution to execution
-          persist_depot_dirs: packages,artifacts,compiled
-          version: '1.6'
-      - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz
-          rootfs_treehash: "${ROOTFS_HASH?}"
-          uid: 1000
-          gid: 1000
-          workspaces:
-            # Include `/cache/repos` so that our `git` version introspection works.
-            - "/cache/repos:/cache/repos"
-    timeout_in_minutes: ${TIMEOUT?}
-    commands: |
-      # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
-      TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS=${MAKE_FLAGS?} source .buildkite/utilities/calc_version_envs.sh
+  - group: ":package: Packaging"
+    key: "package"
+    steps:
+      - label: ":linux: package ${TRIPLET?}"
+        key: "package_${TRIPLET?}"
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: ".buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - JuliaCI/julia#v1:
+              # Drop default "registries" directory, so it is not persisted from execution to execution
+              persist_depot_dirs: packages,artifacts,compiled
+              version: '1.6'
+          - staticfloat/sandbox#v1:
+              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz
+              rootfs_treehash: "${ROOTFS_HASH?}"
+              uid: 1000
+              gid: 1000
+              workspaces:
+                # Include `/cache/repos` so that our `git` version introspection works.
+                - "/cache/repos:/cache/repos"
+        timeout_in_minutes: ${TIMEOUT?}
+        commands: |
+          # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
+          TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS=${MAKE_FLAGS?} source .buildkite/utilities/calc_version_envs.sh
 
-      echo "--- Build Julia from source"
-      rm -rf $${ARTIFACT_FILENAME}
-      make --output-sync -j 8 ${MAKE_FLAGS?}
+          echo "--- Build Julia from source"
+          rm -rf $${ARTIFACT_FILENAME}
+          make --output-sync -j 8 ${MAKE_FLAGS?}
 
-      echo "--- Check that the working directory is clean"
-      if [ -z "$(git status --short)" ]; then
-        echo "INFO: The working directory is clean."
-      else
-        echo "ERROR: The working directory is dirty."
-        echo "Output of git status:"
-        git status
-        exit 1
-      fi
+          echo "--- Check that the working directory is clean"
+          if [ -z "$(git status --short)" ]; then
+            echo "INFO: The working directory is clean."
+          else
+            echo "ERROR: The working directory is dirty."
+            echo "Output of git status:"
+            git status
+            exit 1
+          fi
 
-      echo "--- Print Julia version info"
-      ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
+          echo "--- Print Julia version info"
+          ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
 
-      echo "--- Create build artifacts"
-      make --output-sync -j 8 binary-dist ${MAKE_FLAGS?}
+          echo "--- Create build artifacts"
+          make --output-sync -j 8 binary-dist ${MAKE_FLAGS?}
 
-      # Rename the build artifact in case we want to name it differently, as is the case on `musl`.
-      if [[ "$${JULIA_BINARYDIST}" != "$${ARTIFACT_FILENAME}" ]]; then
-          mv $${JULIA_BINARYDIST} $${ARTIFACT_FILENAME}
-      fi
+          # Rename the build artifact in case we want to name it differently, as is the case on `musl`.
+          if [[ "$${JULIA_BINARYDIST}" != "$${ARTIFACT_FILENAME}" ]]; then
+              mv $${JULIA_BINARYDIST} $${ARTIFACT_FILENAME}
+          fi
 
-      echo "--- Upload build artifacts"
-      buildkite-agent artifact upload $${ARTIFACT_FILENAME}
+          echo "--- Upload build artifacts"
+          buildkite-agent artifact upload $${ARTIFACT_FILENAME}
diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml
index 432c8b46..4d5cf790 100644
--- a/pipelines/main/platforms/upload_linux.yml
+++ b/pipelines/main/platforms/upload_linux.yml
@@ -20,7 +20,7 @@ steps:
           version: '1.6'
       - staticfloat/sandbox#v1:
           rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.1/aws_uploader.x86_64.tar.gz
-          rootfs_treehash: "65becc21c66b853a6057410109a660aafab1a755"
+          rootfs_treehash: "f865a3a68e33de10b3b4c9c5d0c9e5f2ff5a6965"
           uid: 1000
           gid: 1000
     timeout_in_minutes: ${TIMEOUT?}