CI (Buildkite): add package_musl64 and tester_musl64 (Linux, musl…

… libc, 64-bit, x86_64 architecture) (#42476)

(cherry picked from commit 1195f5a)

.buildkite/pipelines/main/launch_unsigned_builders.yml

@@ -13,19 +13,21 @@
 # something about the privileged steps.
 
 steps:
-  - label: ":buildkite: Launch unsigned builders"
+  - label: ":buildkite: Launch unsigned jobs"
     commands: |
-      # Launch the `whitespace` builder first, because we want that builder to finish as quickly as possible.
-      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
-
-      # Launch the miscellaneous builders in alphabetical order.
+      # Launch the miscellaneous jobs in alphabetical order.
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml
 
-      # Launch all of the platform builders.
+      # Launch all of the platform jobs.
       bash .buildkite/pipelines/main/platforms/platforms.sh package_linux
       bash .buildkite/pipelines/main/platforms/platforms.sh tester_linux
+
+      # Launch the `whitespace` job last. Uploading it last actually causes it to start
+      # first. We want this job to start first because we want it to finish as quickly
+      # as possible.
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
     agents:
       queue: julia

.buildkite/pipelines/main/platforms/package_linux.arches

@@ -1,6 +1,7 @@
-# ARCH      ARCH_LABEL      ROOTFS_ARCH      TIMEOUT      ROOTFS_TAG      ROOTFS_TREE
-# aarch64   aarch64         aarch64          60           v3.2            0566841e29f0f9880541c26a6595fd5ce0beb5ff
-# armv7l    armv7l          armv7l           60           v3.2            fb359370b052a47ce5c84cc6b4a7a03ed7053b25
-32          32              i686             60           v3.2            209c4db679a515befd7fb50ecc6bfbecf7ec3d32
-# ppc64le   ppc64le         powerpc64le      60           v3.2            c03a0158b19d48ac84b426834fce0d3584cdd0c7
-64          64              x86_64           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
+# PLATFORM      ARCH        ARCH_LABEL      ROOTFS_ARCH      ALLOW_FAIL      TIMEOUT      ROOTFS_TAG      ROOTFS_TREE
+# linux         aarch64     aarch64         aarch64          false           60           v3.2            0566841e29f0f9880541c26a6595fd5ce0beb5ff
+# linux         armv7l      armv7l          armv7l           false           60           v3.2            fb359370b052a47ce5c84cc6b4a7a03ed7053b25
+linux           32          32              i686             false           60           v3.2            209c4db679a515befd7fb50ecc6bfbecf7ec3d32
+# linux         ppc64le     ppc64le         powerpc64le      false           60           v3.2            c03a0158b19d48ac84b426834fce0d3584cdd0c7
+linux           64          64              x86_64           false           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
+musl            64          64              x86_64           false           60           v3.19           e6a2730e37c386c46915b2650d6aaaa398195152

.buildkite/pipelines/main/platforms/package_linux.yml

@@ -4,15 +4,15 @@ agents:
   sandbox.jl: "true"
   os: "linux"
 steps:
-  - label: "package_linux${ARCH_LABEL?}"
-    key: package_linux${ARCH_LABEL?}
+  - label: "package_${PLATFORM?}${ARCH_LABEL?}"
+    key: package_${PLATFORM?}${ARCH_LABEL?}
     plugins:
       - JuliaCI/julia#v1:
           # Drop default "registries" directory, so it is not persisted from execution to execution
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_linux.${ROOTFS_ARCH?}.tar.gz
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz
           rootfs_treehash: "${ROOTFS_TREE?}"
           uid: 1000
           gid: 1000
@@ -24,9 +24,11 @@ steps:
       echo "--- Print the full and short commit hashes"
       SHORT_COMMIT_LENGTH=10
       SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}`
-      JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-`
       ARTIFACT_FILE_EXTENSION="tar.gz"
-      ARTIFACT_FILENAME="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}"
+      ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}"
+      JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-`
+      JULIA_BINARYDIST="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}"
+
       echo "The full commit is:                     $${BUILDKITE_COMMIT:?}"
       echo "The short commit is:                    $${SHORT_COMMIT:?}"
       echo "The artifact filename will be:    $${ARTIFACT_FILENAME:?}"
@@ -43,7 +45,10 @@ steps:
 
       echo "--- Create build artifacts"
       make -j 8 binary-dist
+      ls -l $${JULIA_BINARYDIST:?}
+      if [[ "$${JULIA_BINARYDIST:?}" != "$${ARTIFACT_FILENAME:?}" ]]; then
+          mv $${JULIA_BINARYDIST:?} $${ARTIFACT_FILENAME:?}
+      fi
       ls -l $${ARTIFACT_FILENAME:?}
-
       echo "--- Upload build artifacts"
       buildkite-agent artifact upload $${ARTIFACT_FILENAME:?}

.buildkite/pipelines/main/platforms/platforms.sh

@@ -2,9 +2,9 @@
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
-PLATFORM="$1"
+OS="$1"
 
-cat "$SCRIPT_DIR/$PLATFORM.arches" | tr -s ' ' | while read _line; do
+cat "$SCRIPT_DIR/$OS.arches" | tr -s ' ' | while read _line; do
   # Remove whitespace from the beginning and end of each line
   line=`echo $_line | tr -s ' '`
 
@@ -13,12 +13,14 @@ cat "$SCRIPT_DIR/$PLATFORM.arches" | tr -s ' ' | while read _line; do
     continue
   fi
 
-  export ARCH=`echo $line        | cut -d ' ' -f 1`
-  export ARCH_LABEL=`echo $line  | cut -d ' ' -f 2`
-  export ROOTFS_ARCH=`echo $line | cut -d ' ' -f 3`
-  export TIMEOUT=`echo $line     | cut -d ' ' -f 4`
-  export ROOTFS_TAG=`echo $line  | cut -d ' ' -f 5`
-  export ROOTFS_TREE=`echo $line | cut -d ' ' -f 6`
-  echo "Launching: $PLATFORM $ARCH $ARCH_LABEL $ROOTFS_ARCH $TIMEOUT"
-  buildkite-agent pipeline upload "$SCRIPT_DIR/$PLATFORM.yml"
+  export PLATFORM=`echo $line    | cut -d ' ' -f 1`
+  export ARCH=`echo $line        | cut -d ' ' -f 2`
+  export ARCH_LABEL=`echo $line  | cut -d ' ' -f 3`
+  export ROOTFS_ARCH=`echo $line | cut -d ' ' -f 4`
+  export ALLOW_FAIL=`echo $line   | cut -d ' ' -f 5`
+  export TIMEOUT=`echo $line     | cut -d ' ' -f 6`
+  export ROOTFS_TAG=`echo $line  | cut -d ' ' -f 7`
+  export ROOTFS_TREE=`echo $line | cut -d ' ' -f 8`
+  echo "Launching: $OS $PLATFORM $ARCH $ARCH_LABEL $ROOTFS_ARCH $ALLOW_FAIL $TIMEOUT"
+  buildkite-agent pipeline upload "$SCRIPT_DIR/$OS.yml"
 done

.buildkite/pipelines/main/platforms/tester_linux.arches

@@ -1,8 +1,9 @@
-# ARCH      ARCH_LABEL      ROOTFS_ARCH      TIMEOUT      ROOTFS_TAG      ROOTFS_TREE
-# aarch64   aarch64         aarch64          60           v3.2            0566841e29f0f9880541c26a6595fd5ce0beb5ff
-# armv7l    armv7l          armv7l           60           v3.2            fb359370b052a47ce5c84cc6b4a7a03ed7053b25
-32          32              i686             60           v3.2            209c4db679a515befd7fb50ecc6bfbecf7ec3d32
-# ppc64le   ppc64le         powerpc64le      60           v3.2            c03a0158b19d48ac84b426834fce0d3584cdd0c7
-64          64_rr           x86_64           180          v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
-64          64_st           x86_64           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
-64          64_mt           x86_64           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
+# PLATFORM      ARCH        ARCH_LABEL      ROOTFS_ARCH      ALLOW_FAIL      TIMEOUT      ROOTFS_TAG      ROOTFS_TREE
+# linux         aarch64     aarch64         aarch64          false           60           v3.2            0566841e29f0f9880541c26a6595fd5ce0beb5ff
+# linux         armv7l      armv7l          armv7l           false           60           v3.2            fb359370b052a47ce5c84cc6b4a7a03ed7053b25
+linux           32          32              i686             false           60           v3.2            209c4db679a515befd7fb50ecc6bfbecf7ec3d32
+# linux         ppc64le     ppc64le         powerpc64le      false           60           v3.2            c03a0158b19d48ac84b426834fce0d3584cdd0c7
+linux           64          64_rr           x86_64           false           180          v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
+linux           64          64_st           x86_64           false           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
+linux           64          64_mt           x86_64           false           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7
+musl            64          64              x86_64           true            60           v3.19           e6a2730e37c386c46915b2650d6aaaa398195152

.buildkite/pipelines/main/platforms/tester_linux.yml

@@ -4,16 +4,17 @@ agents:
   sandbox.jl: "true"
   os: "linux"
 steps:
-  - label: "tester_linux${ARCH_LABEL?}"
-    key: tester_linux${ARCH_LABEL?}
-    depends_on: package_linux${ARCH?}
+  - label: "tester_${PLATFORM?}${ARCH_LABEL?}"
+    key: tester_${PLATFORM?}${ARCH_LABEL?}
+    depends_on: package_${PLATFORM?}${ARCH?}
     plugins:
       - JuliaCI/julia#v1:
           # Drop default "registries" directory, so it is not persisted from execution to execution
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_linux.${ROOTFS_ARCH?}.tar.gz
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz
+          # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz
           rootfs_treehash: "${ROOTFS_TREE?}"
           uid: 1000
           gid: 1000
@@ -23,14 +24,15 @@ steps:
     env:
       JULIA_SHELL: "/bin/bash"
     timeout_in_minutes: ${TIMEOUT?}
+    soft_fail: ${ALLOW_FAIL?}
     commands: |
       echo "--- Print the full and short commit hashes"
       SHORT_COMMIT_LENGTH=10
       SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}`
       JULIA_DIR="julia-$${SHORT_COMMIT:?}"
       JULIA_BINARY="$${JULIA_DIR:?}/bin/julia"
       ARTIFACT_FILE_EXTENSION="tar.gz"
-      ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-linux${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}"
+      ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}"
       echo "The full commit is:                     $${BUILDKITE_COMMIT:?}"
       echo "The short commit is:                    $${SHORT_COMMIT:?}"
       echo "The artifact filename will be:    $${ARTIFACT_FILENAME:?}"