JuliaCI · staticfloat · Apr 6, 2022 · Apr 1, 2022 · Apr 1, 2022 · Apr 2, 2022
diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
@@ -2,7 +2,10 @@
 
 set -euo pipefail
 
-UPSTREAM_URL="https://github.com/JuliaLang/julia.git"
+# Buildkite users can trigger new builds with these values overridden to
+# test specific julia gitshas
+UPSTREAM_URL="${UPSTREAM_URL:-https://github.com/JuliaLang/julia.git}"
+UPSTREAM_BRANCH="${UPSTREAM_BRANCH:-master}"
 
 # All of our workers are required to provide a default for the julia cache dir
 # We're going to cache repositories in here, just like buildkite itself would:
@@ -17,11 +20,14 @@ else
     git -C "${UPSTREAM_CACHE}" fetch
 fi
 
-# Clear out current directory, clone cache into it
+# Clear out current directory
 # The deliciously complicated `rm` invocation is thanks to https://unix.stackexchange.com/a/77313/29688
 rm -rf ..?* .[!.]* *
-git clone --reference "${UPSTREAM_CACHE}" "${UPSTREAM_URL}" "./"
-git reset --hard "$(buildkite-agent meta-data get --default "master" BUILDKITE_JULIA_VERSION)"
+
+# git clone, then force checkout the given gitref
+UPSTREAM_GITREF="$(buildkite-agent meta-data get --default "origin/${UPSTREAM_BRANCH}" BUILDKITE_JULIA_VERSION)"
+git clone --dissociate --reference "${UPSTREAM_CACHE}" "${UPSTREAM_URL}" "./"
+git reset --hard "${UPSTREAM_GITREF}"
 echo
 git --no-pager log -1
 echo

diff --git a/Makefile b/Makefile
@@ -1,3 +1,6 @@
+decrypt:
+	cd .buildkite/cryptic_repo_root && decrypt --repo-root=$$(pwd) --verbose
+
 sign_treehashes:
 	cd .buildkite/cryptic_repo_root && sign_treehashes --repo-root=$$(pwd) --verbose
 

diff --git a/pipelines/main/launch_signed_jobs.yml b/pipelines/main/launch_signed_jobs.yml
@@ -14,6 +14,11 @@ steps:
           .buildkite/pipelines/main/platforms/upload_linux.arches \
           .buildkite/pipelines/main/platforms/upload_linux.yml
 
+      # Launch `upload_macos` jobs to store tarballs into S3 once tests are done
+      bash .buildkite/utilities/arches_pipeline_upload.sh \
+          .buildkite/pipelines/main/platforms/upload_macos.arches \
+          .buildkite/pipelines/main/platforms/upload_macos.yml
+
       # Don't share this with buildkite's env display
       unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
     agents:

diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature
diff --git a/pipelines/main/launch_unsigned_builders.yml b/pipelines/main/launch_unsigned_builders.yml
@@ -44,9 +44,19 @@ steps:
               .buildkite/pipelines/main/platforms/build_linux.arches \
               .buildkite/pipelines/main/platforms/build_linux.yml
 
+          # Launch macos packaging jobs
+          bash .buildkite/utilities/arches_pipeline_upload.sh \
+              .buildkite/pipelines/main/platforms/build_macos.arches \
+              .buildkite/pipelines/main/platforms/build_macos.yml
+
           # Launch Linux testing jobs
           bash .buildkite/utilities/arches_pipeline_upload.sh \
               .buildkite/pipelines/main/platforms/test_linux.arches \
               .buildkite/pipelines/main/platforms/test_linux.yml
+
+          # Launch macos testing jobs
+          bash .buildkite/utilities/arches_pipeline_upload.sh \
+              .buildkite/pipelines/main/platforms/test_macos.arches \
+              .buildkite/pipelines/main/platforms/test_macos.yml
         agents:
           queue: "julia"
diff --git a/pipelines/main/platforms/build_linux.arches b/pipelines/main/platforms/build_linux.arches
@@ -1,11 +1,11 @@
-# OS       TRIPLET                   ALLOW_FAIL    ARCH_ROOTFS    MAKE_FLAGS                                                              TIMEOUT       ROOTFS_TAG    ROOTFS_HASH
-# linux    i686-linux-gnu            .             i686           .                                                                       .             v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
-linux      x86_64-linux-gnu          .             x86_64         CFLAGS=-Werror,CXXFLAGS=-Werror                                         .             v5.1          f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
-linux      x86_64-linux-gnuassert    .             x86_64         FORCE_ASSERTIONS=1,LLVM_ASSERTIONS=1,CFLAGS=-Werror,CXXFLAGS=-Werror    .             v5.1          f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
-# linux    aarch64-linux-gnu         .             aarch64        .                                                                       .             ----          ----------------------------------------
-# linux    armv7l-linux-gnueabihf    .             armv7l         .                                                                       .             ----          ----------------------------------------
-# linux    powerpc64le-linux-gnu     .             powerpc64le    .                                                                       .             ----          ----------------------------------------
-musl       x86_64-linux-musl         .             x86_64         .                                                                       .             v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
+# OS       TRIPLET                   ALLOW_FAIL    ARCH           ARCH_ROOTFS    MAKE_FLAGS                              TIMEOUT       ROOTFS_TAG    ROOTFS_HASH
+# linux    i686-linux-gnu            .             x86_64         i686           .                                                                       .             ----          ----------------------------------------
+linux      x86_64-linux-gnu          .             x86_64         x86_64         CFLAGS=-Werror,CXXFLAGS=-Werror                                         .             v5.8          582b5d44bbc24d2a58ab609c5a520b3fd102e504
+linux      x86_64-linux-gnuassert    .             x86_64         x86_64         FORCE_ASSERTIONS=1,LLVM_ASSERTIONS=1,CFLAGS=-Werror,CXXFLAGS=-Werror    .             v5.8          582b5d44bbc24d2a58ab609c5a520b3fd102e504
+# linux    aarch64-linux-gnu         .             aarch64        aarch64        .                                                                       .             ----          ----------------------------------------
+# linux    armv7l-linux-gnueabihf    .             armv7l         armv7l         .                                                                       .             ----          ----------------------------------------
+# linux    powerpc64le-linux-gnu     .             powerpc64le    powerpc64le    .                                                                       .             ----          ----------------------------------------
+musl       x86_64-linux-musl         .             x86_64         x86_64         .                                                                       .             v5.8          dc23dbeb02f9b85c8c8c7991ee8cfcae8f4c809b
 
 # These special lines allow us to embed default values for the columns above.
 # Any column without a default mapping here will simply substitute a `.` to the empty string

diff --git a/pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/build_linux.yml
@@ -20,53 +20,14 @@ steps:
                 # Include `/cache/repos` so that our `git` version introspection works.
                 - "/cache/repos:/cache/repos"
         timeout_in_minutes: ${TIMEOUT?}
-        commands: |
-          # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
-          TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh
-
-          # export JULIA_CPU_TARGET, which is picked up by the build system
-          ARCH="$${ARCH??}" source .buildkite/utilities/export_julia_cpu_target.sh
-
-          echo "--- Create the Make.user file"
-          echo "${MAKE_FLAGS?}" | tr "," "\n" > Make.user
-          if [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-master" ]] || [[ "$${BUILDKITE_PIPELINE_SLUG:?}" =~ ^julia-release- ]]; then
-            banner="Official https://julialang.org/ release"
-            echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user
-          elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-buildkite" ]]; then
-            banner="Not an official binary; built as part of the CI infrastructure testing"
-            echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user
-          fi
-          echo "VERBOSE=1" >> Make.user
-          cat Make.user
-
-          echo "--- Build Julia"
-          make --output-sync -j 8
-
-          echo "--- Check that the working directory is clean"
-          if [ -z "$(git status --short)" ]; then
-            echo "INFO: The working directory is clean."
-          else
-            echo "ERROR: The working directory is dirty."
-            echo "Output of git status:"
-            git status
-            exit 1
-          fi
-
-          echo "--- Print Julia version info"
-          ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
-
-          echo "--- Create build artifacts"
-          make --output-sync -j 8 binary-dist
-
-          # Rename the build artifact in case we want to name it differently, as is the case on `musl`.
-          if [[ "$${JULIA_BINARYDIST_FILENAME}" != "$${UPLOAD_FILENAME}" ]]; then
-              mv $${JULIA_BINARYDIST_FILENAME} $${UPLOAD_FILENAME}
-          fi
-
-          echo "--- Upload build artifacts"
-          buildkite-agent artifact upload $${UPLOAD_FILENAME}
+        commands: "bash .buildkite/utilities/build_julia.sh"
         agents:
           queue: "julia"
           # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
           sandbox_capable: "true"
           os: "linux"
+          arch: "${ARCH?}"
+        env:
+          # Receive some environment mappings from our templating engine
+          TRIPLET: "${TRIPLET?}"
+          MAKE_FLAGS: "${MAKE_FLAGS?}"
diff --git a/pipelines/main/platforms/build_macos.arches b/pipelines/main/platforms/build_macos.arches
@@ -0,0 +1,13 @@
+# OS       TRIPLET                 ALLOW_FAIL    ARCH          MAKE_FLAGS     TIMEOUT
+macos      x86_64-apple-darwin     .             x86_64        .              .
+macos      aarch64-apple-darwin    .             aarch64       .              .
+
+
+# These special lines allow us to embed default values for the columns above.
+# Any column without a default mapping here will simply substitute a `.` to the empty string
+
+# Of course we do not allow jobs to fail typically
+#default ALLOW_FAIL false
+
+# Most jobs should finish within 1.5 hours, barring exceptionally slow hardware
+#default TIMEOUT 90
diff --git a/pipelines/main/platforms/build_macos.yml b/pipelines/main/platforms/build_macos.yml
@@ -0,0 +1,20 @@
+steps:
+  - group: ":macos: Macos"
+    steps:
+      - label: ":macos: build ${TRIPLET?}"
+        key: "build_${TRIPLET?}"
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: "./.buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+        timeout_in_minutes: ${TIMEOUT?}
+        commands: "bash .buildkite/utilities/build_julia.sh"
+        agents:
+          queue: "julia"
+          os: "macos"
+          arch: "${ARCH?}"
+        env:
+          # Receive some environment mappings from our templating engine
+          TRIPLET: "${TRIPLET?}"
+          MAKE_FLAGS: "${MAKE_FLAGS?}"
+
diff --git a/pipelines/main/platforms/test_linux.arches b/pipelines/main/platforms/test_linux.arches
@@ -1,12 +1,12 @@
-# OS       TRIPLET                   ALLOW_FAIL    ARCH_ROOTFS    TIMEOUT    USE_RR   ROOTFS_TAG    ROOTFS_HASH
-# linux    686-linux-gnu             .             i686           .          .        v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
-linux      x86_64-linux-gnu          .             x86_64         .          .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-linux      x86_64-linux-gnuassert    .             x86_64         360        rr       v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-# linux    aarch64-linux-gnu         true          aarch64        .          .        ----          ----------------------------------------
-# linux    armv7l-linux-gnueabihf    true          armv7l         .          .        ----          ----------------------------------------
-# linux    powerpc64le-linux-gnu     true          powerpc64le    .          .        ----          ----------------------------------------
+# OS       TRIPLET                   ALLOW_FAIL    ARCH           ARCH_ROOTFS    TIMEOUT    USE_RR   ROOTFS_TAG    ROOTFS_HASH
+# linux    i686-linux-gnu            .             x86_64         i686           .          .        ----          ----------------------------------------
+linux      x86_64-linux-gnu          .             x86_64         x86_64         .          .        v5.9          2212d69a3bc8a516ae8731ac20771d90d62206bc
+linux      x86_64-linux-gnuassert    .             x86_64         x86_64         360        rr       v5.9          2212d69a3bc8a516ae8731ac20771d90d62206bc
+# linux    aarch64-linux-gnu         true          aarch64        aarch64        .          .        ----          ----------------------------------------
+# linux    armv7l-linux-gnueabihf    true          armv7l         armv7l         .          .        ----          ----------------------------------------
+# linux    powerpc64le-linux-gnu     true          powerpc64le    powerpc64le    .          .        ----          ----------------------------------------
 
-musl       x86_64-linux-musl         true          x86_64         .          .        v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
+musl       x86_64-linux-musl         true          x86_64         x86_64         .          .        v5.9          417adcf6facf0933d496fa45a831a10632a7313f
 
 # These special lines allow us to embed default values for the columns above.
 # Any column without a default mapping here will simply substitute a `.` to the empty string

diff --git a/pipelines/main/platforms/test_linux.yml b/pipelines/main/platforms/test_linux.yml
@@ -14,69 +14,23 @@ steps:
               persist_depot_dirs: packages,artifacts,compiled
               version: '1.6'
           - staticfloat/sandbox#v1:
-              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz
-              # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${OS?}.${ARCH_ROOTFS?}.tar.gz
+              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester_${OS?}.${ARCH_ROOTFS?}.tar.gz
               rootfs_treehash: "${ROOTFS_HASH?}"
               uid: 1000
               gid: 1000
               workspaces:
                 # Include `/cache/repos` so that our `git` version introspection works.
                 - "/cache/repos:/cache/repos"
-        env:
-          JULIA_SHELL: "/bin/bash"
         timeout_in_minutes: ${TIMEOUT?}
         soft_fail: ${ALLOW_FAIL?}
-        commands: |
-          # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
-          TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh
-
-          echo "--- Download build artifacts"
-          buildkite-agent artifact download "$${UPLOAD_FILENAME}" .
-
-          echo "--- Extract build artifacts"
-          tar xzf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}/"
-
-          echo "--- Print Julia version info"
-          $${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
-          echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS}"
-          $${JULIA_BINARY} -e '@info "" Sys.CPU_THREADS'
-
-          echo "--- Set some environment variables"
-          export OPENBLAS_NUM_THREADS=8
-          unset JULIA_DEPOT_PATH
-          unset JULIA_PKG_SERVER
-
-          # Make sure that temp files and temp directories are created in a location that is
-          # backed by real storage.
-          export TMPDIR="$(pwd)/tmp"
-          mkdir -p $${TMPDIR}
-
-          # If we're running inside of `rr`, limit the number of threads
-          if [[ "${USE_RR?}" == "rr" ]]; then
-            export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY}"
-            export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
-            export JULIA_NUM_THREADS=1
-
-            # Don't run network tests on `rr`, as it causes the trace size to explode.
-            export TESTS="all --ci --skip Artifacts Downloads download LazyArtifacts LibGit2/online Pkg"
-          else
-            export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}"
-            export NCORES_FOR_TESTS="Sys.CPU_THREADS"
-            export JULIA_NUM_THREADS=16
-            export TESTS="all LibGit2/online --ci"
-          fi
-
-          echo "--- Print the list of test sets, and other useful environment variables"
-          echo "JULIA_CMD_FOR_TESTS is:    $${JULIA_CMD_FOR_TESTS:?}"
-          echo "JULIA_NUM_THREADS is:      $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set
-          echo "NCORES_FOR_TESTS is:       $${NCORES_FOR_TESTS:?}"
-          echo "OPENBLAS_NUM_THREADS is:   $${OPENBLAS_NUM_THREADS:?}"
-          echo "TESTS is:                  $${TESTS:?}"
-          echo "USE_RR is:                 ${USE_RR?}"
-
-          echo "--- Run the Julia test suite"
-          $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})"
+        commands: "bash .buildkite/utilities/test_julia.sh"
         agents:
           queue: "julia"
           sandbox_capable: "true"
           os: "linux"
+          arch: "${ARCH?}"
+        env:
+          JULIA_SHELL: "/bin/bash"
+          TRIPLET: "${TRIPLET?}"
+          USE_RR: "${USE_RR?}"
+
diff --git a/pipelines/main/platforms/test_macos.arches b/pipelines/main/platforms/test_macos.arches
@@ -0,0 +1,13 @@
+# OS       TRIPLET                 ALLOW_FAIL    ARCH          MAKE_FLAGS     TIMEOUT
+macos      x86_64-apple-darwin     .             x86_64        .              .
+macos      aarch64-apple-darwin    true             aarch64       .              .
+
+
+# These special lines allow us to embed default values for the columns above.
+# Any column without a default mapping here will simply substitute a `.` to the empty string
+
+# Of course we do not allow jobs to fail typically
+#default ALLOW_FAIL false
+
+# Most jobs should finish within 1.5 hours, barring exceptionally slow hardware
+#default TIMEOUT 90
diff --git a/pipelines/main/platforms/test_macos.yml b/pipelines/main/platforms/test_macos.yml
@@ -0,0 +1,21 @@
+steps:
+  - group: ":macos: Macos"
+    steps:
+      - label: ":macos: :test_tube: test ${TRIPLET?}"
+        key: "test_${TRIPLET?}"
+        depends_on:
+          - "build_${TRIPLET?}"
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: "./.buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+        timeout_in_minutes: ${TIMEOUT?}
+        soft_fail: ${ALLOW_FAIL?}
+        commands: "bash .buildkite/utilities/test_julia.sh"
+        agents:
+          queue: "julia"
+          os: "macos"
+          arch: "${ARCH}"
+        env:
+          JULIA_SHELL: "/bin/bash"
+          TRIPLET: "${TRIPLET?}"