Significant reorganization, massive reduction in repetition

The main idea is to merge our `build`, `test` and `upload` steps into single, shared `.sh` files that get included from our `.yml` files. We try to walk the line correctly between storing Julia-related content (such as `julia.icns`) in the Julia repository, while storing the build rules (such as `build_dmg.sh`) within this repository.
JuliaCI · Apr 2, 2022 · 9d2f0a5 · 9d2f0a5
1 parent 4713828
commit 9d2f0a5
Show file tree

Hide file tree

Showing 19 changed files with 406 additions and 416 deletions.
diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
@@ -26,7 +26,7 @@ rm -rf ..?* .[!.]* *
 
 # git clone, then force checkout the given gitref
 UPSTREAM_GITREF="$(buildkite-agent meta-data get --default "origin/${UPSTREAM_BRANCH}" BUILDKITE_JULIA_VERSION)"
-git clone --reference "${UPSTREAM_CACHE}" "${UPSTREAM_URL}" "./"
+git clone --dissociate --reference "${UPSTREAM_CACHE}" "${UPSTREAM_URL}" "./"
 git reset --hard "${UPSTREAM_GITREF}"
 echo
 git --no-pager log -1

diff --git a/Makefile b/Makefile
@@ -1,3 +1,6 @@
+decrypt:
+	cd .buildkite/cryptic_repo_root && decrypt --repo-root=$$(pwd) --verbose
+
 sign_treehashes:
 	cd .buildkite/cryptic_repo_root && sign_treehashes --repo-root=$$(pwd) --verbose
 

diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature
diff --git a/pipelines/main/platforms/build_linux.arches b/pipelines/main/platforms/build_linux.arches
@@ -1,11 +1,11 @@
-# OS       TRIPLET                   ALLOW_FAIL    ARCH_ROOTFS    MAKE_FLAGS                              TIMEOUT       ROOTFS_TAG    ROOTFS_HASH
-# linux    i686-linux-gnu            .             i686           .                                       .             v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
-linux      x86_64-linux-gnu          .             x86_64         .                                       .             v5.1          f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
-linux      x86_64-linux-gnuassert    .             x86_64         FORCE_ASSERTIONS=1,LLVM_ASSERTIONS=1    .             v5.1          f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
-# linux    aarch64-linux-gnu         .             aarch64        .                                       .             ----          ----------------------------------------
-# linux    armv7l-linux-gnueabihf    .             armv7l         .                                       .             ----          ----------------------------------------
-# linux    powerpc64le-linux-gnu     .             powerpc64le    .                                       .             ----          ----------------------------------------
-musl       x86_64-linux-musl         .             x86_64         .                                       .             v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
+# OS       TRIPLET                   ALLOW_FAIL    ARCH           ARCH_ROOTFS    MAKE_FLAGS                              TIMEOUT       ROOTFS_TAG    ROOTFS_HASH
+# linux    i686-linux-gnu            .             x86_64         i686           .                                       .             ----          ----------------------------------------
+linux      x86_64-linux-gnu          .             x86_64         x86_64         .                                       .             v5.7          7c1509f2b2c14dcabb6721bf40067a104b588a69
+linux      x86_64-linux-gnuassert    .             x86_64         x86_64         FORCE_ASSERTIONS=1,LLVM_ASSERTIONS=1    .             v5.7          7c1509f2b2c14dcabb6721bf40067a104b588a69
+# linux    aarch64-linux-gnu         .             aarch64        aarch64        .                                       .             ----          ----------------------------------------
+# linux    armv7l-linux-gnueabihf    .             armv7l         armv7l         .                                       .             ----          ----------------------------------------
+# linux    powerpc64le-linux-gnu     .             powerpc64le    powerpc64le    .                                       .             ----          ----------------------------------------
+musl       x86_64-linux-musl         .             x86_64         x86_64         .                                       .             v5.7          07de810ec6612f5f0d18bf6adae7cb00fdf53f66
 
 # These special lines allow us to embed default values for the columns above.
 # Any column without a default mapping here will simply substitute a `.` to the empty string

diff --git a/pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/build_linux.yml
@@ -20,53 +20,14 @@ steps:
                 # Include `/cache/repos` so that our `git` version introspection works.
                 - "/cache/repos:/cache/repos"
         timeout_in_minutes: ${TIMEOUT?}
-        commands: |
-          # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
-          TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh
-
-          # export JULIA_CPU_TARGET, which is picked up by the build system
-          ARCH="$${ARCH??}" source .buildkite/utilities/export_julia_cpu_target.sh
-
-          echo "--- Create the Make.user file"
-          echo "${MAKE_FLAGS?}" | tr "," "\n" > Make.user
-          if [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-master" ]] || [[ "$${BUILDKITE_PIPELINE_SLUG:?}" =~ ^julia-release- ]]; then
-            banner="Official https://julialang.org/ release"
-            echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user
-          elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-buildkite" ]]; then
-            banner="Not an official binary; built as part of the CI infrastructure testing"
-            echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user
-          fi
-          echo "VERBOSE=1" >> Make.user
-          cat Make.user
-
-          echo "--- Build Julia"
-          make --output-sync -j 8
-
-          echo "--- Check that the working directory is clean"
-          if [ -z "$(git status --short)" ]; then
-            echo "INFO: The working directory is clean."
-          else
-            echo "ERROR: The working directory is dirty."
-            echo "Output of git status:"
-            git status
-            exit 1
-          fi
-
-          echo "--- Print Julia version info"
-          ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
-
-          echo "--- Create build artifacts"
-          make --output-sync -j 8 binary-dist
-
-          # Rename the build artifact in case we want to name it differently, as is the case on `musl`.
-          if [[ "$${JULIA_BINARYDIST_FILENAME}" != "$${UPLOAD_FILENAME}" ]]; then
-              mv $${JULIA_BINARYDIST_FILENAME} $${UPLOAD_FILENAME}
-          fi
-
-          echo "--- Upload build artifacts"
-          buildkite-agent artifact upload $${UPLOAD_FILENAME}
+        commands: "bash .buildkite/utilities/build_julia.sh"
         agents:
           queue: "julia"
           # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
           sandbox_capable: "true"
           os: "linux"
+          arch: "${ARCH?}"
+        env:
+          # Receive some environment mappings from our templating engine
+          TRIPLET: "${TRIPLET?}"
+          MAKE_FLAGS: "${MAKE_FLAGS?}"
diff --git a/pipelines/main/platforms/build_macos.yml b/pipelines/main/platforms/build_macos.yml
@@ -5,57 +5,16 @@ steps:
         key: "build_${TRIPLET?}"
         plugins:
           - JuliaCI/external-buildkite#v1:
-              version: ".buildkite-external-version"
+              version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
         timeout_in_minutes: ${TIMEOUT?}
-        commands: |
-          # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
-          TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh
-          
-          # export JULIA_CPU_TARGET, which is picked up by the build system
-          ARCH="$${ARCH??}" source .buildkite/utilities/export_julia_cpu_target.sh
-          
-          echo "--- Create the Make.user file"
-          echo "${MAKE_FLAGS?}" | tr "," "\n" > Make.user
-          if [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-master" ]] || [[ "$${BUILDKITE_PIPELINE_SLUG:?}" =~ ^julia-release- ]]; then
-            banner="Official https://julialang.org/ release"
-            echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user
-          elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-buildkite" ]]; then
-            banner="Not an official binary; built as part of the CI infrastructure testing"
-            echo "TAGGED_RELEASE_BANNER=\"$${banner:?}\"" >> Make.user
-          fi
-          echo "VERBOSE=1" >> Make.user
-          cat Make.user
-
-          echo "--- Build Julia"
-          make -j 8 ${MAKE_FLAGS?}
-
-          echo "--- Check that the working directory is clean"
-          if [ -z "$(git status --short)" ]; then
-            echo "INFO: The working directory is clean."
-          else
-            echo "ERROR: The working directory is dirty."
-            echo "Output of git status:"
-            git status
-            exit 1
-          fi
-          echo "--- Print Julia version info"
-          ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
-
-          echo "--- Create build artifacts"
-          make -j 8 binary-dist ${MAKE_FLAGS?}
-
-          # Rename the build artifact in case we want to name it differently, as is the case on `musl`.
-          if [[ "$${JULIA_BINARYDIST_FILENAME}" != "$${UPLOAD_FILENAME}" ]]; then
-              mv $${JULIA_BINARYDIST_FILENAME} $${UPLOAD_FILENAME}
-          fi
-
-          echo "--- Upload build artifacts"
-          buildkite-agent artifact upload $${UPLOAD_FILENAME}
+        commands: "bash .buildkite/utilities/build_julia.sh"
         agents:
           queue: "julia"
-          # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
-          # sandbox_capable: "true"
           os: "macos"
-          arch: "${ARCH}"
+          arch: "${ARCH?}"
+        env:
+          # Receive some environment mappings from our templating engine
+          TRIPLET: "${TRIPLET?}"
+          MAKE_FLAGS: "${MAKE_FLAGS?}"
 
diff --git a/pipelines/main/platforms/test_linux.arches b/pipelines/main/platforms/test_linux.arches
@@ -1,12 +1,12 @@
-# OS       TRIPLET                   ALLOW_FAIL    ARCH_ROOTFS    TIMEOUT    USE_RR   ROOTFS_TAG    ROOTFS_HASH
-# linux    686-linux-gnu             .             i686           .          .        v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
-linux      x86_64-linux-gnu          .             x86_64         .          .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-linux      x86_64-linux-gnuassert    .             x86_64         360        rr       v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-# linux    aarch64-linux-gnu         true          aarch64        .          .        ----          ----------------------------------------
-# linux    armv7l-linux-gnueabihf    true          armv7l         .          .        ----          ----------------------------------------
-# linux    powerpc64le-linux-gnu     true          powerpc64le    .          .        ----          ----------------------------------------
+# OS       TRIPLET                   ALLOW_FAIL    ARCH           ARCH_ROOTFS    TIMEOUT    USE_RR   ROOTFS_TAG    ROOTFS_HASH
+# linux    i686-linux-gnu            .             x86_64         i686           .          .        ----          ----------------------------------------
+linux      x86_64-linux-gnu          .             x86_64         x86_64         .          .        v5.7          3e59f38be28e9c966984631286e77b04515849c2
+linux      x86_64-linux-gnuassert    .             x86_64         x86_64         360        rr       v5.7          3e59f38be28e9c966984631286e77b04515849c2
+# linux    aarch64-linux-gnu         true          aarch64        aarch64        .          .        ----          ----------------------------------------
+# linux    armv7l-linux-gnueabihf    true          armv7l         armv7l         .          .        ----          ----------------------------------------
+# linux    powerpc64le-linux-gnu     true          powerpc64le    powerpc64le    .          .        ----          ----------------------------------------
 
-musl       x86_64-linux-musl         true          x86_64         .          .        v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
+musl       x86_64-linux-musl         true          x86_64         x86_64         .          .        v5.7          efa9a3a1d1a6d5f382518a2935958ae51531e501
 
 # These special lines allow us to embed default values for the columns above.
 # Any column without a default mapping here will simply substitute a `.` to the empty string

diff --git a/pipelines/main/platforms/test_linux.yml b/pipelines/main/platforms/test_linux.yml
@@ -14,69 +14,23 @@ steps:
               persist_depot_dirs: packages,artifacts,compiled
               version: '1.6'
           - staticfloat/sandbox#v1:
-              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz
-              # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${OS?}.${ARCH_ROOTFS?}.tar.gz
+              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester_${OS?}.${ARCH_ROOTFS?}.tar.gz
               rootfs_treehash: "${ROOTFS_HASH?}"
               uid: 1000
               gid: 1000
               workspaces:
                 # Include `/cache/repos` so that our `git` version introspection works.
                 - "/cache/repos:/cache/repos"
-        env:
-          JULIA_SHELL: "/bin/bash"
         timeout_in_minutes: ${TIMEOUT?}
         soft_fail: ${ALLOW_FAIL?}
-        commands: |
-          # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
-          TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh
-
-          echo "--- Download build artifacts"
-          buildkite-agent artifact download "$${UPLOAD_FILENAME}" .
-
-          echo "--- Extract build artifacts"
-          tar xzf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}/"
-
-          echo "--- Print Julia version info"
-          $${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
-          echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS}"
-          $${JULIA_BINARY} -e '@info "" Sys.CPU_THREADS'
-
-          echo "--- Set some environment variables"
-          export OPENBLAS_NUM_THREADS=8
-          unset JULIA_DEPOT_PATH
-          unset JULIA_PKG_SERVER
-
-          # Make sure that temp files and temp directories are created in a location that is
-          # backed by real storage.
-          export TMPDIR="$(pwd)/tmp"
-          mkdir -p $${TMPDIR}
-
-          # If we're running inside of `rr`, limit the number of threads
-          if [[ "${USE_RR?}" == "rr" ]]; then
-            export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY}"
-            export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
-            export JULIA_NUM_THREADS=1
-
-            # Don't run network tests on `rr`, as it causes the trace size to explode.
-            export TESTS="all --ci --skip Artifacts Downloads download LazyArtifacts LibGit2/online Pkg"
-          else
-            export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}"
-            export NCORES_FOR_TESTS="Sys.CPU_THREADS"
-            export JULIA_NUM_THREADS=16
-            export TESTS="all LibGit2/online --ci"
-          fi
-
-          echo "--- Print the list of test sets, and other useful environment variables"
-          echo "JULIA_CMD_FOR_TESTS is:    $${JULIA_CMD_FOR_TESTS:?}"
-          echo "JULIA_NUM_THREADS is:      $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set
-          echo "NCORES_FOR_TESTS is:       $${NCORES_FOR_TESTS:?}"
-          echo "OPENBLAS_NUM_THREADS is:   $${OPENBLAS_NUM_THREADS:?}"
-          echo "TESTS is:                  $${TESTS:?}"
-          echo "USE_RR is:                 ${USE_RR?}"
-
-          echo "--- Run the Julia test suite"
-          $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})"
+        commands: "bash .buildkite/utilities/test_julia.sh"
         agents:
           queue: "julia"
           sandbox_capable: "true"
           os: "linux"
+          arch: "${ARCH?}"
+        env:
+          JULIA_SHELL: "/bin/bash"
+          TRIPLET: "${TRIPLET?}"
+          USE_RR: "${USE_RR?}"
+
diff --git a/pipelines/main/platforms/test_macos.yml b/pipelines/main/platforms/test_macos.yml
@@ -7,52 +7,15 @@ steps:
           - "build_${TRIPLET?}"
         plugins:
           - JuliaCI/external-buildkite#v1:
-              version: ".buildkite-external-version"
+              version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
-        env:
-          JULIA_SHELL: "/bin/bash"
         timeout_in_minutes: ${TIMEOUT?}
         soft_fail: ${ALLOW_FAIL?}
-        commands: |
-          # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
-          TRIPLET="${TRIPLET?}" source .buildkite/utilities/calc_version_envs.sh
-
-          echo "--- Download build artifacts"
-          buildkite-agent artifact download "$${UPLOAD_FILENAME}" .
-
-          echo "--- Extract build artifacts"
-          tar xzf "$${UPLOAD_FILENAME}" "$${JULIA_INSTALL_DIR}/"
-
-          echo "--- Ad-hoc sign for testing"
-          contrib/codesign.sh "-" "$${JULIA_INSTALL_DIR}"
-
-          echo "--- Print Julia version info"
-          $${JULIA_BINARY} -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
-          echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS}"
-          $${JULIA_BINARY} -e '@info "" Sys.CPU_THREADS'
-
-          echo "--- Set some environment variables"
-          export OPENBLAS_NUM_THREADS=4
-          unset JULIA_DEPOT_PATH
-          unset JULIA_PKG_SERVER
-
-          # By default, run all tests.
-          export TESTS="all LibGit2/online --ci"
-
-          export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}"
-          export NCORES_FOR_TESTS="Sys.CPU_THREADS"
-
-
-          echo "--- Print the list of test sets, and other useful environment variables"
-          echo "JULIA_CMD_FOR_TESTS is:    $${JULIA_CMD_FOR_TESTS:?}"
-          echo "JULIA_NUM_THREADS is:      $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set
-          echo "NCORES_FOR_TESTS is:       $${NCORES_FOR_TESTS:?}"
-          echo "OPENBLAS_NUM_THREADS is:   $${OPENBLAS_NUM_THREADS:?}"
-          echo "TESTS is:                  $${TESTS:?}"
-
-          echo "--- Run the Julia test suite"
-          $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})"
+        commands: "bash .buildkite/utilities/test_julia.sh"
         agents:
           queue: "julia"
           os: "macos"
           arch: "${ARCH}"
+        env:
+          JULIA_SHELL: "/bin/bash"
+          TRIPLET: "${TRIPLET?}"