Big cleanup

Makefile

@@ -0,0 +1,5 @@
+sign_treehashes:
+	cd .buildkite/cryptic_repo_root && sign_treehashes --repo-root=$$(pwd)
+
+verify_treehashes:
+	cd .buildkite/cryptic_repo_root && verify_treehashes --repo-root=$$(pwd)

pipelines/main/launch_signed_jobs.yml.signature

@@ -1,3 +1,2 @@
-Salted__QK��I�.��|��a��fr
-��H
-�$�$?�����]$��ԅG0�8B���c!ª�4���=f���1v�0�9�4S���-|
+Salted__}�2|�����o<V\������,������p��o�F7v^s��G��.H�`iŘG���?ҭP�XR"���M�h�
+���ZMa��Ha�{�P

pipelines/main/launch_unsigned_builders.yml

@@ -27,7 +27,7 @@ steps:
 
       # Launch all of the platform jobs.
       bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml
-      bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/main/platforms/tester_linux.arches  .buildkite/pipelines/main/platforms/tester_linux.yml
+      bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/main/platforms/test_linux.arches  .buildkite/pipelines/main/platforms/test_linux.yml
 
       # Launch the `whitespace` job last. Uploading it last actually causes it to start
       # first. We want this job to start first because we want it to finish as quickly

pipelines/main/platforms/package_linux.arches

@@ -1,10 +1,10 @@
-# PLATFORM    ALLOW_FAIL    TAR_ARCH    ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT       ROOTFS_TAG    ROOTFS_HASH
-# linux       .             32          i686           .              .             v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
-linux         .             64          x86_64         .              .             v5.1          f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
-# linux       .             aarch64     aarch64        .              .             ----          ----------------------------------------
-# linux       .             armv7l      armv7l         .              .             ----          ----------------------------------------
-# linux       .             ppc64le     powerpc64le    .              .             ----          ----------------------------------------
-# musl        .             64          x86_64         .              .             v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
+# OS       TRIPLET                 ALLOW_FAIL    TAR_ARCH    ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT       ROOTFS_TAG    ROOTFS_HASH
+# linux    i686-linux-gnu          .             32          i686           .              .             v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+linux      x86_64-linux-gnu        .             64          x86_64         .              .             v5.1          f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
+# linux    aarch64-linux-gnu       .             aarch64     aarch64        .              .             ----          ----------------------------------------
+# linux    armv7l-linux-gnueabihf  .             armv7l      armv7l         .              .             ----          ----------------------------------------
+# linux    powerpc64le-linux-gnu   .             ppc64le     powerpc64le    .              .             ----          ----------------------------------------
+# musl     x86_64-linux-musl       .             64          x86_64         .              .             v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
 
 
 # These special lines allow us to embed default values for the columns above.

pipelines/main/platforms/package_linux.yml

@@ -4,8 +4,8 @@ agents:
   sandbox.jl: "true"
   os: "linux"
 steps:
-  - label: ":linux: package ${PLATFORM?}${TAR_ARCH?}"
-    key: package_${PLATFORM?}${TAR_ARCH?}
+  - label: ":linux: package ${TRIPLET?}"
+    key: "package_${TRIPLET?}"
     plugins:
       - JuliaCI/external-buildkite#v1:
           version: ".buildkite-external-version"
@@ -15,7 +15,7 @@ steps:
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz
           rootfs_treehash: "${ROOTFS_HASH?}"
           uid: 1000
           gid: 1000
@@ -25,7 +25,7 @@ steps:
     timeout_in_minutes: ${TIMEOUT?}
     commands: |
       # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
-      TAR_ARCH=${TAR_ARCH?} PLATFORM=${PLATFORM?} MAKE_FLAGS=${MAKE_FLAGS?} source .buildkite/utilities/calc_version_envs.sh
+      TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS=${MAKE_FLAGS?} source .buildkite/utilities/calc_version_envs.sh
 
       echo "--- Build Julia from source"
       rm -rf $${ARTIFACT_FILENAME}

pipelines/main/platforms/test_linux.arches

@@ -0,0 +1,20 @@
+# OS       TRIPLET                 ALLOW_FAIL    TAR_ARCH    ARCH_ROOTFS    TIMEOUT    IS_RR    ROOTFS_TAG    ROOTFS_HASH
+# linux    686-linux-gnu           .             32          i686           .          .        v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+linux      x86_64-linux-gnu        .             64          x86_64         .          .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux      x86_64-linux-gnu        .             64          x86_64         .          rr       v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+
+# linux    aarch64-linux-gnu       true          aarch64     aarch64        .          .        ----          ----------------------------------------
+# linux    armv7l-linux-gnueabihf  true          armv7l      armv7l         .          .        ----          ----------------------------------------
+# linux    powerpc64le-linux-gnu   true          ppc64le     powerpc64le    .          .        ----          ----------------------------------------
+
+# musl     x86_64-linux-musl       true          64          x86_64         .          .        v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
+
+
+# These special lines allow us to embed default values for the columns above.
+# Any column without a default mapping here will simply substitute a `.` to the empty string
+
+# Of course we do not allow jobs to fail typically
+#default ALLOW_FAIL false
+
+# Most jobs should finish within 1.5 hours, barring exceptionally slow hardware
+#default TIMEOUT 90

pipelines/main/platforms/tester_linux.yml → pipelines/main/platforms/test_linux.yml

@@ -4,10 +4,10 @@ agents:
   sandbox.jl: "true"
   os: "linux"
 steps:
-  - label: ":linux: test ${PLATFORM?}${LABEL?}"
-    key:   "tester_${PLATFORM?}${LABEL?}"
+  - label: ":linux: test ${TRIPLET?}"
+    key: "test_${TRIPLET?}${USE_RR-}"
     depends_on:
-      - "package_${PLATFORM?}${TAR_ARCH?}"
+      - "package_${TRIPLET?}"
     plugins:
       - JuliaCI/external-buildkite#v1:
           version: ".buildkite-external-version"
@@ -17,8 +17,8 @@ steps:
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz
-          # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${OS?}.${ARCH_ROOTFS?}.tar.gz
+          # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${OS?}.${ARCH_ROOTFS?}.tar.gz
           rootfs_treehash: "${ROOTFS_HASH?}"
           uid: 1000
           gid: 1000
@@ -27,11 +27,11 @@ steps:
             - "/cache/repos:/cache/repos"
     env:
       JULIA_SHELL: "/bin/bash"
-    timeout_in_minutes: ${TIMEOUT_BK?}
+    timeout_in_minutes: ${TIMEOUT?}
     soft_fail: ${ALLOW_FAIL?}
     commands: |
       # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
-      TAR_ARCH=${TAR_ARCH?} PLATFORM=${PLATFORM?} MAKE_FLAGS=${MAKE_FLAGS?} source .buildkite/utilities/calc_version_envs.sh
+      TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS= source .buildkite/utilities/calc_version_envs.sh
 
       echo "--- Download build artifacts"
       buildkite-agent artifact download $${ARTIFACT_FILENAME} .
@@ -56,13 +56,15 @@ steps:
 
       # By default, run all tests.
       export TESTS="all LibGit2/online --ci"
-      export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}"
 
       # If we're running inside of `rr`, limit the number of threads
-      if [[ "${IS_RR?}" == "yes" ]]; then
+      if [[ "${IS_RR?}" == "rr" ]]; then
         export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY}"
         export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
         export JULIA_NUM_THREADS=1
+
+        # Give `rr` 60 minutes to run the tests
+        export JULIA_TEST_RR_TIMEOUT="60"
       else
         export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY}"
         export NCORES_FOR_TESTS="Sys.CPU_THREADS"

pipelines/main/platforms/upload_linux.yml

@@ -4,11 +4,12 @@ agents:
   sandbox.jl: "true"
   os: "linux"
 steps:
-  - label: ":linux: upload ${PLATFORM?}${LABEL?}"
-    key:   "upload_${PLATFORM?}${LABEL?}"
+  - label: ":linux: upload ${TRIPLET?}"
+    key:   "upload_${TRIPLET?}"
     depends_on:
-      - "package_${PLATFORM}${TAR_ARCH}"
-      - "tester_${PLATFORM?}${LABEL?}"
+      # Wait for the tester to finish
+      - "test_${TRIPLET?}"
+
     plugins:
       - JuliaCI/external-buildkite#v1:
           version: ".buildkite-external-version"
@@ -22,12 +23,10 @@ steps:
           rootfs_treehash: "65becc21c66b853a6057410109a660aafab1a755"
           uid: 1000
           gid: 1000
-    timeout_in_minutes: ${TIMEOUT_BK?}
+    timeout_in_minutes: ${TIMEOUT?}
     commands: |
-      SHORT_COMMIT_LENGTH=10
-      LONG_COMMIT="$$(git rev-parse HEAD)"
-      SHORT_COMMIT=`echo $${LONG_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}`
-      ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${TAR_ARCH?}.tar.gz"
+      # First, get things like `LONG_COMMIT` and `SHORT_COMMIT`, etc...
+      TAR_ARCH=${TAR_ARCH?} OS=${OS?} MAKE_FLAGS=${MAKE_FLAGS?} source .buildkite/utilities/calc_version_envs.sh
 
-      echo "--- Download $${ARTIFACT_FILENAME:?}"
-      buildkite-agent artifact upload $${ARTIFACT_FILENAME:?}
+      echo "--- Download $${ARTIFACT_FILENAME}"
+      buildkite-agent artifact upload $${ARTIFACT_FILENAME}

pipelines/scheduled/coverage/coverage_linux64.yml.signature

@@ -1 +1 @@
-Salted__+&�6
7�#�:�r��+��Vm@��:\�N��5��
Kz!k��pzk(�����C��q9S3��߶�N�Ţ��M��O�I����Qo�4
+Salted__��#��8�Ыڵk�m��&�o���6��p����I����#����{�*؞oZ0ě�w���ĵd�&�i��kå�1�W��_��<(

pipelines/scheduled/launch_unsigned_jobs.yml

@@ -3,6 +3,6 @@ steps:
     commands: |
       # Launch all of the `USE_BINARYBUILDER=0` jobs.
       bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml
-      bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml
+      bash .buildkite/utilities/arches_pipeline_upload.sh .buildkite/pipelines/scheduled/no_bb/no_bb_test_linux.arches .buildkite/pipelines/main/platforms/test_linux.yml
     agents:
       queue: julia

pipelines/scheduled/no_bb/no_bb_package_linux.arches

@@ -1,2 +1,2 @@
-# PLATFORM    ALLOW_FAIL    TAR_ARCH    ARCH_ROOTFS    MAKE_FLAGS             TIMEOUT    ROOTFS_TAG    ROOTFS_HASH
-linux         .             64src       x86_64         USE_BINARYBUILDER=0    180        v4.8          f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
+# OS    ALLOW_FAIL    TAR_ARCH    ARCH_ROOTFS    MAKE_FLAGS             TIMEOUT    ROOTFS_TAG    ROOTFS_HASH
+linux   .             64src       x86_64         USE_BINARYBUILDER=0    180        v4.8          f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0

pipelines/scheduled/no_bb/no_bb_tester_linux.arches

@@ -1,2 +1,2 @@
-# PLATFORM    LABEL       ALLOW_FAIL    TAR_ARCH ARCH_ROOTFS    TIMEOUT_BK    TIMEOUT_RR     IS_RR    ROOTFS_TAG    ROOTFS_HASH
-linux         64src       .             64src    x86_64         .             .              .        v4.8          f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
+# OS    LABEL       ALLOW_FAIL    TAR_ARCH ARCH_ROOTFS    TIMEOUT_BK    TIMEOUT_RR     IS_RR    ROOTFS_TAG    ROOTFS_HASH
+linux   64src       .             64src    x86_64         .             .              .        v4.8          f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0

utilities/arches_env.sh

@@ -9,8 +9,8 @@ set -eou pipefail
 
 ARCHES_FILE="${1:-}"
 if [[ ! -f "${ARCHES_FILE}" ]] ; then
-  echo "Arches file does not exist: '${ARCHES_FILE}'"
-  exit 1
+    echo "Arches file does not exist: '${ARCHES_FILE}'"
+    exit 1
 fi
 
 
@@ -21,47 +21,47 @@ readarray -d ' ' -s 1 -t var_names < <(head -1 "${ARCHES_FILE}" | tr -s ' ')
 declare -A defaults_map
 readarray -t default_mapping_lines < <(grep "^#default" "${ARCHES_FILE}" | tr -s ' ')
 for idx in "${!default_mapping_lines[@]}"; do
-  key="$(  cut -d' ' -f2 <<<"${default_mapping_lines[${idx}]}" | xargs)"
-  value="$(cut -d' ' -f3 <<<"${default_mapping_lines[${idx}]}" | xargs)"
-  defaults_map["${key}"]="${value}"
+    key="$(  cut -d' ' -f2 <<<"${default_mapping_lines[${idx}]}" | xargs)"
+    value="$(cut -d' ' -f3 <<<"${default_mapping_lines[${idx}]}" | xargs)"
+    defaults_map["${key}"]="${value}"
 done
 
 cat "${ARCHES_FILE}" | while read line; do
-  # Remove whitespace from the beginning and end of each line
-  line="$(tr -s ' ' <<<"${line}")"
+    # Remove whitespace from the beginning and end of each line
+    line="$(tr -s ' ' <<<"${line}")"
 
-  # Skip any line that begins with the `#` character
-  if [[ $line == \#* ]]; then
-    continue
-  fi
+    # Skip any line that begins with the `#` character
+    if [[ $line == \#* ]]; then
+        continue
+    fi
 
-  # Skip any empty line
-  if [[ $line == "" ]]; then
-    continue
-  fi
+    # Skip any empty line
+    if [[ $line == "" ]]; then
+        continue
+    fi
 
-  # Convert line to array
-  readarray -d ' ' -t line_array <<<"${line}"
+    # Convert line to array
+    readarray -d ' ' -t line_array <<<"${line}"
 
-  # Panic if we don't have the same number of items as our column names:
-  if [[ "${#line_array[@]}" != "${#var_names[@]}" ]]; then
-    echo "ERROR: The following line does not contain ${#var_names[@]} columns as we would expect from the header of ${ARCHES_FILE}" >&2
-    echo "${line}"
-    exit 1
-  fi
+    # Panic if we don't have the same number of items as our column names:
+    if [[ "${#line_array[@]}" != "${#var_names[@]}" ]]; then
+        echo "ERROR: The following line does not contain ${#var_names[@]} columns as we would expect from the header of ${ARCHES_FILE}" >&2
+        echo "${line}"
+        exit 1
+    fi
 
-  # Loop over columns, bind values to their column name
-  for idx in "${!var_names[@]}"; do
-    # Get the name and value
-    name="$(xargs <<<"${var_names[${idx}]}")"
-    value="$(xargs <<<"${line_array[${idx}]}")"
+    # Loop over columns, bind values to their column name
+    for idx in "${!var_names[@]}"; do
+        # Get the name and value
+        name="$(xargs <<<"${var_names[${idx}]}")"
+        value="$(xargs <<<"${line_array[${idx}]}")"
 
-    # Apply default values to our special `.` token
-    if [[ "${value}" == "." ]]; then
-      value="${defaults_map[${name}]:-}"
-    fi
+        # Apply default values to our special `.` token
+        if [[ "${value}" == "." ]]; then
+            value="${defaults_map[${name}]:-}"
+        fi
 
-    echo -n "${name}=\"${value}\" "
-  done
-  echo
+        echo -n "${name}=\"${value}\" "
+    done
+    echo
 done

utilities/arches_pipeline_upload.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -eou pipefail
+shopt -s nullglob
 
 # This script reads in an `.arches` file, processes the columns and default value mappings
 # within it, and outputs an environment block (e.g. a line of the form "X=a B= C=123") for
@@ -9,19 +10,19 @@ set -eou pipefail
 
 ARCHES_FILE="${1:-}"
 if [[ ! -f "${ARCHES_FILE}" ]] ; then
-  echo "Arches file does not exist: '${ARCHES_FILE}'"
-  exit 1
+    echo "Arches file does not exist: '${ARCHES_FILE}'"
+    exit 1
 fi
 
 YAML_FILE="${2:-}"
 if [[ ! -f "${YAML_FILE}" ]] ; then
-  echo "YAML file does not exist: '${YAML_FILE}'"
-  exit 1
+    echo "YAML file does not exist: '${YAML_FILE}'"
+    exit 1
 fi
 
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-"${BASH}" "${SCRIPT_DIR}/arches_env.sh" "${ARCHES_FILE:?}" | while read env_map; do
-  # Export the environment mappings, then launch the yaml file
-  eval "export ${env_map}"
-  buildkite-agent pipeline upload "${YAML_FILE}"
+"${BASH}" "${SCRIPT_DIR}/arches_env.sh" "${ARCHES_FILE}" | while read env_map; do
+    # Export the environment mappings, then launch the yaml file
+    eval "export ${env_map}"
+    buildkite-agent pipeline upload "${YAML_FILE}"
 done

utilities/calc_version_envs.sh

@@ -4,7 +4,7 @@ echo "--- Print the full and short commit hashes"
 SHORT_COMMIT_LENGTH=10
 LONG_COMMIT="$(git rev-parse HEAD)"
 SHORT_COMMIT="$(echo ${LONG_COMMIT} | cut -c1-${SHORT_COMMIT_LENGTH})"
-ARTIFACT_FILENAME="julia-${SHORT_COMMIT}-${PLATFORM?}${TAR_ARCH?}.tar.gz"
+ARTIFACT_FILENAME="julia-${SHORT_COMMIT}-${OS?}${TAR_ARCH?}.tar.gz"
 JULIA_BINARYDIST="$(make print-JULIA_BINARYDIST_FILENAME ${MAKE_FLAGS?} | cut -c27- | tr -s ' ').tar.gz"
 JULIA_INSTALL_DIR="julia-${SHORT_COMMIT}"
 JULIA_BINARY="${JULIA_INSTALL_DIR}/bin/julia"