CI (Buildkite): Add package_linux and tester_linux for more Linux…

… architectures
JuliaLang · Aug 7, 2021 · c1ad5ef · c1ad5ef
1 parent ed866b5
commit c1ad5ef
Show file tree

Hide file tree

Showing 8 changed files with 184 additions and 114 deletions.
diff --git a/.buildkite/pipelines/experimental/0_webui.yml b/.buildkite/pipelines/experimental/0_webui.yml
@@ -21,4 +21,4 @@ steps:
           # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
           # signed_pipelines:
           #   - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml
-          #     signature: "my_signature"
+          #     signature_file: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml.signature
diff --git a/.buildkite/pipelines/main/launch_unsigned_builders.yml b/.buildkite/pipelines/main/launch_unsigned_builders.yml
@@ -23,7 +23,7 @@ steps:
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
 
-      # Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order.
-      buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml
+      # Finally, we launch all of the platform builders.
+      bash .buildkite/pipelines/main/platforms/platforms.sh linux
     agents:
       queue: julia
diff --git a/.buildkite/pipelines/main/platforms/.gitignore b/.buildkite/pipelines/main/platforms/.gitignore
@@ -0,0 +1 @@
+linux_rr.yml
diff --git a/.buildkite/pipelines/main/platforms/linux.arches b/.buildkite/pipelines/main/platforms/linux.arches
@@ -0,0 +1,6 @@
+# ARCH   ROOTFS_ARCH    ROOTFS_PACK_TAG ROOTFS_PACK_TREE                            ROOTFS_TEST_TAG ROOTFS_TEST_TREE
+# aarch64  aarch64                 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff               v3.2 d38f34db72317ffa798b18b0c8e193795bb254db
+# armv7l   armv7l                  v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25               v3.2 11c66f1c3a094e031fae6a9de0fa7986459466b6
+32       i686                      v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32               v3.2 b63e7f1002f093dc51390cd8f4a6368ea929560f
+# ppc64le  powerpc64le             v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7               v3.2 0b23a48bb5ed4b3226ebceea18375a2b09caf34d
+64       x86_64                    v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7               v3.2 b1b1fce0b76b2dbe41222093f2f9f3f8e226018d
diff --git a/.buildkite/pipelines/main/platforms/linux.yml b/.buildkite/pipelines/main/platforms/linux.yml
@@ -0,0 +1,146 @@
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+
+steps:
+  - label: "package_linux$ARCH"
+    key: package_linux$ARCH
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_PACK_TAG/package_linux.${ROOTFS_ARCH}.tar.gz
+          rootfs_treehash: "$ROOTFS_PACK_TREE"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    commands: |
+      echo "--- Print the full and short commit hashes"
+      SHORT_COMMIT_LENGTH=10
+      SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
+      JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-`
+      ARTIFACT_FILE_EXTENSION="tar.gz"
+      ARTIFACT_FILENAME="$$JULIA_BINARYDIST_FILENAME.$$ARTIFACT_FILE_EXTENSION"
+      echo "The full commit is:                     $$BUILDKITE_COMMIT"
+      echo "The short commit is:                    $$SHORT_COMMIT"
+      echo "The artifact filename will be:    $$ARTIFACT_FILENAME"
+
+      echo "--- Build Julia from source"
+      rm -rf $$ARTIFACT_FILENAME
+      make -j 8
+
+      echo "--- Make sure that the working directory is clean"
+      if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi
+
+      echo "--- Print Julia version info"
+      ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
+
+      echo "--- Create build artifacts"
+      make -j 8 binary-dist
+      ls -l $$ARTIFACT_FILENAME
+
+      echo "--- Upload build artifacts"
+      buildkite-agent artifact upload $$ARTIFACT_FILENAME
+    timeout_in_minutes: 60
+    notify:
+      - github_commit_status:
+          context: "package_linux$ARCH"
+
+  - label: "tester_linux$ARCH"
+    key: tester_linux$ARCH
+    depends_on: package_linux$ARCH
+    if: false # TODO: delete this line
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_TEST_TREE/package_linux.${ROOTFS_ARCH}.tar.gz
+          rootfs_treehash: "$ROOTFS_TEST_TREE"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    env:
+      JULIA_SHELL: "/bin/bash"
+    commands: |
+      echo "--- Print the full and short commit hashes"
+      SHORT_COMMIT_LENGTH=10
+      SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
+      JULIA_DIRECTORY_NAME="julia-$$SHORT_COMMIT"
+      ARTIFACT_FILE_EXTENSION="tar.gz"
+      ARTIFACT_FILENAME="julia-$$SHORT_COMMIT-linux$ARCH.$$ARTIFACT_FILE_EXTENSION"
+      echo "The full commit is:                     $$BUILDKITE_COMMIT"
+      echo "The short commit is:                    $$SHORT_COMMIT"
+      echo "The artifact filename will be:    $$ARTIFACT_FILENAME"
+      echo "The Julia directory name will be: $$JULIA_DIRECTORY_NAME"
+
+      echo "--- Download build artifacts"
+      rm -rf $$ARTIFACT_FILENAME
+      buildkite-agent artifact download $$ARTIFACT_FILENAME .
+
+      echo "--- Extract build artifacts"
+      rm -rf $$JULIA_DIRECTORY_NAME/
+      tar xzf $$ARTIFACT_FILENAME $$JULIA_DIRECTORY_NAME/
+
+      echo "--- Print Julia version info"
+      $$JULIA_DIRECTORY_NAME/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
+
+      echo "--- Run the Julia test suite"
+      unset JULIA_DEPOT_PATH
+      $$JULIA_DIRECTORY_NAME/bin/julia .buildkite/utilities/rr/rr_capture.jl $$JULIA_DIRECTORY_NAME/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)'
+    timeout_in_minutes: 150
+    notify:
+      - github_commit_status:
+          context: "tester_linux$ARCH"
+
+  - label: "tester2_linux$ARCH"
+    key: tester2_linux$ARCH
+    depends_on: package_linux$ARCH
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_PACK_TAG/package_linux.${ROOTFS_ARCH}.tar.gz
+          rootfs_treehash: "$ROOTFS_PACK_TREE"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    env:
+      JULIA_SHELL: "/bin/bash"
+    commands: |
+      echo "--- Print the full and short commit hashes"
+      SHORT_COMMIT_LENGTH=10
+      SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
+      JULIA_DIRECTORY_NAME="julia-$$SHORT_COMMIT"
+      ARTIFACT_FILE_EXTENSION="tar.gz"
+      ARTIFACT_FILENAME="julia-$$SHORT_COMMIT-linux$ARCH.$$ARTIFACT_FILE_EXTENSION"
+      echo "The full commit is:                     $$BUILDKITE_COMMIT"
+      echo "The short commit is:                    $$SHORT_COMMIT"
+      echo "The artifact filename will be:    $$ARTIFACT_FILENAME"
+      echo "The Julia directory name will be: $$JULIA_DIRECTORY_NAME"
+
+      echo "--- Download build artifacts"
+      rm -rf $$ARTIFACT_FILENAME
+      buildkite-agent artifact download $$ARTIFACT_FILENAME .
+
+      echo "--- Extract build artifacts"
+      rm -rf $$JULIA_DIRECTORY_NAME/
+      tar xzf $$ARTIFACT_FILENAME $$JULIA_DIRECTORY_NAME/
+
+      echo "--- Print Julia version info"
+      $$JULIA_DIRECTORY_NAME/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
+
+      echo "--- Run the Julia test suite"
+      unset JULIA_DEPOT_PATH
+      $$JULIA_DIRECTORY_NAME/bin/julia .buildkite/utilities/rr/rr_capture.jl $$JULIA_DIRECTORY_NAME/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)'
+    timeout_in_minutes: 150
+    notify:
+      - github_commit_status:
+          context: "tester2_linux$ARCH"
diff --git a/.buildkite/pipelines/main/platforms/linux64.yml b/.buildkite/pipelines/main/platforms/linux64.yml
diff --git a/.buildkite/pipelines/main/platforms/platforms.sh b/.buildkite/pipelines/main/platforms/platforms.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+PLATFORM="$1"
+
+cat "$SCRIPT_DIR/$PLATFORM.arches" | tr -s ' ' | while read line; do
+  if [[ $line == \#* ]]; then
+    continue
+  fi
+  export ARCH=`echo $line | cut -d ' ' -f 1`
+  export ROOTFS_ARCH=`echo $line | cut -d ' ' -f 2`
+  export ROOTFS_PACK_TAG=`echo $line | cut -d ' ' -f 3`
+  export ROOTFS_PACK_TREE=`echo $line | cut -d ' ' -f 4`
+  export ROOTFS_TEST_TAG=`echo $line | cut -d ' ' -f 5`
+  export ROOTFS_TEST_TREE=`echo $line | cut -d ' ' -f 6`
+  echo "Launching: PLATFORM=$PLATFORM ARCH=$ARCH ROOTFS_ARCH=$ROOTFS_ARCH"
+  buildkite-agent pipeline upload "$SCRIPT_DIR/$PLATFORM.yml"
+done
diff --git a/.buildkite/utilities/rr/rr_capture.jl b/.buildkite/utilities/rr/rr_capture.jl
@@ -5,35 +5,26 @@ using Tar
 if Base.VERSION < v"1.6"
     throw(ErrorException("The `rr_capture.jl` script requires Julia 1.6 or greater"))
 end
-
 if length(ARGS) < 1
     throw(ErrorException("Usage: rr_capture.jl [command...]"))
 end
 
-const TIMEOUT = 2 * 60 * 60 # timeout in seconds
-
-# We only use `rr` on the `tester_linux64` builder
-const use_rr_if_builder_is = "tester_linux64"
-
-const run_id = get(ENV, "BUILDKITE_JOB_ID", "unknown")
-const shortcommit = get(ENV, "BUILDKITE_COMMIT", "unknown")
-const builder = get(ENV, "BUILDKITE_STEP_KEY", use_rr_if_builder_is)
-const use_rr = builder == use_rr_if_builder_is
-
-@info "" run_id shortcommit builder use_rr
-@info "" ARGS
-
-# if !use_rr # TODO: uncomment this line
-if true # TODO: delete this line
-    @info "We will not run the tests under rr"
+# We only use `rr` on certain builders
+const rr_builder_list = ["tester_linux64", "tester2_linux64"]
+const this_builder = ENV["BUILDKITE_STEP_KEY"]
+if !(this_builder in rr_builder_list)
+    @info "We will not run the tests under rr" this_builder rr_builder_list
     p = run(`$ARGS`)
     exit(p.exitcode)
 end
 
 @info "We will run the tests under rr"
 
+const MINUTE = 60 # number of seconds in one minute
+const TIMEOUT = 120 * MINUTE
+const run_id = ENV["BUILDKITE_JOB_ID"]
+const shortcommit = ENV["BUILDKITE_COMMIT"]
 const num_cores = min(Sys.CPU_THREADS, 8, parse(Int, get(ENV, "JULIA_TEST_NUM_CORES", "8")) + 1)
-@info "" num_cores
 
 proc = nothing