Skip to content

Commit

Permalink
CI (Buildkite): Add package_linux$ARCH and tester_linux$ARCH for …
Browse files Browse the repository at this point in the history
…more Linux architectures, and use a separate rootfs image for the `tester_*` jobs
  • Loading branch information
DilumAluthge committed Jul 26, 2021
1 parent f7f46af commit 986b1c7
Show file tree
Hide file tree
Showing 4 changed files with 148 additions and 93 deletions.
6 changes: 5 additions & 1 deletion .buildkite/pipelines/main/launch_unsigned_builders.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ steps:
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
# Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order.
buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml
# ARCH=aarch64 ROOTFS_ARCH=aarch64 ROOTFS_TAG_PACK=v3.2 ROOTFS_TREE_PACK=0566841e29f0f9880541c26a6595fd5ce0beb5ff ROOTFS_TAG_TEST=v3.2 ROOTFS_TREE_TEST=d38f34db72317ffa798b18b0c8e193795bb254db buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux.yml
# ARCH=armv7l ROOTFS_ARCH=armv7l ROOTFS_TAG_PACK=v3.2 ROOTFS_TREE_PACK=fb359370b052a47ce5c84cc6b4a7a03ed7053b25 ROOTFS_TAG_TEST=v3.2 ROOTFS_TREE_TEST=11c66f1c3a094e031fae6a9de0fa7986459466b6 buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux.yml
ARCH=32 ROOTFS_ARCH=i686 ROOTFS_TAG_PACK=v3.2 ROOTFS_TREE_PACK=209c4db679a515befd7fb50ecc6bfbecf7ec3d32 ROOTFS_TAG_TEST=v3.2 ROOTFS_TREE_TEST=b63e7f1002f093dc51390cd8f4a6368ea929560f buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux.yml
# ARCH=ppc64le ROOTFS_ARCH=powerpc64le ROOTFS_TAG_PACK=v3.2 ROOTFS_TREE_PACK=c03a0158b19d48ac84b426834fce0d3584cdd0c7 ROOTFS_TAG_TEST=v3.2 ROOTFS_TREE_TEST=0b23a48bb5ed4b3226ebceea18375a2b09caf34d buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux.yml
ARCH=64 ROOTFS_ARCH=x86_64 ROOTFS_TAG_PACK=v3.2 ROOTFS_TREE_PACK=474bf61a926b2d7fcf202284d59d4b11a04601d7 ROOTFS_TAG_TEST=v3.2 ROOTFS_TREE_TEST=b1b1fce0b76b2dbe41222093f2f9f3f8e226018d buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux.yml
agents:
queue: julia
142 changes: 142 additions & 0 deletions .buildkite/pipelines/main/platforms/linux.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
agents:
queue: "julia"
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
sandbox.jl: "true"
os: "linux"

steps:
- label: "package_linux$ARCH"
key: package_linux$ARCH
plugins:
- JuliaCI/julia#v1:
version: 1.6
- staticfloat/sandbox#v1:
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_TAG_PACK/package_linux.$ROOTFS_ARCH.tar.gz
rootfs_treehash: "$ROOTFS_TREE_PACK"
uid: 1000
gid: 1000
workspaces:
# Include `/cache/repos` so that our `git` version introspection works.
- "/cache/repos:/cache/repos"
commands: |
echo "--- Print the full and short commit hashes"
SHORT_COMMIT_LENGTH=10
SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-`
ARTIFACT_FILE_EXTENSION="tar.gz"
ARTIFACT_FILENAME="$$JULIA_BINARYDIST_FILENAME.$$ARTIFACT_FILE_EXTENSION"
echo "The full commit is: $$BUILDKITE_COMMIT"
echo "The short commit is: $$SHORT_COMMIT"
echo "The artifact filename will be: $$ARTIFACT_FILENAME"
echo "--- Build Julia from source"
rm -rf $$ARTIFACT_FILENAME
make -j 8
echo "--- Print Julia version info"
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
echo "--- Create build artifacts"
make -j 8 binary-dist
ls -l $$ARTIFACT_FILENAME
echo "--- Upload build artifacts"
buildkite-agent artifact upload $$ARTIFACT_FILENAME
timeout_in_minutes: 60
notify:
- github_commit_status:
context: "package_linux$ARCH"

- label: "tester_linux$ARCH"
key: tester_linux$ARCH
depends_on: package_linux$ARCH
plugins:
- JuliaCI/julia#v1:
version: 1.6
- staticfloat/sandbox#v1:
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_TAG_TEST/tester_linux.$ROOTFS_ARCH.tar.gz
rootfs_treehash: "$ROOTFS_TREE_TEST"
uid: 1000
gid: 1000
workspaces:
# Include `/cache/repos` so that our `git` version introspection works.
- "/cache/repos:/cache/repos"
env:
JULIA_SHELL: "/bin/bash"
commands: |
echo "--- Print the full and short commit hashes"
SHORT_COMMIT_LENGTH=10
SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
JULIA_DIRECTORY_NAME="julia-$$SHORT_COMMIT"
ARTIFACT_FILE_EXTENSION="tar.gz"
ARTIFACT_FILENAME="julia-$$SHORT_COMMIT-linux$ARCH.$$ARTIFACT_FILE_EXTENSION"
echo "The full commit is: $$BUILDKITE_COMMIT"
echo "The short commit is: $$SHORT_COMMIT"
echo "The artifact filename will be: $$ARTIFACT_FILENAME"
echo "The Julia directory name will be: $$JULIA_DIRECTORY_NAME"
echo "--- Download build artifacts"
rm -rf $$ARTIFACT_FILENAME
buildkite-agent artifact download $$ARTIFACT_FILENAME .
echo "--- Extract build artifacts"
rm -rf $$JULIA_DIRECTORY_NAME/
tar xzf $$ARTIFACT_FILENAME $$JULIA_DIRECTORY_NAME/
echo "--- Print Julia version info"
$$JULIA_DIRECTORY_NAME/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
echo "--- Run the Julia test suite"
unset JULIA_DEPOT_PATH
$$JULIA_DIRECTORY_NAME/bin/julia .buildkite/utilities/rr/rr_capture.jl $$JULIA_DIRECTORY_NAME/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)'
timeout_in_minutes: 120
notify:
- github_commit_status:
context: "tester_linux$ARCH"

- label: "tester_linux$ARCH_sameimage"
key: tester_linux$ARCH_sameimage
depends_on: package_linux$ARCH
plugins:
- JuliaCI/julia#v1:
version: 1.6
- staticfloat/sandbox#v1:
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/$ROOTFS_TAG_PACK/tester_linux.$ROOTFS_ARCH.tar.gz
rootfs_treehash: "$ROOTFS_TREE_PACK"
uid: 1000
gid: 1000
workspaces:
# Include `/cache/repos` so that our `git` version introspection works.
- "/cache/repos:/cache/repos"
env:
JULIA_SHELL: "/bin/bash"
commands: |
echo "--- Print the full and short commit hashes"
SHORT_COMMIT_LENGTH=10
SHORT_COMMIT=`echo $$BUILDKITE_COMMIT | cut -c1-$$SHORT_COMMIT_LENGTH`
JULIA_DIRECTORY_NAME="julia-$$SHORT_COMMIT"
ARTIFACT_FILE_EXTENSION="tar.gz"
ARTIFACT_FILENAME="julia-$$SHORT_COMMIT-linux$ARCH.$$ARTIFACT_FILE_EXTENSION"
echo "The full commit is: $$BUILDKITE_COMMIT"
echo "The short commit is: $$SHORT_COMMIT"
echo "The artifact filename will be: $$ARTIFACT_FILENAME"
echo "The Julia directory name will be: $$JULIA_DIRECTORY_NAME"
echo "--- Download build artifacts"
rm -rf $$ARTIFACT_FILENAME
buildkite-agent artifact download $$ARTIFACT_FILENAME .
echo "--- Extract build artifacts"
rm -rf $$JULIA_DIRECTORY_NAME/
tar xzf $$ARTIFACT_FILENAME $$JULIA_DIRECTORY_NAME/
echo "--- Print Julia version info"
$$JULIA_DIRECTORY_NAME/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
echo "--- Run the Julia test suite"
unset JULIA_DEPOT_PATH
$$JULIA_DIRECTORY_NAME/bin/julia .buildkite/utilities/rr/rr_capture.jl $$JULIA_DIRECTORY_NAME/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)'
timeout_in_minutes: 120
notify:
- github_commit_status:
context: "tester_linux$ARCH_sameimage"
90 changes: 0 additions & 90 deletions .buildkite/pipelines/main/platforms/linux64.yml

This file was deleted.

3 changes: 1 addition & 2 deletions .buildkite/utilities/rr/rr_capture.jl
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,7 @@ const use_rr = builder == use_rr_if_builder_is
@info "" run_id shortcommit builder use_rr
@info "" ARGS

# if !use_rr # TODO: uncomment this line
if true # TODO: delete this line
if !use_rr
@info "We will not run the tests under rr"
p = run(`$ARGS`)
exit(p.exitcode)
Expand Down

0 comments on commit 986b1c7

Please sign in to comment.