Skip to content

Commit

Permalink
Significantly simplify the linux64 testing regime
Browse files Browse the repository at this point in the history 
We no longer use "test groups", we're going to move the retry logic into
Base Julia (I hear that Julian has agreed to do this).  Let's boil it
down to only a single variable; whether this is an RR test or not.
  • Loading branch information
@staticfloat
staticfloat committed Mar 1, 2022
1 parent 34f5642 commit e607bd1
Show file tree
Hide file tree
Showing 6 changed files with 25 additions and 95 deletions.
14 changes: 7 additions & 7 deletions pipelines/main/platforms/package_linux.arches
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
# linux 32 . . 32 i686 . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
linux 64 . . 64 x86_64 . . . . . . . v5.1 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
# linux aarch64 . . aarch64 aarch64 . . . . . . . .... ........................................
# linux armv7l . . armv7l armv7l . . . . . . . .... ........................................
# linux ppc64le . . ppc64le powerpc64le . . . . . . . .... ........................................
# musl 64 . . 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR IS_MT ROOTFS_TAG ROOTFS_HASH
# linux 32 . 32 i686 . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
linux 64 . 64 x86_64 . . . . v5.1 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
# linux aarch64 . aarch64 aarch64 . . . . .... ........................................
# linux armv7l . armv7l armv7l . . . . .... ........................................
# linux ppc64le . ppc64le powerpc64le . . . . .... ........................................
# musl 64 . 64 x86_64 . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
31 changes: 8 additions & 23 deletions pipelines/main/platforms/tester_linux.arches
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,10 @@
# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
linux 32_g1 g1 . 32 i686 . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
linux 32_g2 g2 . 32 i686 . . . 3 . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR IS_RR ROOTFS_TAG ROOTFS_HASH
linux 32 . 32 i686 . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
linux 64_mt . 64 x86_64 . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
linux 64_rr . 64 x86_64 . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a

linux 64_g1_mt g1 . 64 x86_64 . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
linux 64_g2_mt g2 . 64 x86_64 . . . 3 . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
# linux aarch64_g1 true aarch64 aarch64 . . . . ---- ----------------------------------------
# linux armv7l_g1 true armv7l armv7l . . . . ---- ----------------------------------------
# linux ppc64le_g1 true ppc64le powerpc64le . . . . ---- ----------------------------------------

linux 64_g1_st g1 . 64 x86_64 . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
linux 64_g2_st g2 . 64 x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a

linux 64_g1_rrst g1 . 64 x86_64 . 300 240 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
linux 64_g2_rrst g2 . 64 x86_64 . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
linux 64_g3_st g3 . 64 x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a

# linux aarch64_g1 g1 true aarch64 aarch64 . . . . . . . ---- ----------------------------------------
# linux aarch64_g2 g2 true aarch64 aarch64 . . . . . . . ---- ----------------------------------------

# linux armv7l_g1 g1 true armv7l armv7l . . . . . . . ---- ----------------------------------------
# linux armv7l_g2 g2 true armv7l armv7l . . . . . . . ---- ----------------------------------------

# linux ppc64le_g1 g1 true ppc64le powerpc64le . . . . . . . ---- ----------------------------------------
# linux ppc64le_g2 g2 true ppc64le powerpc64le . . . . . . . ---- ----------------------------------------

musl 64_g1 g1 true 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
musl 64_g2 g2 true 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
#musl 64 true 64 x86_64 . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
42 changes: 5 additions & 37 deletions pipelines/main/platforms/tester_linux.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,6 @@ steps:
env:
JULIA_SHELL: "/bin/bash"
timeout_in_minutes: ${TIMEOUT_BK?}
retry:
automatic:
- exit_status: "*"
limit: ${RETRIES?}
soft_fail: ${ALLOW_FAIL?}
commands: |
echo "--- Print the full and short commit hashes"
Expand Down Expand Up @@ -66,54 +62,26 @@ steps:
export TMPDIR="$(pwd)/tmp"
mkdir -p $${TMPDIR:?}
export NETWORK_RELATED_TESTS="Artifacts Downloads download LazyArtifacts LibGit2/online Pkg"
if [[ "${GROUP?}" == "all" ]]; then
export TESTS="all LibGit2/online --force-net"
elif [[ "${GROUP?}" == "all_except_pkg" ]]; then
export TESTS="all LibGit2/online --force-net --skip Pkg"
elif [[ "${GROUP?}" == "g1" ]]; then
# Group 1: ALL tests EXCEPT the network-related tests.
export TESTS="all --force-net --skip $${NETWORK_RELATED_TESTS:?}"
elif [[ "${GROUP?}" == "g2" ]]; then
# Group 2: ONLY the network-related tests.
# In Group 2, we use whatever the default setting is with regards to the Pkg server.
export TESTS="$${NETWORK_RELATED_TESTS:?} --force-net"
elif [[ "${GROUP?}" == "g3" ]]; then
# Group 3: only Pkg.
# In Group 3, we explicitly opt-out of the Pkg server.
# The purpose of group 3 is to test the non-Pkg-server codepaths of Pkg.
export TESTS="Pkg --force-net"
export JULIA_PKG_SERVER=""
else
echo "Invalid value for GROUP: ${GROUP?}"
exit 1
fi
# By default, run all tests.
export TESTS="all LibGit2/online --force-net"
export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}"
# If we're running inside of `rr`, limit the number of threads
if [[ "${IS_RR?}" == "yes" ]]; then
export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
export JULIA_NUM_THREADS=1
else
export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?}"
export NCORES_FOR_TESTS="Sys.CPU_THREADS"
fi
if [[ "${IS_ST?}" == "yes" ]]; then
export JULIA_NUM_THREADS=1
fi
if [[ "${IS_MT?}" == "yes" ]]; then
export JULIA_NUM_THREADS=16
fi
echo "--- Print the test group, list of test sets, and other useful environment variables"
echo "--- Print the list of test sets, and other useful environment variables"
echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}"
echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set
echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}"
echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}"
echo "GROUP is: ${GROUP?}"
echo "TESTS is: $${TESTS:?}"
echo "--- Run the Julia test suite"
Expand Down
4 changes: 2 additions & 2 deletions pipelines/scheduled/no_bb/no_bb_package_linux.arches
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
linux 64src . . 64src x86_64 USE_BINARYBUILDER=0 180 . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR IS_RR ROOTFS_TAG ROOTFS_HASH
linux 64src . 64src x86_64 USE_BINARYBUILDER=0 180 . . v4.8 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
12 changes: 2 additions & 10 deletions pipelines/scheduled/no_bb/no_bb_tester_linux.arches
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,2 @@
# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
linux 64src_g1_mt g1 . 64src x86_64 . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
linux 64src_g2_mt g2 . 64src x86_64 . . . 3 . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a

linux 64src_g1_st g1 . 64src x86_64 . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
linux 64src_g2_st g2 . 64src x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a

linux 64src_g1_rrst g1 . 64src x86_64 . 300 240 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
linux 64src_g2_rrst g2 . 64src x86_64 . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
linux 64src_g3_st g3 . 64src x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR IS_RR ROOTFS_TAG ROOTFS_HASH
linux 64src_g1_mt . 64src x86_64 . . . . v4.8 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0
17 changes: 1 addition & 16 deletions utilities/platforms/platforms.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,29 +29,18 @@ cat "${ARCHES:?}" | tr -s ' ' | while read _line; do

export PLATFORM=`echo $line | cut -d ' ' -f 1 | tr -s ' '`
export LABEL=`echo $line | cut -d ' ' -f 2 | tr -s ' '`
export GROUP=`echo $line | cut -d ' ' -f 3 | tr -s ' '`


export ALLOW_FAIL=`echo $line | cut -d ' ' -f 4 | tr -s ' '`
export ARCH=`echo $line | cut -d ' ' -f 5 | tr -s ' '`
export ARCH_ROOTFS=`echo $line | cut -d ' ' -f 6 | tr -s ' '`

export MAKE_FLAGS=`echo $line | cut -d ' ' -f 7 | tr -s ' '`
export TIMEOUT_BK=`echo $line | cut -d ' ' -f 8 | tr -s ' '`
export TIMEOUT_RR=`echo $line | cut -d ' ' -f 9 | tr -s ' '`
export RETRIES=`echo $line | cut -d ' ' -f 10 | tr -s ' '`
export IS_RR=`echo $line | cut -d ' ' -f 11 | tr -s ' '`
export IS_ST=`echo $line | cut -d ' ' -f 12 | tr -s ' '`
export IS_MT=`echo $line | cut -d ' ' -f 13 | tr -s ' '`
export ROOTFS_TAG=`echo $line | cut -d ' ' -f 14 | tr -s ' '`
export ROOTFS_HASH=`echo $line | cut -d ' ' -f 15 | tr -s ' '`

if [[ "${IS_ST:?}" == "yes" ]]; then
if [[ "${IS_MT:?}" == "yes" ]]; then
echo "You cannot set both IS_ST and IS_MT to yes"
exit 1
fi
fi

if [[ "${ALLOW_FAIL:?}" == "." ]]; then
export ALLOW_FAIL="false"
fi
Expand All @@ -68,9 +57,5 @@ cat "${ARCHES:?}" | tr -s ' ' | while read _line; do
export TIMEOUT_RR="60" # minutes
fi

if [[ "${RETRIES:?}" == "." ]]; then
export RETRIES="0"
fi

buildkite-agent pipeline upload "${YAML:?}"
done

0 comments on commit e607bd1

Please sign in to comment.