From e607bd1bb3a02eb234326724514d9c9260e92bdd Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 28 Feb 2022 16:32:14 -0800 Subject: [PATCH] Significantly simplify the linux64 testing regime We no longer use "test groups", we're going to move the retry logic into Base Julia (I hear that Julian has agreed to do this). Let's boil it down to only a single variable; whether this is an RR test or not. --- pipelines/main/platforms/package_linux.arches | 14 +++---- pipelines/main/platforms/tester_linux.arches | 31 ++++---------- pipelines/main/platforms/tester_linux.yml | 42 +++---------------- .../no_bb/no_bb_package_linux.arches | 4 +- .../scheduled/no_bb/no_bb_tester_linux.arches | 12 +----- utilities/platforms/platforms.sh | 17 +------- 6 files changed, 25 insertions(+), 95 deletions(-) diff --git a/pipelines/main/platforms/package_linux.arches b/pipelines/main/platforms/package_linux.arches index b9d369c9d..c2bc0b7b7 100644 --- a/pipelines/main/platforms/package_linux.arches +++ b/pipelines/main/platforms/package_linux.arches @@ -1,7 +1,7 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -# linux 32 . . 32 i686 . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -linux 64 . . 64 x86_64 . . . . . . . v5.1 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0 -# linux aarch64 . . aarch64 aarch64 . . . . . . . .... ........................................ -# linux armv7l . . armv7l armv7l . . . . . . . .... ........................................ -# linux ppc64le . . ppc64le powerpc64le . . . . . . . .... ........................................ -# musl 64 . . 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR IS_MT ROOTFS_TAG ROOTFS_HASH +# linux 32 . 32 i686 . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 +linux 64 . 64 x86_64 . . . . v5.1 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0 +# linux aarch64 . aarch64 aarch64 . . . . .... ........................................ +# linux armv7l . armv7l armv7l . . . . .... ........................................ +# linux ppc64le . ppc64le powerpc64le . . . . .... ........................................ +# musl 64 . 64 x86_64 . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 diff --git a/pipelines/main/platforms/tester_linux.arches b/pipelines/main/platforms/tester_linux.arches index d1304563c..010dad82f 100644 --- a/pipelines/main/platforms/tester_linux.arches +++ b/pipelines/main/platforms/tester_linux.arches @@ -1,25 +1,10 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 32_g1 g1 . 32 i686 . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -linux 32_g2 g2 . 32 i686 . . . 3 . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR IS_RR ROOTFS_TAG ROOTFS_HASH +linux 32 . 32 i686 . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 +linux 64_mt . 64 x86_64 . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +linux 64_rr . 64 x86_64 . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g1_mt g1 . 64 x86_64 . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_mt g2 . 64 x86_64 . . . 3 . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +# linux aarch64_g1 true aarch64 aarch64 . . . . ---- ---------------------------------------- +# linux armv7l_g1 true armv7l armv7l . . . . ---- ---------------------------------------- +# linux ppc64le_g1 true ppc64le powerpc64le . . . . ---- ---------------------------------------- -linux 64_g1_st g1 . 64 x86_64 . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_st g2 . 64 x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64_g1_rrst g1 . 64 x86_64 . 300 240 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g2_rrst g2 . 64 x86_64 . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_g3_st g3 . 64 x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -# linux aarch64_g1 g1 true aarch64 aarch64 . . . . . . . ---- ---------------------------------------- -# linux aarch64_g2 g2 true aarch64 aarch64 . . . . . . . ---- ---------------------------------------- - -# linux armv7l_g1 g1 true armv7l armv7l . . . . . . . ---- ---------------------------------------- -# linux armv7l_g2 g2 true armv7l armv7l . . . . . . . ---- ---------------------------------------- - -# linux ppc64le_g1 g1 true ppc64le powerpc64le . . . . . . . ---- ---------------------------------------- -# linux ppc64le_g2 g2 true ppc64le powerpc64le . . . . . . . ---- ---------------------------------------- - -musl 64_g1 g1 true 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 -musl 64_g2 g2 true 64 x86_64 . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 +#musl 64 true 64 x86_64 . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 diff --git a/pipelines/main/platforms/tester_linux.yml b/pipelines/main/platforms/tester_linux.yml index 18b019e66..282235d51 100644 --- a/pipelines/main/platforms/tester_linux.yml +++ b/pipelines/main/platforms/tester_linux.yml @@ -24,10 +24,6 @@ steps: env: JULIA_SHELL: "/bin/bash" timeout_in_minutes: ${TIMEOUT_BK?} - retry: - automatic: - - exit_status: "*" - limit: ${RETRIES?} soft_fail: ${ALLOW_FAIL?} commands: | echo "--- Print the full and short commit hashes" @@ -66,54 +62,26 @@ steps: export TMPDIR="$(pwd)/tmp" mkdir -p $${TMPDIR:?} - export NETWORK_RELATED_TESTS="Artifacts Downloads download LazyArtifacts LibGit2/online Pkg" - - if [[ "${GROUP?}" == "all" ]]; then - export TESTS="all LibGit2/online --force-net" - elif [[ "${GROUP?}" == "all_except_pkg" ]]; then - export TESTS="all LibGit2/online --force-net --skip Pkg" - elif [[ "${GROUP?}" == "g1" ]]; then - # Group 1: ALL tests EXCEPT the network-related tests. - export TESTS="all --force-net --skip $${NETWORK_RELATED_TESTS:?}" - elif [[ "${GROUP?}" == "g2" ]]; then - # Group 2: ONLY the network-related tests. - # In Group 2, we use whatever the default setting is with regards to the Pkg server. - export TESTS="$${NETWORK_RELATED_TESTS:?} --force-net" - elif [[ "${GROUP?}" == "g3" ]]; then - # Group 3: only Pkg. - # In Group 3, we explicitly opt-out of the Pkg server. - # The purpose of group 3 is to test the non-Pkg-server codepaths of Pkg. - export TESTS="Pkg --force-net" - export JULIA_PKG_SERVER="" - else - echo "Invalid value for GROUP: ${GROUP?}" - exit 1 - fi - + # By default, run all tests. + export TESTS="all LibGit2/online --force-net" export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}" + # If we're running inside of `rr`, limit the number of threads if [[ "${IS_RR?}" == "yes" ]]; then export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}" export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])" + export JULIA_NUM_THREADS=1 else export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?}" export NCORES_FOR_TESTS="Sys.CPU_THREADS" - fi - - if [[ "${IS_ST?}" == "yes" ]]; then - export JULIA_NUM_THREADS=1 - fi - - if [[ "${IS_MT?}" == "yes" ]]; then export JULIA_NUM_THREADS=16 fi - echo "--- Print the test group, list of test sets, and other useful environment variables" + echo "--- Print the list of test sets, and other useful environment variables" echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}" echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}" echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}" - echo "GROUP is: ${GROUP?}" echo "TESTS is: $${TESTS:?}" echo "--- Run the Julia test suite" diff --git a/pipelines/scheduled/no_bb/no_bb_package_linux.arches b/pipelines/scheduled/no_bb/no_bb_package_linux.arches index dff2aab45..c945cab47 100644 --- a/pipelines/scheduled/no_bb/no_bb_package_linux.arches +++ b/pipelines/scheduled/no_bb/no_bb_package_linux.arches @@ -1,2 +1,2 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 64src . . 64src x86_64 USE_BINARYBUILDER=0 180 . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR IS_RR ROOTFS_TAG ROOTFS_HASH +linux 64src . 64src x86_64 USE_BINARYBUILDER=0 180 . . v4.8 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0 diff --git a/pipelines/scheduled/no_bb/no_bb_tester_linux.arches b/pipelines/scheduled/no_bb/no_bb_tester_linux.arches index 0b1fbdf63..30e436e60 100644 --- a/pipelines/scheduled/no_bb/no_bb_tester_linux.arches +++ b/pipelines/scheduled/no_bb/no_bb_tester_linux.arches @@ -1,10 +1,2 @@ -# PLATFORM LABEL GROUP ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 64src_g1_mt g1 . 64src x86_64 . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g2_mt g2 . 64src x86_64 . . . 3 . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64src_g1_st g1 . 64src x86_64 . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g2_st g2 . 64src x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a - -linux 64src_g1_rrst g1 . 64src x86_64 . 300 240 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g2_rrst g2 . 64src x86_64 . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64src_g3_st g3 . 64src x86_64 . . . 3 . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR IS_RR ROOTFS_TAG ROOTFS_HASH +linux 64src_g1_mt . 64src x86_64 . . . . v4.8 f689ba6acab6c13a3da4ff62e3445dd61c3b5ab0 diff --git a/utilities/platforms/platforms.sh b/utilities/platforms/platforms.sh index 9a47c18e9..29fe4e830 100755 --- a/utilities/platforms/platforms.sh +++ b/utilities/platforms/platforms.sh @@ -29,8 +29,7 @@ cat "${ARCHES:?}" | tr -s ' ' | while read _line; do export PLATFORM=`echo $line | cut -d ' ' -f 1 | tr -s ' '` export LABEL=`echo $line | cut -d ' ' -f 2 | tr -s ' '` - export GROUP=`echo $line | cut -d ' ' -f 3 | tr -s ' '` - + export ALLOW_FAIL=`echo $line | cut -d ' ' -f 4 | tr -s ' '` export ARCH=`echo $line | cut -d ' ' -f 5 | tr -s ' '` export ARCH_ROOTFS=`echo $line | cut -d ' ' -f 6 | tr -s ' '` @@ -38,20 +37,10 @@ cat "${ARCHES:?}" | tr -s ' ' | while read _line; do export MAKE_FLAGS=`echo $line | cut -d ' ' -f 7 | tr -s ' '` export TIMEOUT_BK=`echo $line | cut -d ' ' -f 8 | tr -s ' '` export TIMEOUT_RR=`echo $line | cut -d ' ' -f 9 | tr -s ' '` - export RETRIES=`echo $line | cut -d ' ' -f 10 | tr -s ' '` export IS_RR=`echo $line | cut -d ' ' -f 11 | tr -s ' '` - export IS_ST=`echo $line | cut -d ' ' -f 12 | tr -s ' '` - export IS_MT=`echo $line | cut -d ' ' -f 13 | tr -s ' '` export ROOTFS_TAG=`echo $line | cut -d ' ' -f 14 | tr -s ' '` export ROOTFS_HASH=`echo $line | cut -d ' ' -f 15 | tr -s ' '` - if [[ "${IS_ST:?}" == "yes" ]]; then - if [[ "${IS_MT:?}" == "yes" ]]; then - echo "You cannot set both IS_ST and IS_MT to yes" - exit 1 - fi - fi - if [[ "${ALLOW_FAIL:?}" == "." ]]; then export ALLOW_FAIL="false" fi @@ -68,9 +57,5 @@ cat "${ARCHES:?}" | tr -s ' ' | while read _line; do export TIMEOUT_RR="60" # minutes fi - if [[ "${RETRIES:?}" == "." ]]; then - export RETRIES="0" - fi - buildkite-agent pipeline upload "${YAML:?}" done