From 04cc4edf263fb1d8cc4fc306b89e699f667e245b Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Wed, 14 Dec 2016 18:25:47 +0000 Subject: [PATCH 1/7] Parallelise sanity checks in order to speed them up. --- test/sanity_check.sh | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/test/sanity_check.sh b/test/sanity_check.sh index 66cbac499b..e0b679bc01 100755 --- a/test/sanity_check.sh +++ b/test/sanity_check.sh @@ -5,23 +5,42 @@ set -e whitely echo Ping each host from the other + for host in $HOSTS; do for other in $HOSTS; do - [ $host = $other ] || run_on $host $PING $other + if [ "$host" != "$other" ]; then + run_on $host $PING $other & + pids="$pids $!" + fi done done +for pid in $pids; do wait $pid; done +unset pids + whitely echo Check we can reach docker +function check_docker() { + docker_version=$(docker_on $1 version) + docker_info=$(docker_on $1 info) + docker_weave_version=$(docker_on $1 inspect -f {{.Created}} weaveworks/weave:${WEAVE_VERSION:-latest}) + weave_version=$(weave_on $1 version) + cat << EOF + +Host Version Info: $1 +===================================== +# docker version +$docker_version +# docker info +$docker_info +# weave version +$docker_weave_version +$weave_version +EOF +} + for host in $HOSTS; do - echo - echo Host Version Info: $host - echo ===================================== - echo "# docker version" - docker_on $host version - echo "# docker info" - docker_on $host info - echo "# weave version" - docker_on $host inspect -f {{.Created}} weaveworks/weave:${WEAVE_VERSION:-latest} - weave_on $host version + check_docker $host & + pids="$pids $!" done +for pid in $pids; do wait $pid; done From 61375fb785d6d7bf3757fa4fac762688d3cfcc46 Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Thu, 15 Dec 2016 12:08:13 +0000 Subject: [PATCH 2/7] Format ping output as one line. --- test/sanity_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sanity_check.sh b/test/sanity_check.sh index e0b679bc01..217e459780 100755 --- a/test/sanity_check.sh +++ b/test/sanity_check.sh @@ -9,7 +9,7 @@ whitely echo Ping each host from the other for host in $HOSTS; do for other in $HOSTS; do if [ "$host" != "$other" ]; then - run_on $host $PING $other & + echo $(run_on $host $PING $other) & pids="$pids $!" fi done From b171497466b95abf0f7f1243ee1eba14aba9c28b Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Mon, 19 Dec 2016 11:18:05 +0000 Subject: [PATCH 3/7] Ensure sanity checks exit early in case of a ping failure. --- test/sanity_check.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/sanity_check.sh b/test/sanity_check.sh index 217e459780..047d63f6bb 100755 --- a/test/sanity_check.sh +++ b/test/sanity_check.sh @@ -6,10 +6,20 @@ set -e whitely echo Ping each host from the other +# We wrap ping and echo in a function as we want the below parallel for loop +# to exit early in case of a failed ping, and it cannot be done concisely, +# e.g. using a one-liner, without losing the status code for ping. +function check_ping() { + local output=$(run_on $1 $PING $2) + local status=$? + echo $output + return $status +} + for host in $HOSTS; do for other in $HOSTS; do if [ "$host" != "$other" ]; then - echo $(run_on $host $PING $other) & + check_ping "$host" "$other" & pids="$pids $!" fi done From f05169e457b52d337afaeb4add496cc61fe7b489 Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Mon, 19 Dec 2016 11:24:36 +0000 Subject: [PATCH 4/7] Initialise pids before using it, to avoid potential issues with previously set values. --- test/sanity_check.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/sanity_check.sh b/test/sanity_check.sh index 047d63f6bb..fca1d85e75 100755 --- a/test/sanity_check.sh +++ b/test/sanity_check.sh @@ -16,6 +16,7 @@ function check_ping() { return $status } +pids="" for host in $HOSTS; do for other in $HOSTS; do if [ "$host" != "$other" ]; then @@ -25,7 +26,6 @@ for host in $HOSTS; do done done for pid in $pids; do wait $pid; done -unset pids whitely echo Check we can reach docker @@ -49,6 +49,7 @@ $weave_version EOF } +pids="" for host in $HOSTS; do check_docker $host & pids="$pids $!" From 172597a68d8679d9cd68bec5548e1ba5bfbb0431 Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Mon, 19 Dec 2016 14:51:26 +0000 Subject: [PATCH 5/7] Now measure time spent in sanity checks. --- test/sanity_check.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/sanity_check.sh b/test/sanity_check.sh index fca1d85e75..737f9f6e95 100755 --- a/test/sanity_check.sh +++ b/test/sanity_check.sh @@ -4,6 +4,8 @@ set -e +begin=$(date +%s) + whitely echo Ping each host from the other # We wrap ping and echo in a function as we want the below parallel for loop @@ -55,3 +57,5 @@ for host in $HOSTS; do pids="$pids $!" done for pid in $pids; do wait $pid; done + +echo "Sanity checks completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")." From f73fb5307ff0f7d9422962758882c3cda4e461fe Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Wed, 21 Dec 2016 11:12:25 +0000 Subject: [PATCH 6/7] Redirect sanity checks' output to temporary files (and archive them as artifact files under CircleCLI). This should make it easier to find and access relevant debugging information without polluting STDOUT and without any risk of log interleaving. See also: - https://weaveworks.slack.com/archives/net/p1482154505001192 - https://weaveworks.slack.com/archives/net/p1482254780001493 --- bin/circle-test-smoke | 2 ++ test/sanity_check.sh | 22 ++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/bin/circle-test-smoke b/bin/circle-test-smoke index 4164e20aa2..9e59d881c4 100755 --- a/bin/circle-test-smoke +++ b/bin/circle-test-smoke @@ -9,5 +9,7 @@ if [ -n "$TEST_AND_PUBLISH" ] ; then cd $SRCDIR/test eval $(./gce.sh hosts) export COVERAGE=true + export WEAVE_NET_SANITY_CHECKS_FILES="/tmp/weave_net_sanity_check_*.log" ./run_all.sh + cp $WEAVE_NET_SANITY_CHECKS_FILES $CIRCLE_ARTIFACTS fi diff --git a/test/sanity_check.sh b/test/sanity_check.sh index 737f9f6e95..657bdf7193 100755 --- a/test/sanity_check.sh +++ b/test/sanity_check.sh @@ -5,6 +5,7 @@ set -e begin=$(date +%s) +sanity_checks_files=${WEAVE_NET_SANITY_CHECKS_FILES:-"/tmp/weave_net_sanity_check_*.log"} whitely echo Ping each host from the other @@ -14,12 +15,17 @@ whitely echo Ping each host from the other function check_ping() { local output=$(run_on $1 $PING $2) local status=$? - echo $output + echo "$output" >> "${sanity_checks_files/\*/$1}" return $status } pids="" for host in $HOSTS; do + cat >> "${sanity_checks_files/\*/$host}" <> "${sanity_checks_files/\*/$1}" << EOF +# ===================================== +# Host Version Info: $1 +# ===================================== # docker version $docker_version + # docker info $docker_info -# weave version + +# docker inspect -f {{.Created}} weaveworks/weave: $docker_weave_version + +# weave version $weave_version EOF } From edc0cdbc10afa6232b603ff123ee41a78c1119fb Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Mon, 9 Jan 2017 12:25:24 +0000 Subject: [PATCH 7/7] Sanity checks' output is now redirected to $CIRCLE_ARTIFACTS directly when running smoke tests in CircleCI. Previously sanity checks' output was redirected to /tmp and then copied to $CIRCLE_ARTIFACTS, which is more "moving pieces". --- bin/circle-test-smoke | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/circle-test-smoke b/bin/circle-test-smoke index 9e59d881c4..a5fb9a7b61 100755 --- a/bin/circle-test-smoke +++ b/bin/circle-test-smoke @@ -9,7 +9,6 @@ if [ -n "$TEST_AND_PUBLISH" ] ; then cd $SRCDIR/test eval $(./gce.sh hosts) export COVERAGE=true - export WEAVE_NET_SANITY_CHECKS_FILES="/tmp/weave_net_sanity_check_*.log" + export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log" ./run_all.sh - cp $WEAVE_NET_SANITY_CHECKS_FILES $CIRCLE_ARTIFACTS fi