From 9e4ce8c052c1d8568e3a73cbb8fcadc67a71ab2f Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Wed, 7 Dec 2016 17:49:20 +0000 Subject: [PATCH 01/23] Git-ignore Terraform state files. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 5d9b497c3a..3a7f66ad0d 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,5 @@ test/tls/*.pem test/coverage test/coverage.* *qemu-*-static +terraform.tfstate +terraform.tfstate.backup From fde7e88e66860018a34caba2252d8861d1c82452 Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Wed, 21 Dec 2016 09:57:54 +0000 Subject: [PATCH 02/23] Allow RUNNER_ARGS to be passed by calling script. This is required in order to be able to dependency-inject values from layers higher up, e.g.: RUNNER_ARGS="-parallel" from the Makefile. --- test/run_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_all.sh b/test/run_all.sh index db04b50fa7..7d8c6edd97 100755 --- a/test/run_all.sh +++ b/test/run_all.sh @@ -11,7 +11,7 @@ fi whitely echo ...ok TESTS="${@:-$(find "$DIR" -name '*_test.sh')}" -RUNNER_ARGS="" +RUNNER_ARGS=${RUNNER_ARGS:-""} # If running on circle, use the scheduler to work out what tests to run if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then From 487ebfb514e10f3d6a4ca7cec4953bb7888172eb Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Mon, 9 Jan 2017 20:11:18 +0000 Subject: [PATCH 03/23] Increased timeout for test #840. Reason: test is periodically failing because of the timeout being too aggressive, especially when using Google Cloud Platform together with Terraform and Ansible. --- test/840_weave_kube_3_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/840_weave_kube_3_test.sh b/test/840_weave_kube_3_test.sh index 5703b8ced3..7823f92956 100755 --- a/test/840_weave_kube_3_test.sh +++ b/test/840_weave_kube_3_test.sh @@ -31,7 +31,7 @@ sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never$COVERAGE_ARGS%" "$(dirn sleep 5 wait_for_connections() { - for i in $(seq 1 30); do + for i in $(seq 1 45); do if run_on $HOST1 "curl -sS http://127.0.0.1:6784/status | grep \"$SUCCESS\"" ; then return fi @@ -56,7 +56,7 @@ assert_raises "run_on $HOST3 $PING $HOST2EXPIP" run_on $HOST1 "kubectl run hello --image=weaveworks/hello-world --replicas=3" wait_for_pods() { - for i in $(seq 1 30); do + for i in $(seq 1 45); do if run_on $HOST1 "kubectl get pods | grep 'hello.*Running'" ; then return fi From 2aaa400d622117db7f8c835d3fdfb40cb998a399 Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Wed, 11 Jan 2017 10:24:51 +0000 Subject: [PATCH 04/23] Add Terraform and Ansible scripts as part of tools submodule. --- tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools b/tools index 36c1835b49..efcf9d21e0 160000 --- a/tools +++ b/tools @@ -1 +1 @@ -Subproject commit 36c1835b4986c4c4d81a597156e7cd5c1a8e8b23 +Subproject commit efcf9d21e09ccd52e00f973a42c78ad08b33e92c From acd16881d99e593cb56623d8d69a72f5ca6c2b4f Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Wed, 11 Jan 2017 10:23:18 +0000 Subject: [PATCH 05/23] Add make integration-tests based on Terraform/Ansible. --- Makefile | 21 +++ Vagrantfile | 2 +- test/Vagrantfile | 2 +- test/config.sh | 4 +- test/run-integration-tests.sh | 264 ++++++++++++++++++++++++++++++++++ 5 files changed, 289 insertions(+), 4 deletions(-) create mode 100755 test/run-integration-tests.sh diff --git a/Makefile b/Makefile index 73f02fd410..99c694ebb7 100644 --- a/Makefile +++ b/Makefile @@ -359,3 +359,24 @@ build: run-smoketests: all testrunner cd test && ./setup.sh && ./run_all.sh + +integration-tests: all testrunner +# Usage: +# $ make \ +# NAME="" \ +# PROVIDER="" \ +# NUM_HOSTS="<# test machines>" \ +# PLAYBOOK="" \ +# RUNNER_ARGS="<...>" \ +# TESTS="<...>" \ # Can be set to only run one or a few tests instead of the full test suite. +# SKIP_CREATE=1 \ # Skip the provisioning phase before running tests. +# SKIP_CONFIG=1 \ # Skip the configuration phase before running tests. +# SKIP_DESTROY=1 \ # Skip the destroy phase after running tests. +# ONLY_DESTROY=1 \ # Destroy the test cluster (and do not do anything else, like starting the test cluster, configuring it, or running tests). +# DOCKER_VERSION=<...> \ +# KUBERNETES_VERSION=<...> \ +# KUBERNETES_CNI_VERSION=<...> \ +# <...> # See also run-integration-test.sh for all variables. +# integration-tests +# + RUNNER_ARGS="-parallel" ./test/run-integration-tests.sh diff --git a/Vagrantfile b/Vagrantfile index 0f0287bce0..24871afd4e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,6 +1,6 @@ VAGRANTFILE_API_VERSION = "2" -require './vagrant-common.rb' +require File.expand_path(File.join(File.dirname(__FILE__), 'vagrant-common.rb')) vm_ip = "172.16.0.3" # arbitrary private IP diff --git a/test/Vagrantfile b/test/Vagrantfile index eca6475b7f..691c507592 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -9,7 +9,7 @@ n_machines = 3 ip_prefix = "192.168.48" ip_suffix_base = 10 -require '../vagrant-common.rb' +require File.expand_path(File.join(File.dirname(__FILE__), '..', 'vagrant-common.rb')) def configure_docker(host, hostname, ip) pkgs = %w(docker-engine ethtool) diff --git a/test/config.sh b/test/config.sh index 4ec0d1697f..35f26f7a63 100644 --- a/test/config.sh +++ b/test/config.sh @@ -30,9 +30,9 @@ HOST3=$(echo $HOSTS | cut -f 3 -d ' ') . "$DIR/assert.sh" - SSH_DIR=${SSH_DIR:-$DIR} -SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" -o "UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts" -o CheckHostIP=no -o StrictHostKeyChecking=no} +SSH_OPTS=${SSH_OPTS:-"-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o IdentitiesOnly=yes -o LogLevel=ERROR"} +SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" $SSH_OPTS} SMALL_IMAGE="alpine" DNS_IMAGE="aanand/docker-dnsutils" diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh new file mode 100755 index 0000000000..d422f589f3 --- /dev/null +++ b/test/run-integration-tests.sh @@ -0,0 +1,264 @@ +#!/bin/bash +# +# Description: +# This script runs all Weave Net's integration tests on the specified +# provider (default: Google Cloud Platform). +# +# Usage: +# +# Run all integration tests on Google Cloud Platform: +# $ ./run-integration-tests.sh +# +# Run all integration tests on Amazon Web Services: +# PROVIDER=aws ./run-integration-tests.sh +# + +set -e +DIR="$(dirname "$0")" +. "$DIR/../tools/provisioning/setup.sh" # Import gcp_on, do_on, and aws_on. +. "$DIR/config.sh" # Import greenly. + +# Variables: +APP="weave-net" +# shellcheck disable=SC2034 +PROJECT="weave-net-tests" # Only used when PROVIDER is gcp, by tools/provisioning/config.sh. +NAME=${NAME:-"$(whoami | sed -e 's/[\.\_]*//g' | cut -c 1-4)"} +PROVIDER=${PROVIDER:-gcp} # Provision using provided provider, or Google Cloud Platform by default. +NUM_HOSTS=${NUM_HOSTS:-10} +PLAYBOOK=${PLAYBOOK:-setup_weave-net_test.yml} +TESTS=${TESTS:-} +RUNNER_ARGS=${RUNNER_ARGS:-""} +# Dependencies' versions: +DOCKER_VERSION=${DOCKER_VERSION:-1.11.2} +KUBERNETES_VERSION=${KUBERNETES_VERSION:-1.5.2} +KUBERNETES_CNI_VERSION=${KUBERNETES_CNI_VERSION:-0.3.0.1} +# Lifecycle flags: +SKIP_CONFIG=${SKIP_CONFIG:-} + +function print_vars() { + echo "--- Variables: Main ---" + echo "PROVIDER=$PROVIDER" + echo "NUM_HOSTS=$NUM_HOSTS" + echo "PLAYBOOK=$PLAYBOOK" + echo "TESTS=$TESTS" + echo "SSH_OPTS=$SSH_OPTS" + echo "RUNNER_ARGS=$RUNNER_ARGS" + echo "--- Variables: Versions ---" + echo "DOCKER_VERSION=$DOCKER_VERSION" + echo "KUBERNETES_VERSION=$KUBERNETES_VERSION" + echo "KUBERNETES_CNI_VERSION=$KUBERNETES_CNI_VERSION" + echo "--- Variables: Flags ---" + echo "SKIP_CONFIG=$SKIP_CONFIG" +} + +function verify_dependencies() { + local deps=(python terraform ansible-playbook) + for dep in "${deps[@]}"; do + if [ ! "$(which "$dep")" ]; then + echo >&2 "$dep is not installed or not in PATH." + exit 1 + fi + done +} + +# shellcheck disable=SC2155 +function provision_locally() { + export VAGRANT_CWD="$(dirname "${BASH_SOURCE[0]}")" + case "$1" in + on) + vagrant up + local status=$? + + # Set up SSH connection details: + local ssh_config=$(mktemp /tmp/vagrant_ssh_config_XXX) + vagrant ssh-config >"$ssh_config" + export SSH="ssh -F $ssh_config" + # Extract username, SSH private key, and VMs' IP addresses: + ssh_user="$(sed -ne 's/\ *User //p' "$ssh_config" | uniq)" + ssh_id_file="$(sed -ne 's/\ *IdentityFile //p' "$ssh_config" | uniq)" + ssh_hosts=$(sed -ne 's/Host //p' "$ssh_config") + + # Set up /etc/hosts files on this ("local") machine and the ("remote") testing machines, to map hostnames and IP addresses, so that: + # - this machine communicates with the testing machines via their public IPs; + # - testing machines communicate between themselves via their private IPs; + # - we can simply use just the hostname in all scripts to refer to machines, and the difference between public and private IP becomes transparent. + # N.B.: if you decide to use public IPs everywhere, note that some tests may fail (e.g. test #115). + update_local_etc_hosts "$ssh_hosts" "$(for host in $ssh_hosts; do $SSH "$host" "cat /etc/hosts | grep $host"; done)" + + SKIP_CONFIG=1 # Vagrant directly configures virtual machines using Ansible -- see also: Vagrantfile + return $status + ;; + off) + vagrant destroy -f + ;; + *) + echo >&2 "Unknown command $1. Usage: {on|off}." + exit 1 + ;; + esac +} + +function update_local_etc_hosts() { + echo "> Updating local /etc/hosts..." + # Remove old entries (if present): + for host in $1; do sudo sed -i "/$host/d" /etc/hosts; done + # Add new entries: + sudo sh -c "echo \"$2\" >> /etc/hosts" +} + +function upload_etc_hosts() { + # Remove old entries (if present): + # shellcheck disable=SC2016,SC2086 + $SSH $3 'for host in '$1'; do sudo sed -i "/$host/d" /etc/hosts; done' + # Add new entries: + echo "$2" | $SSH "$3" "sudo -- sh -c \"cat >> /etc/hosts\"" +} + +function update_remote_etc_hosts() { + echo "> Updating remote /etc/hosts..." + local pids="" + for host in $1; do + upload_etc_hosts "$1" "$2" "$host" & + local pids="$pids $!" + done + for pid in $pids; do wait "$pid"; done +} + +function provision_remotely() { + case "$1" in + on) + terraform apply -input=false -parallelism="$NUM_HOSTS" -var "app=$APP" -var "name=$NAME" -var "num_hosts=$NUM_HOSTS" "$DIR/../tools/provisioning/$2" + local status=$? + ssh_user=$(terraform output username) + ssh_id_file=$(terraform output private_key_path) + ssh_hosts=$(terraform output hostnames) + export SSH="ssh -l $ssh_user -i $ssh_id_file $SSH_OPTS" + + # Set up /etc/hosts files on this ("local") machine and the ("remote") testing machines, to map hostnames and IP addresses, so that: + # - this machine communicates with the testing machines via their public IPs; + # - testing machines communicate between themselves via their private IPs; + # - we can simply use just the hostname in all scripts to refer to machines, and the difference between public and private IP becomes transparent. + # N.B.: if you decide to use public IPs everywhere, note that some tests may fail (e.g. test #115). + update_local_etc_hosts "$ssh_hosts" "$(terraform output public_etc_hosts)" + update_remote_etc_hosts "$ssh_hosts" "$(terraform output private_etc_hosts)" + + return $status + ;; + off) + terraform destroy -force "$DIR/../tools/provisioning/$2" + ;; + *) + echo >&2 "Unknown command $1. Usage: {on|off}." + exit 1 + ;; + esac +} + +# shellcheck disable=SC2155 +function provision() { + local action=$([ "$1" == "on" ] && echo "Provisioning" || echo "Shutting down") + echo + greenly echo "> $action test host(s) on [$PROVIDER]..." + local begin_prov=$(date +%s) + case "$2" in + 'aws') + aws_on + provision_remotely "$1" "$2" + ;; + 'do') + do_on + provision_remotely "$1" "$2" + ;; + 'gcp') + gcp_on + provision_remotely "$1" "$2" + ;; + 'vagrant') + provision_locally "$1" + ;; + *) + echo >&2 "Unknown provider $2. Usage: PROVIDER={gcp|aws|do|vagrant}." + exit 1 + ;; + esac + + echo + greenly echo "> Provisioning took $(date -u -d @$(($(date +%s) - begin_prov)) +"%T")." +} + +# shellcheck disable=SC2155 +function configure() { + echo + if [ -n "$SKIP_CONFIG" ]; then + greenly echo "> Skipped configuration of test host(s)." + else + greenly echo "> Configuring test host(s)..." + local begin_conf=$(date +%s) + local inventory_file=$(mktemp /tmp/ansible_inventory_XXXXX) + echo "[all]" >"$inventory_file" + # shellcheck disable=SC2001 + echo "$2" | sed "s/$/:$3/" >>"$inventory_file" + + # Configure the provisioned machines using Ansible, allowing up to 3 retries upon failure (e.g. APT connectivity issues, etc.): + for i in $(seq 3); do + ansible-playbook -u "$1" -i "$inventory_file" --private-key="$4" --forks="$NUM_HOSTS" \ + --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ + --extra-vars "docker_version=$DOCKER_VERSION kubernetes_version=$KUBERNETES_VERSION kubernetes_cni_version=$KUBERNETES_CNI_VERSION" \ + "$DIR/../tools/config_management/$PLAYBOOK" \ + && break || echo >&2 "#$i: Ansible failed. Retrying now..." + done + + echo + greenly echo "> Configuration took $(date -u -d @$(($(date +%s) - begin_conf)) +"%T")." + fi +} + +# shellcheck disable=SC2155 +function run_tests() { + export HOSTS="$(echo "$3" | tr '\n' ' ')" + shift 3 # Drop the first 3 arguments, the remainder being, optionally, the list of tests to run. + "$DIR/setup.sh" + echo + greenly echo "> Running tests..." + local begin_tests=$(date +%s) + set +e # Do not fail this script upon test failure, since we need to shut down the test cluster regardless of success or failure. + "$DIR/run_all.sh" "$@" + local status=$? + echo + greenly echo "> Tests took $(date -u -d @$(($(date +%s) - begin_tests)) +"%T")." + return $status +} + +function main() { + begin=$(date +%s) + print_vars + verify_dependencies + + if [ -z "$ONLY_DESTROY" ]; then + provision on "$PROVIDER" + if [ $? -ne 0 ]; then + echo >&2 "> Failed to provision test host(s)." + exit 1 + fi + + if [ -z "$SKIP_CONFIG" ]; then + configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file" + if [ $? -ne 0 ]; then + echo >&2 "Failed to configure test host(s)." + exit 1 + fi + fi + + run_tests "$ssh_user" "$ssh_id_file" "$ssh_hosts" "$TESTS" + status=$? + fi + if [ -z "$SKIP_DESTROY" ]; then + provision off "$PROVIDER" + fi + + echo + greenly echo "> Build took $(date -u -d @$(($(date +%s) - begin)) +"%T")." + exit $status +} + +main From 30f59953d2551d6fdd86e06181bc03a4517a6c0b Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Thu, 26 Jan 2017 18:50:40 +0000 Subject: [PATCH 06/23] Smoke tests are now run using Terraform, Ansible and run-integration-tests.sh. --- bin/circle-dependencies-post | 4 - bin/circle-test-pre | 1 - bin/circle-test-smoke | 13 ++- bin/circle-test-teardown | 1 - bin/setup-circleci-secrets | 115 ------------------- test/gce.sh | 207 ----------------------------------- 6 files changed, 8 insertions(+), 333 deletions(-) delete mode 100755 bin/setup-circleci-secrets delete mode 100755 test/gce.sh diff --git a/bin/circle-dependencies-post b/bin/circle-dependencies-post index e086550a3e..f83a728f4f 100755 --- a/bin/circle-dependencies-post +++ b/bin/circle-dependencies-post @@ -31,11 +31,7 @@ source "$STATE" mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR if [ -n "$TEST_AND_PUBLISH" ] ; then - [ -z "$SECRET_PASSWORD" ] || bin/setup-circleci-secrets "$SECRET_PASSWORD" cd $SRCDIR; git submodule update --init - if [ "$CIRCLE_NODE_INDEX" = "0" -a -n "$SECRET_PASSWORD" ] ; then - cd $SRCDIR/test; ./gce.sh make_template - fi cd $SRCDIR/build ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh touch $SRCDIR/.build.uptodate diff --git a/bin/circle-test-pre b/bin/circle-test-pre index 16a0c0ffae..eafdd29589 100755 --- a/bin/circle-test-pre +++ b/bin/circle-test-pre @@ -6,5 +6,4 @@ source "$STATE" if [ -n "$TEST_AND_PUBLISH" ] ; then cd $SRCDIR/test - [ -z "$SECRET_PASSWORD" ] || (./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh) fi diff --git a/bin/circle-test-smoke b/bin/circle-test-smoke index a5fb9a7b61..672ac34501 100755 --- a/bin/circle-test-smoke +++ b/bin/circle-test-smoke @@ -4,11 +4,14 @@ set -e source "$STATE" -if [ -n "$TEST_AND_PUBLISH" ] ; then - [ -n "$SECRET_PASSWORD" ] || { echo "Cannot run smoke tests: no secret key"; exit 1; } - cd $SRCDIR/test - eval $(./gce.sh hosts) +if [ -n "$TEST_AND_PUBLISH" ]; then + [ -n "$SECRET_KEY" ] || { + echo "Cannot run smoke tests: no secret key" + exit 1 + } export COVERAGE=true export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log" - ./run_all.sh + export NAME="test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX" + cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder. + ./run-integration-tests.sh fi diff --git a/bin/circle-test-teardown b/bin/circle-test-teardown index 7e7bafa25e..c2e6ae25fa 100755 --- a/bin/circle-test-teardown +++ b/bin/circle-test-teardown @@ -6,6 +6,5 @@ source "$STATE" if [ -n "$TEST_AND_PUBLISH" ] ; then cd $SRCDIR/test - [ -z "$SECRET_PASSWORD" ] || ./gce.sh destroy test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh) fi diff --git a/bin/setup-circleci-secrets b/bin/setup-circleci-secrets deleted file mode 100755 index 36e35ca625..0000000000 --- a/bin/setup-circleci-secrets +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash - -set -eu - -# openssl enc -in do-setup-circleci-secrets.orig -out setup-circleci-secrets.orig -e -aes256 -pass stdin -# openssl base64 < setup-circleci-secrets.orig - -openssl base64 -d << EOF \ - | openssl enc \ - -out bin/do-setup-circleci-secrets \ - -d -aes256 -pass pass:$1 -U2FsdGVkX1/3QHnDgpZn0srp+GiEQBTlag0CBdsw6Bp6iaIKy6Jy8gFshrsmvl3+ -c+7hn1nqsBVpzVH0S5QA3Ex5nx9rl7/dcGoogsySFcwMmBFcauT0UTMK1bvX0Hqx -QrshYSndUkww72KA7EsSnjR1BBJWm3AhCDOTIcnO6nQhig1Fc+SBEj5FGS5xTQ5Z -/8hDsKf54Ms49vOO6QeE5w5ndZ+k7IvVJfq9Evhb7htD20RntDhR86foAwMnLV7R -Wh9qAqI3gnRq0GmbeRKaHmB7pcLq9f0OdScqM/Tn1D+q0556fqJjBziXpZiGbxVs -DxEp8zDJ8yNDAi0Wb2NXjoA79SXJ8Q7iTieaST4y2OmXQpY+Vw8RVDXxi6ERyVnH -41OeMzyLm0qvCoMl32pe1sSG5DRCUiAm7wOtDTVvNDRqmGJYUgwwexyayecJI1wQ -LlhYLJavUvw3UGDZOld8BaMQhfOKBVCdFrjmA4HuVxmOf3029qkgzOCSZiQVNqs+ -fvM6J05zBuk+GPy8YInvRkzfxum/x00vmUyGjMs/lHDFfrmYDFK3ux41EoVqvWLG -S5uYxdZJFdR3Xsrs+sMr51XNBQGMmz6KoEEzFYyHs5fZgtdpRdPAuQS1m6FqgZyi -wSz6+EKsALtmGorEp+CF8JN5YzKJCrKzNkg0fILBqsqmJZGP64lmX69950lRLNGM -/l3KVR2zH31b4AnnzTcvldVSCKDXjaMw3JA2Ce/MK6XhOWOcpEv4oDt/62lbhUKP -QUQTPbfpQRvVfBcoIrfAm3ghi/CZc2TDVEuD7DhasLTZD3r7LoK261BWC9O5Wubi -vlXuFIF+uyrpTDf6EJKE/+XywtArppffMP6RkHu1Iqnd8JtsqPaIZsmMmZbjgb6j -BX4ZLb5U9R7jYoSgIke4+u6geFcrMQ+Lny4r7MxXaS0Y1UJ6Jo1bwpYw1TBIobZc -eQ30+xhbSeRC9vEw1NXI6qM2Ai2gIHtAZvEiBkBBXTG5Ld7N/mBAmDIJEzVRxh6F -jRkB3dzESF6eeTBm61OQepwV3aW6lslMaf26drPGv2W2q+K/WOpQOejfggtHUgVs -9SWqtn7hHYSpIASe1Ihi14HAWv3kPhQien0zY1DLKJWf4ilbRaICn1QSJjNuKwtl -A2ZwEw7FFddFcsF4okHPkxEoCre3shq0DEw9UA1gnwEaEF1Q+i9cvLtQg56zK3+R -MfTzFUv2K1z3j1s5tC6Z6LzL8j682EFOll7MkihqiIWh+po1zcNs6Ln0SkjB3v9T -OFRYVJis9GqXYO9PUW1mI69udR4nXbKlBl5tWNyAxn9hufJHLSAmbGMwActKLRM5 -kROapKrN1Ra/sHUmhoSSDUK9AXeYKkqPXVMbhly9xC1Kya4hFaFokjRQIMLPmXdQ -lP46RL3HOq0v1BkXg05poSwDm+BtitZkcIhOzcmeucUqn1whsz39y3VoOhJqQu2j -0dUcj19Fue1CDZfwzACb9mLZr643h0yCIHQaVT7is/yiN2vEt0S0zynvOA1D+Xgy -4bFdjqed9DgBShLwF/lh4/hur5iZPpVr9A8rt3Vq0k5A1tdjsZpXP3qv5jLc7lJv -cAQxdVLEuv8WPwwTV/EQsoH7lpoEdjktw/gu0v5PkUgR/WBxEqJq4liTekViQ+8X -BAmdm/FGWeIxSSEm0a8I+muiBpIuKjuQoJit1f0k+lLUohoVDqpDDfeYHh1TxXHE -xSKAKNmFSc2XOr5Zg2mOInxeH7WoxH6iylXKADBgEgAQ4C55DoAPRjFNtGGz2iJl -WkzhQOfgsDss1VxpPZ4SWDQO+xxuuEnYcCYdJ6Tyi5hAwZ0+RZmXx0g9BCK+5qzc -Ffei066kCItBcayekwCs0i86r8JvDtsMUUkzEKDQTSWHpP8ypIZoz3P+Skdqcs6p -IcZHy2fODIsClctRAsHOjZ9BnWSY2rqf1lysqvRfOkNTViTOUVx2hf+LuL+8rT6g -K4GvS39Q+c9FYSMjCNxv1m37NfHJv2W7AlqVV95gMBEFB97M3388T7PfvmSLsyPv -qAKDE7genTOycQvEX9oeEACxPo08ai2Mgs/YXCig0g0btZoScoKPelQKtD0HLpZ+ -j7ruaKFy4/vySDMostB3AKSkCE/SkGu/W9oI37n1guSYC/9yJjtTcwZULXAClfwt -LyHvLAw4+7BdGJqDe9Tf8z4zqVLax+BcBHkW7pbP+pvxC6fhIA4PO8k5HwwAVlRw -68lG6e3z9+Vst4zrFLewHJvkDRSOTSecs9h9SPOjReTK0ovlDzMHVakfUvqpk9mv -nCj8SWaMPF70z138AuRuKN7X4exNXPBFnOcseKp/NpT1DyvgoHm/oIO65mq9vvo+ -42uSbyyegYiTdVK4SaXBc2hixkhaTteqdEy6GxfWqKCvHtYuRD2LyUSjZn/0Y6CG -3wZapylKL1sxfH+b1gogv2wAA8KtZAfiTlMISuymwQr5RbOFma7UVMVecCa7XrNG -fZwO8R+6++ru0efDzeC/gtLOItjn+yVfsbk2gfu0FikHIV1sogOjnsv3dl6IpH/s -w6WbuK/yZhyrrmhIIueHF30ES512Peogf/Gxp8U/ZAmqBa7p4qsv93XWDR52IPYz -yU1MtboLD02J39daBNrYZuO4Yfo/xAvU7Ijg2KIR9HS0bSWNuSFAZaZXlJ2bSag3 -8l0dZMx2fZa0FALxWvY2+nd1tWcH+Wo99RSxX+4TH3guCpe1x/58yKOKL4d1xRpu -02nGaqerV9bHU6WjCUYTrrZurTp6yqivpSFCALDMhAU5Ihd/Qjz69zg4uZ7iqbNd -WqIYJ3SAOei2tSfoOJbdh8mohotnMFFbupck3Fv+lXAXYDp0V/sSuK32MSeJxan1 -q3t0eenAh6zoBP0Tj53N8B9YmrgZ1i1AljQqYsAdb8N5Z3vecuzQa946U8H4axEc -BL5fgmqSfsbj1EjJVUrw3cE9hgqeqAYBBvZsyW5zSAsLq/RJKYtBxDGADqr0TOOu -TjVqfPObb1u9EAQj15ALBhhPORa2gBAK5gnhJWW5deNgA0UOYoWARFYfATuQvr1T -9xTVbqzZ0VJR4n7X5JyxJo2tZ44QYVzA0fqX0lt+uRkarrkbZMgplKp4MJTFCp2N -tLciWS0YSHPbKTm7svw5mROazUvEoj1N3SYrJT+Ai9UgSL1cA4FmQ5Jsi+GOeQbH -7E4pzjeST4b1aATY9ZZSy4TzK+UNYBcCVqY1dN5Xnk0e5yPMnrkkcYI41s/6oOaW -lQkp/yrHkvpzQWZa7vF2yWg54j3eX8YWx/TsLvmuJmSQqiIHfu9TXcAIPDispCpF -gp4ZbYPZ8RDQLw3ROc+bZ9E0J8e+FHwgXEELsMLW9n4d3PTj9pLFej1vmBKa03o3 -g+JViNxb1gUPNmC6ADk4o7TQzBrjV2pmmuCza16kOmq6oAbJPOeCLtVWA5HJTX8d -AuOzBLO37tWP457dNqoNDaCMik3LCy8cslZjAz02D6VG/QGOAXP9x8bJ3zg9N7vW -EOn1BMtIeMI4yIxpC6HgotNWDNbUBWremAFWYQRgYQqo9L1SdW4hz6X8FqdlBFvK -ml7sdUwjEy2tNW+aiZI4cgAlI/VvMWYr5kPfhvaUu1wGYeS1TD1B+DPcjcXpOGWo -vB1f/uvGc61zZWD0ph9f3S9OnvEko5qV3VhxOnxYUZTrqlM7LVgy/sHrSaPdR7Eo -nb4QGdi5wEGMxVgQlDGwOmEmS5nCaA0RcfHbkQV4SebRleocLB9qcSwj5JzPICAB -sn1qeSv45LnGdTzUnCQErw1EzgrK1FXOknujjZ3GX5dM7a6Gag0E6cAxoQeC8Mh0 -aUWolyBo9aSXltn7Vnywoe4X3v96eVH6tUQtN3wt51LBlfSDTWOQDeXLFyW57oYL -/ErbmJLaSHpANBcC9InI+rxNYePTTba7K8rdPDppmMEmdOXjaL+5aPay2Bay9K8T -FUyZdoyrbzxFZYT9/YyVJV/ukVcT9ru3HpzLTg9Vgc/CyRfIvCZY/Gzh+pPUbRsR -pHYDT09BFS+mjszTkcdbceweYXJvknXpajMiISrPUUkb9rwCTmLS4QR4y7I9zAgJ -9U72mCZB9yhhzS17kyKAANihXvOFrMXCO7QTWZbtSkobMBS2SybDxYP3rM5b6824 -HQaY4MZW5qCndydfz4fa908XbPTDka9Nu3CZfI3fhGiiGDFWViotGEYE4i9BuGZJ -vmtzII7zeIoHwM4t5fb5mYDd9H+Rkct1yATGEIFEx7cu1fj4Ht2177EzhdkF3Q6I -Fccn43yGDW06nFvEjnjb+Lz4EvtGp6dDefUhNBohdzd4Rm2o+etKobm0lZ1xOX74 -2gOGBLfXS6PF+msNkWP76mqKE6hG0+R1iPrN06Arti+ZRBhNGJDp1pnOERl7IyB0 -hiLUqae7xVSMVRDeq4O04EUpMXh5ixs3RJXjthYILt7vX+6qskJC3rMuyThrsvMi -Kk0wErDpzR5U7KFnM2fk4jY5by/0b/jHZ/PWFGnaFrXNOiwK/mV1RIJtg5FCrvMv -xJN5Qo7o2ZYzUL+IBf5mP5O49FCnz3e++9r4ySY/K57ybhQft1l+cQPWS7xuvVAg -wZGM9DPl9AhBus0I3V3XzfkcWWHkMynMRcfTnkBHrmKjMopIOggsq05JPalM9b0V -NfwZngVvTFbQejbGjgQQQ+G7spav/icCIA8Iu8JPHaKZt2IX77L1xMcQOouqMrx8 -JCXnJ7zsuU+8W3GQGm70H2kcen4bf4beA8TSkFmbVsaqDNtGaG+ZopWO8ctZX3CP -jFgIYgEGi137IWvs1KD5QNFHZI6l0JMzglmvcfnViRPDHLi+EQXAoL/98j/YzA9t -a9L4xMO0oucovKkqYOS2cdFaQKVIUM9L5ibcbW7Tol4ugMo5ibIsTgjZoyFohG6t -acg+h2wTTyxtrIapvOOBeAo9V/iQowTAPQf38LKQZ5tApksiCG1qvJerKALTMp1J -yUHjTCPCPsaiXVVO0mfa9OPtzVvi/icLmlz7umk82mew13djedgMS3/ghq5aEHbg -e7rtI5snFlm/GQnV6IaCDytb+jYxqQ0DZ9Mb0CHcMpnLyiBO6nzHfadpZYlrxrpx -dPx9MVqfLavj1UVjKuVj4Mgec9J38IBBRGMJZ4MGhde2hrw/s2jVzG1BzaIzcvFa -bWiJOkKKRC4BY9DBDYfae5c/obwzkxLmvkbWAVzoFXc+46K1IK6bbC2a4EerOAxc -C7B3X8BucD8OtiPORwXgjctd9AJ7cXq0EEVGoZZ8jkILnklsK6RXsxrT4gSOGFpx -cBU7TF4kHpH+o4KGQmjlFA2VYP1tXirjEep57LOHfL+zy+HiSXdsbexf6odkqLYA -1RR0TzSkmqJYvP0GfHPMvp2xredx1kP934fhTIUcWvErrG3NPd/Dod4BzQye1nNK -8Y8aFYF6fACcD4SFZ11ecK9DJmB+WXGlsxtO3w1LeEysny2lVkkH6HuMRIzPyXAM -/sEpwh3hwPJosAh6vGM6JxuL7CXP8hC1oUvK91VR0hxeDcO/LgdcHtDnlNMqt1wC -jvYt9Hg+3aY5GehBnQPRppnIMaGhiBrRC6HjBg8Umt6UTUr5PXNY2aia5wRcx4M6 -Tj0O0dAt8WWZ9ySgzOoq4SRoaAp6m4E7CpRNMgXH031OWEeHqtt8rsYqAcvcQLuW -vIwts5hiXZOfgXyR6SBY3ANE1SjDY46na2G4oiG8FI7eAqYgehjHrXqCnGrzjefE -V1eVcIr7DBYuM8wq7C9x+qdMir6RNV4MrVD+7rO/drivnd3MaqDHlG0LfMZo6GIK -QZGIy0z1W5lOXm9x3RAz7/Ea11ynkZyPDDxBE5NRcV6JfukdjDrvsaQA/Qk2Wo7h -PdUZjv17H4syMdap6tUDTc8Cohut6n1UxnDnhaAYEDGR3TgPqI5vEaJ+yoaM9itv -b2I/K+0uK6GfWwca7PeTtYn92/5etO4czG99xEBFzw7tiEmBTA8KfseZIUe/i3fr -9LO+rKEGalFB4mVTYyCKPYYx4hOIvjdCox/jEBTFyYUEhgGnZvArgw9XabvII7eM -c1FmwnN2CM/MHUzM0Kw7cS/oaoRFcG/PqioO/krL2Bd+xThwSH4Zn+Y1UVO7aXNy -90MOehHSc2nhN984Y2re3ffVsjPhC3KdWxr8jbho/Az65rUkYg64roiO2ZqdqRvf -aYtX9fCe8N8SzgLTlWZt+l8sjY8HP2k9NoajphV1nG4iC/pX0ZJiVQ+VBRNaGhSl -6IhM9gM1ZsI86cULJpOTAhNu1fZBv+Mq8IXYD8SqlNVwVtjvshtlzBN4InSGEKiO -HM7coo7aDuu+Go5AEiIVMbR09jAa0VINsCVScfSbTFhlGRTrZr2DSO/s8WxbV+Mb -EQAfZmyrLF71Z3MfIukg+ps5iuqFhZ+gJSXXSFwumpg= -EOF - -exec sh bin/do-setup-circleci-secrets diff --git a/test/gce.sh b/test/gce.sh deleted file mode 100755 index 6f2046de2d..0000000000 --- a/test/gce.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/bin/bash -# This script has a bunch of GCE-related functions: -# ./gce.sh setup - starts two VMs on GCE and configures them to run our integration tests -# . ./gce.sh; ./run_all.sh - set a bunch of environment variables for the tests -# ./gce.sh destroy - tear down the VMs -# ./gce.sh make_template - make a fresh VM template; update TEMPLATE_NAME first! - -set -e - -: ${KEY_FILE:=/tmp/gce_private_key.json} -: ${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key} -: ${PROJECT:=positive-cocoa-90213} -: ${IMAGE_FAMILY:=ubuntu-1604-lts} -: ${IMAGE_PROJECT:=ubuntu-os-cloud} -: ${TEMPLATE_NAME:=test-template-12} -: ${ZONE:=us-central1-a} -: ${NUM_HOSTS:=5} -SUFFIX="" -if [ -n "$CIRCLECI" ]; then - SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX" -fi - -# Setup authentication -gcloud auth activate-service-account --key-file $KEY_FILE 1>/dev/null -gcloud config set project $PROJECT - -function vm_names { - local names= - for i in $(seq 1 $NUM_HOSTS); do - names="host$i$SUFFIX $names" - done - echo "$names" -} - -# Delete all vms in this account -function destroy { - if [ $(gcloud compute firewall-rules list test-allow-docker$SUFFIX 2>/dev/null | wc -l) -gt 0 ] ; then - gcloud compute firewall-rules delete test-allow-docker$SUFFIX - fi - names="$(vm_names)" - if [ $(gcloud compute instances list --zones $ZONE -q $names | wc -l) -le 1 ] ; then - return 0 - fi - for i in {0..10}; do - # gcloud instances delete can sometimes hang. - case $(set +e; timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names >/dev/null 2>&1"; echo $?) in - 0) - return 0 - ;; - 124) - # 124 means it timed out - break - ;; - *) - return 1 - esac - done -} - -function internal_ip { - jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" $1 -} - -function external_ip { - jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" $1 -} - -function try_connect { - for i in {0..10}; do - ssh -t $1 true && return - sleep 2 - done -} - -function install_docker_on { - name=$1 - ssh -t $name sudo bash -x -s </etc/systemd/system/docker.service.d/override.conf < /etc/apt/sources.list.d/kubernetes.list -apt-get update -qq -apt-get install -q -y kubelet kubeadm kubectl kubernetes-cni -systemctl --now disable kubelet -# Pre-pull images required for Kubernetes -docker pull gcr.io/google_containers/etcd-amd64:2.2.5 -docker pull gcr.io/google_containers/kube-apiserver-amd64:v1.4.0 -docker pull gcr.io/google_containers/kube-controller-manager-amd64:v1.4.0 -docker pull gcr.io/google_containers/kube-proxy-amd64:v1.4.0 -docker pull gcr.io/google_containers/kube-scheduler-amd64:v1.4.0 -docker pull gcr.io/google_containers/kube-discovery-amd64:1.0 -docker pull gcr.io/google_containers/pause-amd64:3.0 -EOF -} - -function copy_hosts { - hostname=$1 - hosts=$2 - cat $hosts | ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" -} - -# Create new set of VMs -function setup { - destroy - - names="$(vm_names)" - gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE --tags test$SUFFIX --network=test - my_ip="$(curl -s http://ipinfo.io/ip)" - gcloud compute firewall-rules create test-allow-docker$SUFFIX --network=test --allow tcp:2375,tcp:12375 --target-tags test$SUFFIX --source-ranges $my_ip - gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE - sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config - - # build an /etc/hosts file for these vms - hosts=$(mktemp hosts.XXXXXXXXXX) - json=$(mktemp json.XXXXXXXXXX) - gcloud compute instances list --format=json >$json - for name in $names; do - echo "$(internal_ip $json $name) $name.$ZONE.$PROJECT" >>$hosts - done - - for name in $names; do - hostname="$name.$ZONE.$PROJECT" - - # Add the remote ip to the local /etc/hosts - sudo sed -i "/$hostname/d" /etc/hosts - sudo sh -c "echo \"$(external_ip $json $name) $hostname\" >>/etc/hosts" - try_connect $hostname - - copy_hosts $hostname $hosts & - done - - wait - - rm $hosts $json -} - -function make_template { - gcloud compute instances create $TEMPLATE_NAME --image-family=$IMAGE_FAMILY --image-project=$IMAGE_PROJECT --zone $ZONE - gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE - name="$TEMPLATE_NAME.$ZONE.$PROJECT" - try_connect $name - install_docker_on $name - install_kubernetes_on $name - gcloud -q compute instances delete $TEMPLATE_NAME --keep-disks boot --zone $ZONE - gcloud compute images create $TEMPLATE_NAME --source-disk $TEMPLATE_NAME --source-disk-zone $ZONE -} - -function hosts { - hosts= - json=$(mktemp json.XXXXXXXXXX) - gcloud compute instances list --format=json >$json - for name in $(vm_names); do - hostname="$name.$ZONE.$PROJECT" - hosts="$hostname $hosts" - done - echo export SSH=\"ssh -l vagrant\" - echo export HOSTS=\"$hosts\" - rm $json -} - -case "$1" in -setup) - setup - ;; - -hosts) - hosts - ;; - -destroy) - destroy - ;; - -make_template) - # see if template exists - if ! gcloud compute images list | grep $PROJECT | grep $TEMPLATE_NAME; then - make_template - fi - ;; - -*) - echo "Unknown command:" $1 >&2 - exit 1 -esac From a2679794b36b8218d0e014c57564c58411dfea50 Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Tue, 10 Jan 2017 18:32:26 +0000 Subject: [PATCH 07/23] Install Terraform and Ansible in CircleCI's containers. Terraform and Ansible are now required to run the smoke tests. --- bin/circle-test-smoke | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/circle-test-smoke b/bin/circle-test-smoke index 672ac34501..ccca722ee1 100755 --- a/bin/circle-test-smoke +++ b/bin/circle-test-smoke @@ -9,6 +9,12 @@ if [ -n "$TEST_AND_PUBLISH" ]; then echo "Cannot run smoke tests: no secret key" exit 1 } + # Install Terraform and Ansible: + curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin + sudo apt-get update || true + sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible + export PATH="$PATH:$HOME/.local/bin" + # Run integration tests: export COVERAGE=true export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log" export NAME="test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX" From e647fe6331415175b7a118d3026353cf64a4ed4a Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Mon, 16 Jan 2017 16:40:00 +0000 Subject: [PATCH 08/23] Highlight setup and sanity checks in CircleCI's logs. --- test/sanity_check.sh | 3 ++- test/setup.sh | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/sanity_check.sh b/test/sanity_check.sh index 657bdf7193..d5b8768dee 100755 --- a/test/sanity_check.sh +++ b/test/sanity_check.sh @@ -7,6 +7,7 @@ set -e begin=$(date +%s) sanity_checks_files=${WEAVE_NET_SANITY_CHECKS_FILES:-"/tmp/weave_net_sanity_check_*.log"} +greenly echo "> Sanity-checking test machines: ping, check Docker, and check Weave..." whitely echo Ping each host from the other # We wrap ping and echo in a function as we want the below parallel for loop @@ -68,4 +69,4 @@ for host in $HOSTS; do done for pid in $pids; do wait $pid; done -echo "Sanity checks completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")." +greenly echo "> Sanity checks completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")." diff --git a/test/setup.sh b/test/setup.sh index 4d21c3de98..3bab725aad 100755 --- a/test/setup.sh +++ b/test/setup.sh @@ -9,8 +9,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" (cd ./tls && ./tls $HOSTS) -echo "Copying weave images, scripts, and certificates to hosts, and" -echo " prefetch test images" +greenly echo "> Setting up test machines: copying weave images, scripts and certificates to hosts, and prefetch test images..." exists_on() { docker_on $1 inspect --format=" " $2 >/dev/null 2>&1 @@ -60,4 +59,4 @@ for ppid in $ppids; do wait $ppid; done -echo "Setup completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")." +greenly echo "> Setup completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")." From 3e5593dc4f4655ac13d4c02423e5af82756731b0 Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Mon, 16 Jan 2017 18:01:33 +0000 Subject: [PATCH 09/23] Add GCP image creation and usage. Provisioning and configuration being expensive time-wise, we create Google Cloud Platform images for each Docker/Kubernetes/Kubernetes CNI versions triple, and configure these once (instead of for each build), and then re-use these in subsequent builds, effectively implementing a 'cache' of images. --- bin/circle-test-smoke | 6 +++ test/README.md | 21 ++------- test/run-integration-tests.sh | 84 ++++++++++++++++++++++++++++++++--- 3 files changed, 89 insertions(+), 22 deletions(-) diff --git a/bin/circle-test-smoke b/bin/circle-test-smoke index ccca722ee1..aaa50de1cf 100755 --- a/bin/circle-test-smoke +++ b/bin/circle-test-smoke @@ -14,10 +14,16 @@ if [ -n "$TEST_AND_PUBLISH" ]; then sudo apt-get update || true sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible export PATH="$PATH:$HOME/.local/bin" + # Run integration tests: export COVERAGE=true export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log" export NAME="test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX" + export DISK_NAME_PREFIX="test-$CIRCLE_BUILD_NUM-0" + export USE_IMAGE=1 + # Only attempt to create GCP image in first container, wait for it to be created otherwise: + [ "$CIRCLE_NODE_INDEX" == "0" ] && export CREATE_IMAGE=1 + cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder. ./run-integration-tests.sh fi diff --git a/test/README.md b/test/README.md index 842579c39f..b0f84238eb 100644 --- a/test/README.md +++ b/test/README.md @@ -60,20 +60,7 @@ to the file `/etc/default/docker`, then restart docker. ## Updating the GCE test image -When a new version of Docker is released, you'll need to update the GCE test image. - -To do this, pick a fresh ```TEMPLATE_NAME``` and update any commands in -```function make_template``` in gce.sh, then run: - -``` -./gce.sh make_template -``` - -For this you'll need the GCE credentials, which can be found in ```bin/setup-circleci-secrets```, -which you'll need to decrypt and run (its echos the secrets into know locations): - -``` -./bin/setup-circleci-secrets "$SECRET_PASSWORD" -``` - -If you don't know the password, ask tom@weave.works. +When a new version of Docker is released, you willneed to update the GCE test image. +To do this, change the Docker version in `run-integration-tests.sh` and push the change. +Next build in CircleCI will detect that there is no template for this version of Docker and will first create the template before running tests. +Subsequent builds will then simply re-use the template. diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh index d422f589f3..4cb1b1a99b 100755 --- a/test/run-integration-tests.sh +++ b/test/run-integration-tests.sh @@ -32,6 +32,12 @@ RUNNER_ARGS=${RUNNER_ARGS:-""} DOCKER_VERSION=${DOCKER_VERSION:-1.11.2} KUBERNETES_VERSION=${KUBERNETES_VERSION:-1.5.2} KUBERNETES_CNI_VERSION=${KUBERNETES_CNI_VERSION:-0.3.0.1} +# Google Cloud Platform image's name & usage (only used when PROVIDER is gcp): +IMAGE_NAME=${IMAGE_NAME:-"$(echo "$APP-docker$DOCKER_VERSION-k8s$KUBERNETES_VERSION-k8scni$KUBERNETES_CNI_VERSION" | sed -e 's/[\.\_]*//g')"} +DISK_NAME_PREFIX=${DISK_NAME_PREFIX:-$NAME} +USE_IMAGE=${USE_IMAGE:-1} +CREATE_IMAGE=${CREATE_IMAGE:-} +CREATE_IMAGE_TIMEOUT_IN_SECS=${CREATE_IMAGE_TIMEOUT_IN_SECS:-600} # Lifecycle flags: SKIP_CONFIG=${SKIP_CONFIG:-} @@ -47,6 +53,11 @@ function print_vars() { echo "DOCKER_VERSION=$DOCKER_VERSION" echo "KUBERNETES_VERSION=$KUBERNETES_VERSION" echo "KUBERNETES_CNI_VERSION=$KUBERNETES_CNI_VERSION" + echo "IMAGE_NAME=$IMAGE_NAME" + echo "DISK_NAME_PREFIX=$DISK_NAME_PREFIX" + echo "USE_IMAGE=$USE_IMAGE" + echo "CREATE_IMAGE=$CREATE_IMAGE" + echo "CREATE_IMAGE_TIMEOUT_IN_SECS=$CREATE_IMAGE_TIMEOUT_IN_SECS" echo "--- Variables: Flags ---" echo "SKIP_CONFIG=$SKIP_CONFIG" } @@ -98,6 +109,65 @@ function provision_locally() { esac } +function setup_gcloud() { + # Authenticate: + gcloud auth activate-service-account --key-file "$GOOGLE_CREDENTIALS_FILE" 1>/dev/null + # Set current project: + gcloud config set project $PROJECT +} + +function image_exists() { + gcloud compute images list | grep "$PROJECT" | grep "$IMAGE_NAME" +} + +function image_ready() { + # GCP images seem to be listed before they are actually ready for use, + # typically failing the build with: "googleapi: Error 400: The resource is not ready". + # We therefore consider the image to be ready once the disk of its template instance has been deleted. + ! gcloud compute disks list | grep "$DISK_NAME_PREFIX" +} + +function wait_for_image() { + greenly echo "> Waiting for GCP image $IMAGE_NAME to be created..." + for i in $(seq "$CREATE_IMAGE_TIMEOUT_IN_SECS"); do + image_exists && image_ready && return 0 + if ! ((i % 60)); then echo "Waited for $i seconds and still waiting..."; fi + sleep 1 + done + redly echo "> Waited $CREATE_IMAGE_TIMEOUT_IN_SECS seconds for GCP image $IMAGE_NAME to be created, but image could not be found." + exit 1 +} + +# shellcheck disable=SC2155 +function create_image() { + if [ -n "$CREATE_IMAGE" ]; then + greenly echo "> Creating GCP image $IMAGE_NAME..." + local begin_img=$(date +%s) + local num_hosts=1 + terraform apply -input=false -var "app=$APP" -var "name=$NAME" -var "num_hosts=$num_hosts" "$DIR/../tools/provisioning/gcp" + configure_with_ansible "$(terraform output username)" "$(terraform output public_ips)," "$(terraform output private_key_path)" $num_hosts + local zone=$(terraform output zone) + local name=$(terraform output instances_names) + gcloud -q compute instances delete "$name" --keep-disks boot --zone "$zone" + gcloud compute images create "$IMAGE_NAME" --source-disk "$name" --source-disk-zone "$zone" \ + --description "Testing image for Weave Net based on $(terraform output image), Docker $DOCKER_VERSION, Kubernetes $KUBERNETES_VERSION and Kubernetes CNI $KUBERNETES_CNI_VERSION." + gcloud compute disks delete "$name" --zone "$zone" + terraform destroy -force "$DIR/../tools/provisioning/gcp" + rm terraform.tfstate* + echo + greenly echo "> Created GCP image $IMAGE_NAME in $(date -u -d @$(($(date +%s) - begin_img)) +"%T")." + else + wait_for_image + fi +} + +function use_or_create_image() { + setup_gcloud + image_exists || create_image + export TF_VAR_gcp_image="$IMAGE_NAME" # Override the default image name. + export SKIP_CONFIG=1 # No need to configure the image, since already done when making the template +} + function update_local_etc_hosts() { echo "> Updating local /etc/hosts..." # Remove old entries (if present): @@ -171,6 +241,7 @@ function provision() { ;; 'gcp') gcp_on + [[ "$USE_IMAGE" == 1 ]] && use_or_create_image provision_remotely "$1" "$2" ;; 'vagrant') @@ -186,6 +257,13 @@ function provision() { greenly echo "> Provisioning took $(date -u -d @$(($(date +%s) - begin_prov)) +"%T")." } +function configure_with_ansible() { + ansible-playbook -u "$1" -i "$2" --private-key="$3" --forks="${4:-$NUM_HOSTS}" \ + --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ + --extra-vars "docker_version=$DOCKER_VERSION kubernetes_version=$KUBERNETES_VERSION kubernetes_cni_version=$KUBERNETES_CNI_VERSION" \ + "$DIR/../tools/config_management/$PLAYBOOK" +} + # shellcheck disable=SC2155 function configure() { echo @@ -201,11 +279,7 @@ function configure() { # Configure the provisioned machines using Ansible, allowing up to 3 retries upon failure (e.g. APT connectivity issues, etc.): for i in $(seq 3); do - ansible-playbook -u "$1" -i "$inventory_file" --private-key="$4" --forks="$NUM_HOSTS" \ - --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ - --extra-vars "docker_version=$DOCKER_VERSION kubernetes_version=$KUBERNETES_VERSION kubernetes_cni_version=$KUBERNETES_CNI_VERSION" \ - "$DIR/../tools/config_management/$PLAYBOOK" \ - && break || echo >&2 "#$i: Ansible failed. Retrying now..." + configure_with_ansible "$1" "$inventory_file" "$4" && break || echo >&2 "#$i: Ansible failed. Retrying now..." done echo From 17ada094f85fc7d191fc94560fdea34c2244fed4 Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Wed, 18 Jan 2017 14:15:58 +0000 Subject: [PATCH 10/23] Updated and improved testing documentation. --- test/README.md | 67 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/test/README.md b/test/README.md index b0f84238eb..664547976c 100644 --- a/test/README.md +++ b/test/README.md @@ -2,40 +2,67 @@ This directory contains integration tests for weave. ## Requirements -You need two VMs with docker >=1.6.0 installed and listening on TCP -port 2375 (see below). You also need to be able to ssh to these VMs, -preferably without having to input anything. +You need three VMs with `docker` (`>=1.6.0`) installed and listening on TCP +port `2375` (see below). You also need to be able to SSH to these VMs without +having to input anything. -The `Vagrantfile` in this directory constructs two such VMs. +The `Vagrantfile` in this directory constructs three such VMs. -To create the VMs, open a shell and in this directory and type +If you are [building Weave Net using Vagrant](https://www.weave.works/docs/net/latest/building/), +it is recommended to run the tests from the build VM and not the host. - vagrant up -To meet the aforementioned ssh requirement you may want to +## Running tests - cp ~/.vagrant.d/insecure_private_key . +**TL;DR**: You can run the steps 2. to 7. below with one command: -## Running tests + make PROVIDER=vagrant integration-tests -If you are [building weave using Vagrant](https://www.weave.works/docs/net/latest/building/), -it is recommended to run the tests from the build VM and not the host. +**Detailed steps**: + + 1. Start the build virtual machine (see above article for more details): + + vagrant up + + 2. Start the three testing VMs: + + cd test + vagrant up + + 3. SSH into the build VM and go to Weave Net's sources: + + cd .. + vagrant ssh + # you are now on the build VM: + cd ~/weave + + 4. Compile all code and dependencies: + + make + make testrunner + cd test + + 5. Upload the weave images from where the `Makefile` puts them (`weave.tar.gz`) to + the three docker hosts, `docker load` these, and copies the `weave` script over: + + ./setup.sh - ./setup.sh + 6. Run individual tests, e.g.: -uploads the weave images from where the Makefile puts them -(`/var/tmp`) to the two docker hosts, and copies the weave script -over. + ./200_dns_test.sh -Then you can use, e.g., + or run all tests (everything named `*_test.sh`): - ./200_dns_test.sh + ./run_all.sh -to run an individual test, or + 7. Stop all VMs: - ./run_all.sh + exit + # you are now on your host machine + vagrant destroy -f + cd test + vagrant destroy -f -to run everything named `*_test.sh`. ## Using other VMs From e5185e73ce3b3df4b1ac55bcebfa49d7c3a6fe8c Mon Sep 17 00:00:00 2001 From: Marc CARRE Date: Tue, 17 Jan 2017 17:16:08 +0000 Subject: [PATCH 11/23] Refactor Vagrant configuration. --- Vagrantfile | 11 ++++++----- test/Vagrantfile | 8 +++----- vagrant-common.rb | 32 ++++++++++++++++++++++---------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 24871afd4e..a7077293ca 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,5 +1,3 @@ -VAGRANTFILE_API_VERSION = "2" - require File.expand_path(File.join(File.dirname(__FILE__), 'vagrant-common.rb')) vm_ip = "172.16.0.3" # arbitrary private IP @@ -19,17 +17,20 @@ pkgs = %w( Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "ubuntu/wily64" + config.vm.box = VAGRANT_IMAGE config.vm.network "private_network", ip: vm_ip config.vm.provider :virtualbox do |vb| vb.memory = 2048 - vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"] - vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"] + configure_nat_dns(vb) end + # Disable default Vagrant shared folder, which we don't need: config.vm.synced_folder ".", "/vagrant", disabled: true + # Keep Weave Net sources' in sync: config.vm.synced_folder ".", "/home/vagrant/src/github.com/weaveworks/weave" + # Create a convenience symlink to $HOME/src/github.com/weaveworks/weave + config.vm.provision :shell, :inline => 'ln -sf ~vagrant/src/github.com/weaveworks/weave ~vagrant/' # Set SSH keys up to be able to run smoke tests straightaway: config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/home/vagrant/src/github.com/weaveworks/weave/test/insecure_private_key" diff --git a/test/Vagrantfile b/test/Vagrantfile index 691c507592..281785f712 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -1,5 +1,3 @@ -VAGRANTFILE_API_VERSION = "2" - # Necessary because the switch disabling unwanted insecure key # replacement in Vagrant 1.7+ is not accepted by earlier versions Vagrant.require_version ">= 1.7.0" @@ -14,11 +12,12 @@ require File.expand_path(File.join(File.dirname(__FILE__), '..', 'vagrant-common def configure_docker(host, hostname, ip) pkgs = %w(docker-engine ethtool) - host.vm.box = "ubuntu/wily64" + host.vm.box = VAGRANT_IMAGE host.vm.provision :shell, :inline => "hostnamectl set-hostname "+hostname host.vm.network "private_network", ip: ip + # Disable default Vagrant shared folder, which we don't need: host.vm.synced_folder ".", "/vagrant", disabled: true host.vm.provision :shell, :inline => "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D" @@ -38,9 +37,8 @@ end Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"] - vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"] vb.memory = 384 + configure_nat_dns(vb) end # Disable Vagrant 1.7 per host insecure key replacement diff --git a/vagrant-common.rb b/vagrant-common.rb index 125abeeace..001dc8ead5 100644 --- a/vagrant-common.rb +++ b/vagrant-common.rb @@ -1,11 +1,23 @@ -$go_regexp = /FROM golang:(\S*).*?/ -$dockerfile_path = File.expand_path(File.join(File.dirname(__FILE__), 'build', 'Dockerfile')) -$go_version = File.readlines($dockerfile_path).first { |line| line.match($go_regexp) }.match($go_regexp).captures.first -if $go_version.nil? - raise ArgumentError.new("Failed to read Go version from Dockerfile.") +VAGRANT_IMAGE = 'ubuntu/wily64' +VAGRANTFILE_API_VERSION = '2' + +def get_go_version_from_build_dockerfile() + go_regexp = /FROM golang:(\S*).*?/ + dockerfile_path = File.expand_path(File.join(File.dirname(__FILE__), 'build', 'Dockerfile')) + go_version = File.readlines(dockerfile_path).first { |line| line.match(go_regexp) }.match(go_regexp).captures.first + if go_version.nil? + raise ArgumentError.new("Failed to read Go version from Dockerfile.") + end + go_version end -$go_path = "/usr/local/go/bin" +GO_BINARY_PATH = '/usr/local/go/bin' +GO_VERSION = get_go_version_from_build_dockerfile() + +def configure_nat_dns(vb) + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"] + vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"] +end def install_packages(vm, pkgs) vm.provision :shell, :inline => <