diff --git a/.gitignore b/.gitignore index 5d9b497c3a..3a7f66ad0d 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,5 @@ test/tls/*.pem test/coverage test/coverage.* *qemu-*-static +terraform.tfstate +terraform.tfstate.backup diff --git a/.lintignore b/.lintignore index 3403a0d123..f6b9efd061 100644 --- a/.lintignore +++ b/.lintignore @@ -1,8 +1,27 @@ # IMPORTANT: Please only use GLOBs to ignore files and directories. -# Do not lint any bash script in Weave Net, in order to +# Do not lint the below bash scripts in Weave Net, in order to # avoid breaking the build on make lint: -*.sh -bin/* +bin/circle-deploy-issues +bin/circle-deploy-master +bin/circle-teardown-pre +bin/circle-test-teardown +bin/circle-test-unit +bin/install-wordepress +bin/multiweave +bin/publish-site +bin/release +build/build.sh +prog/weave-kube/launch.sh prog/weaveexec/symlink +test/*_test.sh +test/600_proxy_docker_py.sh +test/assert.sh +test/config.sh +test/gce.sh +test/gen_coverage_reports.sh +test/run_all.sh +test/sanity_check.sh +test/setup.sh +vendor/*.sh weave diff --git a/Makefile b/Makefile index 73f02fd410..5fc050a5c4 100644 --- a/Makefile +++ b/Makefile @@ -214,7 +214,7 @@ tests: ./tools/test -no-go-get -netgo -timeout 8m lint: - ./tools/lint -nocomment -notestpackage . + ./tools/lint -nocomment -notestpackage endif @@ -359,3 +359,20 @@ build: run-smoketests: all testrunner cd test && ./setup.sh && ./run_all.sh + +integration-tests: all testrunner +# Usage: +# $ make \ +# NAME="" \ +# PROVIDER="" \ +# NUM_HOSTS="<# test machines>" \ +# PLAYBOOK="" \ +# RUNNER_ARGS="<...>" \ +# TESTS="<...>" \ # Can be set to only run one or a few tests instead of the full test suite. +# DOCKER_VERSION=<...> \ +# KUBERNETES_VERSION=<...> \ +# KUBERNETES_CNI_VERSION=<...> \ +# <...> # See also run-integration-test.sh for all variables and individual functions. +# integration-tests +# + RUNNER_ARGS="-parallel" ./test/run-integration-tests.sh diff --git a/Vagrantfile b/Vagrantfile index 0f0287bce0..516827745d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,45 +1,36 @@ -VAGRANTFILE_API_VERSION = "2" - -require './vagrant-common.rb' +require File.expand_path(File.join(File.dirname(__FILE__), 'vagrant-common.rb')) vm_ip = "172.16.0.3" # arbitrary private IP -pkgs = %w( - docker-engine=1.10.2-0~wily - aufs-tools - build-essential - ethtool - iputils-arping - libpcap-dev - git - mercurial - bc - jq -) - Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "ubuntu/wily64" + config.vm.box = VAGRANT_IMAGE config.vm.network "private_network", ip: vm_ip config.vm.provider :virtualbox do |vb| vb.memory = 2048 - vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"] - vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"] + configure_nat_dns(vb) end + # Disable default Vagrant shared folder, which we don't need: config.vm.synced_folder ".", "/vagrant", disabled: true + # Keep Weave Net sources' in sync: config.vm.synced_folder ".", "/home/vagrant/src/github.com/weaveworks/weave" + # Create a convenience symlink to $HOME/src/github.com/weaveworks/weave + config.vm.provision :shell, :inline => 'ln -sf ~vagrant/src/github.com/weaveworks/weave ~vagrant/' # Set SSH keys up to be able to run smoke tests straightaway: config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/home/vagrant/src/github.com/weaveworks/weave/test/insecure_private_key" - - install_build_deps config.vm, pkgs - install_go_toochain config.vm - tweak_user_env config.vm - tweak_docker_daemon config.vm + # Grant permissions on sources: + config.vm.provision :shell, :inline => 'sudo chown -R vagrant:vagrant ~vagrant/src', :privileged => false cleanup config.vm + config.vm.provision 'ansible' do |ansible| + ansible.playbook = 'tools/config_management/setup_weave-net_dev.yml' + ansible.extra_vars = { + go_version: GO_VERSION + } + end end begin diff --git a/bin/circle-dependencies-post b/bin/circle-dependencies-post index e086550a3e..1b705fc638 100755 --- a/bin/circle-dependencies-post +++ b/bin/circle-dependencies-post @@ -2,45 +2,16 @@ set -e -echo "TEST_AND_PUBLISH=1" > "$STATE" - -# Avoid doing a full build if we can. Note if the PR doesn't exist -# when the build is kicked off, we can't run this check so we do a -# full build. Subsequent pushes to the same branch will have the desired behaviour -if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ] ; then - if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$') ; then - echo "GH_PR=$PR" >> "$STATE" - if FILES=$(curl -s https://api.github.com/repos/weaveworks/weave/pulls/$PR/files | jq -r '.[]|.filename') ; then - echo "PR touches [$FILES]" - if ! echo "$FILES" | grep -v ^site/ ; then - echo "PR contains site changes only - skipping tests" - echo "TEST_AND_PUBLISH=" >> "$STATE" - else - echo "PR contains non-site changes - running tests" - fi - else - echo "Unable to get files for PR $PR" >&2 - fi - else - echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2 - fi -fi - source "$STATE" -mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR - -if [ -n "$TEST_AND_PUBLISH" ] ; then - [ -z "$SECRET_PASSWORD" ] || bin/setup-circleci-secrets "$SECRET_PASSWORD" - cd $SRCDIR; git submodule update --init - if [ "$CIRCLE_NODE_INDEX" = "0" -a -n "$SECRET_PASSWORD" ] ; then - cd $SRCDIR/test; ./gce.sh make_template - fi - cd $SRCDIR/build +if [ -n "$TEST_AND_PUBLISH" ]; then + cd "$SRCDIR" + git submodule update --init + cd "$SRCDIR/build" ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh - touch $SRCDIR/.build.uptodate + touch "$SRCDIR/.build.uptodate" sudo chown -R ubuntu /usr/local/go - cd $SRCDIR + cd "$SRCDIR" make testrunner make COVERAGE=true RM= exes all fi diff --git a/bin/circle-dependencies-post-sources b/bin/circle-dependencies-post-sources new file mode 100755 index 0000000000..f6875ced92 --- /dev/null +++ b/bin/circle-dependencies-post-sources @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +echo "TEST_AND_PUBLISH=1" >"$STATE" + +# Avoid doing a full build if we can. Note if the PR doesn't exist +# when the build is kicked off, we can't run this check so we do a +# full build. Subsequent pushes to the same branch will have the desired behaviour +if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ]; then + if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$'); then + echo "GH_PR=$PR" >>"$STATE" + if FILES=$(curl -s "https://api.github.com/repos/weaveworks/weave/pulls/$PR/files" | jq -r '.[]|.filename'); then + echo "PR touches [$FILES]" + if ! echo "$FILES" | grep -v ^site/; then + echo "PR contains site changes only - skipping tests" + echo "TEST_AND_PUBLISH=" >>"$STATE" + else + echo "PR contains non-site changes - running tests" + fi + else + echo "Unable to get files for PR $PR" >&2 + fi + else + echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2 + fi +fi + +source "$STATE" + +mkdir -p "$(dirname "$SRCDIR")" && cp -r "$(pwd)/" "$SRCDIR" + +if [ -n "$TEST_AND_PUBLISH" ]; then + cd "$SRCDIR" + git submodule update --init +fi diff --git a/bin/circle-test-pre b/bin/circle-test-pre index 16a0c0ffae..67254cf927 100755 --- a/bin/circle-test-pre +++ b/bin/circle-test-pre @@ -2,9 +2,29 @@ set -e +# Signal failures in lock file, in order to fail fast: +function signal_failure() { + echo "KO" >"$TEST_VMS_READY_LOCK_FILE" + exit 1 +} +trap signal_failure ERR + source "$STATE" +source "$(dirname "$0")/wait_for.sh" + +if [ -n "$TEST_AND_PUBLISH" ]; then + [ -n "$SECRET_KEY" ] || { + echo "Cannot run smoke tests: no secret key" + exit 1 + } + + # Provisioning from image typically take 90 seconds, + # but if we are creating a GCP image, it can take more than 10 minutes: + wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE" -if [ -n "$TEST_AND_PUBLISH" ] ; then - cd $SRCDIR/test - [ -z "$SECRET_PASSWORD" ] || (./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh) + # Upload built binaries to testing VMs: + cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests. + ./run-integration-tests.sh setup >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 + echo "OK" >"$TEST_VMS_READY_LOCK_FILE" + echo "Test VMs now ready. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE" fi diff --git a/bin/circle-test-smoke b/bin/circle-test-smoke index a5fb9a7b61..250841ebe7 100755 --- a/bin/circle-test-smoke +++ b/bin/circle-test-smoke @@ -3,12 +3,21 @@ set -e source "$STATE" +source "$(dirname "$0")/wait_for.sh" -if [ -n "$TEST_AND_PUBLISH" ] ; then - [ -n "$SECRET_PASSWORD" ] || { echo "Cannot run smoke tests: no secret key"; exit 1; } - cd $SRCDIR/test - eval $(./gce.sh hosts) +if [ -n "$TEST_AND_PUBLISH" ]; then + [ -n "$SECRET_KEY" ] || { + echo "Cannot run smoke tests: no secret key" + exit 1 + } + + # Provisioning from image typically take 90 seconds, + # but if we are creating a GCP image, it can take more than 10 minutes: + wait_for 900 "$TEST_VMS_READY_LOCK_FILE" + + # Run integration tests: export COVERAGE=true export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log" - ./run_all.sh + cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs. + ./run-integration-tests.sh test fi diff --git a/bin/circle-test-teardown b/bin/circle-test-teardown index 7e7bafa25e..c2e6ae25fa 100755 --- a/bin/circle-test-teardown +++ b/bin/circle-test-teardown @@ -6,6 +6,5 @@ source "$STATE" if [ -n "$TEST_AND_PUBLISH" ] ; then cd $SRCDIR/test - [ -z "$SECRET_PASSWORD" ] || ./gce.sh destroy test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh) fi diff --git a/bin/provision_test_vms.sh b/bin/provision_test_vms.sh new file mode 100755 index 0000000000..dfedb14446 --- /dev/null +++ b/bin/provision_test_vms.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +# Signal failures in lock file, in order to fail fast: +function signal_failure() { + echo "KO" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE" + exit 1 +} +trap signal_failure ERR + +source "$STATE" + +function install_terraform() { + curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin +} + +function install_ansible() { + sudo apt-get update || true + sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible + export PATH="$PATH:$HOME/.local/bin" +} + +if [ -n "$TEST_AND_PUBLISH" ]; then + [ -n "$SECRET_KEY" ] || { + echo "Cannot run smoke tests: no secret key" + exit 1 + } + + install_terraform >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 + install_ansible >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 + + # Only attempt to create GCP image in first container, wait for it to be created otherwise: + [ "$CIRCLE_NODE_INDEX" != "0" ] && export CREATE_IMAGE=0 + + # Provision and configure testing VMs: + cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests. + ./run-integration-tests.sh configure >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 + echo "OK" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE" + echo "Test VMs now provisioned and configured. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE" +fi diff --git a/bin/setup-circleci-secrets b/bin/setup-circleci-secrets deleted file mode 100755 index 36e35ca625..0000000000 --- a/bin/setup-circleci-secrets +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash - -set -eu - -# openssl enc -in do-setup-circleci-secrets.orig -out setup-circleci-secrets.orig -e -aes256 -pass stdin -# openssl base64 < setup-circleci-secrets.orig - -openssl base64 -d << EOF \ - | openssl enc \ - -out bin/do-setup-circleci-secrets \ - -d -aes256 -pass pass:$1 -U2FsdGVkX1/3QHnDgpZn0srp+GiEQBTlag0CBdsw6Bp6iaIKy6Jy8gFshrsmvl3+ -c+7hn1nqsBVpzVH0S5QA3Ex5nx9rl7/dcGoogsySFcwMmBFcauT0UTMK1bvX0Hqx -QrshYSndUkww72KA7EsSnjR1BBJWm3AhCDOTIcnO6nQhig1Fc+SBEj5FGS5xTQ5Z -/8hDsKf54Ms49vOO6QeE5w5ndZ+k7IvVJfq9Evhb7htD20RntDhR86foAwMnLV7R -Wh9qAqI3gnRq0GmbeRKaHmB7pcLq9f0OdScqM/Tn1D+q0556fqJjBziXpZiGbxVs -DxEp8zDJ8yNDAi0Wb2NXjoA79SXJ8Q7iTieaST4y2OmXQpY+Vw8RVDXxi6ERyVnH -41OeMzyLm0qvCoMl32pe1sSG5DRCUiAm7wOtDTVvNDRqmGJYUgwwexyayecJI1wQ -LlhYLJavUvw3UGDZOld8BaMQhfOKBVCdFrjmA4HuVxmOf3029qkgzOCSZiQVNqs+ -fvM6J05zBuk+GPy8YInvRkzfxum/x00vmUyGjMs/lHDFfrmYDFK3ux41EoVqvWLG -S5uYxdZJFdR3Xsrs+sMr51XNBQGMmz6KoEEzFYyHs5fZgtdpRdPAuQS1m6FqgZyi -wSz6+EKsALtmGorEp+CF8JN5YzKJCrKzNkg0fILBqsqmJZGP64lmX69950lRLNGM -/l3KVR2zH31b4AnnzTcvldVSCKDXjaMw3JA2Ce/MK6XhOWOcpEv4oDt/62lbhUKP -QUQTPbfpQRvVfBcoIrfAm3ghi/CZc2TDVEuD7DhasLTZD3r7LoK261BWC9O5Wubi -vlXuFIF+uyrpTDf6EJKE/+XywtArppffMP6RkHu1Iqnd8JtsqPaIZsmMmZbjgb6j -BX4ZLb5U9R7jYoSgIke4+u6geFcrMQ+Lny4r7MxXaS0Y1UJ6Jo1bwpYw1TBIobZc -eQ30+xhbSeRC9vEw1NXI6qM2Ai2gIHtAZvEiBkBBXTG5Ld7N/mBAmDIJEzVRxh6F -jRkB3dzESF6eeTBm61OQepwV3aW6lslMaf26drPGv2W2q+K/WOpQOejfggtHUgVs -9SWqtn7hHYSpIASe1Ihi14HAWv3kPhQien0zY1DLKJWf4ilbRaICn1QSJjNuKwtl -A2ZwEw7FFddFcsF4okHPkxEoCre3shq0DEw9UA1gnwEaEF1Q+i9cvLtQg56zK3+R -MfTzFUv2K1z3j1s5tC6Z6LzL8j682EFOll7MkihqiIWh+po1zcNs6Ln0SkjB3v9T -OFRYVJis9GqXYO9PUW1mI69udR4nXbKlBl5tWNyAxn9hufJHLSAmbGMwActKLRM5 -kROapKrN1Ra/sHUmhoSSDUK9AXeYKkqPXVMbhly9xC1Kya4hFaFokjRQIMLPmXdQ -lP46RL3HOq0v1BkXg05poSwDm+BtitZkcIhOzcmeucUqn1whsz39y3VoOhJqQu2j -0dUcj19Fue1CDZfwzACb9mLZr643h0yCIHQaVT7is/yiN2vEt0S0zynvOA1D+Xgy -4bFdjqed9DgBShLwF/lh4/hur5iZPpVr9A8rt3Vq0k5A1tdjsZpXP3qv5jLc7lJv -cAQxdVLEuv8WPwwTV/EQsoH7lpoEdjktw/gu0v5PkUgR/WBxEqJq4liTekViQ+8X -BAmdm/FGWeIxSSEm0a8I+muiBpIuKjuQoJit1f0k+lLUohoVDqpDDfeYHh1TxXHE -xSKAKNmFSc2XOr5Zg2mOInxeH7WoxH6iylXKADBgEgAQ4C55DoAPRjFNtGGz2iJl -WkzhQOfgsDss1VxpPZ4SWDQO+xxuuEnYcCYdJ6Tyi5hAwZ0+RZmXx0g9BCK+5qzc -Ffei066kCItBcayekwCs0i86r8JvDtsMUUkzEKDQTSWHpP8ypIZoz3P+Skdqcs6p -IcZHy2fODIsClctRAsHOjZ9BnWSY2rqf1lysqvRfOkNTViTOUVx2hf+LuL+8rT6g -K4GvS39Q+c9FYSMjCNxv1m37NfHJv2W7AlqVV95gMBEFB97M3388T7PfvmSLsyPv -qAKDE7genTOycQvEX9oeEACxPo08ai2Mgs/YXCig0g0btZoScoKPelQKtD0HLpZ+ -j7ruaKFy4/vySDMostB3AKSkCE/SkGu/W9oI37n1guSYC/9yJjtTcwZULXAClfwt -LyHvLAw4+7BdGJqDe9Tf8z4zqVLax+BcBHkW7pbP+pvxC6fhIA4PO8k5HwwAVlRw -68lG6e3z9+Vst4zrFLewHJvkDRSOTSecs9h9SPOjReTK0ovlDzMHVakfUvqpk9mv -nCj8SWaMPF70z138AuRuKN7X4exNXPBFnOcseKp/NpT1DyvgoHm/oIO65mq9vvo+ -42uSbyyegYiTdVK4SaXBc2hixkhaTteqdEy6GxfWqKCvHtYuRD2LyUSjZn/0Y6CG -3wZapylKL1sxfH+b1gogv2wAA8KtZAfiTlMISuymwQr5RbOFma7UVMVecCa7XrNG -fZwO8R+6++ru0efDzeC/gtLOItjn+yVfsbk2gfu0FikHIV1sogOjnsv3dl6IpH/s -w6WbuK/yZhyrrmhIIueHF30ES512Peogf/Gxp8U/ZAmqBa7p4qsv93XWDR52IPYz -yU1MtboLD02J39daBNrYZuO4Yfo/xAvU7Ijg2KIR9HS0bSWNuSFAZaZXlJ2bSag3 -8l0dZMx2fZa0FALxWvY2+nd1tWcH+Wo99RSxX+4TH3guCpe1x/58yKOKL4d1xRpu -02nGaqerV9bHU6WjCUYTrrZurTp6yqivpSFCALDMhAU5Ihd/Qjz69zg4uZ7iqbNd -WqIYJ3SAOei2tSfoOJbdh8mohotnMFFbupck3Fv+lXAXYDp0V/sSuK32MSeJxan1 -q3t0eenAh6zoBP0Tj53N8B9YmrgZ1i1AljQqYsAdb8N5Z3vecuzQa946U8H4axEc -BL5fgmqSfsbj1EjJVUrw3cE9hgqeqAYBBvZsyW5zSAsLq/RJKYtBxDGADqr0TOOu -TjVqfPObb1u9EAQj15ALBhhPORa2gBAK5gnhJWW5deNgA0UOYoWARFYfATuQvr1T -9xTVbqzZ0VJR4n7X5JyxJo2tZ44QYVzA0fqX0lt+uRkarrkbZMgplKp4MJTFCp2N -tLciWS0YSHPbKTm7svw5mROazUvEoj1N3SYrJT+Ai9UgSL1cA4FmQ5Jsi+GOeQbH -7E4pzjeST4b1aATY9ZZSy4TzK+UNYBcCVqY1dN5Xnk0e5yPMnrkkcYI41s/6oOaW -lQkp/yrHkvpzQWZa7vF2yWg54j3eX8YWx/TsLvmuJmSQqiIHfu9TXcAIPDispCpF -gp4ZbYPZ8RDQLw3ROc+bZ9E0J8e+FHwgXEELsMLW9n4d3PTj9pLFej1vmBKa03o3 -g+JViNxb1gUPNmC6ADk4o7TQzBrjV2pmmuCza16kOmq6oAbJPOeCLtVWA5HJTX8d -AuOzBLO37tWP457dNqoNDaCMik3LCy8cslZjAz02D6VG/QGOAXP9x8bJ3zg9N7vW -EOn1BMtIeMI4yIxpC6HgotNWDNbUBWremAFWYQRgYQqo9L1SdW4hz6X8FqdlBFvK -ml7sdUwjEy2tNW+aiZI4cgAlI/VvMWYr5kPfhvaUu1wGYeS1TD1B+DPcjcXpOGWo -vB1f/uvGc61zZWD0ph9f3S9OnvEko5qV3VhxOnxYUZTrqlM7LVgy/sHrSaPdR7Eo -nb4QGdi5wEGMxVgQlDGwOmEmS5nCaA0RcfHbkQV4SebRleocLB9qcSwj5JzPICAB -sn1qeSv45LnGdTzUnCQErw1EzgrK1FXOknujjZ3GX5dM7a6Gag0E6cAxoQeC8Mh0 -aUWolyBo9aSXltn7Vnywoe4X3v96eVH6tUQtN3wt51LBlfSDTWOQDeXLFyW57oYL -/ErbmJLaSHpANBcC9InI+rxNYePTTba7K8rdPDppmMEmdOXjaL+5aPay2Bay9K8T -FUyZdoyrbzxFZYT9/YyVJV/ukVcT9ru3HpzLTg9Vgc/CyRfIvCZY/Gzh+pPUbRsR -pHYDT09BFS+mjszTkcdbceweYXJvknXpajMiISrPUUkb9rwCTmLS4QR4y7I9zAgJ -9U72mCZB9yhhzS17kyKAANihXvOFrMXCO7QTWZbtSkobMBS2SybDxYP3rM5b6824 -HQaY4MZW5qCndydfz4fa908XbPTDka9Nu3CZfI3fhGiiGDFWViotGEYE4i9BuGZJ -vmtzII7zeIoHwM4t5fb5mYDd9H+Rkct1yATGEIFEx7cu1fj4Ht2177EzhdkF3Q6I -Fccn43yGDW06nFvEjnjb+Lz4EvtGp6dDefUhNBohdzd4Rm2o+etKobm0lZ1xOX74 -2gOGBLfXS6PF+msNkWP76mqKE6hG0+R1iPrN06Arti+ZRBhNGJDp1pnOERl7IyB0 -hiLUqae7xVSMVRDeq4O04EUpMXh5ixs3RJXjthYILt7vX+6qskJC3rMuyThrsvMi -Kk0wErDpzR5U7KFnM2fk4jY5by/0b/jHZ/PWFGnaFrXNOiwK/mV1RIJtg5FCrvMv -xJN5Qo7o2ZYzUL+IBf5mP5O49FCnz3e++9r4ySY/K57ybhQft1l+cQPWS7xuvVAg -wZGM9DPl9AhBus0I3V3XzfkcWWHkMynMRcfTnkBHrmKjMopIOggsq05JPalM9b0V -NfwZngVvTFbQejbGjgQQQ+G7spav/icCIA8Iu8JPHaKZt2IX77L1xMcQOouqMrx8 -JCXnJ7zsuU+8W3GQGm70H2kcen4bf4beA8TSkFmbVsaqDNtGaG+ZopWO8ctZX3CP -jFgIYgEGi137IWvs1KD5QNFHZI6l0JMzglmvcfnViRPDHLi+EQXAoL/98j/YzA9t -a9L4xMO0oucovKkqYOS2cdFaQKVIUM9L5ibcbW7Tol4ugMo5ibIsTgjZoyFohG6t -acg+h2wTTyxtrIapvOOBeAo9V/iQowTAPQf38LKQZ5tApksiCG1qvJerKALTMp1J -yUHjTCPCPsaiXVVO0mfa9OPtzVvi/icLmlz7umk82mew13djedgMS3/ghq5aEHbg -e7rtI5snFlm/GQnV6IaCDytb+jYxqQ0DZ9Mb0CHcMpnLyiBO6nzHfadpZYlrxrpx -dPx9MVqfLavj1UVjKuVj4Mgec9J38IBBRGMJZ4MGhde2hrw/s2jVzG1BzaIzcvFa -bWiJOkKKRC4BY9DBDYfae5c/obwzkxLmvkbWAVzoFXc+46K1IK6bbC2a4EerOAxc -C7B3X8BucD8OtiPORwXgjctd9AJ7cXq0EEVGoZZ8jkILnklsK6RXsxrT4gSOGFpx -cBU7TF4kHpH+o4KGQmjlFA2VYP1tXirjEep57LOHfL+zy+HiSXdsbexf6odkqLYA -1RR0TzSkmqJYvP0GfHPMvp2xredx1kP934fhTIUcWvErrG3NPd/Dod4BzQye1nNK -8Y8aFYF6fACcD4SFZ11ecK9DJmB+WXGlsxtO3w1LeEysny2lVkkH6HuMRIzPyXAM -/sEpwh3hwPJosAh6vGM6JxuL7CXP8hC1oUvK91VR0hxeDcO/LgdcHtDnlNMqt1wC -jvYt9Hg+3aY5GehBnQPRppnIMaGhiBrRC6HjBg8Umt6UTUr5PXNY2aia5wRcx4M6 -Tj0O0dAt8WWZ9ySgzOoq4SRoaAp6m4E7CpRNMgXH031OWEeHqtt8rsYqAcvcQLuW -vIwts5hiXZOfgXyR6SBY3ANE1SjDY46na2G4oiG8FI7eAqYgehjHrXqCnGrzjefE -V1eVcIr7DBYuM8wq7C9x+qdMir6RNV4MrVD+7rO/drivnd3MaqDHlG0LfMZo6GIK -QZGIy0z1W5lOXm9x3RAz7/Ea11ynkZyPDDxBE5NRcV6JfukdjDrvsaQA/Qk2Wo7h -PdUZjv17H4syMdap6tUDTc8Cohut6n1UxnDnhaAYEDGR3TgPqI5vEaJ+yoaM9itv -b2I/K+0uK6GfWwca7PeTtYn92/5etO4czG99xEBFzw7tiEmBTA8KfseZIUe/i3fr -9LO+rKEGalFB4mVTYyCKPYYx4hOIvjdCox/jEBTFyYUEhgGnZvArgw9XabvII7eM -c1FmwnN2CM/MHUzM0Kw7cS/oaoRFcG/PqioO/krL2Bd+xThwSH4Zn+Y1UVO7aXNy -90MOehHSc2nhN984Y2re3ffVsjPhC3KdWxr8jbho/Az65rUkYg64roiO2ZqdqRvf -aYtX9fCe8N8SzgLTlWZt+l8sjY8HP2k9NoajphV1nG4iC/pX0ZJiVQ+VBRNaGhSl -6IhM9gM1ZsI86cULJpOTAhNu1fZBv+Mq8IXYD8SqlNVwVtjvshtlzBN4InSGEKiO -HM7coo7aDuu+Go5AEiIVMbR09jAa0VINsCVScfSbTFhlGRTrZr2DSO/s8WxbV+Mb -EQAfZmyrLF71Z3MfIukg+ps5iuqFhZ+gJSXXSFwumpg= -EOF - -exec sh bin/do-setup-circleci-secrets diff --git a/bin/wait_for.sh b/bin/wait_for.sh new file mode 100755 index 0000000000..d9148cb194 --- /dev/null +++ b/bin/wait_for.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# shellcheck disable=SC2155 +wait_for() { + local timeout=$1 + local lock_file=$2 + for i in $(seq 1 "$timeout"); do + if [ -f "$lock_file" ]; then + local status="$(cat "$lock_file")" + if [ "$status" == "OK" ]; then + echo "[$i seconds]: $lock_file found and status: $status." && return + else + echo "[$i seconds]: $lock_file found and status: $status." && return 1 + fi + fi + if ! ((i % 10)); then echo "[$i seconds]: Waiting for $lock_file to be created..."; fi + sleep 1 + done + echo "Timed out waiting for test VMs to be ready. See details in: $TEST_VMS_SETUP_OUTPUT_FILE" >&2 + return 1 +} diff --git a/build/Dockerfile b/build/Dockerfile index aa8a16f9de..2856739e47 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -45,6 +45,8 @@ RUN apt-get update \ RUN go get \ github.com/golang/lint/golint \ github.com/fzipp/gocyclo \ + gopkg.in/mvdan/sh.v1/cmd/shfmt \ + github.com/fatih/hclfmt \ github.com/client9/misspell/cmd/misspell # First clean, because we're gonna rebuild the std both with -race enabled and disabled diff --git a/circle.yml b/circle.yml index 7463926447..3016ba48cf 100644 --- a/circle.yml +++ b/circle.yml @@ -14,17 +14,32 @@ machine: PATH: $PATH:$HOME/.local/bin CLOUDSDK_CORE_DISABLE_PROMPTS: 1 STATE: /home/ubuntu/state.env + NUM_HOSTS: 16 + # Base name of VMs for integration tests: + NAME: test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX + TEST_VMS_SETUP_OUTPUT_FILE: $CIRCLE_ARTIFACTS/test_vms_setup_output.txt + TEST_VMS_PROV_AND_CONF_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt + TEST_VMS_READY_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_ready.txt + # Google Cloud Platorm images for integration tests: + USE_IMAGE: 1 + DISK_NAME_PREFIX: test-$CIRCLE_BUILD_NUM-0 dependencies: cache_directories: - "~/docker" post: + - bin/circle-dependencies-post-sources + # Provision & configure testing VMs: + - "while [ ! -d $SRCDIR ]; do sleep 1; done; $SRCDIR/bin/provision_test_vms.sh": + parallel: true + background: true - bin/circle-dependencies-post test: pre: - "$SRCDIR/bin/circle-test-pre": parallel: true + background: true override: - "$SRCDIR/bin/circle-test-unit": parallel: true @@ -37,6 +52,10 @@ test: teardown: pre: + # Destroy testing VMs: + - "cd $SRCDIR/test && ./run-integration-tests.sh destroy": + parallel: true + background: true - "$SRCDIR/bin/circle-teardown-pre": parallel: true diff --git a/test/111_fastdp_encryption_2_test.sh b/test/111_fastdp_encryption_2_test.sh index 7059d0654c..19baf308b5 100755 --- a/test/111_fastdp_encryption_2_test.sh +++ b/test/111_fastdp_encryption_2_test.sh @@ -3,7 +3,7 @@ # TODO(mp) 1. test anti-replay 2. test sleeve <-> fastdp # TODO(mp) test with --trusted-subnets -. ./config.sh +. "$(dirname "$0")/config.sh" HOST1_IP=$($SSH $HOST1 getent ahosts $HOST1 | grep "RAW" | cut -d' ' -f1) HOST2_IP=$($SSH $HOST2 getent ahosts $HOST2 | grep "RAW" | cut -d' ' -f1) diff --git a/test/840_weave_kube_3_test.sh b/test/840_weave_kube_3_test.sh index 5703b8ced3..7823f92956 100755 --- a/test/840_weave_kube_3_test.sh +++ b/test/840_weave_kube_3_test.sh @@ -31,7 +31,7 @@ sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never$COVERAGE_ARGS%" "$(dirn sleep 5 wait_for_connections() { - for i in $(seq 1 30); do + for i in $(seq 1 45); do if run_on $HOST1 "curl -sS http://127.0.0.1:6784/status | grep \"$SUCCESS\"" ; then return fi @@ -56,7 +56,7 @@ assert_raises "run_on $HOST3 $PING $HOST2EXPIP" run_on $HOST1 "kubectl run hello --image=weaveworks/hello-world --replicas=3" wait_for_pods() { - for i in $(seq 1 30); do + for i in $(seq 1 45); do if run_on $HOST1 "kubectl get pods | grep 'hello.*Running'" ; then return fi diff --git a/test/README.md b/test/README.md index 842579c39f..664547976c 100644 --- a/test/README.md +++ b/test/README.md @@ -2,40 +2,67 @@ This directory contains integration tests for weave. ## Requirements -You need two VMs with docker >=1.6.0 installed and listening on TCP -port 2375 (see below). You also need to be able to ssh to these VMs, -preferably without having to input anything. +You need three VMs with `docker` (`>=1.6.0`) installed and listening on TCP +port `2375` (see below). You also need to be able to SSH to these VMs without +having to input anything. -The `Vagrantfile` in this directory constructs two such VMs. +The `Vagrantfile` in this directory constructs three such VMs. -To create the VMs, open a shell and in this directory and type +If you are [building Weave Net using Vagrant](https://www.weave.works/docs/net/latest/building/), +it is recommended to run the tests from the build VM and not the host. - vagrant up -To meet the aforementioned ssh requirement you may want to +## Running tests - cp ~/.vagrant.d/insecure_private_key . +**TL;DR**: You can run the steps 2. to 7. below with one command: -## Running tests + make PROVIDER=vagrant integration-tests -If you are [building weave using Vagrant](https://www.weave.works/docs/net/latest/building/), -it is recommended to run the tests from the build VM and not the host. +**Detailed steps**: + + 1. Start the build virtual machine (see above article for more details): + + vagrant up + + 2. Start the three testing VMs: + + cd test + vagrant up + + 3. SSH into the build VM and go to Weave Net's sources: + + cd .. + vagrant ssh + # you are now on the build VM: + cd ~/weave + + 4. Compile all code and dependencies: + + make + make testrunner + cd test - ./setup.sh + 5. Upload the weave images from where the `Makefile` puts them (`weave.tar.gz`) to + the three docker hosts, `docker load` these, and copies the `weave` script over: -uploads the weave images from where the Makefile puts them -(`/var/tmp`) to the two docker hosts, and copies the weave script -over. + ./setup.sh -Then you can use, e.g., + 6. Run individual tests, e.g.: - ./200_dns_test.sh + ./200_dns_test.sh -to run an individual test, or + or run all tests (everything named `*_test.sh`): - ./run_all.sh + ./run_all.sh + + 7. Stop all VMs: + + exit + # you are now on your host machine + vagrant destroy -f + cd test + vagrant destroy -f -to run everything named `*_test.sh`. ## Using other VMs @@ -60,20 +87,7 @@ to the file `/etc/default/docker`, then restart docker. ## Updating the GCE test image -When a new version of Docker is released, you'll need to update the GCE test image. - -To do this, pick a fresh ```TEMPLATE_NAME``` and update any commands in -```function make_template``` in gce.sh, then run: - -``` -./gce.sh make_template -``` - -For this you'll need the GCE credentials, which can be found in ```bin/setup-circleci-secrets```, -which you'll need to decrypt and run (its echos the secrets into know locations): - -``` -./bin/setup-circleci-secrets "$SECRET_PASSWORD" -``` - -If you don't know the password, ask tom@weave.works. +When a new version of Docker is released, you willneed to update the GCE test image. +To do this, change the Docker version in `run-integration-tests.sh` and push the change. +Next build in CircleCI will detect that there is no template for this version of Docker and will first create the template before running tests. +Subsequent builds will then simply re-use the template. diff --git a/test/Vagrantfile b/test/Vagrantfile index eca6475b7f..cae3513434 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -1,59 +1,64 @@ -VAGRANTFILE_API_VERSION = "2" - # Necessary because the switch disabling unwanted insecure key # replacement in Vagrant 1.7+ is not accepted by earlier versions Vagrant.require_version ">= 1.7.0" # these ought to match what is in config.sh -n_machines = 3 -ip_prefix = "192.168.48" -ip_suffix_base = 10 - -require '../vagrant-common.rb' - -def configure_docker(host, hostname, ip) - pkgs = %w(docker-engine ethtool) - - host.vm.box = "ubuntu/wily64" - - host.vm.provision :shell, :inline => "hostnamectl set-hostname "+hostname - host.vm.network "private_network", ip: ip - - host.vm.synced_folder ".", "/vagrant", disabled: true +N_MACHINES = 3 +IP_PREFIX = '192.168.48' +IP_SUFFIX_BASE = 10 - host.vm.provision :shell, :inline => "sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D" - host.vm.provision :shell, :inline => "echo deb https://apt.dockerproject.org/repo ubuntu-wily main > /etc/apt/sources.list.d/docker.list" +require File.expand_path(File.join(File.dirname(__FILE__), '..', 'vagrant-common.rb')) - install_packages host.vm, pkgs - tweak_docker_daemon host.vm - cleanup host.vm +def ip_address(i) + "#{IP_PREFIX}.#{IP_SUFFIX_BASE + i}" end -def configure_resolv_conf(host) - # Fix the resolution errors by using 8.8.8.8 - host.vm.provision :shell, :inline => "echo 'nameserver 8.8.8.8' > /etc/resolv.conf" +def hostname(i) + "host#{i}" end +ETC_HOSTS_ENTRIES = (1..N_MACHINES).map { |i| "#{ip_address(i)} #{hostname(i)}\n"}.join('') Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - config.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"] - vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"] - vb.memory = 384 - end - - # Disable Vagrant 1.7 per host insecure key replacement - config.ssh.insert_key = false - - (1..n_machines).each do |i| - config.vm.define "host#{i}" do |host| - ip_suffix = ip_suffix_base + i - configure_resolv_conf(host) - configure_docker(host, "host#{i}", "#{ip_prefix}.#{ip_suffix}") + (1..N_MACHINES).each do |i| + hostname = hostname(i) + config.vm.define hostname do |host| + host.vm.provider :virtualbox do |vb| + vb.name = hostname + vb.memory = 384 + configure_nat_dns(vb) + end + + # Disable Vagrant 1.7 per host insecure key replacement + host.ssh.insert_key = false + + host.vm.box = VAGRANT_IMAGE + host.vm.network "private_network", ip: ip_address(i) + host.vm.hostname = hostname + + host.vm.provision :shell, :inline => <> /etc/hosts + +# Fix the resolution errors by using 8.8.8.8 +echo 'nameserver 8.8.8.8' > /etc/resolv.conf +END + + # Disable default Vagrant shared folder, which we don't need: + host.vm.synced_folder ".", "/vagrant", disabled: true + + cleanup host.vm + + # Only execute once the Ansible provisioner, when all the machines are up and ready: + if i == N_MACHINES + host.vm.provision 'ansible' do |ansible| + ansible.limit = 'all' # Disable default limit to connect to all the machines + ansible.playbook = '../tools/config_management/setup_weave-net_test.yml' + end + end end end - end begin diff --git a/test/config.sh b/test/config.sh index 4ec0d1697f..35f26f7a63 100644 --- a/test/config.sh +++ b/test/config.sh @@ -30,9 +30,9 @@ HOST3=$(echo $HOSTS | cut -f 3 -d ' ') . "$DIR/assert.sh" - SSH_DIR=${SSH_DIR:-$DIR} -SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" -o "UserKnownHostsFile=$SSH_DIR/.ssh_known_hosts" -o CheckHostIP=no -o StrictHostKeyChecking=no} +SSH_OPTS=${SSH_OPTS:-"-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o IdentitiesOnly=yes -o LogLevel=ERROR"} +SSH=${SSH:-ssh -l vagrant -i "$SSH_DIR/insecure_private_key" $SSH_OPTS} SMALL_IMAGE="alpine" DNS_IMAGE="aanand/docker-dnsutils" diff --git a/test/gce.sh b/test/gce.sh deleted file mode 100755 index 6f2046de2d..0000000000 --- a/test/gce.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/bin/bash -# This script has a bunch of GCE-related functions: -# ./gce.sh setup - starts two VMs on GCE and configures them to run our integration tests -# . ./gce.sh; ./run_all.sh - set a bunch of environment variables for the tests -# ./gce.sh destroy - tear down the VMs -# ./gce.sh make_template - make a fresh VM template; update TEMPLATE_NAME first! - -set -e - -: ${KEY_FILE:=/tmp/gce_private_key.json} -: ${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key} -: ${PROJECT:=positive-cocoa-90213} -: ${IMAGE_FAMILY:=ubuntu-1604-lts} -: ${IMAGE_PROJECT:=ubuntu-os-cloud} -: ${TEMPLATE_NAME:=test-template-12} -: ${ZONE:=us-central1-a} -: ${NUM_HOSTS:=5} -SUFFIX="" -if [ -n "$CIRCLECI" ]; then - SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX" -fi - -# Setup authentication -gcloud auth activate-service-account --key-file $KEY_FILE 1>/dev/null -gcloud config set project $PROJECT - -function vm_names { - local names= - for i in $(seq 1 $NUM_HOSTS); do - names="host$i$SUFFIX $names" - done - echo "$names" -} - -# Delete all vms in this account -function destroy { - if [ $(gcloud compute firewall-rules list test-allow-docker$SUFFIX 2>/dev/null | wc -l) -gt 0 ] ; then - gcloud compute firewall-rules delete test-allow-docker$SUFFIX - fi - names="$(vm_names)" - if [ $(gcloud compute instances list --zones $ZONE -q $names | wc -l) -le 1 ] ; then - return 0 - fi - for i in {0..10}; do - # gcloud instances delete can sometimes hang. - case $(set +e; timeout 60s /bin/bash -c "gcloud compute instances delete --zone $ZONE -q $names >/dev/null 2>&1"; echo $?) in - 0) - return 0 - ;; - 124) - # 124 means it timed out - break - ;; - *) - return 1 - esac - done -} - -function internal_ip { - jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" $1 -} - -function external_ip { - jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" $1 -} - -function try_connect { - for i in {0..10}; do - ssh -t $1 true && return - sleep 2 - done -} - -function install_docker_on { - name=$1 - ssh -t $name sudo bash -x -s </etc/systemd/system/docker.service.d/override.conf < /etc/apt/sources.list.d/kubernetes.list -apt-get update -qq -apt-get install -q -y kubelet kubeadm kubectl kubernetes-cni -systemctl --now disable kubelet -# Pre-pull images required for Kubernetes -docker pull gcr.io/google_containers/etcd-amd64:2.2.5 -docker pull gcr.io/google_containers/kube-apiserver-amd64:v1.4.0 -docker pull gcr.io/google_containers/kube-controller-manager-amd64:v1.4.0 -docker pull gcr.io/google_containers/kube-proxy-amd64:v1.4.0 -docker pull gcr.io/google_containers/kube-scheduler-amd64:v1.4.0 -docker pull gcr.io/google_containers/kube-discovery-amd64:1.0 -docker pull gcr.io/google_containers/pause-amd64:3.0 -EOF -} - -function copy_hosts { - hostname=$1 - hosts=$2 - cat $hosts | ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" -} - -# Create new set of VMs -function setup { - destroy - - names="$(vm_names)" - gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE --tags test$SUFFIX --network=test - my_ip="$(curl -s http://ipinfo.io/ip)" - gcloud compute firewall-rules create test-allow-docker$SUFFIX --network=test --allow tcp:2375,tcp:12375 --target-tags test$SUFFIX --source-ranges $my_ip - gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE - sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config - - # build an /etc/hosts file for these vms - hosts=$(mktemp hosts.XXXXXXXXXX) - json=$(mktemp json.XXXXXXXXXX) - gcloud compute instances list --format=json >$json - for name in $names; do - echo "$(internal_ip $json $name) $name.$ZONE.$PROJECT" >>$hosts - done - - for name in $names; do - hostname="$name.$ZONE.$PROJECT" - - # Add the remote ip to the local /etc/hosts - sudo sed -i "/$hostname/d" /etc/hosts - sudo sh -c "echo \"$(external_ip $json $name) $hostname\" >>/etc/hosts" - try_connect $hostname - - copy_hosts $hostname $hosts & - done - - wait - - rm $hosts $json -} - -function make_template { - gcloud compute instances create $TEMPLATE_NAME --image-family=$IMAGE_FAMILY --image-project=$IMAGE_PROJECT --zone $ZONE - gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE - name="$TEMPLATE_NAME.$ZONE.$PROJECT" - try_connect $name - install_docker_on $name - install_kubernetes_on $name - gcloud -q compute instances delete $TEMPLATE_NAME --keep-disks boot --zone $ZONE - gcloud compute images create $TEMPLATE_NAME --source-disk $TEMPLATE_NAME --source-disk-zone $ZONE -} - -function hosts { - hosts= - json=$(mktemp json.XXXXXXXXXX) - gcloud compute instances list --format=json >$json - for name in $(vm_names); do - hostname="$name.$ZONE.$PROJECT" - hosts="$hostname $hosts" - done - echo export SSH=\"ssh -l vagrant\" - echo export HOSTS=\"$hosts\" - rm $json -} - -case "$1" in -setup) - setup - ;; - -hosts) - hosts - ;; - -destroy) - destroy - ;; - -make_template) - # see if template exists - if ! gcloud compute images list | grep $PROJECT | grep $TEMPLATE_NAME; then - make_template - fi - ;; - -*) - echo "Unknown command:" $1 >&2 - exit 1 -esac diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh new file mode 100755 index 0000000000..178dcd9a66 --- /dev/null +++ b/test/run-integration-tests.sh @@ -0,0 +1,383 @@ +#!/bin/bash +# +# Description: +# This script runs all Weave Net's integration tests on the specified +# provider (default: Google Cloud Platform). +# +# Usage: +# +# Run all integration tests on Google Cloud Platform: +# $ ./run-integration-tests.sh +# +# Run all integration tests on Amazon Web Services: +# PROVIDER=aws ./run-integration-tests.sh +# + +set -e +DIR="$(dirname "$0")" +. "$DIR/../tools/provisioning/setup.sh" # Import gcp_on, do_on, and aws_on. +. "$DIR/config.sh" # Import greenly. + +# Variables: +APP="weave-net" +# shellcheck disable=SC2034 +PROJECT="weave-net-tests" # Only used when PROVIDER is gcp, by tools/provisioning/config.sh. +NAME=${NAME:-"$(whoami | sed -e 's/[\.\_]*//g' | cut -c 1-4)"} +PROVIDER=${PROVIDER:-gcp} # Provision using provided provider, or Google Cloud Platform by default. +NUM_HOSTS=${NUM_HOSTS:-3} +PLAYBOOK=${PLAYBOOK:-setup_weave-net_test.yml} +TESTS=${TESTS:-} +RUNNER_ARGS=${RUNNER_ARGS:-""} +# Dependencies' versions: +DOCKER_VERSION=${DOCKER_VERSION:-1.11.2} +KUBERNETES_VERSION=${KUBERNETES_VERSION:-1.5.2} +KUBERNETES_CNI_VERSION=${KUBERNETES_CNI_VERSION:-0.3.0.1} +# Google Cloud Platform image's name & usage (only used when PROVIDER is gcp): +IMAGE_NAME=${IMAGE_NAME:-"$(echo "$APP-docker$DOCKER_VERSION-k8s$KUBERNETES_VERSION-k8scni$KUBERNETES_CNI_VERSION" | sed -e 's/[\.\_]*//g')"} +DISK_NAME_PREFIX=${DISK_NAME_PREFIX:-$NAME} +USE_IMAGE=${USE_IMAGE:-1} +CREATE_IMAGE=${CREATE_IMAGE:-1} +CREATE_IMAGE_TIMEOUT_IN_SECS=${CREATE_IMAGE_TIMEOUT_IN_SECS:-600} +# Lifecycle flags: +SKIP_CONFIG=${SKIP_CONFIG:-} +SKIP_DESTROY=${SKIP_DESTROY:-} + +function print_vars() { + echo "--- Variables: Main ---" + echo "PROVIDER=$PROVIDER" + echo "NUM_HOSTS=$NUM_HOSTS" + echo "PLAYBOOK=$PLAYBOOK" + echo "TESTS=$TESTS" + echo "SSH_OPTS=$SSH_OPTS" + echo "RUNNER_ARGS=$RUNNER_ARGS" + echo "--- Variables: Versions ---" + echo "DOCKER_VERSION=$DOCKER_VERSION" + echo "KUBERNETES_VERSION=$KUBERNETES_VERSION" + echo "KUBERNETES_CNI_VERSION=$KUBERNETES_CNI_VERSION" + echo "IMAGE_NAME=$IMAGE_NAME" + echo "DISK_NAME_PREFIX=$DISK_NAME_PREFIX" + echo "USE_IMAGE=$USE_IMAGE" + echo "CREATE_IMAGE=$CREATE_IMAGE" + echo "CREATE_IMAGE_TIMEOUT_IN_SECS=$CREATE_IMAGE_TIMEOUT_IN_SECS" + echo "--- Variables: Flags ---" + echo "SKIP_CONFIG=$SKIP_CONFIG" + echo "SKIP_DESTROY=$SKIP_DESTROY" +} + +function verify_dependencies() { + local deps=(python terraform ansible-playbook) + for dep in "${deps[@]}"; do + if [ ! "$(which "$dep")" ]; then + echo >&2 "$dep is not installed or not in PATH." + exit 1 + fi + done +} + +# shellcheck disable=SC2155 +function provision_locally() { + export VAGRANT_CWD="$(dirname "${BASH_SOURCE[0]}")" + case "$1" in + on) + vagrant up + local status=$? + + # Set up SSH connection details: + local ssh_config=$(mktemp /tmp/vagrant_ssh_config_XXX) + vagrant ssh-config >"$ssh_config" + export SSH="ssh -F $ssh_config" + # Extract username, SSH private key, and VMs' IP addresses: + ssh_user="$(sed -ne 's/\ *User //p' "$ssh_config" | uniq)" + ssh_id_file="$(sed -ne 's/\ *IdentityFile //p' "$ssh_config" | uniq)" + ssh_hosts=$(sed -ne 's/Host //p' "$ssh_config") + + # Set up /etc/hosts files on this ("local") machine and the ("remote") testing machines, to map hostnames and IP addresses, so that: + # - this machine communicates with the testing machines via their public IPs; + # - testing machines communicate between themselves via their private IPs; + # - we can simply use just the hostname in all scripts to refer to machines, and the difference between public and private IP becomes transparent. + # N.B.: if you decide to use public IPs everywhere, note that some tests may fail (e.g. test #115). + update_local_etc_hosts "$ssh_hosts" "$(for host in $ssh_hosts; do $SSH "$host" "cat /etc/hosts | grep $host"; done)" + + SKIP_CONFIG=1 # Vagrant directly configures virtual machines using Ansible -- see also: Vagrantfile + return $status + ;; + off) + vagrant destroy -f + ;; + *) + echo >&2 "Unknown command $1. Usage: {on|off}." + exit 1 + ;; + esac +} + +function setup_gcloud() { + # Authenticate: + gcloud auth activate-service-account --key-file "$GOOGLE_CREDENTIALS_FILE" 1>/dev/null + # Set current project: + gcloud config set project $PROJECT +} + +function image_exists() { + gcloud compute images list | grep "$PROJECT" | grep "$IMAGE_NAME" +} + +function image_ready() { + # GCP images seem to be listed before they are actually ready for use, + # typically failing the build with: "googleapi: Error 400: The resource is not ready". + # We therefore consider the image to be ready once the disk of its template instance has been deleted. + ! gcloud compute disks list | grep "$DISK_NAME_PREFIX" +} + +function wait_for_image() { + greenly echo "> Waiting for GCP image $IMAGE_NAME to be created..." + for i in $(seq "$CREATE_IMAGE_TIMEOUT_IN_SECS"); do + image_exists && image_ready && return 0 + if ! ((i % 60)); then echo "Waited for $i seconds and still waiting..."; fi + sleep 1 + done + redly echo "> Waited $CREATE_IMAGE_TIMEOUT_IN_SECS seconds for GCP image $IMAGE_NAME to be created, but image could not be found." + exit 1 +} + +# shellcheck disable=SC2155 +function create_image() { + if [[ "$CREATE_IMAGE" == 1 ]]; then + greenly echo "> Creating GCP image $IMAGE_NAME..." + local begin_img=$(date +%s) + local num_hosts=1 + terraform apply -input=false -var "app=$APP" -var "name=$NAME" -var "num_hosts=$num_hosts" "$DIR/../tools/provisioning/gcp" + configure_with_ansible "$(terraform output username)" "$(terraform output public_ips)," "$(terraform output private_key_path)" $num_hosts + local zone=$(terraform output zone) + local name=$(terraform output instances_names) + gcloud -q compute instances delete "$name" --keep-disks boot --zone "$zone" + gcloud compute images create "$IMAGE_NAME" --source-disk "$name" --source-disk-zone "$zone" \ + --description "Testing image for Weave Net based on $(terraform output image), Docker $DOCKER_VERSION, Kubernetes $KUBERNETES_VERSION and Kubernetes CNI $KUBERNETES_CNI_VERSION." + gcloud compute disks delete "$name" --zone "$zone" + terraform destroy -force "$DIR/../tools/provisioning/gcp" + rm terraform.tfstate* + echo + greenly echo "> Created GCP image $IMAGE_NAME in $(date -u -d @$(($(date +%s) - begin_img)) +"%T")." + else + wait_for_image + fi +} + +function use_or_create_image() { + setup_gcloud + image_exists || create_image + export TF_VAR_gcp_image="$IMAGE_NAME" # Override the default image name. + export SKIP_CONFIG=1 # No need to configure the image, since already done when making the template +} + +function update_local_etc_hosts() { + echo "> Updating local /etc/hosts..." + # Remove old entries (if present): + for host in $1; do sudo sed -i "/$host/d" /etc/hosts; done + # Add new entries: + sudo sh -c "echo \"$2\" >> /etc/hosts" +} + +function upload_etc_hosts() { + # Remove old entries (if present): + # shellcheck disable=SC2016,SC2086 + $SSH $3 'for host in '$1'; do sudo sed -i "/$host/d" /etc/hosts; done' + # Add new entries: + echo "$2" | $SSH "$3" "sudo -- sh -c \"cat >> /etc/hosts\"" +} + +function update_remote_etc_hosts() { + echo "> Updating remote /etc/hosts..." + local pids="" + for host in $1; do + upload_etc_hosts "$1" "$2" "$host" & + local pids="$pids $!" + done + for pid in $pids; do wait "$pid"; done +} + +# shellcheck disable=SC2155 +function set_hosts() { + export HOSTS="$(echo "$ssh_hosts" | tr '\n' ' ')" +} + +function provision_remotely() { + case "$1" in + on) + terraform apply -input=false -parallelism="$NUM_HOSTS" -var "app=$APP" -var "name=$NAME" -var "num_hosts=$NUM_HOSTS" "$DIR/../tools/provisioning/$2" + local status=$? + ssh_user=$(terraform output username) + ssh_id_file=$(terraform output private_key_path) + ssh_hosts=$(terraform output hostnames) + export SSH="ssh -l $ssh_user -i $ssh_id_file $SSH_OPTS" + + # Set up /etc/hosts files on this ("local") machine and the ("remote") testing machines, to map hostnames and IP addresses, so that: + # - this machine communicates with the testing machines via their public IPs; + # - testing machines communicate between themselves via their private IPs; + # - we can simply use just the hostname in all scripts to refer to machines, and the difference between public and private IP becomes transparent. + # N.B.: if you decide to use public IPs everywhere, note that some tests may fail (e.g. test #115). + update_local_etc_hosts "$ssh_hosts" "$(terraform output public_etc_hosts)" + update_remote_etc_hosts "$ssh_hosts" "$(terraform output private_etc_hosts)" + + return $status + ;; + off) + terraform destroy -force "$DIR/../tools/provisioning/$2" + ;; + *) + echo >&2 "Unknown command $1. Usage: {on|off}." + exit 1 + ;; + esac +} + +# shellcheck disable=SC2155 +function provision() { + local action=$([ "$1" == "on" ] && echo "Provisioning" || echo "Shutting down") + echo + greenly echo "> $action test host(s) on [$PROVIDER]..." + local begin_prov=$(date +%s) + case "$2" in + 'aws') + aws_on + provision_remotely "$1" "$2" + ;; + 'do') + do_on + provision_remotely "$1" "$2" + ;; + 'gcp') + gcp_on + [[ "$1" == "on" ]] && [[ "$USE_IMAGE" == 1 ]] && use_or_create_image + provision_remotely "$1" "$2" + ;; + 'vagrant') + provision_locally "$1" + ;; + *) + echo >&2 "Unknown provider $2. Usage: PROVIDER={gcp|aws|do|vagrant}." + exit 1 + ;; + esac + [ "$1" == "on" ] && set_hosts + echo + greenly echo "> Provisioning took $(date -u -d @$(($(date +%s) - begin_prov)) +"%T")." +} + +function configure_with_ansible() { + ansible-playbook -u "$1" -i "$2" --private-key="$3" --forks="${4:-$NUM_HOSTS}" \ + --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ + --extra-vars "docker_version=$DOCKER_VERSION kubernetes_version=$KUBERNETES_VERSION kubernetes_cni_version=$KUBERNETES_CNI_VERSION" \ + "$DIR/../tools/config_management/$PLAYBOOK" +} + +# shellcheck disable=SC2155 +function configure() { + echo + if [ -n "$SKIP_CONFIG" ]; then + greenly echo "> Skipped configuration of test host(s)." + else + greenly echo "> Configuring test host(s)..." + local begin_conf=$(date +%s) + local inventory_file=$(mktemp /tmp/ansible_inventory_XXXXX) + echo "[all]" >"$inventory_file" + # shellcheck disable=SC2001 + echo "$2" | sed "s/$/:$3/" >>"$inventory_file" + + # Configure the provisioned machines using Ansible, allowing up to 3 retries upon failure (e.g. APT connectivity issues, etc.): + for i in $(seq 3); do + configure_with_ansible "$1" "$inventory_file" "$4" && break || echo >&2 "#$i: Ansible failed. Retrying now..." + done + + echo + greenly echo "> Configuration took $(date -u -d @$(($(date +%s) - begin_conf)) +"%T")." + fi +} + +# shellcheck disable=SC2155 +function run_tests() { + echo + greenly echo "> Running tests..." + local begin_tests=$(date +%s) + set +e # Do not fail this script upon test failure, since we need to shut down the test cluster regardless of success or failure. + "$DIR/run_all.sh" "$@" + local status=$? + echo + greenly echo "> Tests took $(date -u -d @$(($(date +%s) - begin_tests)) +"%T")." + return $status +} + +function end() { + echo + echo "> Build took $(date -u -d @$(($(date +%s) - begin)) +"%T")." +} + +function echo_export_hosts() { + exec 1>&111 + # Print a command to set HOSTS in the calling script, so that subsequent calls to + # test scripts can point to the right testing machines while developing: + echo "export HOSTS=\"$HOSTS\"" + exec 1>&2 +} + +function main() { + # Keep a reference to stdout in another file descriptor (FD #111), and then globally redirect all stdout to stderr. + # This is so that HOSTS can be eval'ed in the calling script using: + # $ eval $(./run-integration-tests.sh [provision|configure|setup]) + # and ultimately subsequent calls to test scripts can point to the right testing machines during development. + if [ "$1" == "provision" ] || [ "$1" == "configure" ] || [ "$1" == "setup" ]; then + exec 111>&1 # 111 ought to match the file descriptor used in echo_export_hosts. + exec 1>&2 + fi + + begin=$(date +%s) + trap end EXIT + + print_vars + verify_dependencies + + case "$1" in + "") # Provision, configure, run tests, and destroy test environment: + provision on "$PROVIDER" + configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file" + "$DIR/setup.sh" + run_tests "$TESTS" + status=$? + [ -z "$SKIP_DESTROY" ] && provision off "$PROVIDER" + exit $status + ;; + + provision) + provision on "$PROVIDER" + echo_export_hosts + ;; + + configure) + provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables. + configure "$ssh_user" "$ssh_hosts" "${ssh_port:-22}" "$ssh_id_file" + echo_export_hosts + ;; + + setup) + provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables. + "$DIR/setup.sh" + echo_export_hosts + ;; + + test) + provision on "$PROVIDER" # Vagrant and Terraform do not provision twice if VMs are already provisioned, so we just set environment variables. + run_tests "$TESTS" + ;; + + destroy) + provision off "$PROVIDER" + ;; + + *) + echo "Unknown command: $1" >&2 + exit 1 + ;; + esac +} + +main "$@" diff --git a/test/run_all.sh b/test/run_all.sh index db04b50fa7..7d8c6edd97 100755 --- a/test/run_all.sh +++ b/test/run_all.sh @@ -11,7 +11,7 @@ fi whitely echo ...ok TESTS="${@:-$(find "$DIR" -name '*_test.sh')}" -RUNNER_ARGS="" +RUNNER_ARGS=${RUNNER_ARGS:-""} # If running on circle, use the scheduler to work out what tests to run if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then diff --git a/test/sanity_check.sh b/test/sanity_check.sh index 657bdf7193..d5b8768dee 100755 --- a/test/sanity_check.sh +++ b/test/sanity_check.sh @@ -7,6 +7,7 @@ set -e begin=$(date +%s) sanity_checks_files=${WEAVE_NET_SANITY_CHECKS_FILES:-"/tmp/weave_net_sanity_check_*.log"} +greenly echo "> Sanity-checking test machines: ping, check Docker, and check Weave..." whitely echo Ping each host from the other # We wrap ping and echo in a function as we want the below parallel for loop @@ -68,4 +69,4 @@ for host in $HOSTS; do done for pid in $pids; do wait $pid; done -echo "Sanity checks completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")." +greenly echo "> Sanity checks completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")." diff --git a/test/setup.sh b/test/setup.sh index 4d21c3de98..3bab725aad 100755 --- a/test/setup.sh +++ b/test/setup.sh @@ -9,8 +9,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" (cd ./tls && ./tls $HOSTS) -echo "Copying weave images, scripts, and certificates to hosts, and" -echo " prefetch test images" +greenly echo "> Setting up test machines: copying weave images, scripts and certificates to hosts, and prefetch test images..." exists_on() { docker_on $1 inspect --format=" " $2 >/dev/null 2>&1 @@ -60,4 +59,4 @@ for ppid in $ppids; do wait $ppid; done -echo "Setup completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")." +greenly echo "> Setup completed successfully in $(date -u -d @$(($(date +%s)-$begin)) +"%T")." diff --git a/tools b/tools index 36c1835b49..efcf9d21e0 160000 --- a/tools +++ b/tools @@ -1 +1 @@ -Subproject commit 36c1835b4986c4c4d81a597156e7cd5c1a8e8b23 +Subproject commit efcf9d21e09ccd52e00f973a42c78ad08b33e92c diff --git a/vagrant-common.rb b/vagrant-common.rb index 125abeeace..81afa1a1b0 100644 --- a/vagrant-common.rb +++ b/vagrant-common.rb @@ -1,73 +1,29 @@ -$go_regexp = /FROM golang:(\S*).*?/ -$dockerfile_path = File.expand_path(File.join(File.dirname(__FILE__), 'build', 'Dockerfile')) -$go_version = File.readlines($dockerfile_path).first { |line| line.match($go_regexp) }.match($go_regexp).captures.first -if $go_version.nil? - raise ArgumentError.new("Failed to read Go version from Dockerfile.") -end - -$go_path = "/usr/local/go/bin" +VAGRANT_IMAGE = 'bento/ubuntu-16.04' +VAGRANTFILE_API_VERSION = '2' -def install_packages(vm, pkgs) - vm.provision :shell, :inline => <