From a1774e1b410f09d483016782968182281697f149 Mon Sep 17 00:00:00 2001 From: Alban Crequy Date: Tue, 9 May 2017 15:31:08 +0200 Subject: [PATCH 1/2] shfmt: update test scripts to the new shfmt style --- semaphore.sh | 4 ++-- tests/test.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/semaphore.sh b/semaphore.sh index bfd0baa5..4ba87e77 100755 --- a/semaphore.sh +++ b/semaphore.sh @@ -22,8 +22,8 @@ set -o pipefail readonly kernel_versions=("4.4.45" "4.9.6") readonly rkt_version="1.23.0" -if [[ ! -f "./rkt/rkt" ]] \ - || [[ ! "$(./rkt/rkt version | awk '/rkt Version/{print $3}')" == "${rkt_version}" ]]; then +if [[ ! -f "./rkt/rkt" ]] || + [[ ! "$(./rkt/rkt version | awk '/rkt Version/{print $3}')" == "${rkt_version}" ]]; then curl -LsS "https://github.com/coreos/rkt/releases/download/v${rkt_version}/rkt-v${rkt_version}.tar.gz" \ -o rkt.tgz diff --git a/tests/test.sh b/tests/test.sh index e9e06fa3..fd60034f 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -59,10 +59,10 @@ while [[ $lines_read -lt 4 ]]; do daddr=${BASH_REMATCH[3]} lines_read=$((lines_read + 1)) printf "action: %s program: nc saddr: %s daddr: %s\n" "${action}" "${saddr}" "${daddr}" - if [[ "${action}" == "connect" && "$daddr" == "127.0.0.1:${port}" ]] \ - || [[ "${action}" == "accept" && "$saddr" == "127.0.0.1:${port}" ]] \ - || [[ "${action}" == "close" && "$daddr" == "127.0.0.1:${port}" ]] \ - || [[ "${action}" == "close" && "$saddr" == "127.0.0.1:${port}" ]]; then + if [[ "${action}" == "connect" && "$daddr" == "127.0.0.1:${port}" ]] || + [[ "${action}" == "accept" && "$saddr" == "127.0.0.1:${port}" ]] || + [[ "${action}" == "close" && "$daddr" == "127.0.0.1:${port}" ]] || + [[ "${action}" == "close" && "$saddr" == "127.0.0.1:${port}" ]]; then lines_found=$((lines_found + 1)) else echo "^^^ unexpected values in event" From 8c07abb1547c28353e484f9645303f80ae293307 Mon Sep 17 00:00:00 2001 From: Alban Crequy Date: Tue, 9 May 2017 17:37:17 +0200 Subject: [PATCH 2/2] Squashed 'tools/' changes from 52d695c..571c7d7 571c7d7 Merge pull request #95 from kinvolk/alban/update-shfmt bead6ed Update for latest shfmt b08dc4d Update for latest shfmt (#94) 2ed8aaa Add no-race argument to test script (#92) 80dd78e Merge pull request #91 from weaveworks/upgrade-go-1.8.1 08dcd0d Please ./lint as shfmt changed its rules between 1.0.0 and 1.3.0. a8bc9ab Upgrade default Go version to 1.8.1. 41c5622 Merge pull request #90 from weaveworks/build-golang-service-conf e8ebdd5 broaden imagetag regex to fix haskell build image ba3fbfa Merge pull request #89 from weaveworks/build-golang-service-conf e506f1b Fix up test script for updated shfmt 9216db8 Add stuff for service-conf build to build-goland image 66a9a93 Merge pull request #88 from weaveworks/haskell-image cb3e3a2 shfmt 74a5239 Haskell build image 4ccd42b Trying circle quay login b2c295f Merge branch 'common-build' 0ac746f Trim quay prefix in circle script c405b31 Merge pull request #87 from weaveworks/common-build 9672d7c Push build images to quay as they have sane robot accounts a2bf112 Review feedback fef9b7d Add protobuf tools 10a77ea Update readme 254f266 Don't need the image name in ffb59fc Adding a weaveworks/build-golang image with tags b817368 Update min Weave Net docker version cf87ca3 Merge pull request #86 from weaveworks/lock-kubeadm-version 3ae6919 Add example of custom SSH private key to tf_ssh's usage. cf8bd8a Add example of custom SSH private key to tf_ansi's usage. c7d3370 Lock kubeadm's Kubernetes version. faaaa6f Merge pull request #84 from weaveworks/centos-rhel ef552e7 Select weave-kube YAML URL based on K8S version. b4c1198 Upgrade default kubernetes_version to 1.6.1. b82805e Use a fixed version of kubeadm. f33888b Factorise and make kubeconfig option optional. f7b8b89 Install EPEL repo for CentOS. 615917a Fix error in decrypting AWS access key and secret. 86f97b4 Add CentOS 7 AMI and username for AWS via Terraform. eafd810 Add tf_ansi example with Ansible variables. 2b05787 Skip setup of Docker over TCP for CentOS/RHEL. 84c420b Add docker-ce role for CentOS/RHEL. 00a820c Add setup_weave-net_debug.yml playbook for user issues' debugging. 3eae480 Upgrade default kubernetes_version to 1.5.4. 753921c Allow injection of Docker installation role. e1ff90d Fix kubectl taint command for 1.5. b989e97 Fix typo in kubectl taint for single node K8S cluster. 541f58d Remove 'install_recommends: no' for ethtool. c3f9711 Make Ansible role docker-from-get.docker.com work on RHEL/CentOS. 038c0ae Add frequently used OS images, for convenience. d30649f Add --insecure-registry to docker.conf 1dd9218 shfmt -i 4 -w push-images 6de96ac Add option to not push docker hub images 310f53d Add push-images script from cortex 8641381 Add port 6443 to kubeadm join commands for K8S 1.6+. 50bf0bc Force type of K8S token to string. 08ab1c0 Remove trailing whitespaces. ae9efb8 Enable testing against K8S release candidates. 9e32194 Secure GCP servers for Scope: open port 80. a22536a Secure GCP servers for Scope. 89c3a29 Merge pull request #78 from weaveworks/lint-merge-rebase-issue-in-docs 73ad56d Add linter function to avoid bad merge/rebase artefact git-subtree-dir: tools git-subtree-split: 571c7d71e610c504e2ed4104f90fb09c95164225 --- .gitignore | 1 + README.md | 3 ++ build/Makefile | 46 +++++++++++++++++ build/golang/Dockerfile | 46 +++++++++++++++++ build/golang/build.sh | 22 ++++++++ build/haskell/Dockerfile | 4 ++ build/haskell/build.sh | 12 +++++ build/haskell/copy-libraries | 41 +++++++++++++++ circle.yml | 19 +++++++ config_management/group_vars/all | 9 ++-- .../roles/dev-tools/tasks/main.yml | 2 +- .../docker-configuration/files/docker.conf | 3 ++ .../files/docker_over_tcp.conf | 3 -- .../roles/docker-configuration/tasks/main.yml | 14 +++--- .../docker-from-docker-ce-repo/tasks/main.yml | 29 +++++++++++ .../docker-from-docker-repo/tasks/debian.yml | 8 +-- .../docker-from-docker-repo/tasks/main.yml | 6 --- .../docker-from-docker-repo/tasks/redhat.yml | 4 +- .../tasks/debian.yml | 8 +++ .../docker-from-get.docker.com/tasks/main.yml | 15 ++---- .../tasks/redhat.yml | 11 ++++ .../roles/docker-from-tarball/tasks/main.yml | 2 +- .../roles/docker-install/tasks/main.yml | 30 +++++++++++ .../docker-prerequisites/tasks/debian.yml | 2 +- .../roles/golang-from-tarball/tasks/main.yml | 6 +-- .../roles/kubelet-stop/tasks/main.yml | 2 +- .../kubernetes-docker-images/tasks/main.yml | 14 ++++++ .../roles/kubernetes-install/tasks/debian.yml | 15 ++++-- .../roles/kubernetes-install/tasks/main.yml | 5 +- .../roles/kubernetes-install/tasks/redhat.yml | 9 ++-- .../roles/kubernetes-start/tasks/main.yml | 13 +++-- .../roles/sock-shop/tasks/tasks.yml | 8 +-- .../roles/weave-kube/tasks/main.yml | 14 ++++-- .../roles/weave-net-sources/tasks/main.yml | 2 +- .../roles/weave-net-utilities/tasks/main.yml | 30 +++++------ .../roles/weave-net/tasks/main.yml | 6 +-- config_management/setup_weave-kube.yml | 9 ++-- config_management/setup_weave-net_debug.yml | 18 +++++++ config_management/setup_weave-net_dev.yml | 4 +- config_management/setup_weave-net_test.yml | 7 +-- dependencies/list_os_images.sh | 8 +-- dependencies/list_versions.py | 4 +- integration/assert.sh | 10 ++-- integration/gce.sh | 9 +++- lint | 15 ++++++ provisioning/aws/variables.tf | 5 +- provisioning/gcp/variables.tf | 8 ++- provisioning/setup.sh | 16 ++++-- push-images | 50 +++++++++++++++++++ test | 11 +++- 50 files changed, 522 insertions(+), 116 deletions(-) create mode 100644 build/Makefile create mode 100644 build/golang/Dockerfile create mode 100755 build/golang/build.sh create mode 100644 build/haskell/Dockerfile create mode 100755 build/haskell/build.sh create mode 100755 build/haskell/copy-libraries create mode 100644 config_management/roles/docker-configuration/files/docker.conf delete mode 100644 config_management/roles/docker-configuration/files/docker_over_tcp.conf create mode 100644 config_management/roles/docker-from-docker-ce-repo/tasks/main.yml create mode 100644 config_management/roles/docker-from-get.docker.com/tasks/debian.yml create mode 100644 config_management/roles/docker-from-get.docker.com/tasks/redhat.yml create mode 100644 config_management/roles/docker-install/tasks/main.yml create mode 100644 config_management/roles/kubernetes-docker-images/tasks/main.yml create mode 100644 config_management/setup_weave-net_debug.yml create mode 100755 push-images diff --git a/.gitignore b/.gitignore index c349d5c0..308ae9d3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ runner/runner terraform.tfstate terraform.tfstate.backup *.retry +build/**/.uptodate diff --git a/README.md b/README.md index ae492e2f..9092b8e2 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ Included in this repo are tools shared by weave.git and scope.git. They include +- ```build```: a set of docker base-images for building weave + projects. These should be used instead of giving each project its + own build image. - ```provisioning```: a set of Terraform scripts to provision virtual machines in GCP, AWS or Digital Ocean. - ```config_management```: a set of Ansible playbooks to configure virtual machines for development, testing, etc. - ```cover```: a tool which merges overlapping coverage reports generated by go diff --git a/build/Makefile b/build/Makefile new file mode 100644 index 00000000..cea049be --- /dev/null +++ b/build/Makefile @@ -0,0 +1,46 @@ +.PHONY: all clean images +.DEFAULT_GOAL := all + +# Boiler plate for bulding Docker containers. +# All this must go at top of file I'm afraid. +IMAGE_PREFIX := quay.io/weaveworks/build- +IMAGE_TAG := $(shell ../image-tag) +UPTODATE := .uptodate + +# Every directory with a Dockerfile in it builds an image called +# $(IMAGE_PREFIX). Dependencies (i.e. things that go in the image) +# still need to be explicitly declared. +%/$(UPTODATE): %/Dockerfile %/* + $(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/ + $(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG) + touch $@ + +# Get a list of directories containing Dockerfiles +DOCKERFILES := $(shell find . -name tools -prune -o -name vendor -prune -o -type f -name 'Dockerfile' -print) +UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES)) +DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) +IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir)))) +images: + $(info $(IMAGE_NAMES)) + @echo > /dev/null + +# Define imagetag-golang, etc, for each image, which parses the dockerfile and +# prints an image tag. For example: +# FROM golang:1.8.1-stretch +# in the "foo/Dockerfile" becomes: +# $ make imagetag-foo +# 1.8.1-stretch +define imagetag_dep +.PHONY: imagetag-$(1) +$(patsubst $(IMAGE_PREFIX)%,imagetag-%,$(1)): $(patsubst $(IMAGE_PREFIX)%,%,$(1))/Dockerfile + @cat $$< | grep "^FROM " | head -n1 | sed 's/FROM \(.*\):\(.*\)/\2/' +endef +$(foreach image, $(IMAGE_NAMES), $(eval $(call imagetag_dep, $(image)))) + +all: $(UPTODATE_FILES) + +clean: + $(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true + rm -rf $(UPTODATE_FILES) + + diff --git a/build/golang/Dockerfile b/build/golang/Dockerfile new file mode 100644 index 00000000..6936eec2 --- /dev/null +++ b/build/golang/Dockerfile @@ -0,0 +1,46 @@ +FROM golang:1.8.0-stretch +RUN apt-get update && \ + apt-get install -y \ + curl \ + file \ + git \ + jq \ + libprotobuf-dev \ + make \ + protobuf-compiler \ + python-pip \ + python-requests \ + python-yaml \ + unzip && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN pip install attrs +RUN go clean -i net && \ + go install -tags netgo std && \ + go install -race -tags netgo std +RUN go get -tags netgo \ + github.com/FiloSottile/gvt \ + github.com/client9/misspell/cmd/misspell \ + github.com/fatih/hclfmt \ + github.com/fzipp/gocyclo \ + github.com/gogo/protobuf/gogoproto \ + github.com/gogo/protobuf/protoc-gen-gogoslick \ + github.com/golang/dep/... \ + github.com/golang/lint/golint \ + github.com/golang/protobuf/protoc-gen-go \ + github.com/kisielk/errcheck \ + github.com/mjibson/esc \ + github.com/mvdan/sh/cmd/shfmt \ + github.com/prometheus/prometheus/cmd/promtool && \ + rm -rf /go/pkg /go/src +RUN mkdir protoc && \ + cd protoc && \ + curl -O -L https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip && \ + unzip protoc-3.1.0-linux-x86_64.zip && \ + cp bin/protoc /usr/bin/ && \ + chmod o+x /usr/bin/protoc && \ + cd .. && \ + rm -rf protoc +RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \ + touch /var/run/secrets/kubernetes.io/serviceaccount/token +COPY build.sh / +ENTRYPOINT ["/build.sh"] diff --git a/build/golang/build.sh b/build/golang/build.sh new file mode 100755 index 00000000..cf70e1c5 --- /dev/null +++ b/build/golang/build.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -eu + +if [ -n "${SRC_NAME:-}" ]; then + SRC_PATH=${SRC_PATH:-$GOPATH/src/$SRC_NAME} +elif [ -z "${SRC_PATH:-}" ]; then + echo "Must set either \$SRC_NAME or \$SRC_PATH." + exit 1 +fi + +# If we run make directly, any files created on the bind mount +# will have awkward ownership. So we switch to a user with the +# same user and group IDs as source directory. We have to set a +# few things up so that sudo works without complaining later on. +uid=$(stat --format="%u" "$SRC_PATH") +gid=$(stat --format="%g" "$SRC_PATH") +echo "weave:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd +echo "weave:*:::::::" >>/etc/shadow +echo "weave ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers + +su weave -c "PATH=$PATH make -C $SRC_PATH BUILD_IN_CONTAINER=false $*" diff --git a/build/haskell/Dockerfile b/build/haskell/Dockerfile new file mode 100644 index 00000000..8d40c662 --- /dev/null +++ b/build/haskell/Dockerfile @@ -0,0 +1,4 @@ +FROM fpco/stack-build:lts-8.9 +COPY build.sh / +COPY copy-libraries /usr/local/bin/ +ENTRYPOINT ["/build.sh"] diff --git a/build/haskell/build.sh b/build/haskell/build.sh new file mode 100755 index 00000000..e80d2abb --- /dev/null +++ b/build/haskell/build.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# +# Build a static Haskell binary using stack. + +set -eu + +if [ -z "${SRC_PATH:-}" ]; then + echo "Must set \$SRC_PATH." + exit 1 +fi + +make -C "$SRC_PATH" BUILD_IN_CONTAINER=false "$@" diff --git a/build/haskell/copy-libraries b/build/haskell/copy-libraries new file mode 100755 index 00000000..18cbba60 --- /dev/null +++ b/build/haskell/copy-libraries @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Copy dynamically linked libraries for a binary, so we can assemble a Docker +# image. +# +# Run with: +# copy-libraries /path/to/binary /output/dir +# +# Dependencies: +# - awk +# - cp +# - grep +# - ldd +# - mkdir + +set -o errexit +set -o nounset +set -o pipefail + +# Path to a Linux binary that we're going to run in the container. +binary_path="${1}" +# Path to directory to write the output to. +output_dir="${2}" + +exe_name=$(basename "${binary_path}") + +# Identify linked libraries. +libraries=($(ldd "${binary_path}" | awk '{print $(NF-1)}' | grep -v '=>')) +# Add /bin/sh, which we need for Docker imports. +libraries+=('/bin/sh') + +mkdir -p "${output_dir}" + +# Copy executable and all needed libraries into temporary directory. +cp "${binary_path}" "${output_dir}/${exe_name}" +for lib in "${libraries[@]}"; do + mkdir -p "${output_dir}/$(dirname "$lib")" + # Need -L to make sure we get actual libraries & binaries, not symlinks to + # them. + cp -L "${lib}" "${output_dir}/${lib}" +done diff --git a/circle.yml b/circle.yml index 688f4a0d..57686d69 100644 --- a/circle.yml +++ b/circle.yml @@ -28,4 +28,23 @@ test: - cd $SRCDIR/cover; make - cd $SRCDIR/socks; make - cd $SRCDIR/runner; make + - cd $SRCDIR/build; make +deployment: + snapshot: + branch: master + commands: + - docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASS" "$DOCKER_REGISTRY_URL" + - | + cd $SRCDIR/build; + for image in $(make images); do + # Tag the built images with the revision of this repo. + docker push "${image}:${GIT_TAG}" + + # Tag the built images with something derived from the base images in + # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a + # base image would lead to a tag of "1.8.0-stretch" + IMG_TAG=$(make "imagetag-${image#quay.io/weaveworks/build-}") + docker tag "${image}:latest" "${image}:${IMG_TAG}" + docker push "${image}:${IMG_TAG}" + done diff --git a/config_management/group_vars/all b/config_management/group_vars/all index 96971998..05f5dbc3 100644 --- a/config_management/group_vars/all +++ b/config_management/group_vars/all @@ -1,10 +1,11 @@ --- -go_version: 1.7.4 +go_version: 1.8.1 terraform_version: 0.8.5 docker_version: 1.11.2 -kubernetes_version: 1.5.2 -kubernetes_cni_version: 0.3.0.1 -kubernetes_token: 123456.0123456789123456 +docker_install_role: 'docker-from-get.docker.com' +kubernetes_version: 1.6.1 +kubernetes_cni_version: 0.5.1 +kubernetes_token: '123456.0123456789123456' etcd_container_version: 2.2.5 kube_discovery_container_version: 1.0 pause_container_version: 3.0 diff --git a/config_management/roles/dev-tools/tasks/main.yml b/config_management/roles/dev-tools/tasks/main.yml index 422344df..a9cb99dd 100644 --- a/config_management/roles/dev-tools/tasks/main.yml +++ b/config_management/roles/dev-tools/tasks/main.yml @@ -5,7 +5,7 @@ package: name: "{{ item }}" state: present - with_items: + with_items: # weave net dependencies - make - vagrant diff --git a/config_management/roles/docker-configuration/files/docker.conf b/config_management/roles/docker-configuration/files/docker.conf new file mode 100644 index 00000000..6d02b55e --- /dev/null +++ b/config_management/roles/docker-configuration/files/docker.conf @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000" diff --git a/config_management/roles/docker-configuration/files/docker_over_tcp.conf b/config_management/roles/docker-configuration/files/docker_over_tcp.conf deleted file mode 100644 index c1598245..00000000 --- a/config_management/roles/docker-configuration/files/docker_over_tcp.conf +++ /dev/null @@ -1,3 +0,0 @@ -[Service] -ExecStart= -ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay diff --git a/config_management/roles/docker-configuration/tasks/main.yml b/config_management/roles/docker-configuration/tasks/main.yml index a9d5c199..d6736968 100644 --- a/config_management/roles/docker-configuration/tasks/main.yml +++ b/config_management/roles/docker-configuration/tasks/main.yml @@ -18,17 +18,19 @@ path: /etc/systemd/system/docker.service.d state: directory recurse: yes + when: ansible_os_family != "RedHat" - name: enable docker remote api over tcp copy: - src: "{{ role_path }}/files/docker_over_tcp.conf" - dest: /etc/systemd/system/docker.service.d/docker_over_tcp.conf - register: docker_over_tcp + src: "{{ role_path }}/files/docker.conf" + dest: /etc/systemd/system/docker.service.d/docker.conf + register: docker_conf + when: ansible_os_family != "RedHat" - name: restart docker service - systemd: + systemd: name: docker state: restarted - daemon_reload: yes # ensure docker_over_tcp.conf is picked up. + daemon_reload: yes # ensure docker.conf is picked up. enabled: yes - when: docker_over_tcp.changed + when: docker_conf.changed or ansible_os_family == "RedHat" diff --git a/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml b/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml new file mode 100644 index 00000000..ea9a3fa4 --- /dev/null +++ b/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml @@ -0,0 +1,29 @@ +# Docker installation from Docker's CentOS Community Edition +# See also: https://docs.docker.com/engine/installation/linux/centos/ + +- name: remove all potentially pre existing packages + yum: + name: '{{ item }}' + state: absent + with_items: + - docker + - docker-common + - container-selinux + - docker-selinux + - docker-engine + +- name: install yum-utils + yum: + name: yum-utils + state: present + +- name: add docker ce repo + command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + +# Note that Docker CE versions do not follow regular Docker versions, but look +# like, for example: "17.03.0.el7" +- name: install docker + yum: + name: 'docker-ce-{{ docker_version }}' + update_cache: yes + state: present diff --git a/config_management/roles/docker-from-docker-repo/tasks/debian.yml b/config_management/roles/docker-from-docker-repo/tasks/debian.yml index 28bf8d96..cc33c2c9 100644 --- a/config_management/roles/docker-from-docker-repo/tasks/debian.yml +++ b/config_management/roles/docker-from-docker-repo/tasks/debian.yml @@ -5,12 +5,12 @@ package: name: "{{ item }}" state: present - with_items: + with_items: - apt-transport-https - ca-certificates - name: add apt key for the docker repository - apt_key: + apt_key: keyserver: hkp://ha.pool.sks-keyservers.net:80 id: 58118E89F3A912897C070ADBF76221572C52609D state: present @@ -23,7 +23,7 @@ register: apt_docker_repo - name: update apt's cache - apt: + apt: update_cache: yes when: apt_key_docker_repo.changed or apt_docker_repo.changed @@ -31,5 +31,5 @@ package: name: "{{ item }}" state: present - with_items: + with_items: - docker-engine={{ docker_version }}* diff --git a/config_management/roles/docker-from-docker-repo/tasks/main.yml b/config_management/roles/docker-from-docker-repo/tasks/main.yml index c3707a1c..0acb6d8c 100644 --- a/config_management/roles/docker-from-docker-repo/tasks/main.yml +++ b/config_management/roles/docker-from-docker-repo/tasks/main.yml @@ -2,15 +2,9 @@ # Set up Docker # See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install -- include_role: - name: docker-prerequisites - # Distribution-specific tasks: - include: debian.yml when: ansible_os_family == "Debian" - include: redhat.yml when: ansible_os_family == "RedHat" - -- include_role: - name: docker-configuration diff --git a/config_management/roles/docker-from-docker-repo/tasks/redhat.yml b/config_management/roles/docker-from-docker-repo/tasks/redhat.yml index ae491300..d29964e1 100644 --- a/config_management/roles/docker-from-docker-repo/tasks/redhat.yml +++ b/config_management/roles/docker-from-docker-repo/tasks/redhat.yml @@ -13,7 +13,7 @@ state: present - name: update yum's cache - yum: + yum: name: "*" update_cache: yes @@ -21,5 +21,5 @@ package: name: "{{ item }}" state: present - with_items: + with_items: - docker-engine-{{ docker_version }} diff --git a/config_management/roles/docker-from-get.docker.com/tasks/debian.yml b/config_management/roles/docker-from-get.docker.com/tasks/debian.yml new file mode 100644 index 00000000..97b5b7a3 --- /dev/null +++ b/config_management/roles/docker-from-get.docker.com/tasks/debian.yml @@ -0,0 +1,8 @@ +--- +# Debian / Ubuntu specific: + +- name: apt-import gpg key for the docker repository + shell: curl -sSL https://get.docker.com/gpg | sudo apt-key add - + +- name: install docker + shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ | sh' diff --git a/config_management/roles/docker-from-get.docker.com/tasks/main.yml b/config_management/roles/docker-from-get.docker.com/tasks/main.yml index ef47b6f7..92c497b7 100644 --- a/config_management/roles/docker-from-get.docker.com/tasks/main.yml +++ b/config_management/roles/docker-from-get.docker.com/tasks/main.yml @@ -2,14 +2,9 @@ # Set up Docker # See also: legacy gce.sh script -- include_role: - name: docker-prerequisites +# Distribution-specific tasks: +- include: debian.yml + when: ansible_os_family == "Debian" -- name: add apt key for the docker repository - shell: curl -sSL https://get.docker.com/gpg | sudo apt-key add - - -- name: install docker - shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ | sh' - -- include_role: - name: docker-configuration +- include: redhat.yml + when: ansible_os_family == "RedHat" diff --git a/config_management/roles/docker-from-get.docker.com/tasks/redhat.yml b/config_management/roles/docker-from-get.docker.com/tasks/redhat.yml new file mode 100644 index 00000000..ea7cbfc4 --- /dev/null +++ b/config_management/roles/docker-from-get.docker.com/tasks/redhat.yml @@ -0,0 +1,11 @@ +--- +# RedHat / CentOS specific: + +- name: rpm-import gpg key for the docker repository + shell: curl -sSLo /tmp/docker.gpg https://get.docker.com/gpg && sudo rpm --import /tmp/docker.gpg + +- name: install docker + shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine-{{ docker_version }}*/ | sh' + +- name: wait for docker installation to complete + shell: yum install -y yum-utils && yum-complete-transaction diff --git a/config_management/roles/docker-from-tarball/tasks/main.yml b/config_management/roles/docker-from-tarball/tasks/main.yml index 90d4c8d1..a233d10a 100644 --- a/config_management/roles/docker-from-tarball/tasks/main.yml +++ b/config_management/roles/docker-from-tarball/tasks/main.yml @@ -1,6 +1,6 @@ --- # Set up Docker -# See also: +# See also: # - https://docs.docker.com/engine/installation/linux/ubuntulinux/#install # - https://github.com/docker/docker/releases diff --git a/config_management/roles/docker-install/tasks/main.yml b/config_management/roles/docker-install/tasks/main.yml new file mode 100644 index 00000000..cc803cab --- /dev/null +++ b/config_management/roles/docker-install/tasks/main.yml @@ -0,0 +1,30 @@ +--- +# Set up Docker + +- include_role: + name: docker-prerequisites + +# Dynamically include docker installation role using 'when' as Ansible does not +# allow for include_role's name to be set to a variable. Indeed: +# - include_role: +# name: '{{ docker_install_role }}' +# fails with: +# ERROR! 'docker_install_role' is undefined +- include_role: + name: docker-from-docker-repo + when: docker_install_role == 'docker-from-docker-repo' + +- include_role: + name: docker-from-docker-ce-repo + when: docker_install_role == 'docker-from-docker-ce-repo' + +- include_role: + name: docker-from-get.docker.com + when: docker_install_role == 'docker-from-get.docker.com' + +- include_role: + name: docker-from-tarball + when: docker_install_role == 'docker-from-tarball' + +- include_role: + name: docker-configuration diff --git a/config_management/roles/docker-prerequisites/tasks/debian.yml b/config_management/roles/docker-prerequisites/tasks/debian.yml index ef2ea148..48b0c2e3 100644 --- a/config_management/roles/docker-prerequisites/tasks/debian.yml +++ b/config_management/roles/docker-prerequisites/tasks/debian.yml @@ -6,6 +6,6 @@ package: name: "{{ item }}" state: present - with_items: + with_items: - linux-image-extra-{{ ansible_kernel }} - linux-image-extra-virtual diff --git a/config_management/roles/golang-from-tarball/tasks/main.yml b/config_management/roles/golang-from-tarball/tasks/main.yml index 1b607e1f..55476bf6 100644 --- a/config_management/roles/golang-from-tarball/tasks/main.yml +++ b/config_management/roles/golang-from-tarball/tasks/main.yml @@ -19,7 +19,7 @@ create: yes mode: 0644 become: '{{ item }}' - with_items: + with_items: - true # Run as root - false # Run as SSH user @@ -31,6 +31,6 @@ create: yes mode: 0644 become: '{{ item }}' - with_items: + with_items: - true # Run as root - - false # Run as SSH user + - false # Run as SSH user diff --git a/config_management/roles/kubelet-stop/tasks/main.yml b/config_management/roles/kubelet-stop/tasks/main.yml index 405c1549..6e5f3148 100644 --- a/config_management/roles/kubelet-stop/tasks/main.yml +++ b/config_management/roles/kubelet-stop/tasks/main.yml @@ -6,7 +6,7 @@ register: kubelet # avoids having weave-net and weave-kube conflict in some test cases (e.g. 130_expose_test.sh) -- name: stop kubelet service +- name: stop kubelet service systemd: name: kubelet state: stopped diff --git a/config_management/roles/kubernetes-docker-images/tasks/main.yml b/config_management/roles/kubernetes-docker-images/tasks/main.yml new file mode 100644 index 00000000..801c4637 --- /dev/null +++ b/config_management/roles/kubernetes-docker-images/tasks/main.yml @@ -0,0 +1,14 @@ +--- + +- name: docker pull images used by k8s tests + docker_image: + name: '{{ item }}' + state: present + with_items: + - gcr.io/google_containers/etcd-amd64:{{ etcd_container_version }} + - gcr.io/google_containers/kube-apiserver-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-controller-manager-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-proxy-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-scheduler-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-discovery-amd64:{{ kube_discovery_container_version }} + - gcr.io/google_containers/pause-amd64:{{ pause_container_version }} diff --git a/config_management/roles/kubernetes-install/tasks/debian.yml b/config_management/roles/kubernetes-install/tasks/debian.yml index 532356d1..9f16edfd 100644 --- a/config_management/roles/kubernetes-install/tasks/debian.yml +++ b/config_management/roles/kubernetes-install/tasks/debian.yml @@ -2,7 +2,7 @@ # Debian / Ubuntu specific: - name: add apt key for the kubernetes repository - apt_key: + apt_key: url: https://packages.cloud.google.com/apt/doc/apt-key.gpg state: present register: apt_key_k8s_repo @@ -12,9 +12,17 @@ repo: deb http://apt.kubernetes.io/ kubernetes-{{ ansible_distribution_release }} main state: present register: apt_k8s_repo + when: '"alpha" not in kubernetes_version and "beta" not in kubernetes_version' + +- name: add kubernetes' apt repository (kubernetes-{{ ansible_distribution_release }}-unstable) + apt_repository: + repo: deb http://apt.kubernetes.io/ kubernetes-{{ ansible_distribution_release }}-unstable main + state: present + register: apt_k8s_repo + when: '"alpha" in kubernetes_version or "beta" in kubernetes_version' - name: update apt's cache - apt: + apt: update_cache: yes when: apt_key_k8s_repo.changed or apt_k8s_repo.changed @@ -22,7 +30,8 @@ package: name: "{{ item }}" state: present - with_items: + with_items: - kubelet={{ kubernetes_version }}* - kubectl={{ kubernetes_version }}* + - kubeadm={{ kubernetes_version }}* - kubernetes-cni={{ kubernetes_cni_version }}* diff --git a/config_management/roles/kubernetes-install/tasks/main.yml b/config_management/roles/kubernetes-install/tasks/main.yml index c602b2cd..50dcddaf 100644 --- a/config_management/roles/kubernetes-install/tasks/main.yml +++ b/config_management/roles/kubernetes-install/tasks/main.yml @@ -8,10 +8,9 @@ - include: redhat.yml when: ansible_os_family == "RedHat" -- name: install ebtables and kubeadm +- name: install ebtables package: name: "{{ item }}" state: present - with_items: + with_items: - ebtables - - kubeadm diff --git a/config_management/roles/kubernetes-install/tasks/redhat.yml b/config_management/roles/kubernetes-install/tasks/redhat.yml index f85b872e..293729dc 100644 --- a/config_management/roles/kubernetes-install/tasks/redhat.yml +++ b/config_management/roles/kubernetes-install/tasks/redhat.yml @@ -14,7 +14,7 @@ register: yum_k8s_repo - name: update yum's cache - yum: + yum: name: "*" update_cache: yes when: yum_k8s_repo.changed @@ -23,7 +23,8 @@ package: name: "{{ item }}" state: present - with_items: - - kubelet-{{ kubernetes_version }} - - kubectl-{{ kubernetes_version }} + with_items: + - kubelet-{{ kubernetes_version }}* + - kubectl-{{ kubernetes_version }}* + - kubeadm-{{ kubernetes_version }}* - kubernetes-cni-{{ kubernetes_cni_version }}* diff --git a/config_management/roles/kubernetes-start/tasks/main.yml b/config_management/roles/kubernetes-start/tasks/main.yml index 7910f140..d343b21c 100644 --- a/config_management/roles/kubernetes-start/tasks/main.yml +++ b/config_management/roles/kubernetes-start/tasks/main.yml @@ -10,20 +10,25 @@ state: restarted enabled: yes +- name: optionally set kubeconfig option + set_fact: + kubeconfig: '{{ (kubernetes_version >= "1.5.4") | ternary("--kubeconfig /etc/kubernetes/admin.conf", "") }}' + kubernetes_version_option: '{{ (kubernetes_version >= "1.6") | ternary("kubernetes_version", "use-kubernetes-version") }}' + - name: kubeadm init on the master - command: 'kubeadm init --token={{ kubernetes_token }}' + command: 'kubeadm init --{{ kubernetes_version_option }}=v{{ kubernetes_version }} --token={{ kubernetes_token }}' when: ' {{ play_hosts[0] == inventory_hostname }}' - name: allow pods to be run on the master (if only node) - command: kubectl taint nodes --all dedicated- + command: 'kubectl {{ kubeconfig }} taint nodes --all {{ (kubernetes_version < "1.6") | ternary("dedicated-", "node-role.kubernetes.io/master:NoSchedule-") }}' when: '{{ play_hosts | length }} == 1' - name: kubeadm join on workers - command: "kubeadm join --token={{ kubernetes_token }} {{ hostvars[play_hosts[0]].private_ip }}" + command: 'kubeadm join --token={{ kubernetes_token }} {{ hostvars[play_hosts[0]].private_ip }}{{ (kubernetes_version > "1.6") | ternary(":6443", "") }}' when: ' {{ play_hosts[0] != inventory_hostname }}' - name: list kubernetes' pods - command: kubectl get pods --all-namespaces + command: kubectl {{ kubeconfig }} get pods --all-namespaces when: ' {{ play_hosts[0] == inventory_hostname }}' changed_when: false register: kubectl_get_pods diff --git a/config_management/roles/sock-shop/tasks/tasks.yml b/config_management/roles/sock-shop/tasks/tasks.yml index fc00e809..9667ab04 100644 --- a/config_management/roles/sock-shop/tasks/tasks.yml +++ b/config_management/roles/sock-shop/tasks/tasks.yml @@ -4,13 +4,13 @@ # - kubernetes - name: create sock-shop namespace in k8s - command: kubectl create namespace sock-shop + command: kubectl --kubeconfig /etc/kubernetes/admin.conf create namespace sock-shop - name: create sock-shop in k8s - command: kubectl apply -n sock-shop -f "https://github.com/microservices-demo/microservices-demo/blob/master/deploy/kubernetes/complete-demo.yaml?raw=true" + command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -n sock-shop -f "https://github.com/microservices-demo/microservices-demo/blob/master/deploy/kubernetes/complete-demo.yaml?raw=true" - name: describe front-end service - command: kubectl describe svc front-end -n sock-shop + command: kubectl --kubeconfig /etc/kubernetes/admin.conf describe svc front-end -n sock-shop changed_when: false register: kubectl_describe_svc_frontend tags: @@ -22,7 +22,7 @@ - output - name: list sock-shop k8s' pods - command: kubectl get pods -n sock-shop + command: kubectl --kubeconfig /etc/kubernetes/admin.conf get pods -n sock-shop changed_when: false register: kubectl_get_pods tags: diff --git a/config_management/roles/weave-kube/tasks/main.yml b/config_management/roles/weave-kube/tasks/main.yml index fcaa7e20..e2025eef 100644 --- a/config_management/roles/weave-kube/tasks/main.yml +++ b/config_management/roles/weave-kube/tasks/main.yml @@ -1,13 +1,17 @@ --- # Set up Weave Kube on top of Kubernetes. +- name: set url for weave-kube daemonset + set_fact: + weave_kube_url: '{{ (kubernetes_version < "1.6") | ternary("https://git.io/weave-kube", "https://git.io/weave-kube-1.6") }}' + - name: configure weave net's cni plugin - command: kubectl apply -f https://git.io/weave-kube - when: ' {{ play_hosts[0] == inventory_hostname }}' + command: 'kubectl {{ kubeconfig }} apply -f {{ weave_kube_url }}' + when: '{{ play_hosts[0] == inventory_hostname }}' - name: list kubernetes' pods - command: kubectl get pods --all-namespaces - when: ' {{ play_hosts[0] == inventory_hostname }}' + command: 'kubectl {{ kubeconfig }} get pods --all-namespaces' + when: '{{ play_hosts[0] == inventory_hostname }}' changed_when: false register: kubectl_get_pods tags: @@ -15,6 +19,6 @@ - name: print outpout of `kubectl get pods --all-namespaces` debug: msg="{{ kubectl_get_pods.stdout_lines }}" - when: ' {{ play_hosts[0] == inventory_hostname }}' + when: '{{ play_hosts[0] == inventory_hostname }}' tags: - output diff --git a/config_management/roles/weave-net-sources/tasks/main.yml b/config_management/roles/weave-net-sources/tasks/main.yml index f753cd41..b0a7815c 100644 --- a/config_management/roles/weave-net-sources/tasks/main.yml +++ b/config_management/roles/weave-net-sources/tasks/main.yml @@ -3,7 +3,7 @@ - name: check if weave net has been checked out become: false # Run as SSH-user - stat: + stat: path: $HOME/src/github.com/weaveworks/weave register: weave failed_when: false diff --git a/config_management/roles/weave-net-utilities/tasks/main.yml b/config_management/roles/weave-net-utilities/tasks/main.yml index 8032f6a2..89038597 100644 --- a/config_management/roles/weave-net-utilities/tasks/main.yml +++ b/config_management/roles/weave-net-utilities/tasks/main.yml @@ -1,18 +1,25 @@ --- +- name: install epel-release + package: + name: "{{ item }}" + state: present + with_items: + - epel-release + when: ansible_os_family == "RedHat" + - name: install jq package: name: "{{ item }}" state: present - with_items: + with_items: - jq - name: install ethtool (used by the weave script) package: name: "{{ item }}" - install_recommends: no state: present - with_items: + with_items: - ethtool - name: install nsenter (used by the weave script) @@ -22,7 +29,7 @@ package: name: "{{ item }}" state: present - with_items: + with_items: - python-pip - name: install docker-py (for docker_image) @@ -34,20 +41,7 @@ docker_image: name: '{{ item }}' state: present - with_items: + with_items: - alpine - aanand/docker-dnsutils - weaveworks/hello-world - -- name: docker pull images used by k8s tests - docker_image: - name: '{{ item }}' - state: present - with_items: - - gcr.io/google_containers/etcd-amd64:{{ etcd_container_version }} - - gcr.io/google_containers/kube-apiserver-amd64:v{{ kubernetes_version }} - - gcr.io/google_containers/kube-controller-manager-amd64:v{{ kubernetes_version }} - - gcr.io/google_containers/kube-proxy-amd64:v{{ kubernetes_version }} - - gcr.io/google_containers/kube-scheduler-amd64:v{{ kubernetes_version }} - - gcr.io/google_containers/kube-discovery-amd64:{{ kube_discovery_container_version }} - - gcr.io/google_containers/pause-amd64:{{ pause_container_version }} diff --git a/config_management/roles/weave-net/tasks/main.yml b/config_management/roles/weave-net/tasks/main.yml index dfd37040..0ef5e351 100644 --- a/config_management/roles/weave-net/tasks/main.yml +++ b/config_management/roles/weave-net/tasks/main.yml @@ -8,13 +8,13 @@ mode: 0555 - name: stop weave net - command: weave stop + command: /usr/local/bin/weave stop - name: start weave net - command: weave launch + command: /usr/local/bin/weave launch - name: get weave net's status - command: weave status + command: /usr/local/bin/weave status changed_when: false register: weave_status tags: diff --git a/config_management/setup_weave-kube.yml b/config_management/setup_weave-kube.yml index 14a73124..5c68c978 100644 --- a/config_management/setup_weave-kube.yml +++ b/config_management/setup_weave-kube.yml @@ -1,6 +1,6 @@ --- ################################################################################ -# Install Docker and Kubernetes, and configure Kubernetes to +# Install Docker and Kubernetes, and configure Kubernetes to # use Weave Net's CNI plugin (a.k.a. Weave Kube). # # See also: @@ -13,14 +13,15 @@ gather_facts: false # required in case Python is not available on the host become: true become_user: root - + pre_tasks: - include: library/setup_ansible_dependencies.yml roles: - - docker-from-get.docker.com - - kubernetes-install + - docker-install - weave-net-utilities + - kubernetes-install + - kubernetes-docker-images - kubelet-stop - kubernetes-start - weave-kube diff --git a/config_management/setup_weave-net_debug.yml b/config_management/setup_weave-net_debug.yml new file mode 100644 index 00000000..ff73a527 --- /dev/null +++ b/config_management/setup_weave-net_debug.yml @@ -0,0 +1,18 @@ +--- +################################################################################ +# Install Docker from Docker's official repository and Weave Net. +################################################################################ + +- name: install docker and weave net for development + hosts: all + gather_facts: false # required in case Python is not available on the host + become: true + become_user: root + + pre_tasks: + - include: library/setup_ansible_dependencies.yml + + roles: + - docker-install + - weave-net-utilities + - weave-net diff --git a/config_management/setup_weave-net_dev.yml b/config_management/setup_weave-net_dev.yml index f72149d0..bdfa08e9 100644 --- a/config_management/setup_weave-net_dev.yml +++ b/config_management/setup_weave-net_dev.yml @@ -8,13 +8,13 @@ gather_facts: false # required in case Python is not available on the host become: true become_user: root - + pre_tasks: - include: library/setup_ansible_dependencies.yml roles: - dev-tools - golang-from-tarball - - docker-from-get.docker.com + - docker-install # Do not run this role when building with Vagrant, as sources have been already checked out: - { role: weave-net-sources, when: "ansible_user != 'vagrant'" } diff --git a/config_management/setup_weave-net_test.yml b/config_management/setup_weave-net_test.yml index 9fb46be6..fbd155df 100644 --- a/config_management/setup_weave-net_test.yml +++ b/config_management/setup_weave-net_test.yml @@ -8,12 +8,13 @@ gather_facts: false # required in case Python is not available on the host become: true become_user: root - + pre_tasks: - include: library/setup_ansible_dependencies.yml roles: - - docker-from-get.docker.com - - kubernetes-install + - docker-install - weave-net-utilities + - kubernetes-install + - kubernetes-docker-images - kubelet-stop diff --git a/dependencies/list_os_images.sh b/dependencies/list_os_images.sh index 7d2495da..90d2832a 100755 --- a/dependencies/list_os_images.sh +++ b/dependencies/list_os_images.sh @@ -58,8 +58,8 @@ fi case "$1" in 'gcp') gcloud compute images list --standard-images --regexp=".*?$2.*" \ - --format="csv[no-heading][separator=/](selfLink.map().scope(projects).segment(0),family)" \ - | sort -d + --format="csv[no-heading][separator=/](selfLink.map().scope(projects).segment(0),family)" | + sort -d ;; 'aws') aws --region "${3:-us-east-1}" ec2 describe-images \ @@ -74,8 +74,8 @@ case "$1" in curl -s -X GET \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \ - "https://api.digitalocean.com/v2/images?page=1&per_page=999999" \ - | jq --raw-output ".images | .[] | .slug" | grep "$2" | sort -d + "https://api.digitalocean.com/v2/images?page=1&per_page=999999" | + jq --raw-output ".images | .[] | .slug" | grep "$2" | sort -d ;; *) echo >&2 "Unknown provider [$1]." diff --git a/dependencies/list_versions.py b/dependencies/list_versions.py index 5b0d9c42..3b756cd1 100755 --- a/dependencies/list_versions.py +++ b/dependencies/list_versions.py @@ -45,8 +45,8 @@ 'docker': { 'url': 'https://github.com/docker/docker', 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-rc\d)*)' % _VERSION, - # Weave Net only works with Docker from 1.6.0 onwards, so we ignore all previous versions: - 'min': '1.6.0' + # Weave Net only works with Docker from 1.10.0 onwards, so we ignore all previous versions: + 'min': '1.10.0' }, 'kubernetes': { 'url': 'https://github.com/kubernetes/kubernetes', diff --git a/integration/assert.sh b/integration/assert.sh index 200b393b..8f61fd47 100755 --- a/integration/assert.sh +++ b/integration/assert.sh @@ -24,8 +24,8 @@ export INVARIANT=${INVARIANT:-} export CONTINUE=${CONTINUE:-} args="$(getopt -n "$0" -l \ - verbose,help,stop,discover,invariant,continue vhxdic "$@")" \ - || exit -1 + verbose,help,stop,discover,invariant,continue vhxdic "$@")" || + exit -1 for arg in $args; do case "$arg" in -h) @@ -108,7 +108,7 @@ assert_end() { assert() { # assert [stdin] - ((tests_ran++)) || : + ((tests_ran++))|| : [[ -z "$DISCOVERONLY" ]] || return expected=$(echo -ne "${2:-}") result="$(eval "$1" 2>/dev/null <<<"${3:-}")" || true @@ -124,7 +124,7 @@ assert() { assert_raises() { # assert_raises [stdin] - ((tests_ran++)) || : + ((tests_ran++))|| : [[ -z "$DISCOVERONLY" ]] || return status=0 (eval "$1" <<<"${3:-}") >/dev/null 2>&1 || status=$? @@ -146,7 +146,7 @@ _assert_fail() { exit 1 fi tests_errors[$tests_failed]="$report" - ((tests_failed++)) || : + ((tests_failed++))|| : } skip_if() { diff --git a/integration/gce.sh b/integration/gce.sh index e52a8573..5c394018 100755 --- a/integration/gce.sh +++ b/integration/gce.sh @@ -44,6 +44,10 @@ function vm_names() { # Delete all vms in this account function destroy() { local names + # shellcheck disable=SC2046 + if [ $(gcloud compute firewall-rules list "test-allow-docker$SUFFIX" 2>/dev/null | wc -l) -gt 0 ]; then + gcloud compute firewall-rules delete "test-allow-docker$SUFFIX" + fi names="$(vm_names)" # shellcheck disable=SC2086 if [ "$(gcloud compute instances list --zones "$ZONE" -q $names | wc -l)" -le 1 ]; then @@ -116,7 +120,10 @@ function setup() { destroy names=($(vm_names)) - gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE" + gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE" --tags "test$SUFFIX" --network=test + my_ip="$(curl -s http://ipinfo.io/ip)" + gcloud compute firewall-rules create "test-allow-docker$SUFFIX" --network=test --allow tcp:2375,tcp:12375,tcp:4040,tcp:80 --target-tags "test$SUFFIX" --source-ranges "$my_ip" + gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config diff --git a/lint b/lint index b9428d9f..1589bd9e 100755 --- a/lint +++ b/lint @@ -139,6 +139,20 @@ lint_tf() { return $lint_result } +lint_md() { + local filename="$1" + local lint_result=0 + + for i in '=======' '>>>>>>>'; do + if grep -q "${i}" "${filename}"; then + lint_result=1 + echo "${filename}: bad merge/rebase!" + fi + done + + return $lint_result +} + lint() { filename="$1" ext="${filename##*\.}" @@ -164,6 +178,7 @@ lint() { go) lint_go "${filename}" || lint_result=1 ;; sh) lint_sh "${filename}" || lint_result=1 ;; tf) lint_tf "${filename}" || lint_result=1 ;; + md) lint_md "${filename}" || lint_result=1 ;; esac spell_check "${filename}" || lint_result=1 diff --git a/provisioning/aws/variables.tf b/provisioning/aws/variables.tf index 6abaff0b..ed5b8b4b 100755 --- a/provisioning/aws/variables.tf +++ b/provisioning/aws/variables.tf @@ -43,8 +43,10 @@ variable "aws_amis" { "eu-west-2" = "ami-23d0da47" # Red Hat Enterprise Linux 7.3 (HVM), SSD Volume Type: - #"us-east-1" = "ami-b63769a1" + + # CentOS 7 (x86_64) - with Updates HVM + #"us-east-1" = "ami-6d1c2007" } } @@ -54,6 +56,7 @@ variable "aws_usernames" { default = { "ami-40d28157" = "ubuntu" # Ubuntu Server 16.04 LTS (HVM) "ami-b63769a1" = "ec2-user" # Red Hat Enterprise Linux 7.3 (HVM) + "ami-6d1c2007" = "centos" # CentOS 7 (x86_64) - with Updates HVM } } diff --git a/provisioning/gcp/variables.tf b/provisioning/gcp/variables.tf index 1ee48bd0..6b2027b2 100755 --- a/provisioning/gcp/variables.tf +++ b/provisioning/gcp/variables.tf @@ -38,8 +38,14 @@ variable "gcp_project" { variable "gcp_image" { # See also: https://cloud.google.com/compute/docs/images + # For example: + # - "ubuntu-os-cloud/ubuntu-1604-lts" + # - "debian-cloud/debian-8" + # - "centos-cloud/centos-7" + # - "rhel-cloud/rhel7" description = "Google Cloud Platform OS" - default = "ubuntu-os-cloud/ubuntu-1604-lts" + + default = "ubuntu-os-cloud/ubuntu-1604-lts" } variable "gcp_size" { diff --git a/provisioning/setup.sh b/provisioning/setup.sh index afe5914d..4e696b92 100755 --- a/provisioning/setup.sh +++ b/provisioning/setup.sh @@ -278,8 +278,14 @@ function aws_on() { # $ openssl enc -in /tmp/aws_secret_access_key.txt -e -aes256 -pass stdin | openssl base64 > /tmp/aws_secret_access_key.txt.aes.b64 # The below commands do the reverse, i.e. base64-decode and AES-decrypt the encrypted and encoded strings, and print it to stdout. # N.B.: Ask the password to Marc, or otherwise re-generate the AWS access key ID and secret access key, as per ../tools/provisioning/aws/README.md. - export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-$(decrypt "$SECRET_KEY" "AWS access key ID" "U2FsdGVkX1+MLsvG53ZVSmFhjvQtWio0pXQpG5Ua+5JaoizuZKtJZFJxrSSyx0jb")} - export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-$(decrypt "$SECRET_KEY" "AWS secret access key" "U2FsdGVkX1+VNjgWv5iGKRqBYP7o8MpOIMnd3BOYPiEho1Mjosx++9CknaZJbeR59vSuz4UdgTS6ezH2dnq2Fw==")} + export AWS_ACCESS_KEY_ID="$(decrypt "$SECRET_KEY" "AWS access key ID" "U2FsdGVkX18Txjm2PWSlJsToYm1vv4dMTtVLkRNiQbrC6Y6GuIHb1ao5MmGPJ1wf")" + export AWS_SECRET_ACCESS_KEY="$(decrypt "$SECRET_KEY" "AWS secret access key" "$( + cat <&2 <<<"$(terraform output public_etc_hosts)" @@ -327,7 +334,8 @@ Usage: Examples: $ tf_ansi setup_weave-net_dev $ tf_ansi 1 - $ tf_ansi 1 -vvv + $ tf_ansi 1 -vvv --private-key=~/.ssh/custom_private_key_id_rsa + $ tf_ansi setup_weave-kube --extra-vars "docker_version=1.12.6 kubernetes_version=1.5.6" Available playbooks: EOF cat -n >&2 <<<"$(for file in "$(dirname "${BASH_SOURCE[0]}")"/../../config_management/*.yml; do basename "$file" | sed 's/.yml//'; done)" @@ -340,7 +348,7 @@ function tf_ansi() { shift # Drop the first argument to allow passing other arguments to Ansible using "$@" -- see below. if [[ "$id" =~ ^[0-9]+$ ]]; then local playbooks=(../../config_management/*.yml) - local path="${playbooks[(($id-1))]}" # Select the ith entry in the list of playbooks (0-based). + local path="${playbooks[(($id - 1))]}" # Select the ith entry in the list of playbooks (0-based). else local path="$(dirname "${BASH_SOURCE[0]}")/../../config_management/$id.yml" fi diff --git a/push-images b/push-images new file mode 100755 index 00000000..1871c378 --- /dev/null +++ b/push-images @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +QUAY_PREFIX=quay.io/ +IMAGES=$(make images) +IMAGE_TAG=$(./tools/image-tag) + +usage() { + echo "$0 [-no-docker-hub]" +} + +NO_DOCKER_HUB= +while [ $# -gt 0 ]; do + case "$1" in + -no-docker-hub) + NO_DOCKER_HUB=1 + shift 1 + ;; + *) + usage + exit 2 + ;; + esac +done + +push_image() { + local image="$1" + docker push "${image}:${IMAGE_TAG}" +} + +for image in ${IMAGES}; do + if [[ "$image" == *"build"* ]]; then + continue + fi + echo "Will push ${image}:${IMAGE_TAG}" + push_image "${image}" & + + if [ -z "$NO_DOCKER_HUB" ]; then + # remove the quey prefix and push to docker hub + docker_hub_image=${image#$QUAY_PREFIX} + docker tag "${image}:${IMAGE_TAG}" "${docker_hub_image}:${IMAGE_TAG}" + echo "Will push ${docker_hub_image}:${IMAGE_TAG}" + docker push "${docker_hub_image}:${IMAGE_TAG}" + fi +done + +wait diff --git a/test b/test index 1772b1bd..1497e7a2 100755 --- a/test +++ b/test @@ -7,6 +7,7 @@ SLOW= NO_GO_GET=true TAGS= PARALLEL= +RACE="-race -covermode=atomic" TIMEOUT=1m usage() { @@ -19,6 +20,10 @@ while [ $# -gt 0 ]; do SLOW=true shift 1 ;; + "-no-race") + RACE= + shift 1 + ;; "-no-go-get") NO_GO_GET=true shift 1 @@ -53,7 +58,7 @@ if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then fi if [ -n "$SLOW" ]; then - GO_TEST_ARGS=("${GO_TEST_ARGS[@]}" -race -covermode=atomic) + GO_TEST_ARGS=("${GO_TEST_ARGS[@]}" ${RACE}) # shellcheck disable=SC2153 if [ -n "$COVERDIR" ]; then @@ -112,7 +117,9 @@ run_test() { if ! go test "${GO_TEST_ARGS_RUN[@]}" "$dir"; then fail=1 fi - local RUNTIME=$(($(date +%s) - START)) + local END + END=$(date +%s) + local RUNTIME=$((END - START)) # Report test runtime when running on circle, to help scheduler if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then