update build-tools git-subtree

semaphore.sh

@@ -22,8 +22,8 @@ set -o pipefail
 readonly kernel_versions=("4.4.45" "4.9.6")
 readonly rkt_version="1.23.0"
 
-if [[ ! -f "./rkt/rkt" ]] \
-    || [[ ! "$(./rkt/rkt version | awk '/rkt Version/{print $3}')" == "${rkt_version}" ]]; then
+if [[ ! -f "./rkt/rkt" ]] ||
+    [[ ! "$(./rkt/rkt version | awk '/rkt Version/{print $3}')" == "${rkt_version}" ]]; then
 
     curl -LsS "https://github.com/coreos/rkt/releases/download/v${rkt_version}/rkt-v${rkt_version}.tar.gz" \
         -o rkt.tgz

tests/test.sh

@@ -59,10 +59,10 @@ while [[ $lines_read -lt 4 ]]; do
         daddr=${BASH_REMATCH[3]}
         lines_read=$((lines_read + 1))
         printf "action: %s program: nc saddr: %s daddr: %s\n" "${action}" "${saddr}" "${daddr}"
-        if [[ "${action}" == "connect" && "$daddr" == "127.0.0.1:${port}" ]] \
-            || [[ "${action}" == "accept" && "$saddr" == "127.0.0.1:${port}" ]] \
-            || [[ "${action}" == "close" && "$daddr" == "127.0.0.1:${port}" ]] \
-            || [[ "${action}" == "close" && "$saddr" == "127.0.0.1:${port}" ]]; then
+        if [[ "${action}" == "connect" && "$daddr" == "127.0.0.1:${port}" ]] ||
+            [[ "${action}" == "accept" && "$saddr" == "127.0.0.1:${port}" ]] ||
+            [[ "${action}" == "close" && "$daddr" == "127.0.0.1:${port}" ]] ||
+            [[ "${action}" == "close" && "$saddr" == "127.0.0.1:${port}" ]]; then
             lines_found=$((lines_found + 1))
         else
             echo "^^^ unexpected values in event"

tools/.gitignore

@@ -7,3 +7,4 @@ runner/runner
 terraform.tfstate
 terraform.tfstate.backup
 *.retry
+build/**/.uptodate

tools/README.md

@@ -2,6 +2,9 @@
 
 Included in this repo are tools shared by weave.git and scope.git.  They include
 
+- ```build```: a set of docker base-images for building weave
+  projects. These should be used instead of giving each project its
+  own build image.
 - ```provisioning```: a set of Terraform scripts to provision virtual machines in GCP, AWS or Digital Ocean.
 - ```config_management```: a set of Ansible playbooks to configure virtual machines for development, testing, etc.
 - ```cover```: a tool which merges overlapping coverage reports generated by go

tools/build/Makefile

@@ -0,0 +1,46 @@
+.PHONY: all clean images
+.DEFAULT_GOAL := all
+
+# Boiler plate for bulding Docker containers.
+# All this must go at top of file I'm afraid.
+IMAGE_PREFIX := quay.io/weaveworks/build-
+IMAGE_TAG := $(shell ../image-tag)
+UPTODATE := .uptodate
+
+# Every directory with a Dockerfile in it builds an image called
+# $(IMAGE_PREFIX)<dirname>. Dependencies (i.e. things that go in the image)
+# still need to be explicitly declared.
+%/$(UPTODATE): %/Dockerfile %/*
+	$(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/
+	$(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG)
+	touch $@
+
+# Get a list of directories containing Dockerfiles
+DOCKERFILES := $(shell find . -name tools -prune -o -name vendor -prune -o -type f -name 'Dockerfile' -print)
+UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES))
+DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES))
+IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir))))
+images:
+	$(info $(IMAGE_NAMES))
+	@echo > /dev/null
+
+# Define imagetag-golang, etc, for each image, which parses the dockerfile and
+# prints an image tag. For example:
+#     FROM golang:1.8.1-stretch
+# in the "foo/Dockerfile" becomes:
+#     $ make imagetag-foo
+#     1.8.1-stretch
+define imagetag_dep
+.PHONY: imagetag-$(1)
+$(patsubst $(IMAGE_PREFIX)%,imagetag-%,$(1)): $(patsubst $(IMAGE_PREFIX)%,%,$(1))/Dockerfile
+	@cat $$< | grep "^FROM " | head -n1 | sed 's/FROM \(.*\):\(.*\)/\2/'
+endef
+$(foreach image, $(IMAGE_NAMES), $(eval $(call imagetag_dep, $(image))))
+
+all: $(UPTODATE_FILES)
+
+clean:
+	$(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true
+	rm -rf $(UPTODATE_FILES)
+
+

tools/build/golang/Dockerfile

@@ -0,0 +1,46 @@
+FROM golang:1.8.0-stretch
+RUN apt-get update && \
+    apt-get install -y \
+      curl \
+      file \
+      git \
+      jq \
+      libprotobuf-dev \
+      make \
+      protobuf-compiler \
+      python-pip \
+      python-requests \
+      python-yaml \
+      unzip && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN pip install attrs
+RUN go clean -i net && \
+	go install -tags netgo std && \
+	go install -race -tags netgo std
+RUN go get -tags netgo \
+		github.com/FiloSottile/gvt \
+		github.com/client9/misspell/cmd/misspell \
+		github.com/fatih/hclfmt \
+		github.com/fzipp/gocyclo \
+		github.com/gogo/protobuf/gogoproto \
+		github.com/gogo/protobuf/protoc-gen-gogoslick \
+		github.com/golang/dep/... \
+		github.com/golang/lint/golint \
+		github.com/golang/protobuf/protoc-gen-go \
+		github.com/kisielk/errcheck \
+		github.com/mjibson/esc \
+		github.com/mvdan/sh/cmd/shfmt \
+		github.com/prometheus/prometheus/cmd/promtool && \
+		rm -rf /go/pkg /go/src
+RUN mkdir protoc && \
+	cd protoc && \
+	curl -O -L https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip && \
+	unzip protoc-3.1.0-linux-x86_64.zip && \
+	cp bin/protoc /usr/bin/ && \
+	chmod o+x /usr/bin/protoc && \
+	cd .. && \
+	rm -rf protoc
+RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \
+		touch /var/run/secrets/kubernetes.io/serviceaccount/token
+COPY build.sh /
+ENTRYPOINT ["/build.sh"]

tools/build/golang/build.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -eu
+
+if [ -n "${SRC_NAME:-}" ]; then
+    SRC_PATH=${SRC_PATH:-$GOPATH/src/$SRC_NAME}
+elif [ -z "${SRC_PATH:-}" ]; then
+    echo "Must set either \$SRC_NAME or \$SRC_PATH."
+    exit 1
+fi
+
+# If we run make directly, any files created on the bind mount
+# will have awkward ownership.  So we switch to a user with the
+# same user and group IDs as source directory.  We have to set a
+# few things up so that sudo works without complaining later on.
+uid=$(stat --format="%u" "$SRC_PATH")
+gid=$(stat --format="%g" "$SRC_PATH")
+echo "weave:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd
+echo "weave:*:::::::" >>/etc/shadow
+echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
+
+su weave -c "PATH=$PATH make -C $SRC_PATH BUILD_IN_CONTAINER=false $*"

tools/build/haskell/Dockerfile

@@ -0,0 +1,4 @@
+FROM fpco/stack-build:lts-8.9
+COPY build.sh /
+COPY copy-libraries /usr/local/bin/
+ENTRYPOINT ["/build.sh"]

tools/build/haskell/build.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+#
+# Build a static Haskell binary using stack.
+
+set -eu
+
+if [ -z "${SRC_PATH:-}" ]; then
+    echo "Must set \$SRC_PATH."
+    exit 1
+fi
+
+make -C "$SRC_PATH" BUILD_IN_CONTAINER=false "$@"

tools/build/haskell/copy-libraries

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Copy dynamically linked libraries for a binary, so we can assemble a Docker
+# image.
+#
+# Run with:
+#   copy-libraries /path/to/binary /output/dir
+#
+# Dependencies:
+# - awk
+# - cp
+# - grep
+# - ldd
+# - mkdir
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Path to a Linux binary that we're going to run in the container.
+binary_path="${1}"
+# Path to directory to write the output to.
+output_dir="${2}"
+
+exe_name=$(basename "${binary_path}")
+
+# Identify linked libraries.
+libraries=($(ldd "${binary_path}" | awk '{print $(NF-1)}' | grep -v '=>'))
+# Add /bin/sh, which we need for Docker imports.
+libraries+=('/bin/sh')
+
+mkdir -p "${output_dir}"
+
+# Copy executable and all needed libraries into temporary directory.
+cp "${binary_path}" "${output_dir}/${exe_name}"
+for lib in "${libraries[@]}"; do
+    mkdir -p "${output_dir}/$(dirname "$lib")"
+    # Need -L to make sure we get actual libraries & binaries, not symlinks to
+    # them.
+    cp -L "${lib}" "${output_dir}/${lib}"
+done

tools/circle.yml

@@ -28,4 +28,23 @@ test:
     - cd $SRCDIR/cover; make
     - cd $SRCDIR/socks; make
     - cd $SRCDIR/runner; make
+    - cd $SRCDIR/build; make
 
+deployment:
+  snapshot:
+    branch: master
+    commands:
+      - docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASS" "$DOCKER_REGISTRY_URL"
+      - |
+        cd $SRCDIR/build;
+        for image in $(make images); do
+          # Tag the built images with the revision of this repo.
+          docker push "${image}:${GIT_TAG}"
+
+          # Tag the built images with something derived from the base images in
+          # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a
+          # base image would lead to a tag of "1.8.0-stretch"
+          IMG_TAG=$(make "imagetag-${image#quay.io/weaveworks/build-}")
+          docker tag "${image}:latest" "${image}:${IMG_TAG}"
+          docker push "${image}:${IMG_TAG}"
+        done

tools/config_management/group_vars/all

@@ -1,10 +1,11 @@
 ---
-go_version: 1.7.4
+go_version: 1.8.1
 terraform_version: 0.8.5
 docker_version: 1.11.2
-kubernetes_version: 1.5.2
-kubernetes_cni_version: 0.3.0.1
-kubernetes_token: 123456.0123456789123456
+docker_install_role: 'docker-from-get.docker.com'
+kubernetes_version: 1.6.1
+kubernetes_cni_version: 0.5.1
+kubernetes_token: '123456.0123456789123456'
 etcd_container_version: 2.2.5
 kube_discovery_container_version: 1.0
 pause_container_version: 3.0

tools/config_management/roles/dev-tools/tasks/main.yml

@@ -5,7 +5,7 @@
   package:
     name: "{{ item }}"
     state: present
-  with_items: 
+  with_items:
     # weave net dependencies
     - make
     - vagrant

tools/config_management/roles/docker-configuration/files/docker.conf

@@ -0,0 +1,3 @@
+[Service]
+ExecStart=
+ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000"

tools/config_management/roles/docker-configuration/tasks/main.yml

@@ -18,17 +18,19 @@
     path: /etc/systemd/system/docker.service.d
     state: directory
     recurse: yes
+  when: ansible_os_family != "RedHat"
 
 - name: enable docker remote api over tcp
   copy:
-    src: "{{ role_path }}/files/docker_over_tcp.conf"
-    dest: /etc/systemd/system/docker.service.d/docker_over_tcp.conf
-  register: docker_over_tcp
+    src: "{{ role_path }}/files/docker.conf"
+    dest: /etc/systemd/system/docker.service.d/docker.conf
+  register: docker_conf
+  when: ansible_os_family != "RedHat"
 
 - name: restart docker service
-  systemd: 
+  systemd:
     name: docker
     state: restarted
-    daemon_reload: yes  # ensure docker_over_tcp.conf is picked up.
+    daemon_reload: yes  # ensure docker.conf is picked up.
     enabled: yes
-  when: docker_over_tcp.changed
+  when: docker_conf.changed or ansible_os_family == "RedHat"

tools/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml

@@ -0,0 +1,29 @@
+# Docker installation from Docker's CentOS Community Edition
+# See also: https://docs.docker.com/engine/installation/linux/centos/
+
+- name: remove all potentially pre existing packages
+  yum:
+    name: '{{ item }}'
+    state: absent
+  with_items:
+    - docker
+    - docker-common
+    - container-selinux
+    - docker-selinux
+    - docker-engine
+
+- name: install yum-utils
+  yum:
+    name: yum-utils
+    state: present
+
+- name: add docker ce repo
+  command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+
+# Note that Docker CE versions do not follow regular Docker versions, but look
+# like, for example: "17.03.0.el7"
+- name: install docker
+  yum:
+    name: 'docker-ce-{{ docker_version }}'
+    update_cache: yes
+    state: present

tools/config_management/roles/docker-from-docker-repo/tasks/debian.yml

@@ -5,12 +5,12 @@
   package:
     name: "{{ item }}"
     state: present
-  with_items: 
+  with_items:
     - apt-transport-https
     - ca-certificates
 
 - name: add apt key for the docker repository
-  apt_key: 
+  apt_key:
     keyserver: hkp://ha.pool.sks-keyservers.net:80
     id: 58118E89F3A912897C070ADBF76221572C52609D
     state: present
@@ -23,13 +23,13 @@
   register: apt_docker_repo
 
 - name: update apt's cache
-  apt: 
+  apt:
     update_cache: yes
   when: apt_key_docker_repo.changed or apt_docker_repo.changed
 
 - name: install docker-engine
   package:
     name: "{{ item }}"
     state: present
-  with_items: 
+  with_items:
     - docker-engine={{ docker_version }}*

tools/config_management/roles/docker-from-docker-repo/tasks/main.yml

@@ -2,15 +2,9 @@
 # Set up Docker
 # See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install
 
-- include_role:
-    name: docker-prerequisites
-
 # Distribution-specific tasks:
 - include: debian.yml
   when: ansible_os_family == "Debian"
 
 - include: redhat.yml
   when: ansible_os_family == "RedHat"
-
-- include_role:
-    name: docker-configuration

tools/config_management/roles/docker-from-docker-repo/tasks/redhat.yml

@@ -13,13 +13,13 @@
     state: present
 
 - name: update yum's cache
-  yum: 
+  yum:
     name: "*"
     update_cache: yes
 
 - name: install docker-engine
   package:
     name: "{{ item }}"
     state: present
-  with_items: 
+  with_items:
     - docker-engine-{{ docker_version }}