Merge pull request #584 from weaveworks/535-vendor

Build in a container, use go1.5, vendor Dependancies, build for Darwin and Arm on Circle.

Makefile

@@ -1,8 +1,7 @@
 .PHONY: all deps static clean client-lint client-test client-sync backend frontend
 
 # If you can use Docker without being root, you can `make SUDO= <target>`
-SUDO=sudo
-DOCKER_SQUASH=$(shell which docker-squash 2>/dev/null)
+SUDO=sudo -E
 DOCKERHUB_USER=weaveworks
 APP_EXE=app/scope-app
 PROBE_EXE=probe/scope-probe
@@ -17,8 +16,9 @@ SCOPE_VERSION=$(shell git rev-parse --short HEAD)
 DOCKER_VERSION=1.3.1
 DOCKER_DISTRIB=docker/docker-$(DOCKER_VERSION).tgz
 DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(DOCKER_VERSION).tgz
-RUNSVINIT=docker/runsvinit
+RUNSVINIT=vendor/runsvinit/runsvinit
 RM=--rm
+BUILD_IN_CONTAINER=true
 
 all: $(SCOPE_EXPORT)
 
@@ -29,25 +29,24 @@ docker/weave:
 	curl -L git.io/weave -o docker/weave
 	chmod u+x docker/weave
 
-$(SCOPE_EXPORT): $(APP_EXE) $(PROBE_EXE) $(DOCKER_DISTRIB) docker/weave $(RUNSVINIT) docker/Dockerfile docker/run-app docker/run-probe docker/entrypoint.sh docker/ca-certificates.crt
-	@if [ -z '$(DOCKER_SQUASH)' ] ; then echo "Please install docker-squash by running 'make deps' (and make sure GOPATH/bin is in your PATH)." && exit 1 ; fi
-	cp $(APP_EXE) $(PROBE_EXE) docker/
+$(SCOPE_EXPORT): $(APP_EXE) $(PROBE_EXE) $(DOCKER_DISTRIB) docker/weave $(RUNSVINIT) docker/Dockerfile docker/run-app docker/run-probe docker/entrypoint.sh
+	cp $(APP_EXE) $(PROBE_EXE) $(RUNSVINIT) docker/
 	cp $(DOCKER_DISTRIB) docker/docker.tgz
 	$(SUDO) docker build -t $(SCOPE_IMAGE) docker/
-	$(SUDO) docker save $(SCOPE_IMAGE):latest | sudo $(DOCKER_SQUASH) -t $(SCOPE_IMAGE) | tee $@ | $(SUDO) docker load
-
-docker/ca-certificates.crt: /etc/ssl/certs/ca-certificates.crt
-	cp $? $@
+	$(SUDO) docker save $(SCOPE_IMAGE):latest > $@
 
 $(RUNSVINIT): vendor/runsvinit/*.go
-	go build -o $@ github.com/weaveworks/scope/vendor/runsvinit
 
 $(APP_EXE): app/*.go render/*.go report/*.go xfer/*.go common/sanitize/*.go
 
 $(PROBE_EXE): probe/*.go probe/docker/*.go probe/kubernetes/*.go probe/endpoint/*.go probe/host/*.go probe/process/*.go probe/overlay/*.go report/*.go xfer/*.go common/sanitize/*.go common/exec/*.go
 
+ifeq ($(BUILD_IN_CONTAINER),true)
+$(APP_EXE) $(PROBE_EXE) $(RUNSVINIT): $(SCOPE_BACKEND_BUILD_UPTODATE)
+	$(SUDO) docker run -ti $(RM) -v $(shell pwd):/go/src/github.com/weaveworks/scope -e GOARCH -e GOOS \
+		$(SCOPE_BACKEND_BUILD_IMAGE) $@
+else
 $(APP_EXE) $(PROBE_EXE):
-	go get -d -tags netgo ./$(@D)
 	go build -ldflags "-extldflags \"-static\" -X main.version $(SCOPE_VERSION)" -tags netgo -o $@ ./$(@D)
 	@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 	        rm $@; \
@@ -58,56 +57,67 @@ $(APP_EXE) $(PROBE_EXE):
 	        false; \
 	    }
 
+$(RUNSVINIT):
+	go build -ldflags "-extldflags \"-static\"" -o $@ ./$(@D)
+endif
+
 static: client/build/app.js
 	esc -o app/static.go -prefix client/build client/build
 
+ifeq ($(BUILD_IN_CONTAINER),true)
 client/build/app.js: client/app/scripts/*
 	mkdir -p client/build
-	docker run -ti $(RM) -v $(shell pwd)/client/app:/home/weave/app \
+	$(SUDO) docker run -ti $(RM) -v $(shell pwd)/client/app:/home/weave/app \
 		-v $(shell pwd)/client/build:/home/weave/build \
 		$(SCOPE_UI_BUILD_IMAGE) npm run build
 
 client-test: client/test/*
-	docker run -ti $(RM) -v $(shell pwd)/client/app:/home/weave/app \
+	$(SUDO) docker run -ti $(RM) -v $(shell pwd)/client/app:/home/weave/app \
 		-v $(shell pwd)/client/test:/home/weave/test \
 		$(SCOPE_UI_BUILD_IMAGE) npm test
 
 client-lint:
-	docker run -ti $(RM) -v $(shell pwd)/client/app:/home/weave/app \
+	$(SUDO) docker run -ti $(RM) -v $(shell pwd)/client/app:/home/weave/app \
 		-v $(shell pwd)/client/test:/home/weave/test \
 		$(SCOPE_UI_BUILD_IMAGE) npm run lint
 
 client-start:
-	docker run -ti $(RM) --net=host -v $(shell pwd)/client/app:/home/weave/app \
+	$(SUDO) docker run -ti $(RM) --net=host -v $(shell pwd)/client/app:/home/weave/app \
 		-v $(shell pwd)/client/build:/home/weave/build \
 		$(SCOPE_UI_BUILD_IMAGE) npm start
+endif
 
 $(SCOPE_UI_BUILD_UPTODATE): client/Dockerfile client/package.json client/webpack.local.config.js client/webpack.production.config.js client/server.js client/.eslintrc
-	docker build -t $(SCOPE_UI_BUILD_IMAGE) client
+	$(SUDO) docker build -t $(SCOPE_UI_BUILD_IMAGE) client
 	touch $@
 
 $(SCOPE_BACKEND_BUILD_UPTODATE): backend/*
-	docker build -t $(SCOPE_BACKEND_BUILD_IMAGE) backend
+	$(SUDO) docker build -t $(SCOPE_BACKEND_BUILD_IMAGE) backend
 	touch $@
 
-backend: $(SCOPE_BACKEND_BUILD_UPTODATE)
-	docker run -ti $(RM) -v $(shell pwd):/go/src/github.com/weaveworks/scope $(SCOPE_BACKEND_BUILD_IMAGE) /build.bash
-
 frontend: $(SCOPE_UI_BUILD_UPTODATE)
 
 clean:
 	go clean ./...
-	rm -rf $(SCOPE_EXPORT) $(SCOPE_UI_BUILD_EXPORT) $(APP_EXE) $(PROBE_EXE) client/build/app.js docker/weave
+	$(SUDO) docker rmi $(SCOPE_UI_BUILD_IMAGE) $(SCOPE_BACKEND_BUILD_IMAGE) >/dev/null 2>&1 || true
+	rm -rf $(SCOPE_EXPORT) $(SCOPE_UI_BUILD_UPTODATE) $(SCOPE_BACKEND_BUILD_UPTODATE) \
+		$(APP_EXE) $(PROBE_EXE) $(RUNSVINIT) client/build/app.js docker/weave
+
+ifeq ($(BUILD_IN_CONTAINER),true)
+tests:
+	$(SUDO) docker run -ti $(RM) -v $(shell pwd):/go/src/github.com/weaveworks/scope \
+		-e GOARCH -e GOOS -e CIRCLECI --entrypoint=/bin/sh $(SCOPE_BACKEND_BUILD_IMAGE) -c \
+		"cd /go/src/github.com/weaveworks/scope && ./tools/test -no-go-get"
+else
+tests:
+	./tools/test -no-go-get
+endif
 
 deps:
 	go get -u -f -tags netgo \
-		github.com/jwilder/docker-squash \
 		github.com/golang/lint/golint \
 		github.com/fzipp/gocyclo \
 		github.com/mattn/goveralls \
 		github.com/mjibson/esc \
 		github.com/kisielk/errcheck \
-		github.com/aktau/github-release
-
-update:
-	go get -u -f -v -tags netgo ./...
+		github.com/weaveworks/github-release

README.md

@@ -142,15 +142,13 @@ sudo scope launch --service-token=<token>
 The build is in five stages. `make deps` installs some tools we use later in
 the build. `make frontend` builds a UI build image with all NPM dependencies.
 `make static` compiles the UI into `static.go` which is part of the repository
-for convenience. `make backend` builds the backend Go app which then includes
-the static files. The final `make` pushes the app into a Docker image called
-**weaveworks/scope**.
+for convenience. The final `make` builds the app and probe, in a container,
+and pushes the lot into a Docker image called **weaveworks/scope**.
 
 ```
 make deps
 make frontend
 make static
-make backend
 make
 ```
 

backend/Dockerfile

@@ -1,4 +1,5 @@
 FROM golang:1.5.1
+ENV GO15VENDOREXPERIMENT 1
 RUN apt-get update && apt-get install -y libpcap-dev
-COPY build.bash /
-
+COPY build.sh /
+ENTRYPOINT ["/build.sh"]

backend/build.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -eux
+
+# Mount the scope repo:
+#  -v $(pwd):/go/src/github.com/weaveworks/scope
+
+cd $GOPATH/src/github.com/weaveworks/scope
+rm $1 2>/dev/null || true
+make BUILD_IN_CONTAINER=false $@
+

circle.yml

@@ -7,8 +7,7 @@ machine:
   services:
     - docker
   environment:
-    GOPATH: /home/ubuntu:$GOPATH
-    TOOLS: /home/ubuntu/src/github.com/weaveworks/tools
+    GOPATH: /home/ubuntu
     SRCDIR: /home/ubuntu/src/github.com/weaveworks/scope
     PATH: $PATH:$HOME/.local/bin
     CLOUDSDK_CORE_DISABLE_PROMPTS: 1
@@ -20,31 +19,30 @@ dependencies:
   cache_directories:
     - "~/docker"
   post:
-    - mkdir -p $TOOLS
-    - git clone https://github.com/weaveworks/tools.git $TOOLS
     - sudo apt-get update
-    - sudo apt-get --only-upgrade install tar libpcap0.8-dev
     - sudo apt-get install jq pv
     - curl https://sdk.cloud.google.com | bash
     - test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD"
     - go get $WEAVE_REPO/...
     - make -C $WEAVE_ROOT testing/runner/runner
-    - go version
-    - go clean -i net
-    - go install -tags netgo std
     - make deps
     - mkdir -p $(dirname $SRCDIR)
     - cp -r $(pwd)/ $SRCDIR
-    - cd $SRCDIR/client; $TOOLS/rebuild-image weaveworks/scope-ui-build . Dockerfile package.json webpack.production.config.js .eslintrc
+    - cd $SRCDIR/client; ../tools/rebuild-image weaveworks/scope-ui-build . Dockerfile package.json webpack.production.config.js .eslintrc
+    - touch $SRCDIR/.scope_ui_build.uptodate
+    - cd $SRCDIR/backend; ../tools/rebuild-image weaveworks/scope-backend-build . Dockerfile build.sh
+    - touch $SRCDIR/.scope_backend_build.uptodate
 
 test:
   override:
-    - cd $SRCDIR; $TOOLS/lint .
+    - cd $SRCDIR; ./tools/lint .
+    - cd $SRCDIR; make RM= tests
     - cd $SRCDIR; make RM= client-test
     - cd $SRCDIR; make RM= static
-    - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; make
-    - cd $SRCDIR; $TOOLS/test -slow
-    - cd $SRCDIR/experimental; make
+    - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; GOARCH=arm make RM= app/scope-app probe/scope-probe
+    - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; GOOS=darwin make RM= app/scope-app probe/scope-probe
+    - cd $SRCDIR; rm -f app/scope-app probe/scope-probe; make RM=
+    - cd $SRCDIR/experimental; make RM=
     - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh setup)
     - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./setup.sh)
     - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./run_all.sh):

docker/Dockerfile

@@ -10,6 +10,5 @@ ADD ./weave /usr/bin/
 COPY ./scope-app ./scope-probe ./runsvinit ./entrypoint.sh /home/weave/
 COPY ./run-app /etc/service/app/run
 COPY ./run-probe /etc/service/probe/run
-COPY ./ca-certificates.crt /etc/ssl/certs/
 EXPOSE 4040
 ENTRYPOINT ["/home/weave/entrypoint.sh"]

experimental/Makefile

@@ -1,13 +1,20 @@
 .PHONY: all test clean
 
-DIRS=$(shell find . -maxdepth 2 -name *.go -printf "%h\n" | uniq)
+DIRS=$(shell find . -maxdepth 2 -name *.go -printf "%h\n" | sort -u)
 TARGETS=$(join $(patsubst %,%/,$(DIRS)),$(DIRS))
+BUILD_IN_CONTAINER=true
+RM=--rm
 
 all: $(TARGETS)
 
+ifeq ($(BUILD_IN_CONTAINER),true)
+$(TARGETS):
+	$(SUDO) docker run -ti $(RM) -v $(shell pwd)/../:/go/src/github.com/weaveworks/scope -e GOARCH -e GOOS \
+		weaveworks/scope-backend-build -C experimental $@
+else
 $(TARGETS):
-	go get -tags netgo ./$(@D)
 	go build -ldflags "-extldflags \"-static\"" -tags netgo -o $@ ./$(@D)
+endif
 
 test:
 	go test ./...

integration/310_container_to_container_edge_test.sh

@@ -4,11 +4,11 @@
 
 start_suite "Test short lived connections between containers"
 
-weave_on $HOST1 launch
+WEAVE_NO_FASTDP=true weave_on $HOST1 launch
 scope_on $HOST1 launch
 weave_on $HOST1 run -d --name nginx nginx
 weave_on $HOST1 run -d --name client alpine /bin/sh -c "while true; do \
-	wget http://nginx.weave.local:80/ >/dev/null || true; \
+	wget http://nginx.weave.local:80/ -O - >/dev/null || true; \
 	sleep 1; \
 done"
 

integration/320_container_edge_cross_host_2_test.sh

@@ -4,8 +4,8 @@
 
 start_suite "Test short lived connections between containers on different hosts"
 
-weave_on $HOST1 launch $HOST1 $HOST2
-weave_on $HOST2 launch $HOST1 $HOST2
+WEAVE_NO_FASTDP=true weave_on $HOST1 launch $HOST1 $HOST2
+WEAVE_NO_FASTDP=true weave_on $HOST2 launch $HOST1 $HOST2
 
 scope_on $HOST1 launch
 scope_on $HOST2 launch

tools/.gitignore

@@ -0,0 +1,4 @@
+cover/cover
+socks/proxy
+socks/image.tar
+runner/runner

tools/README.md

@@ -0,0 +1,36 @@
+# Weaveworks Build Tools
+
+Included in this repo are tools shared by weave.git and scope.git.  They include
+
+- ```cover```: a tool which merges overlapping coverage reports generated by go
+  test
+- ```lint```: a script to lint Go project; runs various tools like golint, go
+  vet, errcheck etc
+- ```rebuild-image```: a script to rebuild docker images when their input files
+  change; useful when you using docker images to build your software, but you
+  don't want to build the image every time.
+- ```socks```: a simple, dockerised SOCKS proxy for getting your laptop onto
+  the Weave network
+- ```test```: a script to run all go unit tests in subdirectories, gather the
+  coverage results, and merge them into a single report.
+- ```runner```: a tool for running tests in parallel; given each test is
+  suffixed with the number of hosts it requires, and the hosts available are
+  contained in the environment variable HOSTS, the tool will run tests in
+  parallel, on different hosts.
+
+## Using build-tools.git
+
+To allow you to tie your code to a specific version of build-tools.git, such
+that future changes don't break you, we recommendation that you [`git subtree`]()
+this repository into your own repository:
+
+[`git subtree`]: http://blogs.atlassian.com/2013/05/alternatives-to-git-submodule-git-subtree/
+
+```
+git subtree add --prefix tools https://github.com/weaveworks/build-tools.git master --squash
+````
+
+To update the code in build-tools.git, the process is therefore:
+- PR into build-tools.git, go through normal review process etc.
+- Do `git subtree pull --prefix tools https://github.com/weaveworks/build-tools.git master --squash`
+  in your repo, and PR that.

tools/circle.yml

@@ -0,0 +1,23 @@
+machine:
+  services:
+    - docker
+  environment:
+    GOPATH: /home/ubuntu
+    SRCDIR: /home/ubuntu/src/github.com/weaveworks/tools
+    PATH: $PATH:$HOME/bin
+
+dependencies:
+  post:
+    - go clean -i net
+    - go install -tags netgo std
+    - mkdir -p $(dirname $SRCDIR)
+    - cp -r $(pwd)/ $SRCDIR
+    - go get github.com/golang/lint/golint github.com/fzipp/gocyclo github.com/kisielk/errcheck
+
+test:
+  override:
+    - cd $SRCDIR; ./lint .
+    - cd $SRCDIR/cover; make
+    - cd $SRCDIR/socks; make
+    - cd $SRCDIR/runner; make
+

tools/cover/Makefile

@@ -0,0 +1,11 @@
+.PHONY: all clean
+
+all: cover
+
+cover: *.go
+	go get -tags netgo ./$(@D)
+	go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D)
+
+clean:
+	rm -rf cover
+	go clean ./...