Skip to content

Commit

Permalink
Consolidate build tools in to ./tools/src/, Telepresence-style
Browse files Browse the repository at this point in the history
Signed-off-by: Luke Shumaker <[email protected]>
  • Loading branch information
LukeShu committed Sep 17, 2021
1 parent 7cbac83 commit a65c0a6
Show file tree
Hide file tree
Showing 55 changed files with 227 additions and 372 deletions.
4 changes: 2 additions & 2 deletions _cxx/envoy.mk
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,13 @@ $(OSS_HOME)/_cxx/go-control-plane: FORCE
git checkout $(ENVOY_GO_CONTROL_PLANE_COMMIT); \
}

$(OSS_HOME)/_cxx/envoy-build-image.txt: $(OSS_HOME)/_cxx/envoy $(WRITE_IFCHANGED) FORCE
$(OSS_HOME)/_cxx/envoy-build-image.txt: $(OSS_HOME)/_cxx/envoy $(tools/write-ifchanged) FORCE
@PS4=; set -ex -o pipefail; { \
pushd $</ci; \
echo "$$(pwd)"; \
. envoy_build_sha.sh; \
popd; \
echo docker.io/envoyproxy/envoy-build-ubuntu:$$ENVOY_BUILD_SHA | $(WRITE_IFCHANGED) $@; \
echo docker.io/envoyproxy/envoy-build-ubuntu:$$ENVOY_BUILD_SHA | $(tools/write-ifchanged) $@; \
}

$(OSS_HOME)/_cxx/envoy-build-container.txt: $(OSS_HOME)/_cxx/envoy-build-image.txt FORCE
Expand Down
60 changes: 0 additions & 60 deletions build-aux-local/generate.mk
Original file line number Diff line number Diff line change
Expand Up @@ -76,66 +76,6 @@ comma=,

gomoddir = $(shell cd $(OSS_HOME); go list $1/... >/dev/null 2>/dev/null; go list -m -f='{{.Dir}}' $1)

#
# Tools we need to install for `make generate`

clobber: _makefile_clobber
_makefile_clobber:
rm -rf $(OSS_HOME)/bin_*/
.PHONY: _makefile_clobber

GOHOSTOS=$(call lazyonce,GOHOSTOS,$(shell go env GOHOSTOS))
GOHOSTARCH=$(call lazyonce,GOHOSTARCH,$(shell go env GOHOSTARCH))

# PROTOC_VERSION must be at least 3.8.0 in order to contain the fix so that it doesn't generate
# invalid Python if you name an Enum member the same as a Python keyword.
PROTOC_VERSION = 3.8.0
PROTOC_PLATFORM = $(patsubst darwin,osx,$(GOHOSTOS))-$(patsubst amd64,x86_64,$(patsubst 386,x86_32,$(GOHOSTARCH)))
tools/protoc = $(OSS_HOME)/bin_$(GOHOSTOS)_$(GOHOSTARCH)/bin/protoc
$(tools/protoc): $(OSS_HOME)/build-aux-local/generate.mk
mkdir -p $(dir $(@D))
set -o pipefail; curl --fail -L https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_PLATFORM).zip | bsdtar -C $(dir $(@D)) -xf -
chmod 755 $@

# The version number of protoc-gen-go is controlled by `./go.mod`. Additionally, the package name is
# mentioned in `./pkg/ignore/pin.go`, so that `go mod tidy` won't make the `go.mod` file forget about
# it.
tools/protoc-gen-go = $(OSS_HOME)/bin_$(GOHOSTOS)_$(GOHOSTARCH)/protoc-gen-go
$(tools/protoc-gen-go): $(OSS_HOME)/go.mod
mkdir -p $(@D)
cd $(OSS_HOME) && go build -o $@ github.com/golang/protobuf/protoc-gen-go

GRPC_WEB_VERSION = 1.0.3
GRPC_WEB_PLATFORM = $(GOHOSTOS)-x86_64
tools/protoc-gen-grpc-web = $(OSS_HOME)/bin_$(GOHOSTOS)_$(GOHOSTARCH)/protoc-gen-grpc-web
$(tools/protoc-gen-grpc-web): $(OSS_HOME)/build-aux-local/generate.mk
mkdir -p $(@D)
curl -o $@ -L --fail https://github.com/grpc/grpc-web/releases/download/$(GRPC_WEB_VERSION)/protoc-gen-grpc-web-$(GRPC_WEB_VERSION)-$(GRPC_WEB_PLATFORM)
chmod 755 $@

# The version number of protoc-gen-validate is controlled by `./go.mod`. Additionally, the package
# name is mentioned in `./pkg/ignore/pin.go`, so that `go mod tidy` won't make the `go.mod` file
# forget about it.
tools/controller-gen = $(OSS_HOME)/bin_$(GOHOSTOS)_$(GOHOSTARCH)/controller-gen
$(tools/controller-gen): $(OSS_HOME)/go.mod
mkdir -p $(@D)
cd $(OSS_HOME) && go build -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen

tools/fix-crds = $(OSS_HOME)/bin_$(GOHOSTOS)_$(GOHOSTARCH)/fix-crds
$(tools/fix-crds): FORCE
mkdir -p $(@D)
cd $(OSS_HOME) && go build -o $@ github.com/datawire/ambassador/v2/cmd/fix-crds

tools/go-mkopensource = $(OSS_HOME)/bin_$(GOHOSTOS)_$(GOHOSTARCH)/go-mkopensource
$(tools/go-mkopensource): FORCE
mkdir -p $(@D)
cd $(OSS_HOME) && go build -o $@ github.com/datawire/ambassador/v2/cmd/go-mkopensource

tools/py-mkopensource = $(OSS_HOME)/bin_$(GOHOSTOS)_$(GOHOSTARCH)/py-mkopensource
$(tools/py-mkopensource): FORCE
mkdir -p $(@D)
cd $(OSS_HOME) && go build -o $@ github.com/datawire/ambassador/v2/cmd/py-mkopensource

#
# `make generate` vendor rules

Expand Down
6 changes: 0 additions & 6 deletions build-aux-local/lint.mk
Original file line number Diff line number Diff line change
@@ -1,11 +1,5 @@
# Must be included *after* generate.mk, because we use GOHOSTOS and GOHOSTARCH defined there.

# The version number of golangci-lint is controllers in a go.mod file
tools/golangci-lint = $(OSS_HOME)/bin_$(GOHOSTOS)_$(GOHOSTARCH)/golangci-lint
$(tools/golangci-lint): $(OSS_HOME)/build-aux/bin-go/golangci-lint/go.mod
mkdir -p $(@D)
cd $(<D) && go build -o $@ github.com/golangci/golangci-lint/cmd/golangci-lint

lint/go-dirs = $(OSS_HOME)

lint:
Expand Down
9 changes: 0 additions & 9 deletions build-aux/bin-go/flock/go.mod

This file was deleted.

Empty file removed build-aux/bin-go/flock/go.sum
Empty file.
6 changes: 0 additions & 6 deletions build-aux/bin-go/go-bindata/pin.go

This file was deleted.

6 changes: 0 additions & 6 deletions build-aux/bin-go/golangci-lint/pin.go

This file was deleted.

3 changes: 0 additions & 3 deletions build-aux/bin-go/gotest2tap/go.mod

This file was deleted.

Empty file removed build-aux/bin-go/gotest2tap/go.sum
Empty file.
19 changes: 8 additions & 11 deletions build-aux/common.mk
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@
# - Alias: bin/% -> bin_$(GOHOSTOS)_$(GOHOSTARCH)/%
## common.mk targets ##
# (N/A)
#
# Dependencies of `clobber` MUST NOT depend on programs in
# `$(build-aux.bindir)/`.
ifeq ($(words $(filter $(abspath $(lastword $(MAKEFILE_LIST))),$(abspath $(MAKEFILE_LIST)))),1)
_common.mk := $(lastword $(MAKEFILE_LIST))
include $(dir $(_common.mk))prelude.mk
Expand Down Expand Up @@ -87,22 +84,22 @@ bin:

check: lint build
$(MAKE) -f $(firstword $(MAKEFILE_LIST)) test-suite.tap.summary
test-suite.tap: $(TAP_DRIVER)
@$(TAP_DRIVER) cat $(sort $(filter %.tap,$^)) > $@
test-suite.tap: $(tools/tap-driver)
@$(tools/tap-driver) cat $(sort $(filter %.tap,$^)) > $@

%.tap.summary: %.tap $(TAP_DRIVER)
@$(TAP_DRIVER) summarize $<
%.tap.summary: %.tap $(tools/tap-driver)
@$(tools/tap-driver) summarize $<

%.tap: %.tap.gen $(TAP_DRIVER) FORCE
@{ $(abspath $<) || true; } 2>&1 | tee $@ | $(TAP_DRIVER) stream -n $<
%.tap: %.tap.gen $(tools/tap-driver) FORCE
@{ $(abspath $<) || true; } 2>&1 | tee $@ | $(tools/tap-driver) stream -n $<
%.log: %.test FORCE
@$(abspath $<) >$@ 2>&1; echo :exit-status: $$? >>$@
%.tap: %.log %.test $(TAP_DRIVER)
%.tap: %.log %.test $(tools/tap-driver)
@{ \
printf '%s\n' 'TAP version 13' '1..1' && \
sed 's/^/#/' < $< && \
sed -n '$${ s/^:exit-status: 0$$/ok 1/; s/^:exit-status: 77$$/ok 1 # SKIP/; s/^:exit-status: .*/not ok 1/; p; }' < $<; \
} | tee $@ | $(TAP_DRIVER) stream -n $*.test
} | tee $@ | $(tools/tap-driver) stream -n $*.test

#
# Configure how Make works
Expand Down
31 changes: 12 additions & 19 deletions build-aux/docker.mk
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@
# - Target: : $(NAME).docker # build untagged image; define this for each image $(NAME)
## Outputs ##
#
# - Executable : WRITE_DOCKERTAGFILE ?= $(CURDIR)/build-aux/bin/write-dockertagfile
#
# - Variable : HAVE_DOCKER # non-empty if true, empty if false
# - Variable : docker.LOCALHOST # "host.docker.internal" on Docker for Desktop, "localhost" on Docker CE
#
Expand Down Expand Up @@ -46,16 +44,16 @@
#
# The simplest version of that is:
#
# IMAGE.docker: $(MOVE_IFCHANGED) FORCE
# IMAGE.docker: $(tools/move-ifchanged) FORCE
# docker build [email protected] .
# $(MOVE_IFCHANGED) [email protected] $@
# $(tools/move-ifchanged) [email protected] $@
#
# If you have multiple `Dockerfile` at `IMAGE/Dockerfile`, you
# might write a pattern rule:
#
# %.docker: %/Dockerfile $(MOVE_IFCHANGED) FORCE
# %.docker: %/Dockerfile $(tools/move-ifchanged) FORCE
# docker build [email protected] $*
# $(MOVE_IFCHANGED) [email protected] $@
# $(tools/move-ifchanged) [email protected] $@
#
# Unless you have a good reason to, you shouldn't concern yourself
# with tagging the image in this rule.
Expand Down Expand Up @@ -149,9 +147,9 @@
# `--build-arg=` or `-f`:
#
# # Set a custom --build-arg, and use a funny Dockerfile name
# myimage.docker: $(MOVE_IFCHANGED) FORCE
# myimage.docker: $(tools/move-ifchanged) FORCE
# docker build --iidfile=$(@D)/.tmp.$(@F).tmp --build-arg=FOO=BAR -f Dockerfile.myimage .
# $(MOVE_IFCHANGED) $(@D)/.tmp.$(@F).tmp $@
# $(tools/move-ifchanged) $(@D)/.tmp.$(@F).tmp $@
#
# 2. In `ambassador.git`, building the Envoy binary is slow, so we
# might want to try pulling it from a build-cache Docker
Expand All @@ -160,22 +158,22 @@
# # Building this is expensive, so try grabbing a cached
# # version before trying to build it. This goes ahead and
# # tags the image, for caching purposes.
# base-envoy.docker: $(WRITE_IFCHANGED) $(var.)BASE_ENVOY_IMAGE_CACHE
# base-envoy.docker: $(tools/write-ifchanged) $(var.)BASE_ENVOY_IMAGE_CACHE
# if ! docker run --rm --entrypoint=true $(BASE_ENVOY_IMAGE_CACHE); then \
# $(MAKE) envoy-bin/envoy-static-stripped
# docker build -t $(BASE_ENVOY_IMAGE_CACHE) -f Dockerfile.base-envoy; \
# fi
# docker image inspect $(BASE_ENVOY_IMAGE_CACHE) --format='{{.Id}}' | $(WRITE_IFCHANGED) $@
# docker image inspect $(BASE_ENVOY_IMAGE_CACHE) --format='{{.Id}}' | $(tools/write-ifchanged) $@
#
# 3. In `apro.git`, we have many Docker images to build; each with
# a Dockerfile at `docker/NAME/Dockerfile`. We accomplish this
# with a simple pattern rule only slightly more complex than
# the one given in the "Building" section:
#
# %.docker: %/Dockerfile $(MOVE_IFCHANGED) FORCE
# %.docker: %/Dockerfile $(tools/move-ifchanged) FORCE
# # Try with --pull, fall back to without --pull
# docker build --iidfile=$(@D)/.tmp.$(@F).tmp --pull $* || docker build --iidfile=$(@D)/.tmp.$(@F).tmp $*
# $(MOVE_IFCHANGED) $(@D)/.tmp.$(@F).tmp $@
# $(tools/move-ifchanged) $(@D)/.tmp.$(@F).tmp $@
#
# The `--pull` is a good way to ensure that we incorporate any
# patches to the base images that we build ours FROM. However,
Expand Down Expand Up @@ -211,11 +209,6 @@ _docker.tag.groups = $(patsubst docker.tag.%,%,$(filter docker.tag.%,$(.VARIABLE
# clean.groups is separate from tag.groups as a special-case for docker-cluster.mk
_docker.clean.groups += $(_docker.tag.groups)

#
# Executables

WRITE_DOCKERTAGFILE ?= $(build-aux.bindir)/write-dockertagfile

#
# Output variables

Expand All @@ -242,9 +235,9 @@ define _docker.tag.rule
# line 2: tag 1
# line 3: tag 2
# ...
%.docker.tag.$(_docker.tag.group): %.docker $$(WRITE_DOCKERTAGFILE) FORCE
%.docker.tag.$(_docker.tag.group): %.docker $$(tools/write-dockertagfile) FORCE
# The 'foreach' is to handle newlines as normal whitespace
printf '%s\n' $$$$(cat $$<) $$(foreach v,$$(docker.tag.$(_docker.tag.group)),$$v) | $$(WRITE_DOCKERTAGFILE) $$@
printf '%s\n' $$$$(cat $$<) $$(foreach v,$$(docker.tag.$(_docker.tag.group)),$$v) | $$(tools/write-dockertagfile) $$@

# file contents:
# line 1: image ID
Expand Down
16 changes: 8 additions & 8 deletions build-aux/docs/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,22 +86,22 @@ directory, you could write:
## Adding your own test runners

To add a new test runner, you just need a command that emits TAP:
`tee` it to a `.tap` file, and pipe that to `$(TAP_DRIVER) stream -n
`tee` it to a `.tap` file, and pipe that to `$(tools/tap-driver) stream -n
TEST_GROUP_NAME` to pretty-print the results as they happen:

# Tell Make how to run the test command, and stream the results to
# `$(TAP_DRIVER) stream` to pretty-print the results as they happen.
my-test.tap: my-test.input $(TAP_DRIVER) FORCE
@SOME_COMMAND_THAT_EMITS_TAP 2>&1 | tee $@ | $(TAP_DRIVER) stream -n my-test
# `$(tools/tap-driver) stream` to pretty-print the results as they happen.
my-test.tap: my-test.input $(tools/tap-driver) FORCE
@SOME_COMMAND_THAT_EMITS_TAP 2>&1 | tee $@ | $(tools/tap-driver) stream -n my-test

# Tell Make to include 'my-test' in `make check`
test-suite.tap: my-test.tap

For example, to use [BATS (Bash Automated Testing System)][BATS], you
would write:

%.tap: %.bats $(TAP_DRIVER) FORCE
@bats --tap $< | tee $@ | $(TAP_DRIVER) stream -n $<
%.tap: %.bats $(tools/tap-driver) FORCE
@bats --tap $< | tee $@ | $(tools/tap-driver) stream -n $<

# Automatically include `./tests/*.bats`
test-suite.tap: $(patsubst %.bats,%.tap,$(wildcard tests/*.bats))
Expand All @@ -117,8 +117,8 @@ cons of which I won't comment on here), you should be sure that if
your test-runner indicates success or failure with an exit code, that
you ignore that exit code:

%.tap: %.bats $(TAP_DRIVER) FORCE
@{ bats --tap $< || true; } | tee $@ | $(TAP_DRIVER) stream -n $<
%.tap: %.bats $(tools/tap-driver) FORCE
@{ bats --tap $< || true; } | tee $@ | $(tools/tap-driver) stream -n $<

## Adding dependencies of tests

Expand Down
52 changes: 0 additions & 52 deletions build-aux/prelude.mk
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,6 @@
# - Variable: export GOHOSTOS
# - Variable: export GOHOSTARCH
# - Variable: build-aux.dir
# - Variable: build-aux.bindir
# - Function: build-aux.bin-go.rule
# - Executable: FLOCK ?= $(CURDIR)/build-aux/bin/flock # or /usr/bin/flock
# - Executable: WRITE_IFCHANGED ?= $(CURDIR)/build-aux/bin/write-ifchanged
# - Executable: COPY_IFCHANGED ?= $(CURDIR)/build-aux/bin/copy-ifchanged
# - Executable: MOVE_IFCHANGED ?= $(CURDIR)/build-aux/bin/move-ifchanged
# - Executable: TAP_DRIVER ?= $(CURDIR)/build-aux/bin/tap-driver
#
# Other support:
# - Function: joinlist
Expand All @@ -53,19 +46,10 @@
# Internal use:
# - Variable: _prelude.go.VERSION (exposed as go-mod.mk:go.goversion)
# - Function: _prelude.go.VERSION.HAVE (exposed as go-mod.mk:go.goversion.HAVE)
# - Variable: _prelude.go.lock (exposed as go-mod.mk:go.lock)
# - Variable: _prelude.go.ensure (used by go-mod.mk)
#
## common.mk targets ##
# - clobber
#
# `include`ing this file on its own does not introduce a dependency on
# `go`, but:
# - Calling the `build-aux.bin-go.rule` function introduces a hard
# dependency Go 1.11.4+.
# - Using the $(FLOCK) executable introduces a hard dependency on Go
# 1.11+ on systems that don't have a native `flock(1)` program
# (macOS).
ifeq ($(words $(filter $(abspath $(lastword $(MAKEFILE_LIST))),$(abspath $(MAKEFILE_LIST)))),1)
_prelude.mk := $(lastword $(MAKEFILE_LIST))

Expand Down Expand Up @@ -125,42 +109,6 @@ endif
# Variable constants

build-aux.dir = $(patsubst %/,%,$(dir $(_prelude.mk)))
build-aux.bindir = $(abspath $(build-aux.dir)/bin)

#
# Executables
#
# Have this section toward the end, so that it can eagerly use stuff
# defined above.

FLOCK ?= $(call lazyonce,FLOCK,$(or $(shell which flock 2>/dev/null),$(_prelude.go.ensure)$(build-aux.bindir)/flock))
COPY_IFCHANGED ?= $(build-aux.bindir)/copy-ifchanged
MOVE_IFCHANGED ?= $(build-aux.bindir)/move-ifchanged
WRITE_IFCHANGED ?= $(build-aux.bindir)/write-ifchanged
TAP_DRIVER ?= $(build-aux.bindir)/tap-driver

$(build-aux.bindir):
mkdir $@

clobber: _clobber-prelude
_clobber-prelude:
rm -rf $(build-aux.bindir)
.PHONY: _clobber-prelude

$(build-aux.bindir)/%: $(build-aux.dir)/bin-sh/%.sh | $(build-aux.bindir)
install $< $@

# Usage: $(eval $(call build-aux.bin-go.rule,BINARY_NAME,GO_PACKAGE))
define build-aux.bin-go.rule
$$(build-aux.bindir)/.$(strip $1).stamp: $$(build-aux.bindir)/.%.stamp: $$(build-aux.dir)/bin-go/%/go.mod $$(_prelude.go.lock) FORCE | $$(build-aux.bindir)
cd $$(<D) && GO111MODULE=on $$(_prelude.go.lock)go build -o $$(abspath $$@) $2
endef
$(build-aux.bindir)/%: $(build-aux.bindir)/.%.stamp $(COPY_IFCHANGED)
$(COPY_IFCHANGED) $< $@

# bin/.flock.stamp doesn't use build-aux.bin-go.rule, because bootstrapping
$(build-aux.bindir)/.flock.stamp: $(build-aux.bindir)/.%.stamp: $(build-aux.dir)/bin-go/%/go.mod $(shell find $(build-aux.dir)/bin-go/flock) | $(build-aux.bindir)
cd $(<D) && GO111MODULE=on go build -o $(abspath $@) github.com/datawire/build-aux/bin-go/flock

#
# Targets
Expand Down
Loading

0 comments on commit a65c0a6

Please sign in to comment.