diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..e1edc05d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +.git +.github +.gitignore +.travis +.travis.yml + +.dockerignore +scm/build/Dockerfile + +build_danm.sh +run_uts.sh + +integration/ +example/ +vendor/ +ut_logs/ + +*.md +*.sh +*.png +*.svg diff --git a/.gitignore b/.gitignore index 2d09a253..4d168407 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.DS_Store bin/ vendor/ +ut_logs/ .idea/ diff --git a/build_buildah.sh b/build_buildah.sh deleted file mode 100644 index b30a4186..00000000 --- a/build_buildah.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -echo 'Building DANM builder container' -buildah bud --no-cache -t danm_builder:1.0 scm/build - -echo 'Running DANM build' -build_container=$(buildah from danm_builder:1.0) -buildah run --net=host -v $GOPATH/bin:/go/bin -v $GOPATH/src:/go/src $build_container /bin/sh -c /build.sh -buildah rm $build_container - -echo 'Cleaning up DANM builder container' -buildah rmi -f danm_builder:1.0 - -echo 'DANM libraries successfully built!' diff --git a/build_danm.sh b/build_danm.sh index 58b59729..fdf23fe4 100755 --- a/build_danm.sh +++ b/build_danm.sh @@ -1,7 +1,30 @@ -#!/bin/bash - +#!/bin/bash -e #ERR pseudo-signal is only supported by bash. +# +# Build script to create DANM container images. +# +# This script supports the following environment variables: +# +# - `TAG_PREFIX`. A string that will be prepended to all +# built image tags. This can, for example, be a registry +# name. Note that the string is prepended without any additional +# separators, so if it is eg. a registry name, it MUST end with +# a "/". +# +# - `EXTRA_BUILD_ARGS`. Any additional arguments that you want to +# provide to the container build. +# +# - `USE_CACHE`. If defined, use cache during image build process. +# For compativility with earlier versions of the build process, +# the default is to NOT use the cache. +# +# - `KEEP_BUILDER`. If defined, keep the builder image and tag +# it. This can be useful if the image is to be re-used for +# unit testing, or if a developer wants to run an instance +# off the builder image as a work environment. +# + # error handling with trap taken from https://unix.stackexchange.com/questions/79648/how-to-trigger-error-using-trap-command/157327 unset killer_sig for sig in SIGHUP SIGINT SIGQUIT SIGTERM; do @@ -30,12 +53,78 @@ error_handler() exit $1 } -if [[ ( "$TRAVIS_PIPELINE" = "buildah" ) || ( "$TRAVIS_PIPELINE" = "" && -x "$(command -v buildah)" ) ]]; then - source ./build_buildah.sh -elif [[ ( "$TRAVIS_PIPELINE" = "docker" ) || ( "$TRAVIS_PIPELINE" = "" && -x "$(command -v docker)" ) ]]; then - source ./build_docker.sh + +# +# Don't use cache unless we're told otherwise. +# +if [ -z "${USE_CACHE}" ] +then + FIRST_BUILD_EXTRA_BUILD_ARGS="--no-cache" +fi + + +# +# Identify if we need to run docker or buildah +# +if [[ ( "$TRAVIS_PIPELINE" = "buildah" ) || ( "$TRAVIS_PIPELINE" = "" && -x "$(command -v buildah)" ) ]] +then + BUILD_COMMAND="buildah bud" + TAG_COMMAND="buildah tag" +elif [[ ( "$TRAVIS_PIPELINE" = "docker" ) || ( "$TRAVIS_PIPELINE" = "" && -x "$(command -v docker)" ) ]] +then + BUILD_COMMAND="docker image build" + TAG_COMMAND="docker image tag" else echo 'The build process requires docker or buildah/podman installed. Please install any of these and make sure these are executable' exit 1 fi + +# +# Construct a unique version number from the git commit +# hash that is being build. If the workspace isn't +# clean (ie. if "git status" would say anything other than +# "working tree clean"), add a _dirty suffix to that +# version number. +# +LATEST_TAG=$(git describe --tags) +COMMIT_HASH=$(git rev-parse --short=8 HEAD) +if [ -n "$(git status --porcelain)" ] +then + COMMIT_HASH="${COMMIT_HASH}_dirty" +fi + + +# +# Determine which build stages we want to tag as an image. +# +build_targets=(netwatcher svcwatcher webhook danm-cni-plugins) + +if [ -n "${KEEP_BUILDER}" ] +then + build_targets+=(builder) +fi + +# +# Build the various images. Each image is represented +# by one target in the multi-stage Dockerfile. +# +for plugin in ${build_targets[@]} +do + echo Building: ${plugin}, version ${COMMIT_HASH} + ${BUILD_COMMAND} \ + ${EXTRA_BUILD_ARGS} \ + ${FIRST_BUILD_EXTRA_BUILD_ARGS} \ + --build-arg LATEST_TAG=${LATEST_TAG} \ + --build-arg COMMIT_HASH=${COMMIT_HASH} \ + --tag ${TAG_PREFIX}${plugin}:${COMMIT_HASH} \ + --target ${plugin} \ + --file scm/build/Dockerfile \ + . + + # Tag image as "latest", too + ${TAG_COMMAND} ${TAG_PREFIX}${plugin}:${COMMIT_HASH} ${TAG_PREFIX}${plugin}:latest + + # Make sure we use the cache on the 2nd and subsequent iterations. + unset FIRST_BUILD_EXTRA_BUILD_ARGS +done diff --git a/build_docker.sh b/build_docker.sh deleted file mode 100644 index 8229dba5..00000000 --- a/build_docker.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -e - -echo 'Updating alpine base image' -docker pull golang:1.13-alpine3.10 - -echo 'Building DANM builder container' -docker build --no-cache --tag=danm_builder:1.0 scm/build - -echo 'Running DANM build' -docker run --rm --net=host --name=danm_build -v ${GOPATH}/bin:/go/bin -v ${GOPATH}/src:/go/src -v ${GOPATH}/pkg:/go/pkg danm_builder:1.0 - -echo 'Cleaning up DANM builder container' -docker rmi -f danm_builder:1.0 - -echo 'DANM libraries successfully built!' diff --git a/deployment-guide.md b/deployment-guide.md index e86f2313..9db2ff4b 100644 --- a/deployment-guide.md +++ b/deployment-guide.md @@ -20,44 +20,24 @@ To be able to do that, your development environment shall already have Docker da Note, that the project itself depends on Golang 1.10+ and glide being available, but we packaged these dependencies into an automatically created builder container, so you don't have to worry about them! ### Building the binaries -It is actually as easy as go get-ting the repository from GitHub, and executing the build_danm.sh script from the root of the project! +It is actually as easy as cloning the repository from GitHub, and executing the build_danm.sh script from the root of the project! ``` -go get -d github.com/nokia/danm -cd $GOPATH/src/github.com/nokia/danm +git clone github.com/nokia/danm +cd danm ./build_danm.sh ``` -This will first build the Alpine based builder container, mount the $GOPATH/src, $GOPATH/bin and $GOPATH/pkg directory into it, and invoke the necessary script to build all binaries inside the container. -The builder container destroys itself once its purpose has been fulfilled. -The result will be 6, statically linked binaries put into your $GOPATH/bin directory. +The result will four container images: -- danm +- `danm-cni-plugins`: This image contains the core CNI plugins (`danm`, `fakeipam`). + It will be deployed as a DaemonSet that puts these binaries in place in each Kubernetes node. -- fakeipam +- `netwatcher`: This image will be used by the `netwatcher` DaemonSet -- netwatcher +- `webhook`: This image will be used by the `webhook` pod -- svcwatcher +- `svcwatcher`: This image will be used by the `svcwatcher` DaemonSet if you choose to install it. -- webhook - -### Building the containers -Netwatcher, svcwatcher, and webhook binaries are built into their own containers. -The project contains example Dockerfiles for all of these components under the integration/docker directory. - -**Copy the respective binary into the right folder (netwatcher into integration/docker/netwatcher, svcwatcher into integration/docker/svcwatcher, webhook into integration/docker/webhook), then execute:** -``` -docker build -t netwatcher:latest integration/docker/netwatcher -docker build -t svcwatcher:latest integration/docker/svcwatcher -docker build -t webhook:latest integration/docker/webhook -``` -or -``` -buildah bud -t netwatcher:latest integration/docker/netwatcher -buildah bud -t svcwatcher:latest integration/docker/svcwatcher -buildah bud -t webhook:latest integration/docker/webhook -``` -This builds the respective containers. Afterwards, these containers can be directly integrated into a running Kubernetes cluster! ## Deployment The method of deploying the whole DANM suite into a Kubernetes cluster is the following. @@ -111,35 +91,36 @@ Also provision the necessary RBAC rules so DANM can do its job: kubectl create -f integration/cni_config/danm_rbac.yaml ``` -**4. Copy the "danm" binary into the configured CNI plugin directory of all your kubelet nodes' (by default it is /opt/cni/bin/):** -``` -/ # ls /opt/cni/bin -bridge dhcp flannel host-local loopback portmap sample tuning -**danm** host-device ipvlan macvlan ptp sriov vlan -``` -**5. Copy the "fakeipam" binary into the configured CNI plugin directory of all your kubelet nodes' (by default it is /opt/cni/bin/):** +**4. Onboard the netwatcher, svcwatcher, and webhook containers into the image registry of your cluster** + +**5. Create the cni-plugin DaemonSet by executing the following command from the project's root directory:** + ``` -/ # ls /opt/cni/bin -bridge dhcp flannel host-local loopback portmap sample tuning -danm **fakeipam** host-device ipvlan macvlan ptp sriov vlan +kubectl create -f integration/cni_config/danm_rbac.yaml +kubectl create -f integration/manifests/cni_plugins ``` -**6. OPTIONAL: Copy any CNI binaries (flannel, sriov, macvlan etc.) you would like to use in your cluster into the configured CNI plugin directory of all your kubelet nodes' (by default it is /opt/cni/bin/)** -**7. Onboard the netwatcher, svcwatcher, and webhook containers into the image registry of your cluster** +This DaemonSet will copy the `danm` and `fakeipam` binaries into the +`/opt/cni/bin` directory of each node. - **8. Create the netwatcher DaemonSet by executing the following command from the project's root directory:** - ``` +**6. OPTIONAL: Install any other CNI plugins (flannel, sriov etc.) you would like to use in your cluster** + +Specific installation steps depend on the CNI plugin; some require copying into `/opt/cni/bin` on +all nodes in your cluster, whereas others are installed using a DaemonSet (or a combination of both). + +**7. Create the netwatcher DaemonSet by executing the following command from the project's root directory:** +``` kubectl create -f integration/manifests/netwatcher/ ``` Note1: you should take a look at the example manifest, and possibly tailor it to your own environment first Note2: we assume RBAC is configured for the Kubernetes API, so the manifests include the required Role and ServiceAccount for this case. - **9. Create at least one DANM network to bootstrap your infrastructure Pods!** + **8. Create at least one DANM network to bootstrap your infrastructure Pods!** Otherwise you can easily fall into a catch 22 situation - you won't be able to bring-up Pods because you don't have network, but you cannot create networks because you cannot bring-up a Pod to validate them. Your bootstrap networking solution can be really anything you fancy! We use Flannel or Calico for the purpose in our environments, and connect Pods to it with such simple network descriptors like what you can find in **integration/bootstrap_networks**. - **10. Create the webhook Deployment and provide it with certificates by executing the following commands from the project's root directory:** + **9. Create the webhook Deployment and provide it with certificates by executing the following commands from the project's root directory:** Below scripts require the `jq` tool and `openssl`; please make sure you have them installed. diff --git a/integration/bootstrap_networks/lightweight/flannel.conf b/integration/bootstrap_networks/lightweight/flannel.conf new file mode 120000 index 00000000..4f0e20d7 --- /dev/null +++ b/integration/bootstrap_networks/lightweight/flannel.conf @@ -0,0 +1 @@ +../production/flannel.conf \ No newline at end of file diff --git a/integration/bootstrap_networks/production/flannel.conf b/integration/bootstrap_networks/production/flannel.conf new file mode 100644 index 00000000..63f44b78 --- /dev/null +++ b/integration/bootstrap_networks/production/flannel.conf @@ -0,0 +1,8 @@ +{ + "cniVersion": "0.3.1", + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } +} diff --git a/integration/docker/netwatcher/Dockerfile b/integration/docker/netwatcher/Dockerfile deleted file mode 100644 index afabe10d..00000000 --- a/integration/docker/netwatcher/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM alpine:latest -MAINTAINER Levente Kale - -COPY netwatcher /usr/local/bin/netwatcher - -RUN apk add --no-cache --virtual .tools curl libcap iputils \ -&& adduser -u 147 -D -H -s /sbin/nologin danm \ -&& chown root:danm /usr/local/bin/netwatcher \ -&& chmod 750 /usr/local/bin/netwatcher \ -&& setcap cap_sys_ptrace,cap_sys_admin,cap_net_admin=eip /usr/local/bin/netwatcher \ -&& setcap cap_net_raw=eip /usr/sbin/arping \ -&& apk del .tools - -USER danm - -WORKDIR / -ENTRYPOINT ["/usr/local/bin/netwatcher"] diff --git a/integration/docker/svcwatcher/Dockerfile b/integration/docker/svcwatcher/Dockerfile deleted file mode 100644 index fd5618b4..00000000 --- a/integration/docker/svcwatcher/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM alpine:latest -MAINTAINER Levente Kale - -COPY svcwatcher /usr/local/bin/svcwatcher - -RUN apk add --no-cache --virtual .tools curl libcap iputils \ -&& adduser -u 147 -D -H -s /sbin/nologin danm \ -&& chown root:danm /usr/local/bin/svcwatcher \ -&& chmod 750 /usr/local/bin/svcwatcher \ -&& apk del .tools - -USER danm - -WORKDIR / -ENTRYPOINT ["/usr/local/bin/svcwatcher"] diff --git a/integration/docker/webhook/Dockerfile b/integration/docker/webhook/Dockerfile deleted file mode 100644 index 6111d583..00000000 --- a/integration/docker/webhook/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM alpine:latest -MAINTAINER Levente Kale - -COPY webhook /usr/local/bin/webhook - -RUN apk add --no-cache --virtual .tools curl libcap iputils \ -&& adduser -u 147 -D -H -s /sbin/nologin danm \ -&& chown root:danm /usr/local/bin/webhook \ -&& chmod 750 /usr/local/bin/webhook \ -&& apk del .tools - -USER danm - -WORKDIR / - -ENTRYPOINT ["/usr/local/bin/webhook"] diff --git a/integration/manifests/cni_plugins/cni_plugins_ds.yml b/integration/manifests/cni_plugins/cni_plugins_ds.yml new file mode 100644 index 00000000..eba76fa4 --- /dev/null +++ b/integration/manifests/cni_plugins/cni_plugins_ds.yml @@ -0,0 +1,27 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: danm-cni + namespace: kube-system +spec: + selector: + matchLabels: + danm.k8s.io: danm-cni + template: + metadata: + labels: + danm.k8s.io: danm-cni + spec: + containers: + - name: danm-cni + image: danm-cni-plugins + volumeMounts: + - name: cni + mountPath: /host/cni + hostNetwork: true + terminationGracePeriodSeconds: 0 + volumes: + - name: cni + hostPath: + path: /opt/cni/bin diff --git a/run_uts.sh b/run_uts.sh index d8dba892..302ed802 100755 --- a/run_uts.sh +++ b/run_uts.sh @@ -1,15 +1,17 @@ -#!/bin/sh -e +#!/bin/bash -e +DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -echo 'Updating alpine base image' -docker pull golang:1.13-alpine3.10 +# Build DANM container images and keep the builder container. Use the cache, which is especially +# useful in a pipeline case where we may just have built the image a moment ago. +USE_CACHE=1 KEEP_BUILDER=1 "${DIR}"/build_danm.sh -echo 'Building DANM UT container' -docker build --no-cache --tag=danm_ut:1.0 scm/ut +COMMIT_HASH=$(git rev-parse --short=8 HEAD) +if [ -n "$(git status --porcelain)" ] +then + COMMIT_HASH="${COMMIT_HASH}_dirty" +fi echo 'Running DANM UT' -docker run --rm --net=host --name=danm_ut -v ${GOPATH}/bin:/go/bin -v ${GOPATH}/src:/go/src -v ${GOPATH}/pkg:/go/pkg -v /var/log:/var/log danm_ut:1.0 - -echo 'Cleaning up DANM UT container' -docker rmi -f danm_ut:1.0 +docker run --rm -v ${DIR}/ut_logs:/var/log ${TAG_PREFIX}builder:${COMMIT_HASH} scm/ut/run_uts.sh echo 'DANM tests were successfully executed!' diff --git a/scm/build/Dockerfile b/scm/build/Dockerfile index 578d4b50..580835a9 100644 --- a/scm/build/Dockerfile +++ b/scm/build/Dockerfile @@ -1,7 +1,91 @@ -FROM golang:1.13-alpine3.10 +# Set defaults for USERNAME, UID, GID +ARG USERNAME=danm +ARG UID=147 +ARG GID=147 + +# +# Stage: Build container. Used to build all binaries, once +# +FROM golang:1.13-alpine3.10 AS builder MAINTAINER Levente Kale +ARG USERNAME +ARG UID +ARG GID +ARG COMMIT_HASH +ARG LATEST_TAG + RUN apk add --no-cache ca-certificates make git bash -COPY build.sh /build.sh -ENTRYPOINT /build.sh +RUN mkdir -p ${GOPATH}/src/github.com/nokia/danm +COPY / ${GOPATH}/src/github.com/nokia/danm/ + +WORKDIR ${GOPATH}/src/github.com/nokia/danm + +RUN scm/build/build.sh \ + && adduser -u ${UID} -D -H -s /sbin/nologin ${USERNAME} \ + && chown root:${USERNAME} /go/bin/netwatcher /go/bin/svcwatcher /go/bin/webhook \ + && chmod 0750 /go/bin/* + + +# +# (Intermediate) Stage: Alping base image. Includes common user account and environment +# for netwatcher, svcwatcher, webhook +# +FROM alpine:latest AS base-alpine +ARG USERNAME +ARG UID +ARG GID + +RUN adduser -u ${UID} -D -H -s /sbin/nologin ${USERNAME} +WORKDIR / +USER ${USER} + +# +# Stage: Netwatcher +# +FROM base-alpine AS netwatcher +MAINTAINER Levente Kale + +COPY --from=builder /go/bin/netwatcher /usr/local/bin/netwatcher +RUN apk add --no-cache --virtual .tools libcap \ + && setcap cap_sys_ptrace,cap_sys_admin,cap_net_admin=eip /usr/local/bin/netwatcher \ + && apk del .tools +ENTRYPOINT ["/usr/local/bin/netwatcher"] + + +# +# Stage: Svcwatcher +# +FROM base-alpine AS svcwatcher +MAINTAINER Levente Kale + +COPY --from=builder /go/bin/svcwatcher /usr/local/bin/svcwatcher +ENTRYPOINT ["/usr/local/bin/svcwatcher"] + + +# +# Stage: Webhook +# +FROM base-alpine AS webhook +MAINTAINER Levente Kale + +COPY --from=builder /go/bin/webhook /usr/local/bin/webhook +ENTRYPOINT ["/usr/local/bin/webhook"] + + +# +# Stage: CNI plugins daemonset +# +# Note that unlike the other containers, this needs to run as root as +# it places CNI plugins into the host's filesystem. +# +FROM alpine:latest AS danm-cni-plugins + +VOLUME ["/host/cni"] + +RUN mkdir /cni +COPY --from=builder /go/bin/danm /go/bin/fakeipam /cni/ + +COPY scm/build/cni_ds/entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/scm/build/build.sh b/scm/build/build.sh index f76f0d90..7d7ed83c 100755 --- a/scm/build/build.sh +++ b/scm/build/build.sh @@ -6,6 +6,20 @@ export GOOS=linux export CGO_ENABLED=0 cd "${GOPATH}/src/github.com/nokia/danm" go mod vendor -LATEST_TAG=$(git describe --tags) -COMMIT_HASH=$(git rev-parse HEAD) -go install -mod=vendor -a -ldflags "-extldflags '-static' -X main.version=${LATEST_TAG} -X main.commitHash=${COMMIT_HASH}" github.com/nokia/danm/cmd/... \ No newline at end of file + +# +# If we're being invoked inside Docker by the /build_danm script, use the +# COMMIT_HASH and LATEST_TAG values that were already set by the invoking script. +# Otherwise (eg if build.sh invoked directly during development cycle), set them +# here. +# +if [ -z "${COMMIT_HASH}" ] +then + COMMIT_HASH=$(git rev-parse HEAD) +fi +if [ -z "${LATEST_TAG}" ] +then + LATEST_TAG=$(git describe --tags) +fi + +go install -mod=vendor -a -ldflags "-extldflags '-static' -X main.version=${LATEST_TAG} -X main.commitHash=${COMMIT_HASH}" github.com/nokia/danm/cmd/... diff --git a/scm/build/cni_ds/entrypoint.sh b/scm/build/cni_ds/entrypoint.sh new file mode 100755 index 00000000..73a2c9ab --- /dev/null +++ b/scm/build/cni_ds/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh -e + +echo "Copying plugins..." + +# Copy files into place and atomically move into final binary name +for plugin in /cni/* +do + plugin=$(basename "${plugin}") + + echo " - ${plugin}" + cp -fa /cni/"${plugin}" /host/cni/_"${plugin}" + mv -f /host/cni/_"${plugin}" /host/cni/"${plugin}" +done + +echo "Done. Sleeping..." +sleep infinity diff --git a/scm/ut/Dockerfile b/scm/ut/Dockerfile deleted file mode 100644 index c324c55c..00000000 --- a/scm/ut/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM golang:1.13-alpine3.10 -MAINTAINER Levente Kale - -RUN apk add --no-cache ca-certificates make git bash - -COPY run_uts.sh /run_uts.sh -ENTRYPOINT /run_uts.sh diff --git a/scm/ut/run_uts.sh b/scm/ut/run_uts.sh index 81465d3e..c99d49ff 100755 --- a/scm/ut/run_uts.sh +++ b/scm/ut/run_uts.sh @@ -26,7 +26,7 @@ function check_codegen_tampering { set -e export CGO_ENABLED=0 cd ${GOPATH}/src/github.com/nokia/danm -go mod vendor + run_uts check_codegen_tampering if [ -n "${DID_YOU_TAMPER}" ] @@ -34,4 +34,4 @@ then echo "${DID_YOU_TAMPER}" echo "Generated DANM client code was manually modified in the working copy, failing tests!" exit 1 -fi \ No newline at end of file +fi