From 5adddfb814bf72ce4ea164c3ec2034177ed85e3b Mon Sep 17 00:00:00 2001 From: Claes Mogren Date: Wed, 29 Apr 2020 12:17:08 -0700 Subject: [PATCH] Add init container --- .dockerignore | 1 + Makefile | 41 ++++-- config/master/aws-k8s-cni.yaml | 178 +++++++++++++++++++++++++ scripts/dockerfiles/Dockerfile.init | 30 +++++ scripts/dockerfiles/Dockerfile.release | 5 +- scripts/entrypoint.sh | 25 ++-- scripts/init.sh | 36 +++++ 7 files changed, 292 insertions(+), 24 deletions(-) create mode 100644 config/master/aws-k8s-cni.yaml create mode 100644 scripts/dockerfiles/Dockerfile.init create mode 100755 scripts/init.sh diff --git a/.dockerignore b/.dockerignore index 7e2cc191ac..d224bb3edd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,5 @@ aws-k8s-agent cni-metrics-helper grpc-health-probe portmap +loopback routed-eni-cni-plugin diff --git a/Makefile b/Makefile index de93ce33bf..e9b58e9c39 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ .PHONY: all dist check clean \ lint format check-format vet docker-vet \ - build-linux docker \ + build-linux docker init \ unit-test unit-test-race build-docker-test docker-func-test \ build-metrics docker-metrics \ metrics-unit-test docker-metrics-test @@ -29,6 +29,10 @@ DESTDIR = . IMAGE = amazon/amazon-k8s-cni IMAGE_NAME = $(IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION) IMAGE_DIST = $(DESTDIR)/$(subst /,_,$(IMAGE_NAME)).tar.gz +# INIT_IMAGE is the init container for AWS VPC CNI. +INIT_IMAGE = amazon/amazon-k8s-cni-init +INIT_IMAGE_NAME = $(INIT_IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION) +INIT_IMAGE_DIST = $(DESTDIR)/$(subst /,_,$(INIT_IMAGE_NAME)).tar.gz # METRICS_IMAGE is the CNI metrics publisher sidecar container image. METRICS_IMAGE = amazon/cni-metrics-helper METRICS_IMAGE_NAME = $(METRICS_IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION) @@ -69,6 +73,11 @@ LDFLAGS = -X main.version=$(VERSION) ALLPKGS = $(shell go list ./...) # BINS is the set of built command executables. BINS = aws-k8s-agent aws-cni grpc-health-probe cni-metrics-helper +# Plugin binaries +# Not copied: bandwidth bridge dhcp firewall flannel host-device host-local ipvlan macvlan ptp sbr static tuning vlan +# For gnu tar, the full path in the tar file is required +PLUGIN_BINS = ./loopback ./portmap + # DOCKER_ARGS is extra arguments passed during container image build. DOCKER_ARGS = # DOCKER_RUN_FLAGS is set the flags passed during runs of containers. @@ -85,11 +94,12 @@ DOCKER_BUILD_FLAGS = --build-arg GOARCH="$(ARCH)" \ .DEFAULT_GOAL = build-linux # Build both CNI and metrics helper container images. -all: docker docker-metrics +all: docker init docker-metrics dist: all mkdir -p $(DESTDIR) docker save $(IMAGE_NAME) | gzip > $(IMAGE_DIST) + docker save $(INIT_IMAGE_NAME) | gzip > $(INIT_IMAGE_DIST) docker save $(METRICS_IMAGE_NAME) | gzip > $(METRICS_IMAGE_DIST) # Build the VPC CNI plugin agent using the host's Go toolchain. @@ -108,7 +118,14 @@ docker: . @echo "Built Docker image \"$(IMAGE_NAME)\"" -# Run the built cni container image to use in functional testing +init: + docker build $(DOCKER_BUILD_FLAGS) \ + -f scripts/dockerfiles/Dockerfile.init \ + -t "$(INIT_IMAGE_NAME)" \ + . + @echo "Built Docker image \"$(INIT_IMAGE_NAME)\"" + +# Run the built CNI container image to use in functional testing docker-func-test: docker docker run $(DOCKER_RUN_FLAGS) \ "$(IMAGE_NAME)" @@ -178,17 +195,17 @@ generate: generate-limits: go run pkg/awsutils/gen_vpc_ip_limits.go -# Fetch portmap the port-forwarding management CNI plugin -portmap: FETCH_VERSION=0.8.5 -portmap: FETCH_URL=https://github.com/containernetworking/plugins/releases/download/v$(FETCH_VERSION)/cni-plugins-$(GOOS)-$(GOARCH)-v$(FETCH_VERSION).tgz -portmap: VISIT_URL=https://github.com/containernetworking/plugins/tree/v$(FETCH_VERSION)/plugins/meta/portmap -portmap: - @echo "Fetching portmap CNI plugin v$(FETCH_VERSION) from upstream release" +# Fetch the CNI plugins +plugins: FETCH_VERSION=0.8.5 +plugins: FETCH_URL=https://github.com/containernetworking/plugins/releases/download/v$(FETCH_VERSION)/cni-plugins-$(GOOS)-$(GOARCH)-v$(FETCH_VERSION).tgz +plugins: VISIT_URL=https://github.com/containernetworking/plugins/tree/v$(FETCH_VERSION)/plugins/ +plugins: + @echo "Fetching Container networking plugins v$(FETCH_VERSION) from upstream release" @echo - @echo "Visit upstream project for portmap plugin details:" + @echo "Visit upstream project for plugin details:" @echo "$(VISIT_URL)" @echo - curl -L $(FETCH_URL) | tar -z -x ./portmap + curl -L $(FETCH_URL) | tar -zx $(PLUGIN_BINS) debug-script: FETCH_URL=https://raw.githubusercontent.com/awslabs/amazon-eks-ami/master/log-collector-script/linux/eks-log-collector.sh debug-script: VISIT_URL=https://github.com/awslabs/amazon-eks-ami/tree/master/log-collector-script/linux @@ -244,5 +261,5 @@ check-format: format # Clean temporary files and build artifacts from the project. clean: @rm -f -- $(BINS) - @rm -f -- portmap + @rm -f -- $(PLUGIN_BINS) @rm -f -- coverage.txt diff --git a/config/master/aws-k8s-cni.yaml b/config/master/aws-k8s-cni.yaml new file mode 100644 index 0000000000..eb19a04ad2 --- /dev/null +++ b/config/master/aws-k8s-cni.yaml @@ -0,0 +1,178 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: aws-node +rules: + - apiGroups: + - crd.k8s.amazonaws.com + resources: + - "*" + verbs: + - "*" + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: ["list", "watch", "get"] + - apiGroups: ["extensions"] + resources: + - daemonsets + verbs: ["list", "watch"] + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: aws-node + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: aws-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: aws-node +subjects: + - kind: ServiceAccount + name: aws-node + namespace: kube-system + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: aws-node + namespace: kube-system + labels: + k8s-app: aws-node +spec: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: "10%" + selector: + matchLabels: + k8s-app: aws-node + template: + metadata: + labels: + k8s-app: aws-node + spec: + priorityClassName: system-node-critical + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/os" + operator: In + values: + - linux + - key: "kubernetes.io/arch" + operator: In + values: + - amd64 + - key: "eks.amazonaws.com/compute-type" + operator: NotIn + values: + - fargate + - matchExpressions: + - key: "beta.kubernetes.io/os" + operator: In + values: + - linux + - key: "beta.kubernetes.io/arch" + operator: In + values: + - amd64 + - key: "eks.amazonaws.com/compute-type" + operator: NotIn + values: + - fargate + serviceAccountName: aws-node + hostNetwork: true + tolerations: + - operator: Exists + containers: + - name: aws-node + image: 973117571331.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:master + imagePullPolicy: Always + ports: + - containerPort: 61678 + name: metrics + livenessProbe: + exec: + command: ["/app/grpc-health-probe", "-addr=:50051"] + initialDelaySeconds: 35 + env: + - name: AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER + value: "false" + - name: AWS_VPC_K8S_CNI_LOGLEVEL + value: DEBUG + - name: AWS_VPC_K8S_CNI_VETHPREFIX + value: eni + - name: AWS_VPC_ENI_MTU + value: "9001" + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 10m + securityContext: + capabilities: + add: ["NET_ADMIN"] + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + - mountPath: /host/var/log + name: log-dir + - mountPath: /var/run/dockershim.sock + name: dockershim + initContainers: + - name: aws-vpc-cni-init + image: 973117571331.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:master + imagePullPolicy: Always + securityContext: + privileged: true + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + volumes: + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + - name: log-dir + hostPath: + path: /var/log + - name: dockershim + hostPath: + path: /var/run/dockershim.sock + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: eniconfigs.crd.k8s.amazonaws.com +spec: + scope: Cluster + group: crd.k8s.amazonaws.com + versions: + - name: v1alpha1 + served: true + storage: true + names: + plural: eniconfigs + singular: eniconfig + kind: ENIConfig diff --git a/scripts/dockerfiles/Dockerfile.init b/scripts/dockerfiles/Dockerfile.init new file mode 100644 index 0000000000..f922e34bf9 --- /dev/null +++ b/scripts/dockerfiles/Dockerfile.init @@ -0,0 +1,30 @@ +ARG docker_arch +ARG golang_image=golang:1.13-stretch + +FROM $golang_image as builder +WORKDIR /go/src/github.com/aws/amazon-vpc-cni-k8s +ARG GOARCH +# Configure build with Go modules +ENV GO111MODULE=on +ENV GOPROXY=direct + +COPY Makefile ./ +RUN make plugins && make debug-script + +COPY . ./ + +# Build the architecture specific container image: +FROM $docker_arch/amazonlinux:2 +RUN yum update -y && \ + yum install -y iproute procps-ng && \ + yum clean all + +WORKDIR /init + +COPY --from=builder \ + /go/src/github.com/aws/amazon-vpc-cni-k8s/loopback \ + /go/src/github.com/aws/amazon-vpc-cni-k8s/portmap \ + /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-cni-support.sh \ + /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/init.sh /init/ + +ENTRYPOINT /init/init.sh diff --git a/scripts/dockerfiles/Dockerfile.release b/scripts/dockerfiles/Dockerfile.release index ba6eebddc3..94b3387a94 100644 --- a/scripts/dockerfiles/Dockerfile.release +++ b/scripts/dockerfiles/Dockerfile.release @@ -8,13 +8,12 @@ ARG GOARCH ENV GO111MODULE=on ENV GOPROXY=direct -# Copy modules in before the rest of the source to only expire cache on module -# changes: +# Copy modules in before the rest of the source to only expire cache on module changes: COPY go.mod go.sum ./ RUN go mod download COPY Makefile ./ -RUN make portmap && make debug-script +RUN make plugins && make debug-script COPY . ./ RUN make build-linux diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 4fd976519a..535f944cd2 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -44,13 +44,15 @@ AWS_VPC_ENI_MTU=${AWS_VPC_ENI_MTU:-"9001"} AWS_VPC_K8S_PLUGIN_LOG_FILE=${AWS_VPC_K8S_PLUGIN_LOG_FILE:-"/var/log/aws-routed-eni/plugin.log"} AWS_VPC_K8S_PLUGIN_LOG_LEVEL=${AWS_VPC_K8S_PLUGIN_LOG_LEVEL:-"Debug"} +AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER=${AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:-"true"} + # Checks for IPAM connectivity on localhost port 50051, retrying connectivity # check with a timeout of 36 seconds wait_for_ipam() { local __sleep_time=0 until [ $__sleep_time -eq 8 ]; do - sleep $(( __sleep_time++ )) + sleep $((__sleep_time++)) if ./grpc-health-probe -addr 127.0.0.1:50051 >/dev/null 2>&1; then return 0 fi @@ -58,13 +60,19 @@ wait_for_ipam() { return 1 } -echo -n "Copying portmap binary ... " - -HOST_PORTMAP="$HOST_CNI_BIN_PATH/portmap" -if [[ -f "$HOST_PORTMAP" ]]; then - rm "$HOST_PORTMAP" +# If there is no init container, copy the required files +if [[ "$AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER" != "false" ]]; then + # Copy files + echo "Copying CNI plugin binaries ... " + PLUGIN_BINS="portmap aws-cni-support.sh" + for b in $PLUGIN_BINS; do + # If the file exist, delete it first + if [[ -f "$HOST_CNI_BIN_PATH/$b" ]]; then + rm "$HOST_CNI_BIN_PATH/$b" + fi + cp "$b" "$HOST_CNI_BIN_PATH" + done fi -cp portmap "$HOST_CNI_BIN_PATH" echo -n "Starting IPAM daemon in the background ... " ./aws-k8s-agent | tee -i "$AGENT_LOG_PATH" 2>&1 & @@ -81,10 +89,9 @@ fi echo "ok." -echo -n "Copying additional CNI plugin binaries and config files ... " +echo -n "Copying CNI plugin binary and config file ... " cp aws-cni "$HOST_CNI_BIN_PATH" -cp aws-cni-support.sh "$HOST_CNI_BIN_PATH" sed -i s~__VETHPREFIX__~"${AWS_VPC_K8S_CNI_VETHPREFIX}"~g 10-aws.conflist sed -i s~__MTU__~"${AWS_VPC_ENI_MTU}"~g 10-aws.conflist diff --git a/scripts/init.sh b/scripts/init.sh new file mode 100755 index 0000000000..8ce1d079f1 --- /dev/null +++ b/scripts/init.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +PLUGIN_BINS="loopback portmap aws-cni-support.sh" + +for b in $PLUGIN_BINS; do + if [ ! -f "$b" ]; then + echo "Required $b executable not found." + exit 1 + fi +done + +HOST_CNI_BIN_PATH=${HOST_CNI_BIN_PATH:-"/host/opt/cni/bin"} + +# Copy files +echo "Copying CNI plugin binaries ... " + +for b in $PLUGIN_BINS; do + # If the file exist, delete it first + if [[ -f "$HOST_CNI_BIN_PATH/$b" ]]; then + rm "$HOST_CNI_BIN_PATH/$b" + fi + cp "$b" "$HOST_CNI_BIN_PATH" +done + +# Configure rp_filter +echo "Configure rp_filter loose... " +TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60") +HOST_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/local-ipv4) +PRIMARY_IF=$(ip -4 -o a | grep "$HOST_IP" | awk '{print $2}') +sysctl -w "net.ipv4.conf.$PRIMARY_IF.rp_filter=2" + +cat "/proc/sys/net/ipv4/conf/$PRIMARY_IF/rp_filter" + +echo "CNI init container done"