Add init container

.dockerignore

@@ -3,4 +3,5 @@ aws-k8s-agent
 cni-metrics-helper
 grpc-health-probe
 portmap
+loopback
 routed-eni-cni-plugin

Makefile

@@ -14,7 +14,7 @@
 
 .PHONY: all dist check clean \
 		lint format check-format vet docker-vet \
-		build-linux docker \
+		build-linux docker init \
 		unit-test unit-test-race build-docker-test docker-func-test \
 		build-metrics docker-metrics \
 		metrics-unit-test docker-metrics-test
@@ -29,6 +29,10 @@ DESTDIR = .
 IMAGE = amazon/amazon-k8s-cni
 IMAGE_NAME = $(IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION)
 IMAGE_DIST = $(DESTDIR)/$(subst /,_,$(IMAGE_NAME)).tar.gz
+# INIT_IMAGE is the init container for AWS VPC CNI.
+INIT_IMAGE = amazon/amazon-k8s-cni-init
+INIT_IMAGE_NAME = $(INIT_IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION)
+INIT_IMAGE_DIST = $(DESTDIR)/$(subst /,_,$(INIT_IMAGE_NAME)).tar.gz
 # METRICS_IMAGE is the CNI metrics publisher sidecar container image.
 METRICS_IMAGE = amazon/cni-metrics-helper
 METRICS_IMAGE_NAME = $(METRICS_IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION)
@@ -69,6 +73,11 @@ LDFLAGS = -X main.version=$(VERSION)
 ALLPKGS = $(shell go list ./...)
 # BINS is the set of built command executables.
 BINS = aws-k8s-agent aws-cni grpc-health-probe cni-metrics-helper
+# Plugin binaries
+# Not copied: bandwidth bridge dhcp firewall flannel host-device host-local ipvlan macvlan ptp sbr static tuning vlan
+# For gnu tar, the full path in the tar file is required
+PLUGIN_BINS = ./loopback ./portmap
+
 # DOCKER_ARGS is extra arguments passed during container image build.
 DOCKER_ARGS =
 # DOCKER_RUN_FLAGS is set the flags passed during runs of containers.
@@ -85,11 +94,12 @@ DOCKER_BUILD_FLAGS = --build-arg GOARCH="$(ARCH)" \
 .DEFAULT_GOAL = build-linux
 
 # Build both CNI and metrics helper container images.
-all: docker docker-metrics
+all: docker init docker-metrics
 
 dist: all
 	mkdir -p $(DESTDIR)
 	docker save $(IMAGE_NAME) | gzip > $(IMAGE_DIST)
+	docker save $(INIT_IMAGE_NAME) | gzip > $(INIT_IMAGE_DIST)
 	docker save $(METRICS_IMAGE_NAME) | gzip > $(METRICS_IMAGE_DIST)
 
 # Build the VPC CNI plugin agent using the host's Go toolchain.
@@ -108,7 +118,14 @@ docker:
 		.
 	@echo "Built Docker image \"$(IMAGE_NAME)\""
 
-# Run the built cni container image to use in functional testing
+init:
+	docker build $(DOCKER_BUILD_FLAGS) \
+		-f scripts/dockerfiles/Dockerfile.init \
+		-t "$(INIT_IMAGE_NAME)" \
+		.
+	@echo "Built Docker image \"$(INIT_IMAGE_NAME)\""
+
+# Run the built CNI container image to use in functional testing
 docker-func-test: docker
 	docker run $(DOCKER_RUN_FLAGS) \
 		"$(IMAGE_NAME)"
@@ -178,17 +195,17 @@ generate:
 generate-limits:
 	go run pkg/awsutils/gen_vpc_ip_limits.go
 
-# Fetch portmap the port-forwarding management CNI plugin
-portmap: FETCH_VERSION=0.8.5
-portmap: FETCH_URL=https://github.com/containernetworking/plugins/releases/download/v$(FETCH_VERSION)/cni-plugins-$(GOOS)-$(GOARCH)-v$(FETCH_VERSION).tgz
-portmap: VISIT_URL=https://github.com/containernetworking/plugins/tree/v$(FETCH_VERSION)/plugins/meta/portmap
-portmap:
-	@echo "Fetching portmap CNI plugin v$(FETCH_VERSION) from upstream release"
+# Fetch the CNI plugins
+plugins: FETCH_VERSION=0.8.5
+plugins: FETCH_URL=https://github.com/containernetworking/plugins/releases/download/v$(FETCH_VERSION)/cni-plugins-$(GOOS)-$(GOARCH)-v$(FETCH_VERSION).tgz
+plugins: VISIT_URL=https://github.com/containernetworking/plugins/tree/v$(FETCH_VERSION)/plugins/
+plugins:
+	@echo "Fetching Container networking plugins v$(FETCH_VERSION) from upstream release"
 	@echo
-	@echo "Visit upstream project for portmap plugin details:"
+	@echo "Visit upstream project for plugin details:"
 	@echo "$(VISIT_URL)"
 	@echo
-	curl -L $(FETCH_URL) | tar -z -x ./portmap
+	curl -L $(FETCH_URL) | tar -zx $(PLUGIN_BINS)
 
 debug-script: FETCH_URL=https://raw.githubusercontent.com/awslabs/amazon-eks-ami/master/log-collector-script/linux/eks-log-collector.sh
 debug-script: VISIT_URL=https://github.com/awslabs/amazon-eks-ami/tree/master/log-collector-script/linux
@@ -244,5 +261,5 @@ check-format: format
 # Clean temporary files and build artifacts from the project.
 clean:
 	@rm -f -- $(BINS)
-	@rm -f -- portmap
+	@rm -f -- $(PLUGIN_BINS)
 	@rm -f -- coverage.txt

config/master/aws-k8s-cni.yaml

@@ -0,0 +1,178 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aws-node
+rules:
+  - apiGroups:
+      - crd.k8s.amazonaws.com
+    resources:
+      - "*"
+    verbs:
+      - "*"
+  - apiGroups: [""]
+    resources:
+      - pods
+      - nodes
+      - namespaces
+    verbs: ["list", "watch", "get"]
+  - apiGroups: ["extensions"]
+    resources:
+      - daemonsets
+    verbs: ["list", "watch"]
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aws-node
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: aws-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node
+subjects:
+  - kind: ServiceAccount
+    name: aws-node
+    namespace: kube-system
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: aws-node
+  namespace: kube-system
+  labels:
+    k8s-app: aws-node
+spec:
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: "10%"
+  selector:
+    matchLabels:
+      k8s-app: aws-node
+  template:
+    metadata:
+      labels:
+        k8s-app: aws-node
+    spec:
+      priorityClassName: system-node-critical
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: "kubernetes.io/os"
+                    operator: In
+                    values:
+                      - linux
+                  - key: "kubernetes.io/arch"
+                    operator: In
+                    values:
+                      - amd64
+                  - key: "eks.amazonaws.com/compute-type"
+                    operator: NotIn
+                    values:
+                      - fargate
+              - matchExpressions:
+                  - key: "beta.kubernetes.io/os"
+                    operator: In
+                    values:
+                      - linux
+                  - key: "beta.kubernetes.io/arch"
+                    operator: In
+                    values:
+                      - amd64
+                  - key: "eks.amazonaws.com/compute-type"
+                    operator: NotIn
+                    values:
+                      - fargate
+      serviceAccountName: aws-node
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: aws-node
+          image: 973117571331.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:master
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 61678
+              name: metrics
+          livenessProbe:
+            exec:
+              command: ["/app/grpc-health-probe", "-addr=:50051"]
+            initialDelaySeconds: 35
+          env:
+            - name: AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER
+              value: "false"
+            - name: AWS_VPC_K8S_CNI_LOGLEVEL
+              value: DEBUG
+            - name: AWS_VPC_K8S_CNI_VETHPREFIX
+              value: eni
+            - name: AWS_VPC_ENI_MTU
+              value: "9001"
+            - name: MY_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          resources:
+            requests:
+              cpu: 10m
+          securityContext:
+            capabilities:
+              add: ["NET_ADMIN"]
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+            - mountPath: /host/var/log
+              name: log-dir
+            - mountPath: /var/run/dockershim.sock
+              name: dockershim
+      initContainers:
+        - name: aws-vpc-cni-init
+          image: 973117571331.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:master
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+      volumes:
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/cni/net.d
+        - name: log-dir
+          hostPath:
+            path: /var/log
+        - name: dockershim
+          hostPath:
+            path: /var/run/dockershim.sock
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: eniconfigs.crd.k8s.amazonaws.com
+spec:
+  scope: Cluster
+  group: crd.k8s.amazonaws.com
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+  names:
+    plural: eniconfigs
+    singular: eniconfig
+    kind: ENIConfig

scripts/dockerfiles/Dockerfile.init

@@ -0,0 +1,30 @@
+ARG docker_arch
+ARG golang_image=golang:1.13-stretch
+
+FROM $golang_image as builder
+WORKDIR /go/src/github.com/aws/amazon-vpc-cni-k8s
+ARG GOARCH
+# Configure build with Go modules
+ENV GO111MODULE=on
+ENV GOPROXY=direct
+
+COPY Makefile ./
+RUN make plugins && make debug-script
+
+COPY . ./
+
+# Build the architecture specific container image:
+FROM $docker_arch/amazonlinux:2
+RUN yum update -y && \
+    yum install -y iproute procps-ng && \
+    yum clean all
+
+WORKDIR /init
+
+COPY --from=builder \
+    /go/src/github.com/aws/amazon-vpc-cni-k8s/loopback \
+    /go/src/github.com/aws/amazon-vpc-cni-k8s/portmap \
+    /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-cni-support.sh \
+    /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/init.sh /init/
+
+ENTRYPOINT /init/init.sh

scripts/dockerfiles/Dockerfile.release

@@ -8,13 +8,12 @@ ARG GOARCH
 ENV GO111MODULE=on
 ENV GOPROXY=direct
 
-# Copy modules in before the rest of the source to only expire cache on module
-# changes:
+# Copy modules in before the rest of the source to only expire cache on module changes:
 COPY go.mod go.sum ./
 RUN go mod download
 
 COPY Makefile ./
-RUN make portmap && make debug-script
+RUN make plugins && make debug-script
 
 COPY . ./
 RUN make build-linux

scripts/entrypoint.sh

@@ -44,27 +44,35 @@ AWS_VPC_ENI_MTU=${AWS_VPC_ENI_MTU:-"9001"}
 AWS_VPC_K8S_PLUGIN_LOG_FILE=${AWS_VPC_K8S_PLUGIN_LOG_FILE:-"/var/log/aws-routed-eni/plugin.log"}
 AWS_VPC_K8S_PLUGIN_LOG_LEVEL=${AWS_VPC_K8S_PLUGIN_LOG_LEVEL:-"Debug"}
 
+AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER=${AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:-"true"}
+
 # Checks for IPAM connectivity on localhost port 50051, retrying connectivity
 # check with a timeout of 36 seconds
 wait_for_ipam() {
     local __sleep_time=0
 
     until [ $__sleep_time -eq 8 ]; do
-        sleep $(( __sleep_time++ ))
+        sleep $((__sleep_time++))
         if ./grpc-health-probe -addr 127.0.0.1:50051 >/dev/null 2>&1; then
             return 0
         fi
     done
     return 1
 }
 
-echo -n "Copying portmap binary ... "
-
-HOST_PORTMAP="$HOST_CNI_BIN_PATH/portmap"
-if [[ -f "$HOST_PORTMAP" ]]; then
-    rm "$HOST_PORTMAP"
+# If there is no init container, copy the required files
+if [[ "$AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER" != "false" ]]; then
+    # Copy files
+    echo "Copying CNI plugin binaries ... "
+    PLUGIN_BINS="portmap aws-cni-support.sh"
+    for b in $PLUGIN_BINS; do
+        # If the file exist, delete it first
+        if [[ -f "$HOST_CNI_BIN_PATH/$b" ]]; then
+            rm "$HOST_CNI_BIN_PATH/$b"
+        fi
+        cp "$b" "$HOST_CNI_BIN_PATH"
+    done
 fi
-cp portmap "$HOST_CNI_BIN_PATH"
 
 echo -n "Starting IPAM daemon in the background ... "
 ./aws-k8s-agent | tee -i "$AGENT_LOG_PATH" 2>&1 &
@@ -81,10 +89,9 @@ fi
 
 echo "ok."
 
-echo -n "Copying additional CNI plugin binaries and config files ... "
+echo -n "Copying CNI plugin binary and config file ... "
 
 cp aws-cni "$HOST_CNI_BIN_PATH"
-cp aws-cni-support.sh "$HOST_CNI_BIN_PATH"
 
 sed -i s~__VETHPREFIX__~"${AWS_VPC_K8S_CNI_VETHPREFIX}"~g 10-aws.conflist
 sed -i s~__MTU__~"${AWS_VPC_ENI_MTU}"~g 10-aws.conflist