Makefile

@@ -14,10 +14,13 @@
 
 .PHONY: build-linux clean docker docker-build lint unit-test vet
 
+VERSION ?= $(shell git describe --tags --always --dirty)
+LDFLAGS ?= -X main.version=$(VERSION)
+
 # Default to build the Linux binary
 build-linux:
-	GOOS=linux CGO_ENABLED=0 go build -o aws-k8s-agent
-	GOOS=linux CGO_ENABLED=0 go build -o aws-cni ./plugins/routed-eni/
+	GOOS=linux CGO_ENABLED=0 go build -o aws-k8s-agent -ldflags "$(LDFLAGS)"
+	GOOS=linux CGO_ENABLED=0 go build -o aws-cni -ldflags "$(LDFLAGS)" ./plugins/routed-eni/
 
 docker-build:
 	docker run -v $(shell pwd):/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \
@@ -28,8 +31,8 @@ docker-build:
 
 # Build docker image
 docker: docker-build
-	@docker build -f scripts/dockerfiles/Dockerfile.release -t "amazon/amazon-k8s-cni:latest" .
-	@echo "Built Docker image \"amazon/amazon-k8s-cni:latest\""
+	@docker build -f scripts/dockerfiles/Dockerfile.release -t "amazon/amazon-k8s-cni:$(VERSION)" .
+	@echo "Built Docker image \"amazon/amazon-k8s-cni:$(VERSION)\""
 
 # unit-test
 unit-test:

README.md

@@ -13,16 +13,18 @@ Networking plugin for pod networking in [Kubernetes](https://kubernetes.io/) usi
 
 ## Setup
 Download the latest version of the [yaml](./config/) and apply it the cluster.
+
 ```
 kubectl apply -f aws-k8s-cni.yaml
 ```
+
 Launch kubelet with network plugins set to cni (`--network-plugin=cni`), the cni directories configured (`--cni-config-dir` and `--cni-bin-dir`) and node ip set to the primary IPv4 address of the primary ENI for the instance (`--node-ip=$(curl http://169.254.169.254/latest/meta-data/local-ipv4)`).  It is also recommended to set `--max-pods` equal to the number of ENIs for the instance type * (the number of IPs per ENI - 1) [see](./pkg/awsutils/vpc_ip_resource_limit.go) to prevent scheduling that exceeds the IP resources available to the kubelet.
 
 The default manifest expects `--cni-conf-dir=/etc/cni/net.d` and `--cni-bin-dir=/opt/cni/bin`.
 
 L-IPAM requires following [IAM policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html):
 
- ```       
+```      
  {
      "Effect": "Allow",
      "Action": [
@@ -58,11 +60,32 @@ L-IPAM requires following [IAM policy](https://docs.aws.amazon.com/IAM/latest/Us
   There are 2 components:
 
   * [CNI Plugin](https://kubernetes.io/docs/concepts/cluster-administration/network-plugins/#cni), which will wire up host's and pod's network stack when called.
-  * L-IPAM, which is a long running node-Local IP Address Management (IPAM) daemon, is responsible for:
+  * `L-IPAMD`, which is a long running node-Local IP Address Management (IPAM) daemon, is responsible for:
     * maintaining a warm-pool of available IP addresses, and
     * assigning an IP address to a Pod.
 
-The details can be found in [Proposal: CNI plugin for Kubernetes networking over AWS VPC](https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/cni-proposal.md)
+The details can be found in [Proposal: CNI plugin for Kubernetes networking over AWS VPC](https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/cni-proposal.md).
+
+[Troubleshooting Guide](https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/troubleshooting.md) provides tips on how to debug and troubleshoot CNI.
+
+### Notes
+
+`L-IPAMD`(aws-node daemonSet) running on every worker node requires access to kubernetes API server.  If it can **not** reach kubernetes API server, ipamD will exit and CNI will not be able to get any IP address for Pods.  Here is a way to confirm if `L-IPAMD` has access to the kubernetes API server.
+
+```
+
+# find out kubernetes service IP, e.g. 10.0.0.1
+kubectl get svc kubernetes
+NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
+kubernetes   ClusterIP   10.0.0.1   <none>        443/TCP   29d
+
+# ssh into worker node, check if worker node can reach API server
+telnet 10.0.0.1 443
+Trying 10.0.0.1...
+Connected to 10.0.0.1.
+Escape character is '^]'.  <-------- kubernetes API server is reachable
+
+``` 
 
 ## Contributing
 [See CONTRIBUTING.md](./CONTRIBUTING.md)

config/v1.0/aws-k8s-cni.yaml

@@ -58,10 +58,7 @@ spec:
       serviceAccountName: aws-node
       hostNetwork: true
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
-      - key: CriticalAddonsOnly
-        operator: Exists
+      - operator: Exists
       containers:
       - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:1.0.0
         name: aws-node

config/v1.0/calico.yaml

@@ -114,13 +114,7 @@ spec:
             path: /var/run/calico
       tolerations:
         # Make sure calico/node gets scheduled on all nodes.
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
-        # Mark the pod as a critical add-on for rescheduling.
-        - key: CriticalAddonsOnly
-          operator: Exists
+      - operator: Exists
 
 ---
 
@@ -375,8 +369,7 @@ spec:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
+      - operator: Exists
       hostNetwork: true
       serviceAccountName: calico-node
       containers:

ipamd/ipamd.go

@@ -84,6 +84,19 @@ var (
 		},
 		[]string{"fn"},
 	)
+	addIPCnt = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "add_ip_req_count",
+			Help: "The number of add IP address request",
+		},
+	)
+	delIPCnt = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "del_ip_req_count",
+			Help: "The number of delete IP address request",
+		},
+		[]string{"reason"},
+	)
 	prometheusRegistered = false
 )
 
@@ -112,6 +125,8 @@ func prometheusRegister() {
 		prometheus.MustRegister(enisMax)
 		prometheus.MustRegister(ipMax)
 		prometheus.MustRegister(reconcileCnt)
+		prometheus.MustRegister(addIPCnt)
+		prometheus.MustRegister(delIPCnt)
 		prometheusRegistered = true
 	}
 }

ipamd/rpc_handler.go

@@ -19,6 +19,7 @@ import (
 	"github.com/pkg/errors"
 
 	pb "github.com/aws/amazon-vpc-cni-k8s/rpc"
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
@@ -47,12 +48,14 @@ func (s *server) AddNetwork(ctx context.Context, in *pb.AddNetworkRequest) (*pb.
 		Namespace: in.K8S_POD_NAMESPACE,
 		Container: in.K8S_POD_INFRA_CONTAINER_ID})
 	log.Infof("Send AddNetworkReply: IPv4Addr %s, DeviceNumber: %d, err: %v", addr, deviceNumber, err)
+	addIPCnt.Inc()
 	return &pb.AddNetworkReply{Success: err == nil, IPv4Addr: addr, IPv4Subnet: "", DeviceNumber: int32(deviceNumber)}, nil
 }
 
 func (s *server) DelNetwork(ctx context.Context, in *pb.DelNetworkRequest) (*pb.DelNetworkReply, error) {
 	log.Infof("Received DelNetwork for IP %s, Pod %s, Namespace %s, Container %s",
 		in.IPv4Addr, in.K8S_POD_NAME, in.K8S_POD_NAMESPACE, in.K8S_POD_INFRA_CONTAINER_ID)
+	delIPCnt.With(prometheus.Labels{"reason": in.Reason}).Inc()
 
 	var err error
 

main.go

@@ -26,7 +26,10 @@ import (
 
 const (
 	defaultLogFilePath = "/host/var/log/aws-routed-eni/ipamd.log"
-	version            = "1.0.0"
+)
+
+var (
+	version string
 )
 
 func main() {

pkg/awsutils/awsutils.go

@@ -15,6 +15,7 @@ package awsutils
 
 import (
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -26,13 +27,10 @@ import (
 
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/ec2metadata"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper"
-	"github.com/aws/amazon-vpc-cni-k8s/pkg/resourcegroupstaggingapiwrapper"
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/arn"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 )
 
 const (
@@ -53,8 +51,10 @@ const (
 	eniDescriptionPrefix = "aws-K8S-"
 	metadataOwnerID      = "/owner-id"
 	// AllocENI need to choose a first free device number between 0 and maxENI
-	maxENIs   = 128
-	eniTagKey = "k8s-eni-key"
+	maxENIs           = 128
+	clusterNameEnvVar = "CLUSTER_NAME"
+	eniNodeTagKey     = "node.k8s.amazonaws.com/instance_id"
+	eniClusterTagKey  = "cluster.k8s.amazonaws.com/name"
 
 	retryDeleteENIInternal = 5 * time.Second
 
@@ -147,7 +147,6 @@ type EC2InstanceMetadataCache struct {
 
 	ec2Metadata ec2metadata.EC2Metadata
 	ec2SVC      ec2wrapper.EC2
-	tagSVC      resourcegroupstaggingapiwrapper.ResourceGroupsTaggingAPI
 }
 
 // ENIMetadata contains ENI information retrieved from EC2 meta data service
@@ -208,9 +207,6 @@ func New() (*EC2InstanceMetadataCache, error) {
 	ec2SVC := ec2wrapper.New(sess)
 	cache.ec2SVC = ec2SVC
 
-	tagSVC := resourcegroupstaggingapiwrapper.New(sess)
-	cache.tagSVC = tagSVC
-
 	err = cache.initWithEC2Metadata()
 	if err != nil {
 		return nil, err
@@ -628,37 +624,44 @@ func (cache *EC2InstanceMetadataCache) createENI() (string, error) {
 }
 
 func (cache *EC2InstanceMetadataCache) tagENI(eniID string) {
-	tagSvc := cache.tagSVC
-	arns := make([]*string, 0)
 
-	eniARN := &arn.ARN{
-		Partition: "aws",
-		Service:   "ec2",
-		Region:    cache.region,
-		AccountID: cache.accountID,
-		Resource:  "network-interface/" + eniID}
-	arnString := eniARN.String()
-	arns = append(arns, aws.String(arnString))
+	// Tag the ENI with "node.k8s.amazonaws.com/instance_id=<instance_id>"
+	tags := []*ec2.Tag{
+		{
+			Key:   aws.String(eniNodeTagKey),
+			Value: aws.String(cache.instanceID),
+		},
+	}
 
-	tags := make(map[string]*string)
+	// If the CLUSTER_NAME env var is present,
+	// tag the ENI with "cluster.k8s.amazonaws.com/name=<cluster_name>"
+	clusterName := os.Getenv(clusterNameEnvVar)
+	if clusterName != "" {
+		tags = append(tags, &ec2.Tag{
+			Key:   aws.String(eniClusterTagKey),
+			Value: aws.String(clusterName),
+		})
+	}
 
-	tagValue := cache.instanceID
-	tags[eniTagKey] = aws.String(tagValue)
-	log.Debugf("Trying to tag newly created eni: keys=%s, value=%s", eniTagKey, tagValue)
+	for _, tag := range tags {
+		log.Debugf("Trying to tag newly created eni: key=%s, value=%s", aws.StringValue(tag.Key), aws.StringValue(tag.Value))
+	}
 
-	tagInput := &resourcegroupstaggingapi.TagResourcesInput{
-		ResourceARNList: arns,
-		Tags:            tags,
+	input := &ec2.CreateTagsInput{
+		Resources: []*string{
+			aws.String(eniID),
+		},
+		Tags: tags,
 	}
 
 	start := time.Now()
-	_, err := tagSvc.TagResources(tagInput)
-	awsAPILatency.WithLabelValues("TagResources", fmt.Sprint(err != nil)).Observe(msSince(start))
+	_, err := cache.ec2SVC.CreateTags(input)
+	awsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil)).Observe(msSince(start))
 	if err != nil {
-		awsAPIErrInc("TagResources", err)
-		log.Warnf("Fail to tag the newly created eni %s  %v", eniID, err)
+		awsAPIErrInc("CreateTags", err)
+		log.Warnf("Failed to tag the newly created eni %s:  %v", eniID, err)
 	} else {
-		log.Debugf("Tag the newly created eni with arn: %s", arnString)
+		log.Debugf("Successfully tagged ENI: %s", eniID)
 	}
 }