Add routes to all peered VPCs (config option) (#24)

If the key `routeToVpcPeers` is set to `true` on the IPAM
configuration, all known peered VPC CIDRs will be added to the IPvlan
route table allowing for direct VPC<->VPC communication.

Fixes #21 / #23

README.md

@@ -193,7 +193,8 @@ groups.
 Routes are automatically formed for the VPC on the `ipvlan` adapter.
 
 ipMasq is enabled to use the host-IP for egress to the Internet as
-well as providing access to services such as `kube2iam`.
+well as providing access to services such as `kube2iam`. `kube2iam` is
+not a dependency of this software.
 
 ```
 {
@@ -209,12 +210,12 @@ well as providing access to services such as `kube2iam`.
           "type": "cni-ipvlan-vpc-k8s-ipam",
           "interfaceIndex": 1,
 	      "subnetTags": {
-	      "kubernetes_kubelet": "true"
-	  },
-	  "secGroupIds": [
-	      "sg-1234",
-	      "sg-5678"
-	      ]
+	          "kubernetes_kubelet": "true"
+     	  },
+	      "secGroupIds": [
+	          "sg-1234",
+	          "sg-5678"
+	          ]
           }
     },
     {
@@ -228,6 +229,75 @@ well as providing access to services such as `kube2iam`.
 }
 ```
 
+### Other IPAM configuration flags
+
+In the above `ipam` block, several options are available:
+
+ - `interfaceIndex`: We also recommend never using the boot ENI
+   adapter with this plugin (though it is possible). By setting
+   `interfaceIndex` to 1, the plugin will only allocate IPs (and add
+   new adapters) starting at `eth1`.
+ - `subnetTags`: When allocating new adapters, by default the plugin
+   will use all available subnets within the availability zone. You
+   can restrict which subnets the plugin will use by specifying key /
+   value tag names that must be matched in order for the plugin to be
+   considered. These tags are set via the AWS API or in the AWS
+   Console on the subnet object.
+ - `secGroupIds`: When allocating a new ENI adapter, these interface
+   groups will be assigned to the adapter. Specify the `sg-xxxx`
+   interface group ID.
+ - `skipDeallocation`: `true` or `false` - when set to `true`, this
+   plugin will never remove a secondary IP address from an
+   adapter. Useful in workloads that churn many pods to reduce the AWS
+   ratelimits for configuring the VPC (which are low and cannot be
+   raised above a certain threshold). 
+ - `routeToVpcPeers`: `true` or `false` - When set to `true`, the
+   plugin will make a (cached) call to `DescribeVpcPeeringConnections`
+   to enumerate all peered VPCs. Routes will be added so connections
+   to these VPCs will be sourced from the IPvlan adapter in the pod
+   and not through the host masquerade. 
+
+## The CLI Tool
+
+This plugin ships a CLI tool which can be useful to inspect the state
+of the system or perform certain actions (such as provisioning an
+adapter at instance cloud-init time).
+
+Run `cni-ipvlan-vpc-k8s-tool --help` for a complete listing of
+options.
+
+    NAME:
+       cni-ipvlan-vpc-k8s-tool - Interface with ENI adapters and CNI bindings for those
+
+    USAGE:
+       cni-ipvlan-vpc-k8s-tool [global options] command [command options] [arguments...]
+
+    VERSION:
+       v-next
+
+    COMMANDS:
+         new-interface             Create a new interface
+         remove-interface          Remove an existing interface
+         deallocate                Deallocate a private IP
+         allocate-first-available  Allocate a private IP on the first available interface
+         free-ips                  List all currently unassigned AWS IP addresses
+         eniif                     List all ENI interfaces and their setup with addresses
+         addr                      List all bound IP addresses
+         subnets                   Show available subnets for this host
+         limits                    Display limits for ENI for this instance type
+         bugs                      Show any bugs associated with this instance
+         vpccidr                   Show the VPC CIDRs associated with current interfaces
+         vpcpeercidr               Show the peered VPC CIDRs associated with current interfaces
+         help, h                   Shows a list of commands or help for one command
+
+    GLOBAL OPTIONS:
+       --help, -h     show help
+       --version, -v  print the version
+
+    COPYRIGHT:
+       (c) 2017-2018 Lyft Inc.
+
+
 ## Security Considerations
 
 In Kubernetes, pods and kubelets are assumed to have static IP addresses that

aws/vpc.go

@@ -14,6 +14,7 @@ import (
 // VPCClient provides a view into a VPC
 type VPCClient interface {
 	DescribeVPCCIDRs(vpcID string) ([]*net.IPNet, error)
+	DescribeVPCPeerCIDRs(vpcID string) ([]*net.IPNet, error)
 }
 
 type vpcCacheClient struct {
@@ -35,6 +36,21 @@ func (v *vpcCacheClient) DescribeVPCCIDRs(vpcID string) (cidrs []*net.IPNet, err
 	return
 }
 
+func (v *vpcCacheClient) DescribeVPCPeerCIDRs(vpcID string) (cidrs []*net.IPNet, err error) {
+	key := fmt.Sprintf("vpc-peers-%v", vpcID)
+	state := cache.Get(key, &cidrs)
+	if state == cache.CacheFound {
+		return
+	}
+	cidrs, err = v.vpc.DescribeVPCPeerCIDRs(vpcID)
+	if err != nil {
+		return nil, err
+	}
+	cache.Store(key, v.expiration, &cidrs)
+	return
+
+}
+
 type vpcclient struct {
 	aws *awsclient
 }
@@ -66,3 +82,50 @@ func (v *vpcclient) DescribeVPCCIDRs(vpcID string) ([]*net.IPNet, error) {
 	}
 	return cidrs, nil
 }
+
+// DescribeVPCPeerCIDRs returns a list of CIDRs for all peered VPCs to the given VPC
+func (v *vpcclient) DescribeVPCPeerCIDRs(vpcID string) ([]*net.IPNet, error) {
+	ec2c, err := v.aws.newEC2()
+	if err != nil {
+		return nil, err
+	}
+
+	req := &ec2.DescribeVpcPeeringConnectionsInput{}
+
+	res, err := ec2c.DescribeVpcPeeringConnections(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// In certain peering situations, a CIDR may be duplicated
+	// and visible to the API, even if the CIDR is not active in
+	// one of the peered VPCs. We store all of the CIDRs in a map
+	// to de-duplicate them.
+	cidrs := make(map[string]bool, 0)
+
+	for _, peering := range res.VpcPeeringConnections {
+		var peer *ec2.VpcPeeringConnectionVpcInfo
+
+		if vpcID == *peering.AccepterVpcInfo.VpcId {
+			peer = peering.RequesterVpcInfo
+		} else if vpcID == *peering.RequesterVpcInfo.VpcId {
+			peer = peering.AccepterVpcInfo
+		}
+
+		for _, cidrBlock := range peer.CidrBlockSet {
+			_, _, err := net.ParseCIDR(*cidrBlock.CidrBlock)
+			if err == nil {
+				cidrs[*cidrBlock.CidrBlock] = true
+			}
+		}
+	}
+
+	var returnCidrs []*net.IPNet
+	for cidrString := range cidrs {
+		_, cidr, err := net.ParseCIDR(cidrString)
+		if err == nil && cidr != nil {
+			returnCidrs = append(returnCidrs, cidr)
+		}
+	}
+	return returnCidrs, nil
+}

cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool.go

@@ -228,6 +228,26 @@ func actionVpcCidr(c *cli.Context) error {
 	return nil
 }
 
+func actionVpcPeerCidr(c *cli.Context) error {
+	interfaces, err := aws.DefaultClient.GetInterfaces()
+	if err != nil {
+		fmt.Println(err)
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
+	fmt.Fprintln(w, "iface\tpeer_dcidr\t")
+	for _, iface := range interfaces {
+		apiCidrs, _ := aws.DefaultClient.DescribeVPCPeerCIDRs(iface.VpcID)
+
+		fmt.Fprintf(w, "%s\t%v\t\n",
+			iface.LocalName(),
+			apiCidrs)
+	}
+	w.Flush()
+	return nil
+}
+
 func actionSubnets(c *cli.Context) error {
 	subnets, err := aws.DefaultClient.GetSubnetsForInstance()
 	if err != nil {
@@ -330,9 +350,14 @@ func main() {
 			Usage:  "Show the VPC CIDRs associated with current interfaces",
 			Action: actionVpcCidr,
 		},
+		{
+			Name:   "vpcpeercidr",
+			Usage:  "Show the peered VPC CIDRs associated with current interfaces",
+			Action: actionVpcPeerCidr,
+		},
 	}
 	app.Version = version
-	app.Copyright = "(c) 2017 Lyft Inc."
+	app.Copyright = "(c) 2017-2018 Lyft Inc."
 	app.Usage = "Interface with ENI adapters and CNI bindings for those"
 	app.Run(os.Args)
 }

plugin/ipam/main.go

@@ -48,6 +48,7 @@ type IPAMConfig struct {
 	SubnetTags       map[string]string `json:"subnetTags"`
 	IfaceIndex       int               `json:"interfaceIndex"`
 	SkipDeallocation bool              `json:"skipDeallocation"`
+	RouteToVPCPeers  bool              `json:"routeToVpcPeers"`
 }
 
 func init() {
@@ -156,6 +157,15 @@ func cmdAdd(args *skel.CmdArgs) error {
 			return fmt.Errorf("Unable to enumerate CIDRs from the AWS API due to a specific meta-data bug %v", err)
 		}
 	}
+
+	if conf.IPAM.RouteToVPCPeers {
+		peerCidr, err := aws.DefaultClient.DescribeVPCPeerCIDRs(alloc.Interface.VpcID)
+		if err != nil {
+			return fmt.Errorf("unable to enumerate peer CIDrs %v", err)
+		}
+		cidrs = append(cidrs, peerCidr...)
+	}
+
 	// add routes for all VPC cidrs via the subnet gateway
 	for _, dst := range cidrs {
 		result.Routes = append(result.Routes, &types.Route{*dst, gw})