From b79d93e127ae1b8f5aaaf2ee3db81e5dcbd42321 Mon Sep 17 00:00:00 2001 From: Yann Ramin Date: Fri, 26 Jan 2018 18:16:43 -0800 Subject: [PATCH] Add routes to all peered VPCs (config option) (#24) If the key `routeToVpcPeers` is set to `true` on the IPAM configuration, all known peered VPC CIDRs will be added to the IPvlan route table allowing for direct VPC<->VPC communication. Fixes #21 / #23 --- README.md | 84 +++++++++++++++++-- aws/vpc.go | 63 ++++++++++++++ .../cni-ipvlan-vpc-k8s-tool.go | 27 +++++- plugin/ipam/main.go | 10 +++ 4 files changed, 176 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5d37911..8dede92 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,8 @@ groups. Routes are automatically formed for the VPC on the `ipvlan` adapter. ipMasq is enabled to use the host-IP for egress to the Internet as -well as providing access to services such as `kube2iam`. +well as providing access to services such as `kube2iam`. `kube2iam` is +not a dependency of this software. ``` { @@ -209,12 +210,12 @@ well as providing access to services such as `kube2iam`. "type": "cni-ipvlan-vpc-k8s-ipam", "interfaceIndex": 1, "subnetTags": { - "kubernetes_kubelet": "true" - }, - "secGroupIds": [ - "sg-1234", - "sg-5678" - ] + "kubernetes_kubelet": "true" + }, + "secGroupIds": [ + "sg-1234", + "sg-5678" + ] } }, { @@ -228,6 +229,75 @@ well as providing access to services such as `kube2iam`. } ``` +### Other IPAM configuration flags + +In the above `ipam` block, several options are available: + + - `interfaceIndex`: We also recommend never using the boot ENI + adapter with this plugin (though it is possible). By setting + `interfaceIndex` to 1, the plugin will only allocate IPs (and add + new adapters) starting at `eth1`. + - `subnetTags`: When allocating new adapters, by default the plugin + will use all available subnets within the availability zone. You + can restrict which subnets the plugin will use by specifying key / + value tag names that must be matched in order for the plugin to be + considered. These tags are set via the AWS API or in the AWS + Console on the subnet object. + - `secGroupIds`: When allocating a new ENI adapter, these interface + groups will be assigned to the adapter. Specify the `sg-xxxx` + interface group ID. + - `skipDeallocation`: `true` or `false` - when set to `true`, this + plugin will never remove a secondary IP address from an + adapter. Useful in workloads that churn many pods to reduce the AWS + ratelimits for configuring the VPC (which are low and cannot be + raised above a certain threshold). + - `routeToVpcPeers`: `true` or `false` - When set to `true`, the + plugin will make a (cached) call to `DescribeVpcPeeringConnections` + to enumerate all peered VPCs. Routes will be added so connections + to these VPCs will be sourced from the IPvlan adapter in the pod + and not through the host masquerade. + +## The CLI Tool + +This plugin ships a CLI tool which can be useful to inspect the state +of the system or perform certain actions (such as provisioning an +adapter at instance cloud-init time). + +Run `cni-ipvlan-vpc-k8s-tool --help` for a complete listing of +options. + + NAME: + cni-ipvlan-vpc-k8s-tool - Interface with ENI adapters and CNI bindings for those + + USAGE: + cni-ipvlan-vpc-k8s-tool [global options] command [command options] [arguments...] + + VERSION: + v-next + + COMMANDS: + new-interface Create a new interface + remove-interface Remove an existing interface + deallocate Deallocate a private IP + allocate-first-available Allocate a private IP on the first available interface + free-ips List all currently unassigned AWS IP addresses + eniif List all ENI interfaces and their setup with addresses + addr List all bound IP addresses + subnets Show available subnets for this host + limits Display limits for ENI for this instance type + bugs Show any bugs associated with this instance + vpccidr Show the VPC CIDRs associated with current interfaces + vpcpeercidr Show the peered VPC CIDRs associated with current interfaces + help, h Shows a list of commands or help for one command + + GLOBAL OPTIONS: + --help, -h show help + --version, -v print the version + + COPYRIGHT: + (c) 2017-2018 Lyft Inc. + + ## Security Considerations In Kubernetes, pods and kubelets are assumed to have static IP addresses that diff --git a/aws/vpc.go b/aws/vpc.go index 13c2337..bb2aaaf 100644 --- a/aws/vpc.go +++ b/aws/vpc.go @@ -14,6 +14,7 @@ import ( // VPCClient provides a view into a VPC type VPCClient interface { DescribeVPCCIDRs(vpcID string) ([]*net.IPNet, error) + DescribeVPCPeerCIDRs(vpcID string) ([]*net.IPNet, error) } type vpcCacheClient struct { @@ -35,6 +36,21 @@ func (v *vpcCacheClient) DescribeVPCCIDRs(vpcID string) (cidrs []*net.IPNet, err return } +func (v *vpcCacheClient) DescribeVPCPeerCIDRs(vpcID string) (cidrs []*net.IPNet, err error) { + key := fmt.Sprintf("vpc-peers-%v", vpcID) + state := cache.Get(key, &cidrs) + if state == cache.CacheFound { + return + } + cidrs, err = v.vpc.DescribeVPCPeerCIDRs(vpcID) + if err != nil { + return nil, err + } + cache.Store(key, v.expiration, &cidrs) + return + +} + type vpcclient struct { aws *awsclient } @@ -66,3 +82,50 @@ func (v *vpcclient) DescribeVPCCIDRs(vpcID string) ([]*net.IPNet, error) { } return cidrs, nil } + +// DescribeVPCPeerCIDRs returns a list of CIDRs for all peered VPCs to the given VPC +func (v *vpcclient) DescribeVPCPeerCIDRs(vpcID string) ([]*net.IPNet, error) { + ec2c, err := v.aws.newEC2() + if err != nil { + return nil, err + } + + req := &ec2.DescribeVpcPeeringConnectionsInput{} + + res, err := ec2c.DescribeVpcPeeringConnections(req) + if err != nil { + return nil, err + } + + // In certain peering situations, a CIDR may be duplicated + // and visible to the API, even if the CIDR is not active in + // one of the peered VPCs. We store all of the CIDRs in a map + // to de-duplicate them. + cidrs := make(map[string]bool, 0) + + for _, peering := range res.VpcPeeringConnections { + var peer *ec2.VpcPeeringConnectionVpcInfo + + if vpcID == *peering.AccepterVpcInfo.VpcId { + peer = peering.RequesterVpcInfo + } else if vpcID == *peering.RequesterVpcInfo.VpcId { + peer = peering.AccepterVpcInfo + } + + for _, cidrBlock := range peer.CidrBlockSet { + _, _, err := net.ParseCIDR(*cidrBlock.CidrBlock) + if err == nil { + cidrs[*cidrBlock.CidrBlock] = true + } + } + } + + var returnCidrs []*net.IPNet + for cidrString := range cidrs { + _, cidr, err := net.ParseCIDR(cidrString) + if err == nil && cidr != nil { + returnCidrs = append(returnCidrs, cidr) + } + } + return returnCidrs, nil +} diff --git a/cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool.go b/cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool.go index d8ea2e5..7483916 100644 --- a/cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool.go +++ b/cmd/cni-ipvlan-vpc-k8s-tool/cni-ipvlan-vpc-k8s-tool.go @@ -228,6 +228,26 @@ func actionVpcCidr(c *cli.Context) error { return nil } +func actionVpcPeerCidr(c *cli.Context) error { + interfaces, err := aws.DefaultClient.GetInterfaces() + if err != nil { + fmt.Println(err) + return err + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "iface\tpeer_dcidr\t") + for _, iface := range interfaces { + apiCidrs, _ := aws.DefaultClient.DescribeVPCPeerCIDRs(iface.VpcID) + + fmt.Fprintf(w, "%s\t%v\t\n", + iface.LocalName(), + apiCidrs) + } + w.Flush() + return nil +} + func actionSubnets(c *cli.Context) error { subnets, err := aws.DefaultClient.GetSubnetsForInstance() if err != nil { @@ -330,9 +350,14 @@ func main() { Usage: "Show the VPC CIDRs associated with current interfaces", Action: actionVpcCidr, }, + { + Name: "vpcpeercidr", + Usage: "Show the peered VPC CIDRs associated with current interfaces", + Action: actionVpcPeerCidr, + }, } app.Version = version - app.Copyright = "(c) 2017 Lyft Inc." + app.Copyright = "(c) 2017-2018 Lyft Inc." app.Usage = "Interface with ENI adapters and CNI bindings for those" app.Run(os.Args) } diff --git a/plugin/ipam/main.go b/plugin/ipam/main.go index 1f6f87b..6fcb900 100644 --- a/plugin/ipam/main.go +++ b/plugin/ipam/main.go @@ -48,6 +48,7 @@ type IPAMConfig struct { SubnetTags map[string]string `json:"subnetTags"` IfaceIndex int `json:"interfaceIndex"` SkipDeallocation bool `json:"skipDeallocation"` + RouteToVPCPeers bool `json:"routeToVpcPeers"` } func init() { @@ -156,6 +157,15 @@ func cmdAdd(args *skel.CmdArgs) error { return fmt.Errorf("Unable to enumerate CIDRs from the AWS API due to a specific meta-data bug %v", err) } } + + if conf.IPAM.RouteToVPCPeers { + peerCidr, err := aws.DefaultClient.DescribeVPCPeerCIDRs(alloc.Interface.VpcID) + if err != nil { + return fmt.Errorf("unable to enumerate peer CIDrs %v", err) + } + cidrs = append(cidrs, peerCidr...) + } + // add routes for all VPC cidrs via the subnet gateway for _, dst := range cidrs { result.Routes = append(result.Routes, &types.Route{*dst, gw})