Merge pull request #1061 from ksubrmnn/overlay

Remote subnet feature

backend/vxlan/vxlan_network_windows.go

@@ -22,7 +22,10 @@ import (
 	"github.com/coreos/flannel/backend"
 	"github.com/coreos/flannel/subnet"
 
+	"encoding/json"
+	"github.com/Microsoft/hcsshim/hcn"
 	"github.com/coreos/flannel/pkg/ip"
+	"net"
 	"strings"
 )
 
@@ -32,6 +35,11 @@ type network struct {
 	subnetMgr subnet.Manager
 }
 
+type vxlanLeaseAttrs struct {
+	VNI     uint16
+	VtepMAC hardwareAddr
+}
+
 const (
 	encapOverhead = 50
 )
@@ -87,36 +95,66 @@ func (nw *network) handleSubnetEvents(batch []subnet.Event) {
 			continue
 		}
 
-		publicIP := leaseAttrs.PublicIP.String()
-		remoteIP := leaseSubnet.IP + 2
-		lastIP := leaseSubnet.Next().IP - 1
+		var vxlanAttrs vxlanLeaseAttrs
+		if err := json.Unmarshal(leaseAttrs.BackendData, &vxlanAttrs); err != nil {
+			log.Error("error decoding subnet lease JSON: ", err)
+			continue
+		}
+
+		hnsnetwork, err := hcn.GetNetworkByName(nw.dev.link.Name)
+		if err != nil {
+			log.Errorf("Unable to find network %v, error: %v", nw.dev.link.Name, err)
+			continue
+		}
+		managementIp := event.Lease.Attrs.PublicIP.String()
+
+		networkPolicySettings := hcn.RemoteSubnetRoutePolicySetting{
+			IsolationId:                 4096,
+			DistributedRouterMacAddress: net.HardwareAddr(vxlanAttrs.VtepMAC).String(),
+			ProviderAddress:             managementIp,
+			DestinationPrefix:           event.Lease.Subnet.String(),
+		}
+		rawJSON, err := json.Marshal(networkPolicySettings)
+		networkPolicy := hcn.NetworkPolicy{
+			Type:     hcn.RemoteSubnetRoute,
+			Settings: rawJSON,
+		}
+
+		policyNetworkRequest := hcn.PolicyNetworkRequest{
+			Policies: []hcn.NetworkPolicy{networkPolicy},
+		}
 
 		switch event.Type {
 		case subnet.EventAdded:
-			for ; remoteIP < lastIP; remoteIP++ {
-				n := &neighbor{
-					IP:                remoteIP,
-					MAC:               nw.dev.ConjureMac(remoteIP),
-					ManagementAddress: publicIP,
-				}
-
-				log.V(2).Infof("adding subnet: %v publicIP: %s vtepMAC: %s", leaseSubnet, n.ManagementAddress, n.MAC)
-				if err := nw.dev.AddEndpoint(n); err != nil {
-					log.Error(err)
+			for _, policy := range hnsnetwork.Policies {
+				if policy.Type == hcn.RemoteSubnetRoute {
+					existingPolicySettings := hcn.RemoteSubnetRoutePolicySetting{}
+					err = json.Unmarshal(policy.Settings, &existingPolicySettings)
+					if err != nil {
+						log.Error("Failed to unmarshal settings")
+					}
+					if existingPolicySettings.DestinationPrefix == networkPolicySettings.DestinationPrefix {
+						existingJson, err := json.Marshal(existingPolicySettings)
+						if err != nil {
+							log.Error("Failed to marshal settings")
+						}
+						existingPolicy := hcn.NetworkPolicy{
+							Type:     hcn.RemoteSubnetRoute,
+							Settings: existingJson,
+						}
+						existingPolicyNetworkRequest := hcn.PolicyNetworkRequest{
+							Policies: []hcn.NetworkPolicy{existingPolicy},
+						}
+						hnsnetwork.RemovePolicy(existingPolicyNetworkRequest)
+					}
 				}
 			}
+			if networkPolicySettings.DistributedRouterMacAddress != "" {
+				hnsnetwork.AddPolicy(policyNetworkRequest)
+			}
 		case subnet.EventRemoved:
-			for ; remoteIP < lastIP; remoteIP++ {
-				n := &neighbor{
-					IP:                remoteIP,
-					MAC:               nw.dev.ConjureMac(remoteIP),
-					ManagementAddress: publicIP,
-				}
-
-				log.V(2).Infof("removing subnet: %v publicIP: %s vtepMAC: %s", leaseSubnet, n.ManagementAddress, n.MAC)
-				if err := nw.dev.DelEndpoint(n); err != nil {
-					log.Error(err)
-				}
+			if networkPolicySettings.DistributedRouterMacAddress != "" {
+				hnsnetwork.RemovePolicy(policyNetworkRequest)
 			}
 		default:
 			log.Error("internal error: unknown event type: ", int(event.Type))

backend/vxlan/vxlan_windows.go

@@ -33,6 +33,8 @@ import (
 
 	"golang.org/x/net/context"
 
+	"github.com/Microsoft/hcsshim"
+	"github.com/Microsoft/hcsshim/hcn"
 	"github.com/coreos/flannel/backend"
 	"github.com/coreos/flannel/pkg/ip"
 	"github.com/coreos/flannel/subnet"
@@ -62,10 +64,20 @@ func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backen
 	return backend, nil
 }
 
-func newSubnetAttrs(publicIP net.IP) (*subnet.LeaseAttrs, error) {
+func newSubnetAttrs(publicIP net.IP, vnid uint16, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
+	leaseAttrs := &vxlanLeaseAttrs{
+		VNI:     vnid,
+		VtepMAC: hardwareAddr(mac),
+	}
+	data, err := json.Marshal(&leaseAttrs)
+	if err != nil {
+		return nil, err
+	}
+
 	return &subnet.LeaseAttrs{
 		PublicIP:    ip.FromIP(publicIP),
 		BackendType: "vxlan",
+		BackendData: json.RawMessage(data),
 	}, nil
 }
 
@@ -111,7 +123,33 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup,
 	}
 	log.Infof("VXLAN config: Name=%s MacPrefix=%s VNI=%d Port=%d GBP=%v DirectRouting=%v", cfg.Name, cfg.MacPrefix, cfg.VNI, cfg.Port, cfg.GBP, cfg.DirectRouting)
 
-	subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr)
+	hnsNetworks, err := hcsshim.HNSListNetworkRequest("GET", "", "")
+	if err != nil {
+		log.Infof("Cannot get HNS networks [%+v]", err)
+	}
+
+	var remoteDrMac string
+	for _, hnsnetwork := range hnsNetworks {
+		if hnsnetwork.ManagementIP == be.extIface.ExtAddr.String() {
+			hcnnetwork, err := hcn.GetNetworkByID(hnsnetwork.Id)
+			policies := hcnnetwork.Policies
+			for _, policy := range policies {
+				if policy.Type == hcn.DrMacAddress {
+					policySettings := hcn.DrMacAddressNetworkPolicySetting{}
+					err = json.Unmarshal(policy.Settings, &policySettings)
+					if err != nil {
+						return nil, fmt.Errorf("Failed to unmarshal settings")
+					}
+					remoteDrMac = policySettings.Address
+				}
+			}
+		}
+	}
+	mac, err := net.ParseMAC(string(remoteDrMac))
+	if err != nil {
+		return nil, err
+	}
+	subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, uint16(cfg.VNI), mac)
 	if err != nil {
 		return nil, err
 	}
@@ -140,3 +178,26 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup,
 
 	return newNetwork(be.subnetMgr, be.extIface, dev, ip.IP4Net{}, lease)
 }
+
+// So we can make it JSON (un)marshalable
+type hardwareAddr net.HardwareAddr
+
+func (hw hardwareAddr) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf("%q", net.HardwareAddr(hw))), nil
+}
+
+func (hw *hardwareAddr) UnmarshalJSON(bytes []byte) error {
+	if len(bytes) < 2 || bytes[0] != '"' || bytes[len(bytes)-1] != '"' {
+		return fmt.Errorf("error parsing hardware addr")
+	}
+
+	bytes = bytes[1 : len(bytes)-1]
+
+	mac, err := net.ParseMAC(string(bytes))
+	if err != nil {
+		return err
+	}
+
+	*hw = hardwareAddr(mac)
+	return nil
+}

glide.yaml

@@ -63,7 +63,7 @@ import:
 - package: github.com/bronze1man/goStrongswanVici
   version: 4d72634a2f113aa48347dbc7dcb14adb806b6534
 - package: github.com/Microsoft/hcsshim
-  version: v0.7.4
+  version: v0.8.3
 - package: github.com/Microsoft/go-winio
   version: v0.4.11
 - pacakge: github.com/sirupsen/logrus