From ee48092c5e124886287cdda3bf6f02c378fe5665 Mon Sep 17 00:00:00 2001 From: ksubrmnn Date: Fri, 14 Dec 2018 09:58:25 -0800 Subject: [PATCH] Use remote subnet feature --- backend/vxlan/vxlan_network_windows.go | 86 +++++++++++++++++++------- backend/vxlan/vxlan_windows.go | 65 ++++++++++++++++++- 2 files changed, 125 insertions(+), 26 deletions(-) diff --git a/backend/vxlan/vxlan_network_windows.go b/backend/vxlan/vxlan_network_windows.go index 74c4248a81..c4f1b5e6c4 100644 --- a/backend/vxlan/vxlan_network_windows.go +++ b/backend/vxlan/vxlan_network_windows.go @@ -22,7 +22,10 @@ import ( "github.com/coreos/flannel/backend" "github.com/coreos/flannel/subnet" + "encoding/json" + "github.com/Microsoft/hcsshim/hcn" "github.com/coreos/flannel/pkg/ip" + "net" "strings" ) @@ -32,6 +35,11 @@ type network struct { subnetMgr subnet.Manager } +type vxlanLeaseAttrs struct { + VNI uint16 + VtepMAC hardwareAddr +} + const ( encapOverhead = 50 ) @@ -87,36 +95,66 @@ func (nw *network) handleSubnetEvents(batch []subnet.Event) { continue } - publicIP := leaseAttrs.PublicIP.String() - remoteIP := leaseSubnet.IP + 2 - lastIP := leaseSubnet.Next().IP - 1 + var vxlanAttrs vxlanLeaseAttrs + if err := json.Unmarshal(leaseAttrs.BackendData, &vxlanAttrs); err != nil { + log.Error("error decoding subnet lease JSON: ", err) + continue + } + + hnsnetwork, err := hcn.GetNetworkByName(nw.dev.link.Name) + if err != nil { + log.Errorf("Unable to find network %v, error: %v", nw.dev.link.Name, err) + continue + } + managementIp := event.Lease.Attrs.PublicIP.String() + + networkPolicySettings := hcn.RemoteSubnetRoutePolicySetting{ + IsolationId: 4096, + DistributedRouterMacAddress: net.HardwareAddr(vxlanAttrs.VtepMAC).String(), + ProviderAddress: managementIp, + DestinationPrefix: event.Lease.Subnet.String(), + } + rawJSON, err := json.Marshal(networkPolicySettings) + networkPolicy := hcn.NetworkPolicy{ + Type: hcn.RemoteSubnetRoute, + Settings: rawJSON, + } + + policyNetworkRequest := hcn.PolicyNetworkRequest{ + Policies: []hcn.NetworkPolicy{networkPolicy}, + } switch event.Type { case subnet.EventAdded: - for ; remoteIP < lastIP; remoteIP++ { - n := &neighbor{ - IP: remoteIP, - MAC: nw.dev.ConjureMac(remoteIP), - ManagementAddress: publicIP, - } - - log.V(2).Infof("adding subnet: %v publicIP: %s vtepMAC: %s", leaseSubnet, n.ManagementAddress, n.MAC) - if err := nw.dev.AddEndpoint(n); err != nil { - log.Error(err) + for _, policy := range hnsnetwork.Policies { + if policy.Type == hcn.RemoteSubnetRoute { + existingPolicySettings := hcn.RemoteSubnetRoutePolicySetting{} + err = json.Unmarshal(policy.Settings, &existingPolicySettings) + if err != nil { + log.Error("Failed to unmarshal settings") + } + if existingPolicySettings.DestinationPrefix == networkPolicySettings.DestinationPrefix { + existingJson, err := json.Marshal(existingPolicySettings) + if err != nil { + log.Error("Failed to marshal settings") + } + existingPolicy := hcn.NetworkPolicy{ + Type: hcn.RemoteSubnetRoute, + Settings: existingJson, + } + existingPolicyNetworkRequest := hcn.PolicyNetworkRequest{ + Policies: []hcn.NetworkPolicy{existingPolicy}, + } + hnsnetwork.RemovePolicy(existingPolicyNetworkRequest) + } } } + if networkPolicySettings.DistributedRouterMacAddress != "" { + hnsnetwork.AddPolicy(policyNetworkRequest) + } case subnet.EventRemoved: - for ; remoteIP < lastIP; remoteIP++ { - n := &neighbor{ - IP: remoteIP, - MAC: nw.dev.ConjureMac(remoteIP), - ManagementAddress: publicIP, - } - - log.V(2).Infof("removing subnet: %v publicIP: %s vtepMAC: %s", leaseSubnet, n.ManagementAddress, n.MAC) - if err := nw.dev.DelEndpoint(n); err != nil { - log.Error(err) - } + if networkPolicySettings.DistributedRouterMacAddress != "" { + hnsnetwork.RemovePolicy(policyNetworkRequest) } default: log.Error("internal error: unknown event type: ", int(event.Type)) diff --git a/backend/vxlan/vxlan_windows.go b/backend/vxlan/vxlan_windows.go index b35d09f407..fd72f4c9f3 100644 --- a/backend/vxlan/vxlan_windows.go +++ b/backend/vxlan/vxlan_windows.go @@ -33,6 +33,8 @@ import ( "golang.org/x/net/context" + "github.com/Microsoft/hcsshim" + "github.com/Microsoft/hcsshim/hcn" "github.com/coreos/flannel/backend" "github.com/coreos/flannel/pkg/ip" "github.com/coreos/flannel/subnet" @@ -62,10 +64,20 @@ func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backen return backend, nil } -func newSubnetAttrs(publicIP net.IP) (*subnet.LeaseAttrs, error) { +func newSubnetAttrs(publicIP net.IP, vnid uint16, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) { + leaseAttrs := &vxlanLeaseAttrs{ + VNI: vnid, + VtepMAC: hardwareAddr(mac), + } + data, err := json.Marshal(&leaseAttrs) + if err != nil { + return nil, err + } + return &subnet.LeaseAttrs{ PublicIP: ip.FromIP(publicIP), BackendType: "vxlan", + BackendData: json.RawMessage(data), }, nil } @@ -111,7 +123,33 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup, } log.Infof("VXLAN config: Name=%s MacPrefix=%s VNI=%d Port=%d GBP=%v DirectRouting=%v", cfg.Name, cfg.MacPrefix, cfg.VNI, cfg.Port, cfg.GBP, cfg.DirectRouting) - subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr) + hnsNetworks, err := hcsshim.HNSListNetworkRequest("GET", "", "") + if err != nil { + log.Infof("Cannot get HNS networks [%+v]", err) + } + + var remoteDrMac string + for _, hnsnetwork := range hnsNetworks { + if hnsnetwork.ManagementIP == be.extIface.ExtAddr.String() { + hcnnetwork, err := hcn.GetNetworkByID(hnsnetwork.Id) + policies := hcnnetwork.Policies + for _, policy := range policies { + if policy.Type == hcn.DrMacAddress { + policySettings := hcn.DrMacAddressNetworkPolicySetting{} + err = json.Unmarshal(policy.Settings, &policySettings) + if err != nil { + return nil, fmt.Errorf("Failed to unmarshal settings") + } + remoteDrMac = policySettings.Address + } + } + } + } + mac, err := net.ParseMAC(string(remoteDrMac)) + if err != nil { + return nil, err + } + subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, uint16(cfg.VNI), mac) if err != nil { return nil, err } @@ -140,3 +178,26 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup, return newNetwork(be.subnetMgr, be.extIface, dev, ip.IP4Net{}, lease) } + +// So we can make it JSON (un)marshalable +type hardwareAddr net.HardwareAddr + +func (hw hardwareAddr) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("%q", net.HardwareAddr(hw))), nil +} + +func (hw *hardwareAddr) UnmarshalJSON(bytes []byte) error { + if len(bytes) < 2 || bytes[0] != '"' || bytes[len(bytes)-1] != '"' { + return fmt.Errorf("error parsing hardware addr") + } + + bytes = bytes[1 : len(bytes)-1] + + mac, err := net.ParseMAC(string(bytes)) + if err != nil { + return err + } + + *hw = hardwareAddr(mac) + return nil +}