Merge pull request #9662 from caseydavenport/casey-vxlan-borrow

Program connected routes for borrowed VXLAN tunnel addresses

felix/calc/l3_route_resolver.go

@@ -747,6 +747,8 @@ func (c *L3RouteResolver) flush() {
 			Dst:        cidr.String(),
 		}
 		poolAllowsCrossSubnet := false
+		var blockSeen bool
+		var blockNodeName string
 		for _, entry := range buf {
 			ri := entry.Data.(RouteInfo)
 			if len(ri.Pools) > 0 {
@@ -766,6 +768,13 @@ func (c *L3RouteResolver) flush() {
 			}
 			if len(ri.Blocks) > 0 {
 				// We only expect one Block entry for any given CIDR. This constraint is upheld by the datastore.
+				if blockSeen && blockNodeName != ri.Blocks[0].NodeName {
+					logCxt.Debug("Borrowed block IP")
+					rt.Borrowed = true
+				} else {
+					blockSeen = true
+					blockNodeName = ri.Blocks[0].NodeName
+				}
 				rt.DstNodeName = ri.Blocks[0].NodeName
 				if rt.DstNodeName == c.myNodeName {
 					logCxt.Debug("Local workload route.")
@@ -794,6 +803,10 @@ func (c *L3RouteResolver) flush() {
 				// multiple workload, or workload and tunnel, or multiple node Refs with the same IP. Since this will be
 				// transient, we can always just use the first entry (and related tunnel entries)
 				rt.DstNodeName = ri.Refs[0].NodeName
+				if blockSeen && blockNodeName != rt.DstNodeName {
+					logCxt.Debug("Borrowed ref IP")
+					rt.Borrowed = true
+				}
 				if ri.Refs[0].RefType == RefTypeWEP {
 					// This is not a tunnel ref, so must be a workload.
 					if ri.Refs[0].NodeName == c.myNodeName {

felix/calc/states_for_test.go

@@ -1661,6 +1661,7 @@ var vxlanWithBlockAndBorrows = vxlanWithBlock.withKVUpdates(
 		DstNodeName: remoteHostname2,
 		DstNodeIp:   remoteHost2IP.String(),
 		NatOutgoing: true,
+		Borrowed:    true,
 	},
 )
 
@@ -1731,6 +1732,7 @@ var vxlanBlockOwnerSwitch = vxlanWithBlockAndBorrows.withKVUpdates(
 		DstNodeName: remoteHostname,
 		DstNodeIp:   remoteHostIP.String(),
 		NatOutgoing: true,
+		Borrowed:    true,
 	},
 ).withName("VXLAN owner switch")
 
@@ -1784,6 +1786,7 @@ var vxlanLocalBlockWithBorrows = empty.withKVUpdates(
 		DstNodeName: remoteHostname,
 		DstNodeIp:   remoteHostIP.String(),
 		NatOutgoing: true,
+		Borrowed:    true,
 	},
 ).withExpectedEncapsulation(
 	&proto.Encapsulation{IpipEnabled: false, VxlanEnabled: true, VxlanEnabledV6: false},
@@ -1807,6 +1810,7 @@ var localVXLANWep1Route2 = types.RouteUpdate{
 	DstNodeIp:     localHostIP.String(),
 	NatOutgoing:   true,
 	LocalWorkload: true,
+	Borrowed:      true,
 }
 
 // As vxlanLocalBlockWithBorrows but with a local workload.  The local workload has an IP that overlaps with
@@ -1905,6 +1909,7 @@ var vxlanLocalBlockWithBorrowsCrossSubnetNodeRes = vxlanLocalBlockWithBorrowsNod
 		DstNodeName: remoteHostname,
 		DstNodeIp:   remoteHostIP.String(),
 		SameSubnet:  true, // cross subnet.
+		Borrowed:    true,
 	},
 ).withName("VXLAN local with borrows cross subnet (node resources)")
 
@@ -1961,6 +1966,7 @@ var vxlanLocalBlockWithBorrowsDifferentSubnetNodeRes = vxlanLocalBlockWithBorrow
 		DstNodeName: remoteHostname,
 		DstNodeIp:   remoteHostIP.String(),
 		SameSubnet:  false, // subnets don't match.
+		Borrowed:    true,
 	},
 ).withName("VXLAN cross subnet different subnet (node resources)")
 
@@ -1992,6 +1998,7 @@ var vxlanWithBlockAndBorrowsAndMissingFirstVTEP = vxlanWithBlockAndBorrows.withK
 		DstNodeName: remoteHostname2,
 		DstNodeIp:   remoteHost2IP.String(),
 		NatOutgoing: true,
+		Borrowed:    true,
 	},
 )
 

felix/dataplane/linux/vxlan_mgr.go

@@ -198,6 +198,14 @@ func (m *vxlanManager) OnUpdate(protoBufMsg interface{}) {
 			m.routesDirty = true
 		}
 
+		// Process routes for remote tunnel endpoints as well. This is necessary to ensure hosts can
+		// communicate with tunnel endpoints that whose IP address has been borrowed.
+		if msg.Type == proto.RouteType_REMOTE_TUNNEL && msg.IpPoolType == proto.IPPoolType_VXLAN && msg.Borrowed {
+			m.logCtx.WithField("msg", msg).Debug("VXLAN data plane received route update for borrowed VXLAN tunnel IP")
+			m.routesByDest[msg.Dst] = msg
+			m.routesDirty = true
+		}
+
 		// Process IPAM blocks that aren't associated to a single or /32 local workload
 		if routeIsLocalVXLANBlock(msg) {
 			m.logCtx.WithField("msg", msg).Debug("VXLAN data plane received route update for IPAM block")
@@ -491,23 +499,27 @@ func (m *vxlanManager) noEncapRoute(cidr ip.CIDR, r *proto.RouteUpdate) *routeta
 }
 
 func (m *vxlanManager) tunneledRoute(cidr ip.CIDR, r *proto.RouteUpdate) *routetable.Target {
+	if r.Type == proto.RouteType_REMOTE_TUNNEL {
+		// We treat remote tunnel routes as directly connected. They don't have a gateway of
+		// the VTEP because they ARE the VTEP!
+		return &routetable.Target{CIDR: cidr}
+	}
+
 	// Extract the gateway addr for this route based on its remote VTEP.
 	vtep, ok := m.vtepsByNode[r.DstNodeName]
 	if !ok {
 		// When the VTEP arrives, it'll set routesDirty=true so this loop will execute again.
 		return nil
 	}
-
 	vtepAddr := vtep.Ipv4Addr
 	if m.ipVersion == 6 {
 		vtepAddr = vtep.Ipv6Addr
 	}
-	vxlanRoute := routetable.Target{
+	return &routetable.Target{
 		Type: routetable.TargetTypeVXLAN,
 		CIDR: cidr,
 		GW:   ip.FromString(vtepAddr),
 	}
-	return &vxlanRoute
 }
 
 func (m *vxlanManager) OnParentNameUpdate(name string) {

felix/dataplane/linux/vxlan_mgr_test.go

@@ -461,4 +461,64 @@ var _ = Describe("VXLANManager", func() {
 		Expect(rt.currentRoutes["eth0"]).To(HaveLen(1))
 		Expect(rt.currentRoutes[dataplanedefs.VXLANIfaceNameV6]).To(HaveLen(0))
 	})
+
+	It("should program directly connected routes for remote VTEPs with borrowed IP addresses", func() {
+		By("Sending a borrowed tunnel IP address")
+		manager.OnUpdate(&proto.RouteUpdate{
+			Type:        proto.RouteType_REMOTE_TUNNEL,
+			IpPoolType:  proto.IPPoolType_VXLAN,
+			Dst:         "10.0.1.1/32",
+			DstNodeName: "node2",
+			DstNodeIp:   "172.16.0.1",
+			Borrowed:    true,
+		})
+
+		err := manager.CompleteDeferredWork()
+		Expect(err).NotTo(HaveOccurred())
+
+		// Expect a directly connected route to the borrowed IP.
+		Expect(rt.currentRoutes[dataplanedefs.VXLANIfaceNameV4]).To(HaveLen(1))
+		Expect(rt.currentRoutes[dataplanedefs.VXLANIfaceNameV4][0]).To(Equal(routetable.Target{CIDR: ip.MustParseCIDROrIP("10.0.1.1/32")}))
+
+		// Delete the route.
+		manager.OnUpdate(&proto.RouteRemove{
+			Dst: "10.0.1.1/32",
+		})
+
+		err = manager.CompleteDeferredWork()
+		Expect(err).NotTo(HaveOccurred())
+
+		// Expect no routes.
+		Expect(rt.currentRoutes["vxlan.calico"]).To(HaveLen(0))
+	})
+
+	It("IPv6: should program directly connected routes for remote VTEPs with borrowed IP addresses", func() {
+		By("Sending a borrowed tunnel IP address")
+		managerV6.OnUpdate(&proto.RouteUpdate{
+			Type:        proto.RouteType_REMOTE_TUNNEL,
+			IpPoolType:  proto.IPPoolType_VXLAN,
+			Dst:         "fc00:10:244::1/112",
+			DstNodeName: "node2",
+			DstNodeIp:   "fc00:10:10::8",
+			Borrowed:    true,
+		})
+
+		err := managerV6.CompleteDeferredWork()
+		Expect(err).NotTo(HaveOccurred())
+
+		// Expect a directly connected route to the borrowed IP.
+		Expect(rt.currentRoutes[dataplanedefs.VXLANIfaceNameV6]).To(HaveLen(1))
+		Expect(rt.currentRoutes[dataplanedefs.VXLANIfaceNameV6][0]).To(Equal(routetable.Target{CIDR: ip.MustParseCIDROrIP("fc00:10:244::1/112")}))
+
+		// Delete the route.
+		managerV6.OnUpdate(&proto.RouteRemove{
+			Dst: "fc00:10:244::1/112",
+		})
+
+		err = managerV6.CompleteDeferredWork()
+		Expect(err).NotTo(HaveOccurred())
+
+		// Expect no routes.
+		Expect(rt.currentRoutes["vxlan.calico"]).To(HaveLen(0))
+	})
 })