EVPN L2VNI/L3VNI Optimize inline Global walk for remote route installations

bgpd/bgp_evpn.h

@@ -200,4 +200,6 @@ bool bgp_evpn_skip_vrf_import_of_local_es(struct bgp *bgp_vrf, const struct pref
 int uninstall_evpn_route_entry_in_vrf(struct bgp *bgp_vrf, const struct prefix_evpn *evp,
 				      struct bgp_path_info *parent_pi);
 extern void bgp_zebra_evpn_pop_items_from_announce_fifo(struct bgpevpn *vpn);
+extern int install_uninstall_routes_for_vni(struct bgp *bgp, struct bgpevpn *vpn, bool install);
+extern int install_uninstall_routes_for_vrf(struct bgp *bgp_vrf, bool install);
 #endif /* _QUAGGA_BGP_EVPN_H */

bgpd/bgp_evpn_private.h

@@ -32,6 +32,15 @@
 #define BGP_EVPN_TYPE4_V4_PSIZE 23
 #define BGP_EVPN_TYPE4_V6_PSIZE 34
 
+static const struct message bgp_evpn_route_type_str[] = {
+	{BGP_EVPN_UNKN_ROUTE, "UNKNOWN"},
+	{BGP_EVPN_AD_ROUTE, "AD"},
+	{BGP_EVPN_MAC_IP_ROUTE, "MACIP"},
+	{BGP_EVPN_IMET_ROUTE, "IMET"},
+	{BGP_EVPN_ES_ROUTE, "ES"},
+	{BGP_EVPN_IP_PREFIX_ROUTE, "IP-PREFIX"},
+	{0}};
+
 RB_HEAD(bgp_es_evi_rb_head, bgp_evpn_es_evi);
 RB_PROTOTYPE(bgp_es_evi_rb_head, bgp_evpn_es_evi, rb_node,
 		bgp_es_evi_rb_cmp);
@@ -53,8 +62,9 @@ struct bgpevpn {
 #define VNI_FLAG_RD_CFGD           0x4  /* RD is user configured. */
 #define VNI_FLAG_IMPRT_CFGD        0x8  /* Import RT is user configured */
 #define VNI_FLAG_EXPRT_CFGD        0x10 /* Export RT is user configured */
-#define VNI_FLAG_USE_TWO_LABELS    0x20 /* Attach both L2-VNI and L3-VNI if
-					   needed for this VPN */
+/* Attach both L2-VNI and L3-VNI if needed for this VPN */
+#define VNI_FLAG_USE_TWO_LABELS 0x20
+#define VNI_FLAG_ADD		0x40 /* L2VNI Add */
 
 	struct bgp *bgp_vrf; /* back pointer to the vrf instance */
 
@@ -108,11 +118,15 @@ struct bgpevpn {
 	/* List of local ESs */
 	struct list *local_es_evi_list;
 
+	struct zebra_l2_vni_item zl2vni;
+
 	QOBJ_FIELDS;
 };
 
 DECLARE_QOBJ_TYPE(bgpevpn);
 
+DECLARE_LIST(zebra_l2_vni, struct bgpevpn, zl2vni);
+
 /* Mapping of Import RT to VNIs.
  * The Import RTs of all VNIs are maintained in a hash table with each
  * RT linking to all VNIs that will import routes matching this RT.

bgpd/bgp_main.c

@@ -207,6 +207,8 @@ static __attribute__((__noreturn__)) void bgp_exit(int status)
 	bgp_nhg_finish();
 
 	zebra_announce_fini(&bm->zebra_announce_head);
+	zebra_l2_vni_fini(&bm->zebra_l2_vni_head);
+	zebra_l3_vni_fini(&bm->zebra_l3_vni_head);
 
 	/* reverse bgp_dump_init */
 	bgp_dump_finish();

bgpd/bgp_zebra.c

@@ -3025,6 +3025,48 @@ static void bgp_zebra_connected(struct zclient *zclient)
 	BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(bgp, bgp->peer);
 }
 
+void bgp_zebra_process_remote_routes_for_l2vni(struct event *e)
+{
+	/*
+	 * If we have learnt and retained remote routes (VTEPs, MACs)
+	 * for this VNI, install them.
+	 */
+	install_uninstall_routes_for_vni(NULL, NULL, true);
+
+	/*
+	 * If there are VNIs still pending to be processed, schedule them
+	 * after a small sleep so that CPU can be used for other purposes.
+	 */
+	if (zebra_l2_vni_count(&bm->zebra_l2_vni_head))
+		event_add_timer_msec(bm->master, bgp_zebra_process_remote_routes_for_l2vni, NULL,
+				     20, &bm->t_bgp_zebra_l2_vni);
+}
+
+void bgp_zebra_process_remote_routes_for_l3vrf(struct event *e)
+{
+	/*
+	 * Install/Uninstall all remote routes belonging to l3vni
+	 *
+	 * NOTE:
+	 *  - At this point it does not matter whether we call
+	 *    install_routes_for_vrf/uninstall_routes_for_vrf.
+	 *  - Since we pass struct bgp as NULL,
+	 *      * we iterate the bm FIFO list
+	 *      * the second variable (true) is ignored as well and
+	 *        calculated based on the BGP-VRFs flags for ADD/DELETE.
+	 */
+	install_uninstall_routes_for_vrf(NULL, true);
+
+	/*
+	 * If there are L3VNIs still pending to be processed, schedule them
+	 * after a small sleep so that CPU can be used for other purposes.
+	 */
+	if (zebra_l3_vni_count(&bm->zebra_l3_vni_head)) {
+		event_add_timer_msec(bm->master, bgp_zebra_process_remote_routes_for_l3vrf, NULL,
+				     20, &bm->t_bgp_zebra_l3_vni);
+	}
+}
+
 static int bgp_zebra_process_local_es_add(ZAPI_CALLBACK_ARGS)
 {
 	esi_t esi;

bgpd/bgp_zebra.h

@@ -135,4 +135,6 @@ extern void bgp_zebra_release_label_range(uint32_t start, uint32_t end);
 extern enum zclient_send_status
 bgp_zebra_withdraw_actual(struct bgp_dest *dest, struct bgp_path_info *info,
 			  struct bgp *bgp);
+extern void bgp_zebra_process_remote_routes_for_l2vni(struct event *e);
+extern void bgp_zebra_process_remote_routes_for_l3vrf(struct event *e);
 #endif /* _QUAGGA_BGP_ZEBRA_H */

bgpd/bgpd.c

@@ -3966,19 +3966,24 @@ int bgp_delete(struct bgp *bgp)
 	afi_t afi;
 	safi_t safi;
 	int i;
+	uint32_t vni_count;
+	struct bgpevpn *vpn = NULL;
 	struct bgp_dest *dest = NULL;
 	struct bgp_dest *dest_next = NULL;
 	struct bgp_table *dest_table = NULL;
 	struct graceful_restart_info *gr_info;
-	uint32_t cnt_before, cnt_after;
+	uint32_t b_ann_cnt = 0, b_l2_cnt = 0, b_l3_cnt = 0;
+	uint32_t a_ann_cnt = 0, a_l2_cnt = 0, a_l3_cnt = 0;
+	struct bgp *bgp_to_proc = NULL;
+	struct bgp *bgp_to_proc_next = NULL;
 
 	assert(bgp);
 
 	/*
 	 * Iterate the pending dest list and remove all the dest pertaininig to
 	 * the bgp under delete.
 	 */
-	cnt_before = zebra_announce_count(&bm->zebra_announce_head);
+	b_ann_cnt = zebra_announce_count(&bm->zebra_announce_head);
 	for (dest = zebra_announce_first(&bm->zebra_announce_head); dest;
 	     dest = dest_next) {
 		dest_next = zebra_announce_next(&bm->zebra_announce_head, dest);
@@ -3990,10 +3995,36 @@ int bgp_delete(struct bgp *bgp)
 		}
 	}
 
-	cnt_after = zebra_announce_count(&bm->zebra_announce_head);
-	if (BGP_DEBUG(zebra, ZEBRA))
-		zlog_debug("Zebra Announce Fifo cleanup count before %u and after %u during BGP %s deletion",
-			   cnt_before, cnt_after, bgp->name_pretty);
+	/*
+	 * Pop all VPNs yet to be processed for remote routes install if the
+	 * bgp-evpn instance is getting deleted
+	 */
+	if (bgp == bgp_get_evpn()) {
+		b_l2_cnt = zebra_l2_vni_count(&bm->zebra_l2_vni_head);
+		vni_count = b_l2_cnt;
+		while (vni_count) {
+			vpn = zebra_l2_vni_pop(&bm->zebra_l2_vni_head);
+			UNSET_FLAG(vpn->flags, VNI_FLAG_ADD);
+			vni_count--;
+		}
+	}
+
+	b_l3_cnt = zebra_l3_vni_count(&bm->zebra_l3_vni_head);
+	for (bgp_to_proc = zebra_l3_vni_first(&bm->zebra_l3_vni_head); bgp_to_proc;
+	     bgp_to_proc = bgp_to_proc_next) {
+		bgp_to_proc_next = zebra_l3_vni_next(&bm->zebra_l3_vni_head, bgp_to_proc);
+		if (bgp_to_proc == bgp)
+			zebra_l3_vni_del(&bm->zebra_l3_vni_head, bgp_to_proc);
+	}
+
+	if (BGP_DEBUG(zebra, ZEBRA)) {
+		a_ann_cnt = zebra_announce_count(&bm->zebra_announce_head);
+		a_l2_cnt = zebra_l2_vni_count(&bm->zebra_l2_vni_head);
+		a_l3_cnt = zebra_l3_vni_count(&bm->zebra_l3_vni_head);
+		zlog_debug("BGP %s deletion FIFO cnt Zebra_Ann before %u after %u, L2_VNI before %u after, %u L3_VNI before %u after %u",
+			   bgp->name_pretty, b_ann_cnt, a_ann_cnt, b_l2_cnt, a_l2_cnt, b_l3_cnt,
+			   a_l3_cnt);
+	}
 
 	bgp_soft_reconfig_table_task_cancel(bgp, NULL, NULL);
 
@@ -8492,6 +8523,8 @@ void bgp_master_init(struct event_loop *master, const int buffer_size,
 	bm = &bgp_master;
 
 	zebra_announce_init(&bm->zebra_announce_head);
+	zebra_l2_vni_init(&bm->zebra_l2_vni_head);
+	zebra_l3_vni_init(&bm->zebra_l3_vni_head);
 	bm->bgp = list_new();
 	bm->listen_sockets = list_new();
 	bm->port = BGP_PORT_DEFAULT;
@@ -8515,6 +8548,8 @@ void bgp_master_init(struct event_loop *master, const int buffer_size,
 	bm->stalepath_time = BGP_DEFAULT_STALEPATH_TIME;
 	bm->select_defer_time = BGP_DEFAULT_SELECT_DEFERRAL_TIME;
 	bm->rib_stale_time = BGP_DEFAULT_RIB_STALE_TIME;
+	bm->t_bgp_zebra_l2_vni = NULL;
+	bm->t_bgp_zebra_l3_vni = NULL;
 
 	bgp_mac_init();
 	/* init the rd id space.
@@ -8762,6 +8797,8 @@ void bgp_terminate(void)
 	EVENT_OFF(bm->t_bgp_sync_label_manager);
 	EVENT_OFF(bm->t_bgp_start_label_manager);
 	EVENT_OFF(bm->t_bgp_zebra_route);
+	EVENT_OFF(bm->t_bgp_zebra_l2_vni);
+	EVENT_OFF(bm->t_bgp_zebra_l3_vni);
 
 	bgp_mac_finish();
 }

bgpd/bgpd.h

@@ -19,6 +19,8 @@
 #include "asn.h"
 
 PREDECL_LIST(zebra_announce);
+PREDECL_LIST(zebra_l2_vni);
+PREDECL_LIST(zebra_l3_vni);
 
 /* For union sockunion.  */
 #include "queue.h"
@@ -204,6 +206,14 @@ struct bgp_master {
 	/* To preserve ordering of installations into zebra across all Vrfs */
 	struct zebra_announce_head zebra_announce_head;
 
+	struct event *t_bgp_zebra_l2_vni;
+	/* To preserve ordering of processing of L2 VNIs in BGP */
+	struct zebra_l2_vni_head zebra_l2_vni_head;
+
+	struct event *t_bgp_zebra_l3_vni;
+	/* To preserve ordering of processing of BGP-VRFs for L3 VNIs */
+	struct zebra_l3_vni_head zebra_l3_vni_head;
+
 	QOBJ_FIELDS;
 };
 DECLARE_QOBJ_TYPE(bgp_master);
@@ -554,6 +564,8 @@ struct bgp {
 #define BGP_FLAG_INSTANCE_HIDDEN	 (1ULL << 39)
 /* Prohibit BGP from enabling IPv6 RA on interfaces */
 #define BGP_FLAG_IPV6_NO_AUTO_RA (1ULL << 40)
+#define BGP_FLAG_L3VNI_SCHEDULE_FOR_INSTALL (1ULL << 41)
+#define BGP_FLAG_L3VNI_SCHEDULE_FOR_DELETE  (1ULL << 42)
 
 	/* BGP default address-families.
 	 * New peers inherit enabled afi/safis from bgp instance.
@@ -868,10 +880,14 @@ struct bgp {
 	uint64_t node_already_on_queue;
 	uint64_t node_deferred_on_queue;
 
+	struct zebra_l3_vni_item zl3vni;
+
 	QOBJ_FIELDS;
 };
 DECLARE_QOBJ_TYPE(bgp);
 
+DECLARE_LIST(zebra_l3_vni, struct bgp, zl3vni);
+
 struct bgp_interface {
 #define BGP_INTERFACE_MPLS_BGP_FORWARDING (1 << 0)
 /* L3VPN multi domain switching */

lib/prefix.h

@@ -26,6 +26,7 @@ extern "C" {
 
 /* EVPN route types. */
 typedef enum {
+	BGP_EVPN_UNKN_ROUTE = 0,
 	BGP_EVPN_AD_ROUTE = 1,    /* Ethernet Auto-Discovery (A-D) route */
 	BGP_EVPN_MAC_IP_ROUTE,    /* MAC/IP Advertisement route */
 	BGP_EVPN_IMET_ROUTE,      /* Inclusive Multicast Ethernet Tag route */

lib/zclient.h

@@ -47,6 +47,11 @@ typedef uint16_t zebra_size_t;
 #define ZEBRA_MAX_PACKET_SIZ          16384U
 #define ZEBRA_SMALL_PACKET_SIZE       200U
 
+/* Only for L2/L3 VNI Add/Del */
+#define ZEBRA_VNI_MAX_PACKET_SIZE   80U
+#define ZEBRA_VNI_SMALL_PACKET_SIZE 50U
+#define ZEBRA_VNI_MIN_PACKET_SIZE   25U
+
 /* Zebra header size. */
 #define ZEBRA_HEADER_SIZE             10
 

zebra/zebra_evpn.c

@@ -1105,7 +1105,7 @@ int zebra_evpn_send_add_to_client(struct zebra_evpn *zevpn)
 
 	svi_index = zevpn->svi_if ? zevpn->svi_if->ifindex : 0;
 
-	s = stream_new(ZEBRA_SMALL_PACKET_SIZE);
+	s = stream_new(ZEBRA_VNI_SMALL_PACKET_SIZE);
 
 	zclient_create_header(s, ZEBRA_VNI_ADD, zebra_vrf_get_evpn_id());
 	stream_putl(s, zevpn->vni);
@@ -1157,7 +1157,7 @@ int zebra_evpn_send_del_to_client(struct zebra_evpn *zevpn)
 		zebra_evpn_update_all_es(zevpn);
 	}
 
-	s = stream_new(ZEBRA_SMALL_PACKET_SIZE);
+	s = stream_new(ZEBRA_VNI_MIN_PACKET_SIZE);
 	stream_reset(s);
 
 	zclient_create_header(s, ZEBRA_VNI_DEL, zebra_vrf_get_evpn_id());

zebra/zebra_vxlan.c

@@ -2203,7 +2203,7 @@ static int zl3vni_send_add_to_client(struct zebra_l3vni *zl3vni)
 		is_anycast_mac = false;
 	}
 
-	s = stream_new(ZEBRA_MAX_PACKET_SIZ);
+	s = stream_new(ZEBRA_VNI_MAX_PACKET_SIZE);
 
 	/* The message is used for both vni add and/or update like
 	 * vrr mac is added for l3vni SVI.
@@ -2246,7 +2246,7 @@ static int zl3vni_send_del_to_client(struct zebra_l3vni *zl3vni)
 	if (!client)
 		return 0;
 
-	s = stream_new(ZEBRA_MAX_PACKET_SIZ);
+	s = stream_new(ZEBRA_VNI_MIN_PACKET_SIZE);
 
 	zclient_create_header(s, ZEBRA_L3VNI_DEL, zl3vni_vrf_id(zl3vni));
 	stream_putl(s, zl3vni->vni);