Merge remote-tracking branch 'azure/master' into counters

Signed-off-by: SuvarnaMeenakshi <[email protected]>

src/sonic_ax_impl/mibs/__init__.py

@@ -48,6 +48,15 @@
 
 redis_kwargs = {'unix_socket_path': '/var/run/redis/redis.sock'}
 
+
+def get_neigh_info(neigh_key):
+    """
+    split neigh_key string of the format:
+    NEIGH_TABLE:device:ipv4_address
+    """
+    _, device, ip = neigh_key.split(':')
+    return device, ip
+
 def chassis_info_table(chassis_name):
     """
     :param: chassis_name: chassis name
@@ -521,6 +530,21 @@ def dbs_keys(dbs, db_name, pattern='*'):
                 result_keys.extend(keys)
         return result_keys
 
+    @staticmethod
+    def dbs_keys_namespace(dbs, db_name, pattern='*'):
+        """
+        dbs_keys_namespace function execute on global
+        and all namespace DBs. Provides a map of keys
+        and namespace(db index).
+        """
+        result_keys = {}
+        for db_index in range(len(dbs)):
+            keys = dbs[db_index].keys(db_name, pattern)
+            if keys is not None:
+                keys_ns = dict.fromkeys(keys, db_index)
+                result_keys.update(keys_ns)
+        return result_keys
+
     @staticmethod
     def dbs_get_all(dbs, db_name, _hash, *args, **kwargs):
         """
@@ -555,83 +579,22 @@ def get_non_host_dbs(dbs):
         else:
             return dbs[1:]
 
-
-    @staticmethod
-    def init_namespace_sync_d_interface_tables(dbs):
-        if_name_map = {}
-        if_alias_map = {}
-        if_id_map = {}
-        oid_sai_map = {}
-        oid_name_map = {}
-
-        """
-        all_ns_db - will have db_conn to all namespace DBs and
-        global db. First db in the list is global db.
-        Ignore first global db to get interface tables if there
-        are multiple namespaces.
-        if_id_map = {db_index: {sai_oid : if_name}}
-        """
-        Namespace.connect_namespace_dbs(dbs)
-        for db_conn in Namespace.get_non_host_dbs(dbs):
-            if_name_map_ns, \
-            if_alias_map_ns, \
-            if_id_map_ns, \
-            oid_sai_map_ns, \
-            oid_name_map_ns = init_sync_d_interface_tables(db_conn)
-            if_name_map.update(if_name_map_ns)
-            if_alias_map.update(if_alias_map_ns)
-            if_id_map.update(if_id_map_ns)
-            oid_sai_map.update(oid_sai_map_ns)
-            oid_name_map.update(oid_name_map_ns)
-
-        return if_name_map, if_alias_map, if_id_map, oid_sai_map, oid_name_map
-
-    @staticmethod
-    def init_namespace_sync_d_lag_tables(dbs):
-
-        lag_name_if_name_map = {}
-        if_name_lag_name_map = {}
-        oid_lag_name_map = {}
-
-        """
-        all_ns_db - will have db_conn to all namespace DBs and
-        global db. First db in the list is global db.
-        Ignore first global db to get lag tables if
-        there are multiple namespaces.
-        """
-        Namespace.connect_namespace_dbs(dbs)
-        for db_conn in Namespace.get_non_host_dbs(dbs):
-            lag_name_if_name_map_ns, \
-            if_name_lag_name_map_ns, \
-            oid_lag_name_map_ns = init_sync_d_lag_tables(db_conn)
-            lag_name_if_name_map.update(lag_name_if_name_map_ns)
-            if_name_lag_name_map.update(if_name_lag_name_map_ns)
-            oid_lag_name_map.update(oid_lag_name_map_ns)
-
-        return lag_name_if_name_map, if_name_lag_name_map, oid_lag_name_map
-
-    @staticmethod
-    def init_namespace_sync_d_queue_tables(dbs):
-        port_queues_map = {}
-        queue_stat_map = {}
-        port_queue_list_map = {}
-
-        """
-        all_ns_db - will have db_conn to all namespace DBs and
-        global db. First db in the list is global db.
-        Ignore first global db to get queue tables if there
-        are multiple namespaces.
-        """
-        Namespace.connect_namespace_dbs(dbs)
+    def get_sync_d_from_all_namespace(per_namespace_func, dbs):
+        # return merged tuple of dictionaries retrieved from per
+        # namespace functions.
+        result_map = {}
+        # list of return values
+        result_list = []
         for db_conn in Namespace.get_non_host_dbs(dbs):
-            port_queues_map_ns, \
-            queue_stat_map_ns, \
-            port_queue_list_map_ns = init_sync_d_queue_tables(db_conn)
-            port_queues_map.update(port_queues_map_ns)
-            queue_stat_map.update(queue_stat_map_ns)
-            port_queue_list_map.update(port_queue_list_map_ns)
-
-        return port_queues_map, queue_stat_map, port_queue_list_map
+            ns_tuple = per_namespace_func(db_conn)
+            for idx in range(len(ns_tuple)):
+                if idx not in result_map:
+                    result_map[idx] = ns_tuple[idx]
+                else:
+                    result_map[idx].update(ns_tuple[idx])
+        for idx, ns_tuple_dict in result_map.items():
+            result_list.append(ns_tuple_dict)
+        return result_list
 
     @staticmethod
     def dbs_get_bridge_port_map(dbs, db_name):

src/sonic_ax_impl/mibs/ieee802_1ab.py

@@ -166,7 +166,7 @@ def reinit_data(self):
         self.if_alias_map, \
         self.if_id_map, \
         self.oid_sai_map, \
-        self.oid_name_map = Namespace.init_namespace_sync_d_interface_tables(self.db_conn)
+        self.oid_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_interface_tables, self.db_conn)
 
         self.mgmt_oid_name_map, \
         self.mgmt_alias_map = mibs.init_mgmt_interface_tables(self.db_conn[0])
@@ -401,7 +401,7 @@ def reinit_data(self):
         self.if_alias_map, \
         self.if_id_map, \
         self.oid_sai_map, \
-        self.oid_name_map = Namespace.init_namespace_sync_d_interface_tables(self.db_conn)
+        self.oid_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_interface_tables, self.db_conn)
 
         self.mgmt_oid_name_map, _ = mibs.init_mgmt_interface_tables(self.db_conn[0])
 
@@ -430,7 +430,12 @@ def update_data(self):
             if not lldp_kvs:
                 continue
             try:
-                time_mark = int(lldp_kvs[b'lldp_rem_time_mark'])
+                # OID index for this MIB consists of remote time mark, if_oid, remote_index.
+                # For multi-asic platform, it can happen that same interface index result 
+                # is seen in SNMP walk, with a different remote time mark.
+                # To avoid repeating the data of same interface index with different remote 
+                # time mark, remote time mark is made as 0 in the OID indexing.
+                time_mark = 0
                 remote_index = int(lldp_kvs[b'lldp_rem_index'])
                 self.if_range.append((time_mark,
                                       if_oid,
@@ -561,7 +566,7 @@ def reinit_data(self):
         """
         Subclass reinit data routine.
         """
-        _, _, _, _, self.oid_name_map = Namespace.init_namespace_sync_d_interface_tables(self.db_conn)
+        _, _, _, _, self.oid_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_interface_tables, self.db_conn)
 
         self.mgmt_oid_name_map, _ = mibs.init_mgmt_interface_tables(self.db_conn[0])
 

src/sonic_ax_impl/mibs/ietf/rfc1213.py

@@ -54,32 +54,66 @@ class IfTypes(int, Enum):
 class ArpUpdater(MIBUpdater):
     def __init__(self):
         super().__init__()
+        self.db_conn = Namespace.init_namespace_dbs()
         self.arp_dest_map = {}
         self.arp_dest_list = []
         self.arp_dest_map = {}
         self.arp_dest_list = []
+        self.neigh_key_list = {}
 
-    def update_data(self):
-        self.arp_dest_map = {}
-        self.arp_dest_list = []
+    def reinit_data(self):
+        Namespace.connect_all_dbs(self.db_conn, mibs.APPL_DB)
+        self.neigh_key_list = Namespace.dbs_keys_namespace(self.db_conn, mibs.APPL_DB, "NEIGH_TABLE:*")
+
+    def _update_from_arptable(self):
         for entry in python_arptable.get_arp_table():
             dev = entry['Device']
             mac = entry['HW address']
             ip = entry['IP address']
+            self._update_arp_info(dev, mac, ip)
+
+    def _update_from_db(self):
+        for neigh_key in self.neigh_key_list:
+            neigh_str = neigh_key.decode()
+            db_index = self.neigh_key_list[neigh_key]
+            neigh_info = self.db_conn[db_index].get_all(mibs.APPL_DB, neigh_key, blocking=False)
+            if neigh_info is None:
+                continue
+            ip_family = neigh_info[b'family'].decode()
+            if ip_family == "IPv4":
+                dev, ip = mibs.get_neigh_info(neigh_str)
+                mac = neigh_info[b'neigh'].decode()
+                # eth0 interface in a namespace is not management interface
+                # but is a part of docker0 bridge. Ignore this interface.
+                if len(self.db_conn) > 1 and dev == "eth0":
+                    continue
+                self._update_arp_info(dev, mac, ip)
+
+    def _update_arp_info(self, dev, mac, ip):
+        if_index = mibs.get_index_from_str(dev)
+        if if_index is None: return
+
+        mactuple = mac_decimals(mac)
+        machex = ''.join(chr(b) for b in mactuple)
+        # if MAC is all zero
+        #if not any(mac): continue
+
+        iptuple = ip2tuple_v4(ip)
+
+        subid = (if_index,) + iptuple
+        self.arp_dest_map[subid] = machex
+        self.arp_dest_list.append(subid)
 
-            if_index = mibs.get_index_from_str(dev)
-            if if_index is None: continue
-
-            mactuple = mac_decimals(mac)
-            machex = ''.join(chr(b) for b in mactuple)
-            # if MAC is all zero
-            #if not any(mac): continue
-
-            iptuple = ip2tuple_v4(ip)
-
-            subid = (if_index,) + iptuple
-            self.arp_dest_map[subid] = machex
-            self.arp_dest_list.append(subid)
+    def update_data(self):
+        self.arp_dest_map = {}
+        self.arp_dest_list = []
+        # Update arp table of host.
+        # In case of multi-asic platform, get host arp table
+        # from kernel and namespace arp table from NEIGH_TABLE in APP_DB
+        # in each namespace.
+        self._update_from_db()
+        if len(self.db_conn) > 1:
+            self._update_from_arptable()
         self.arp_dest_list.sort()
 
     def arp_dest(self, sub_id):
@@ -178,7 +212,7 @@ def reinit_data(self):
         self.if_alias_map, \
         self.if_id_map, \
         self.oid_sai_map, \
-        self.oid_name_map = Namespace.init_namespace_sync_d_interface_tables(self.db_conn)
+        self.oid_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_interface_tables, self.db_conn)
         """
         db_conn - will have db_conn to all namespace DBs and
         global db. First db in the list is global db.
@@ -200,7 +234,7 @@ def update_data(self):
 
         self.lag_name_if_name_map, \
         self.if_name_lag_name_map, \
-        self.oid_lag_name_map = Namespace.init_namespace_sync_d_lag_tables(self.db_conn)
+        self.oid_lag_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_lag_tables, self.db_conn)
 
         self.if_range = sorted(list(self.oid_sai_map.keys()) +
                                list(self.oid_lag_name_map.keys()) +

src/sonic_ax_impl/mibs/ietf/rfc2737.py

@@ -153,7 +153,7 @@ def reinit_data(self):
 
         # update interface maps
         _, self.if_alias_map, _, _, _ = \
-            Namespace.init_namespace_sync_d_interface_tables(Namespace.init_namespace_dbs())
+            Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_interface_tables, Namespace.init_namespace_dbs())
 
         device_metadata = mibs.get_device_metadata(self.statedb[0])
         chassis_sub_id = (self.CHASSIS_ID, )

src/sonic_ax_impl/mibs/ietf/rfc2863.py

@@ -77,11 +77,11 @@ def reinit_data(self):
         self.if_alias_map, \
         self.if_id_map, \
         self.oid_sai_map, \
-        self.oid_name_map = Namespace.init_namespace_sync_d_interface_tables(self.db_conn)
+        self.oid_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_interface_tables, self.db_conn)
 
         self.lag_name_if_name_map, \
         self.if_name_lag_name_map, \
-        self.oid_lag_name_map = Namespace.init_namespace_sync_d_lag_tables(self.db_conn)
+        self.oid_lag_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_lag_tables, self.db_conn)
         """
         db_conn - will have db_conn to all namespace DBs and
         global db. First db in the list is global db.

src/sonic_ax_impl/mibs/ietf/rfc4363.py

@@ -41,7 +41,7 @@ def reinit_data(self):
         self.if_alias_map, \
         self.if_id_map, \
         self.oid_sai_map, \
-        self.oid_name_map = Namespace.init_namespace_sync_d_interface_tables(self.db_conn)
+        self.oid_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_interface_tables, self.db_conn)
 
         self.if_bpid_map = Namespace.dbs_get_bridge_port_map(self.db_conn, mibs.ASIC_DB)
         self.bvid_vlan_map.clear()

src/sonic_ax_impl/mibs/vendor/cisco/ciscoPfcExtMIB.py

@@ -37,7 +37,7 @@ def reinit_data(self):
         self.if_alias_map, \
         self.if_id_map, \
         self.oid_sai_map, \
-        self.oid_name_map = Namespace.init_namespace_sync_d_interface_tables(self.db_conn)
+        self.oid_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_interface_tables, self.db_conn)
 
         self.update_data()
 
@@ -54,7 +54,7 @@ def update_data(self):
 
         self.lag_name_if_name_map, \
         self.if_name_lag_name_map, \
-        self.oid_lag_name_map = Namespace.init_namespace_sync_d_lag_tables(self.db_conn)
+        self.oid_lag_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_lag_tables, self.db_conn)
 
         self.if_range = sorted(list(self.oid_sai_map.keys()) + list(self.oid_lag_name_map.keys()))
         self.if_range = [(i,) for i in self.if_range]

src/sonic_ax_impl/mibs/vendor/cisco/ciscoSwitchQosMIB.py

@@ -75,14 +75,14 @@ def reinit_data(self):
         self.if_alias_map, \
         self.if_id_map, \
         self.oid_sai_map, \
-        self.oid_name_map = Namespace.init_namespace_sync_d_interface_tables(self.db_conn)
+        self.oid_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_interface_tables, self.db_conn)
 
         for sai_id_key, if_name in self.if_id_map.items():
             namespace, sai_id = mibs.split_sai_id_key(sai_id_key)
             self.if_id_namespace[mibs.get_index_from_str(if_name)] = namespace
 
         self.port_queues_map, self.queue_stat_map, self.port_queue_list_map = \
-            Namespace.init_namespace_sync_d_queue_tables(self.db_conn)
+            Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_queue_tables, self.db_conn)
 
         for db_conn in Namespace.get_non_host_dbs(self.db_conn):
             self.queue_type_map[db_conn.namespace] = db_conn.get_all(mibs.COUNTERS_DB, "COUNTERS_QUEUE_TYPE_MAP", blocking=False)