NAS-127697 / 24.04.0 / Allow RDMA capable interfaces to be exposed fo…

…r regular networking (by bmeagherix) (#13300)

* Allow RDMA capable interfaces to be exposed for regular networking

When JBOF support was added, we added *all* RDMA interfaces to
rdma.interface.internal_interfaces.  Instead only add those currently
in use for JBOF purposes.

Likewise, in rdma.get_link_choices omit any interfaces that have been
configured for general networking.

(cherry picked from commit b2d72ac)

* Address review

Move rdma.get_configured_interfaces to interface.get_configured_interfaces and
reimplement

(cherry picked from commit f9a330c)

---------

Co-authored-by: Brian Meagher <[email protected]>

src/middlewared/middlewared/plugins/interface/configure.py

@@ -173,3 +173,24 @@ def alias_to_addr(self, alias):
         if 'vhid' in alias:
             addr.vhid = alias['vhid']
         return addr
+
+    @private
+    async def get_configured_interfaces(self):
+        """
+        Return a list of configured interfaces.
+
+        This will include names of regular interfaces that have been configured,
+        plus any higher-order interfaces and their constituents."""
+        ds = await self.middleware.call('interface.get_datastores')
+        # Interfaces
+        result = set([i['int_interface'] for i in ds['interfaces']])
+        # Bridges
+        for bridge in ds['bridge']:
+            result.update(bridge['members'])
+        # VLAN
+        for vlan in ds['vlan']:
+            result.add(vlan['vlan_pint'])
+        # Link Aggregation
+        for lag in ds['laggmembers']:
+            result.add(lag['lagg_physnic'])
+        return list(result)

src/middlewared/middlewared/plugins/rdma/interface/crud.py

@@ -238,14 +238,16 @@ async def local_ping(self, ifname, ip, mac=None):
                 return False
         return True
 
-    async def internal_interfaces(self):
-        links = await self.middleware.call('rdma.get_link_choices')
+    async def internal_interfaces(self, all=False):
+        # We must fetch all link choices.  If we did not there would
+        # be a circular call chain between interface and rdma
+        links = await self.middleware.call('rdma._get_link_choices')
         ifname_to_netdev = {}
         for link in links:
             ifname_to_netdev[link['rdma']] = link['netdev']
 
-        if await self.middleware.call('system.is_enterprise'):
-            # For Enterprise we treat all RDMA interfaces as internal
+        if all:
+            # Treat all RDMA interfaces as internal
             return list(ifname_to_netdev.values())
         else:
             # Otherwise we only treat used RDMA interfaces as internal

src/middlewared/middlewared/plugins/rdma/rdma.py

@@ -2,7 +2,7 @@
 import subprocess
 from pathlib import Path
 
-from middlewared.schema import Dict, List, Ref, Str, accepts, returns
+from middlewared.schema import Bool, Dict, List, Ref, Str, accepts, returns
 from middlewared.service import Service, private
 from middlewared.service_exception import CallError
 from middlewared.utils.functools import cache
@@ -40,15 +40,27 @@ def get_pci_vpd(self, pci_addr):
         return result
 
     @private
-    @accepts()
+    @accepts(Bool('all', default=False))
     @returns(List(items=[Dict(
         'rdma_link_config',
         Str('rdma', required=True),
         Str('netdev', required=True),
         register=True
     )]))
+    async def get_link_choices(self, all):
+        """Return a list containing dictionaries with keys 'rdma' and 'netdev'.
+
+        Unless all is set to True, configured interfaces will be excluded."""
+        all_links = await self.middleware.call('rdma._get_link_choices')
+        if all:
+            return all_links
+
+        existing = await self.middleware.call('interface.get_configured_interfaces')
+        return list(filter(lambda x: x['netdev'] not in existing, all_links))
+
+    @private
     @cache
-    def get_link_choices(self):
+    def _get_link_choices(self):
         """Return a list containing dictionaries with keys 'rdma' and 'netdev'.
 
         Since these are just the hardware present in the system, we cache the result."""