NAS-131091 / 25.04 / Enable NFS over RDMA

src/middlewared/middlewared/alembic/versions/25.04/2024-10-04_18-09_add_nfs_rdma.py

@@ -0,0 +1,25 @@
+"""Add NFS RDMA configuration setting
+
+Revision ID: 85e5d349cdb1
+Revises: 5fe28eada969
+Create Date: 2024-10-04 18:09:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '85e5d349cdb1'
+down_revision = '5fe28eada969'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('services_nfs', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('nfs_srv_rdma', sa.Boolean(), nullable=False, server_default='0'))
+
+
+def downgrade():
+    pass

src/middlewared/middlewared/etc_files/nfs.conf.d/local.conf.mako

@@ -13,6 +13,10 @@
     # the number of mountd be 1/4 the number of nfsd.
     num_mountd = max(num_nfsd // 4, 1)
     manage_gids = 'y' if config["userd_manage_gids"] else 'n'
+
+    # RDMA Notes: 
+    #    * The INET default NFS RDMA (nfsrdma) port is 20049
+    #    * Including 'insecure' appears to not be requried on the share settings
 %>
 [nfsd]
 syslog = 1
@@ -31,6 +35,10 @@ vers4 = n
 % if config['bindip']:
 host = ${','.join(config['bindip'])}
 % endif
+% if config['rdma']:
+rdma = y
+rdma-port = 20049
+% endif
 
 [exportd]
 state-directory-path = ${state_path}

src/middlewared/middlewared/plugins/nfs.py

@@ -9,14 +9,16 @@
 from middlewared.common.listen import SystemServiceListenMultipleDelegate
 from middlewared.schema import accepts, Bool, Dict, Dir, Int, IPAddr, List, Patch, returns, Str
 from middlewared.async_validators import check_path_resides_within_volume, validate_port
-from middlewared.validators import Match, NotMatch, Range, IpAddress
+from middlewared.validators import Match, NotMatch, Port, Range, IpAddress
 from middlewared.service import private, SharingService, SystemServiceService
 from middlewared.service import CallError, ValidationError, ValidationErrors
 import middlewared.sqlalchemy as sa
 from middlewared.utils.asyncio_ import asyncio_map
 from middlewared.plugins.nfs_.utils import get_domain, leftmost_has_wildcards, get_wildcard_domain
 from middlewared.plugins.system_dataset.utils import SYSDATASET_PATH
 
+NFS_RDMA_DEFAULT_PORT = 20049
+
 
 class NFSServicePathInfo(enum.Enum):
     # nfs conf sections that use STATEDIR: exportd, mountd, statd
@@ -66,6 +68,7 @@ class NFSModel(sa.Model):
     nfs_srv_statd_lockd_log = sa.Column(sa.Boolean(), default=False)
     nfs_srv_v4_domain = sa.Column(sa.String(120))
     nfs_srv_v4_owner_major = sa.Column(sa.String(1023), default='')
+    nfs_srv_rdma = sa.Column(sa.Boolean(), default=False)
 
 
 class NFSService(SystemServiceService):
@@ -88,15 +91,16 @@ class Config:
         Bool('v4_krb', required=True),
         Str('v4_domain', required=True),
         List('bindip', items=[IPAddr('ip')], required=True),
-        Int('mountd_port', null=True, validators=[Range(min_=1, max_=65535)], required=True),
-        Int('rpcstatd_port', null=True, validators=[Range(min_=1, max_=65535)], required=True),
-        Int('rpclockd_port', null=True, validators=[Range(min_=1, max_=65535)], required=True),
+        Int('mountd_port', null=True, validators=[Port(exclude=[NFS_RDMA_DEFAULT_PORT])], required=True),
+        Int('rpcstatd_port', null=True, validators=[Port(exclude=[NFS_RDMA_DEFAULT_PORT])], required=True),
+        Int('rpclockd_port', null=True, validators=[Port(exclude=[NFS_RDMA_DEFAULT_PORT])], required=True),
         Bool('mountd_log', required=True),
         Bool('statd_lockd_log', required=True),
         Bool('v4_krb_enabled', required=True),
         Bool('userd_manage_gids', required=True),
         Bool('keytab_has_nfs_spn', required=True),
         Bool('managed_nfsd', default=True),
+        Bool('rdma', default=False),
     )
 
     @private
@@ -317,6 +321,13 @@ async def do_update(self, data):
                     INPUT: unset or an integer between 1 .. 65535
                     Default: unset
 
+        `rdma` -    Enable/Disable NFS over RDMA support
+                    Available on supported platforms and requires an installed and RDMA capable NIC.
+                    NFS over RDMA uses port 20040.
+
+                    INPUT: Enable/Disable
+                    Default: Disable
+
         .. examples(websocket)::
 
           Update NFS Service Configuration to listen on 192.168.0.10 and use NFSv4
@@ -391,6 +402,14 @@ async def do_update(self, data):
         if NFSProtocol.NFSv4 not in new["protocols"] and new["v4_domain"]:
             verrors.add("nfs_update.v4_domain", "This option does not apply to NFSv3")
 
+        if new["rdma"]:
+            available_rdma_services = await self.middleware.call('rdma.capable_services')
+            if "NFS" not in available_rdma_services:
+                verrors.add(
+                    "nfs_update.rdma",
+                    "This platform cannot support NFS over RDMA or is missing an RDMA capable NIC."
+                )
+
         verrors.check()
 
         await self.nfs_compress(new)

src/middlewared/middlewared/validators.py

@@ -178,15 +178,20 @@ def __call__(self, value):
 
 
 class Range(ValidatorBase):
-    def __init__(self, min_=None, max_=None):
+    def __init__(self, min_=None, max_=None, exclude=None):
         self.min = min_
         self.max = max_
+        self.exclude = exclude or []
 
     def __call__(self, value):
         if value is None:
             return
         if isinstance(value, str):
             value = len(value)
+        if value in self.exclude:
+            raise ValueError(
+                f'{value} is a reserved for internal use. Please select another value.'
+            )
 
         error = {
             (True, True): f"between {self.min} and {self.max}",
@@ -203,8 +208,11 @@ def __call__(self, value):
 
 
 class Port(Range):
-    def __init__(self):
-        super().__init__(min_=1, max_=65535)
+    ''' Example usage with exclude:
+    validators=[Port(exclude=[NFS_RDMA_DEFAULT_PORT])]
+    '''
+    def __init__(self, exclude=None):
+        super().__init__(min_=1, max_=65535, exclude=exclude)
 
 
 class QueryFilters(ValidatorBase):
@@ -412,7 +420,6 @@ def check_path_resides_within_volume_sync(verrors, schema_name, path, vol_names)
         inode = None
 
     rp = Path(os.path.realpath(path))
-    is_mountpoint = os.path.ismount(path)
 
     vol_paths = [os.path.join("/mnt", vol_name) for vol_name in vol_names]
     if not path.startswith("/mnt/") or not any(

tests/api2/test_300_nfs.py

@@ -72,7 +72,8 @@ class NFS_CONFIG:
         "v4_krb_enabled": False,
         "userd_manage_gids": False,
         "keytab_has_nfs_spn": False,
-        "managed_nfsd": True
+        "managed_nfsd": True,
+        "rdma": False,
     }
 
     initial_service_state = {}
@@ -1243,22 +1244,38 @@ def test_service_udp(self, start_nfs):
         s = parse_server_config()
         assert s.get('nfsd', {}).get('udp') is None, s
 
-    def test_service_ports(self, start_nfs):
+    @pytest.mark.parametrize('test_port', [
+        pp([["mountd", 618, None], ["rpcstatd", 871, None], ["rpclockd", 32803, None]], id="valid ports"),
+        pp([["rpcstatd", -21, 0], ["rpclockd", 328031, 0]], id="invalid ports"),
+        pp([["mountd", 20049, 1]], id="excluded ports"),
+    ])
+    def test_service_ports(self, start_nfs, test_port):
         """
         This test verifies that we can set custom port and the
         settings are reflected in the relevant files and are active.
+        This also tests the port range and exclude.
         """
         assert start_nfs is True
 
-        # Make custom port selections
-        nfs_conf = call("nfs.update", {
-            "mountd_port": 618,
-            "rpcstatd_port": 871,
-            "rpclockd_port": 32803,
-        })
-        assert nfs_conf['mountd_port'] == 618
-        assert nfs_conf['rpcstatd_port'] == 871
-        assert nfs_conf['rpclockd_port'] == 32803
+        # Friendly index names
+        name = 0
+        value = 1
+        err = 2
+
+        # Error message snippets
+        errmsg = ["Should be between", "reserved for internal use"]
+
+        # Test ports
+        for port in test_port:
+            port_name = port[name] + "_port"
+            if port[err] is None:
+                nfs_conf = call("nfs.update", {port_name: port[value]})
+                assert nfs_conf[port_name] == port[value]
+            else:
+                with pytest.raises(ValidationErrors) as ve:
+                    nfs_conf = call("nfs.update", {port_name: port[value]})
+                errStr = str(ve.value.errors[0])
+                assert errmsg[port[err]] in errStr
 
         # Compare DB with setting in /etc/nfs.conf.d/local.conf
         with nfs_config() as config_db: