NAS-128640 / 24.10 / Add ctypes wrapper for NSS operations (#13630)

This allows us to do module-specific NSS operations and provide
information to API users about whether the user is local, from AD,
or from LDAP server.

The changes are necessary for providing user/group cache for
UI / middleware purposes after switching from nslcd to sssd for
LDAP integration.

src/middlewared/middlewared/plugins/account.py

@@ -1049,6 +1049,7 @@ def shell_choices(self, group_ids):
         Int('pw_gid'),
         List('grouplist'),
         Dict('sid_info'),
+        Bool('local'),
         register=True,
     ))
     async def get_user_obj(self, data):

src/middlewared/middlewared/plugins/activedirectory_/cache.py

@@ -1,6 +1,3 @@
-import grp
-import pwd
-
 from middlewared.plugins.idmap_.utils import (
     IDType,
     SID_LOCAL_USER_PREFIX,
@@ -9,6 +6,8 @@
 )
 from middlewared.service import Service, private, job
 from middlewared.service_exception import CallError
+from middlewared.utils.nss import pwd, grp
+from middlewared.utils.nss.nss_common import NssModule
 from time import sleep
 
 
@@ -31,27 +30,13 @@ def get_entries(self, data):
         dom_by_sid = {x['domain_info']['sid']: x for x in domain_info}
 
         if entry_type == 'USER':
-            entries = WBClient().users()
+            entries = pwd.getpwall(module=NssModule.WINBIND.name)[NssModule.WINBIND.name]
+            for i in entries:
+                ret.append({"id": i.pw_uid, "sid": None, "nss": i, "id_type": entry_type})
         else:
-            entries = WBClient().groups()
-
-        for i in entries:
-            entry = {"id": -1, "sid": None, "nss": None, "id_type": entry_type}
-            if entry_type == 'USER':
-                try:
-                    entry["nss"] = pwd.getpwnam(i)
-                except KeyError:
-                    continue
-                entry["id"] = entry["nss"].pw_uid
-
-            else:
-                try:
-                    entry["nss"] = grp.getgrnam(i)
-                except KeyError:
-                    continue
-                entry["id"] = entry["nss"].gr_gid
-
-            ret.append(entry)
+            entries = grp.getgrall(module=NssModule.WINBIND.name)[NssModule.WINBIND.name]
+            for i in entries:
+                ret.append({"id": i.gr_gid, "sid": None, "nss": i, "id_type": entry_type})
 
         idmaps = self.middleware.call_sync('idmap.convert_unixids', ret)
         to_remove = []

src/middlewared/middlewared/plugins/cache.py

@@ -1,15 +1,14 @@
 from middlewared.schema import Any, Str, Ref, Int, Dict, Bool, accepts
 from middlewared.service import Service, private, job, filterable
 from middlewared.utils import filter_list
+from middlewared.utils.nss import pwd, grp
 from middlewared.service_exception import CallError, MatchNotFound
 from middlewared.plugins.idmap_.utils import SID_LOCAL_USER_PREFIX, SID_LOCAL_GROUP_PREFIX
 
 from collections import namedtuple
 import errno
 import os
 import time
-import pwd
-import grp
 
 
 class CacheService(Service):
@@ -234,42 +233,37 @@ def get_uncached_user(self, username=None, uid=None, getgroups=False, sid_info=F
         for user validation.
         """
         if username:
-            u = pwd.getpwnam(username)
+            user_obj = pwd.getpwnam(username, module='ALL', as_dict=True)
         elif uid is not None:
-            u = pwd.getpwuid(uid)
+            user_obj = pwd.getpwuid(uid, module='ALL', as_dict=True)
         else:
             return {}
 
-        user_obj = {
-            'pw_name': u.pw_name,
-            'pw_uid': u.pw_uid,
-            'pw_gid': u.pw_gid,
-            'pw_gecos': u.pw_gecos,
-            'pw_dir': u.pw_dir,
-            'pw_shell': u.pw_shell,
-        }
+        source = user_obj.pop('source')
+        user_obj['local'] = source == 'FILES'
+
         if getgroups:
-            user_obj['grouplist'] = os.getgrouplist(u.pw_name, u.pw_gid)
+            user_obj['grouplist'] = os.getgrouplist(user_obj['pw_name'], user_obj['pw_gid'])
 
         if sid_info:
             try:
                 if (idmap := self.middleware.call_sync('idmap.convert_unixids', [{
                     'id_type': 'USER',
-                    'id': u.pw_uid,
+                    'id': user_obj['pw_uid'],
                 }])['mapped']):
-                    sid = idmap[f'UID:{u.pw_uid}']['sid']
+                    sid = idmap[f'UID:{user_obj["pw_uid"]}']['sid']
                 else:
-                    sid = SID_LOCAL_USER_PREFIX + str(u.pw_uid)
+                    sid = SID_LOCAL_USER_PREFIX + str(user_obj['pw_uid'])
             except CallError as e:
                 # ENOENT means no winbindd entry for user
                 # ENOTCONN means winbindd is stopped / can't be started
                 # EAGAIN means the system dataset is hosed and needs to be fixed,
                 # but we need to let it through so that it's very clear in logs
                 if e.errno not in (errno.ENOENT, errno.ENOTCONN):
-                    self.logger.error('Failed to retrieve SID for uid: %d', u.pw_uid, exc_info=True)
+                    self.logger.error('Failed to retrieve SID for uid: %d', user_obj['pw_uid'], exc_info=True)
                 sid = None
             except Exception:
-                self.logger.error('Failed to retrieve SID for uid: %d', u.pw_uid, exc_info=True)
+                self.logger.error('Failed to retrieve SID for uid: %d', user_obj['pw_uid'], exc_info=True)
                 sid = None
 
             if sid:
@@ -290,33 +284,30 @@ def get_uncached_group(self, groupname=None, gid=None, sid_info=False):
         for group validation.
         """
         if groupname:
-            g = grp.getgrnam(groupname)
+            grp_obj = grp.getgrnam(groupname, module='ALL', as_dict=True)
         elif gid is not None:
-            g = grp.getgrgid(gid)
+            grp_obj = grp.getgrgid(gid, module='ALL', as_dict=True)
         else:
             return {}
 
-        grp_obj = {
-            'gr_name': g.gr_name,
-            'gr_gid': g.gr_gid,
-            'gr_mem': g.gr_mem
-        }
+        source = grp_obj.pop('source')
+        grp_obj['local'] = source == 'FILES'
 
         if sid_info:
             try:
                 if (idmap := self.middleware.call_sync('idmap.convert_unixids', [{
                     'id_type': 'GROUP',
-                    'id': g.gr_gid,
+                    'id': grp_obj['gr_gid'],
                 }])['mapped']):
-                    sid = idmap[f'GID:{g.gr_gid}']['sid']
+                    sid = idmap[f'GID:{grp_obj["gr_gid"]}']['sid']
                 else:
-                    sid = SID_LOCAL_GROUP_PREFIX + str(g.gr_gid)
+                    sid = SID_LOCAL_GROUP_PREFIX + str(grp_obj['gr_gid'])
             except CallError as e:
                 if e.errno not in (errno.ENOENT, errno.ENOTCONN):
-                    self.logger.error('Failed to retrieve SID for gid: %d', grp.gr_gid, exc_info=True)
+                    self.logger.error('Failed to retrieve SID for gid: %d', grp_obj['gr_gid'], exc_info=True)
                 sid = None
             except Exception:
-                self.logger.error('Failed to retrieve SID for gid: %d', grp.gr_gid, exc_info=True)
+                self.logger.error('Failed to retrieve SID for gid: %d', grp['gr_gid'], exc_info=True)
                 sid = None
 
             if sid:

src/middlewared/middlewared/plugins/filesystem.py

@@ -1,10 +1,8 @@
 import binascii
 import errno
 import functools
-import grp
 import os
 import pathlib
-import pwd
 import shutil
 import stat as statlib
 import time
@@ -19,6 +17,7 @@
 from middlewared.service import private, CallError, filterable_returns, filterable, Service, job
 from middlewared.utils import filter_list
 from middlewared.utils.mount import getmntinfo
+from middlewared.utils.nss import pwd, grp
 from middlewared.utils.path import FSLocation, path_location, strip_location_prefix, is_child_realpath
 from middlewared.plugins.filesystem_.utils import ACLType
 from middlewared.plugins.zfs_.utils import ZFSCTL

src/middlewared/middlewared/plugins/filesystem_/perm_check.py

@@ -1,12 +1,11 @@
 import errno
-import grp
 import os
 import pathlib
-import pwd
 
 from middlewared.schema import accepts, Bool, Dict, returns, Str
 from middlewared.service import CallError, Service, private
 
+from middlewared.utils.nss import pwd, grp
 from middlewared.utils.user_context import run_with_user_context, set_user_context
 
 # This should be a sufficiently high UID to never be used explicitly

src/middlewared/middlewared/plugins/ldap.py

@@ -14,6 +14,8 @@
 from middlewared.plugins.idmap import DSType
 from middlewared.plugins.ldap_.ldap_client import LdapClient
 from middlewared.plugins.ldap_ import constants
+from middlewared.utils.nss import pwd, grp
+from middlewared.utils.nss.nss_common import NssModule
 from middlewared.validators import Range
 
 LDAP_SMBCONF_PARAMS = {
@@ -1047,15 +1049,12 @@ async def __stop(self, job):
     @job(lock='fill_ldap_cache')
     def fill_cache(self, job, force=False):
         user_next_index = group_next_index = 100000000
-        if self.middleware.call_sync('cache.has_key', 'LDAP_cache') and not force:
-            raise CallError('LDAP cache already exists. Refusing to generate cache.')
-
         if (self.middleware.call_sync('ldap.config'))['disable_freenas_cache']:
             self.logger.debug('LDAP cache is disabled. Bypassing cache fill.')
             return
 
-        pwd_list = []
-        grp_list = []
+        pwd_list = pwd.getpwall(module=NssModule.SSS.name, as_dict=True)[NssModule.SSS.name]
+        grp_list = grp.getgrall(module=NssModule.SSS.name, as_dict=True)[NssModule.SSS.name]
 
         for u in pwd_list:
             entry = {