From 0155796f207b79bb6f7fb17fb45acae0081635a8 Mon Sep 17 00:00:00 2001 From: Andrew Walker Date: Fri, 6 Sep 2024 05:33:06 -0600 Subject: [PATCH] Be more selective about file metdata in listdir The directory iterator we use is configurable about what file metdata to retrieve. This can in some situations significantly improve performance of generating directory listing. --- .../middlewared/plugins/filesystem.py | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/middlewared/middlewared/plugins/filesystem.py b/src/middlewared/middlewared/plugins/filesystem.py index 94a59f31f668d..ca4320b289764 100644 --- a/src/middlewared/middlewared/plugins/filesystem.py +++ b/src/middlewared/middlewared/plugins/filesystem.py @@ -20,7 +20,7 @@ from middlewared.utils.filesystem import attrs, stat_x from middlewared.utils.filesystem.acl import acl_is_present from middlewared.utils.filesystem.constants import FileType -from middlewared.utils.filesystem.directory import DirectoryIterator +from middlewared.utils.filesystem.directory import DirectoryIterator, DirectoryRequestMask from middlewared.utils.filesystem.utils import timespec_convert from middlewared.utils.mount import getmntinfo from middlewared.utils.nss import pwd, grp @@ -214,6 +214,31 @@ def mkdir(self, data): 'zfs_attrs': ['ARCHIVE'] } + @private + def listdir_request_mask(self, select): + """ create request mask for directory listing """ + if select is None: + # request_mask=None means ALL in the directory iterator + return None + + request_mask = 0 + for i in select: + selected = i[0] if isinstance(i, list) else i + + match selected: + case 'realpath': + request_mask |= DirectoryRequestMask.REALPATH + case 'acl': + request_mask |= DirectoryRequestMask.ACL + case 'zfs_attrs': + request_mask |= DirectoryRequestMask.ZFS_ATTRS + case 'is_ctldir': + request_mask |= DirectoryRequestMask.CTLDIR + case 'xattrs': + request_mask |= DirectoryRequestMask.XATTRS + + return request_mask + @accepts( Str('path', required=True), Ref('query-filters'), @@ -274,6 +299,19 @@ def listdir(self, path, filters, options): if not path.is_dir(): raise CallError(f'Path {path} is not a directory', errno.ENOTDIR) + if options.get('count') is True: + # We're just getting count, drop any unnecessary info + request_mask = 0 + else: + request_mask = self.listdir_request_mask(options.get('select', None)) + + if request_mask and (request_mask & DirectoryRequestMask.ZFS_ATTRS): + # Make sure this is actually ZFS before issuing FS ioctls + try: + self.get_zfs_attributes(path) + except Exception: + raise CallError(f'{path}: ZFS attributes are not supported.') + file_type = None for filter_ in filters: if filter_[0] not in ['type']: @@ -300,7 +338,7 @@ def listdir(self, path, filters, options): # filter these here. filters.extend([['is_mountpoint', '=', True], ['name', '!=', IX_APPS_DIR_NAME]]) - with DirectoryIterator(path, file_type=file_type) as d_iter: + with DirectoryIterator(path, file_type=file_type, request_mask=request_mask) as d_iter: return filter_list(d_iter, filters, options) @accepts(Str('path'), roles=['FILESYSTEM_ATTRS_READ'])