Added paging support for storage list commands. (Azure#5419)

* check if dir exists instead of path

* added paging, broke

* resolved problems with non-listgenerator provided to transformer

* history, setup version, tests

* review feedback

src/command_modules/azure-cli-storage/HISTORY.rst

@@ -5,6 +5,7 @@ Release History
 
 2.0.25
 ++++++
+* Adding paging support for storage list commands.
 * Minor fixes.
 
 2.0.24

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/_format.py

@@ -134,16 +134,14 @@ def transform_file_directory_result(cli_ctx):
     list.
     """
     def transformer(result):
+        from ._transformers import transform_storage_list_output
+        result_list = transform_storage_list_output(result)
         t_file, t_dir = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'File', 'Directory', mod='file.models')
-        return_list = []
-        for each in result:
+        for each in result_list:
             if isinstance(each, t_file):
                 delattr(each, 'content')
                 setattr(each, 'type', 'file')
             elif isinstance(each, t_dir):
                 setattr(each, 'type', 'dir')
-
-            return_list.append(each)
-
-        return return_list
+        return result_list
     return transformer

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/_help.py

@@ -482,7 +482,7 @@
     parameters:
         - name: --exclude-dir
           type: bool
-          short-summary: List only files in the given share.
+          short-summary: List only files in the given share. If the --num-results parameter is also used, this may cause the output to have less results than expected.
 """
 
 helps['storage file copy'] = """

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/_params.py

@@ -72,6 +72,7 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
                    help='Metadata in space-separated key=value pairs. This overwrites any existing metadata.',
                    validator=validate_metadata)
         c.argument('timeout', help='Request timeout in seconds. Applies to each call to the service.', type=int)
+        c.argument('num_results', type=int)
 
     with self.argument_context('storage', arg_group='Precondition') as c:
         c.argument('if_modified_since', help='Alter only if modified since supplied UTC datetime (Y-m-d\'T\'H:M\'Z\')',
@@ -188,7 +189,6 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
     with self.argument_context('storage blob list') as c:
         c.argument('include', validator=validate_included_datasets)
         c.argument('num_results', type=int)
-        c.ignore('marker')  # https://github.com/Azure/azure-cli/issues/3745
 
     with self.argument_context('storage blob generate-sas') as c:
         from .completers import get_storage_acl_name_completion_list
@@ -357,9 +357,6 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
     with self.argument_context('storage container lease') as c:
         c.argument('container_name', container_name_type)
 
-    with self.argument_context('storage container list') as c:
-        c.ignore('marker')  # https://github.com/Azure/azure-cli/issues/3745
-
     with self.argument_context('storage container policy') as c:
         from .completers import get_storage_acl_name_completion_list
         t_container_permissions = self.get_sdk('blob.models#ContainerPermissions')
@@ -399,9 +396,6 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
     with self.argument_context('storage share') as c:
         c.argument('share_name', share_name_type, options_list=('--name', '-n'))
 
-    with self.argument_context('storage share list') as c:
-        c.ignore('marker')  # https://github.com/Azure/azure-cli/issues/3745
-
     with self.argument_context('storage share exists') as c:
         c.ignore('directory_name', 'file_name')
 

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/_transformers.py

@@ -102,7 +102,17 @@ def _transformer(result):
 
 
 def transform_storage_list_output(result):
-    return list(result)
+    if isinstance(result, dict):
+        next_marker = result['next_marker']
+        result = result['generator']
+    else:
+        next_marker = result.next_marker
+
+    def set_marker(item):
+        item.next_marker = next_marker
+        return item
+
+    return [set_marker(item) for item in result]
 
 
 def transform_url(result):

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/_validators.py

@@ -652,7 +652,7 @@ def process_blob_download_batch_parameters(namespace, cmd):
     import os
 
     # 1. quick check
-    if not os.path.exists(namespace.destination) or not os.path.isdir(namespace.destination):
+    if not os.path.isdir(namespace.destination):
         raise ValueError('incorrect usage: destination must be an existing directory')
 
     # 2. try to extract account name and container name from source string
@@ -773,7 +773,7 @@ def process_file_download_batch_parameters(cmd, namespace):
     import os
 
     # 1. quick check
-    if not os.path.exists(namespace.destination) or not os.path.isdir(namespace.destination):
+    if not os.path.isdir(namespace.destination):
         raise ValueError('incorrect usage: destination must be an existing directory')
 
     # 2. try to extract account name and share name from source string

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/operations/blob.py

@@ -100,7 +100,7 @@ def _download_blob(blob_service, container, destination_folder, blob_name):
         # TODO: try catch IO exception
         destination_path = os.path.join(destination_folder, blob_name)
         destination_folder = os.path.dirname(destination_path)
-        if not os.path.exists(destination_folder):
+        if not os.path.isdir(destination_folder):
             mkdir_p(destination_folder)
 
         blob = blob_service.get_blob_to_path(container, blob_name, destination_path, max_connections=max_connections,

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/operations/directory.py

@@ -4,8 +4,12 @@
 # --------------------------------------------------------------------------------------------
 
 
-def list_share_directories(cmd, client, share_name, directory_name=None, timeout=None):
+def list_share_directories(cmd, client, share_name, directory_name=None, num_results=None, marker=None, timeout=None):
     t_dir_properties = cmd.get_models('file.models#DirectoryProperties')
 
-    generator = client.list_directories_and_files(share_name, directory_name, timeout=timeout)
-    return list(f for f in generator if isinstance(f.properties, t_dir_properties))
+    generator = client.list_directories_and_files(share_name, directory_name, num_results=num_results,
+                                                  marker=marker, timeout=timeout)
+    return {
+        'generator': (f for f in generator if isinstance(f.properties, t_dir_properties)),
+        'next_marker': generator.next_marker
+    }

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/operations/file.py

@@ -16,16 +16,27 @@
 from azure.cli.command_modules.storage.url_quote_util import encode_for_url, make_encoded_file_url_and_params
 
 
-def list_share_files(cmd, client, share_name, directory_name=None, timeout=None, exclude_dir=False, snapshot=None):
+def list_share_files(cmd, client, share_name, directory_name=None, timeout=None, exclude_dir=False, num_results=None,
+                     marker=None, snapshot=None):
+    file_list_args = {
+        "share_name": share_name,
+        "directory_name": directory_name,
+        "timeout": timeout,
+        "num_results": num_results,
+        "marker": marker
+    }
     if cmd.supported_api_version(min_api='2017-04-17'):
-        generator = client.list_directories_and_files(share_name, directory_name, timeout=timeout, snapshot=snapshot)
-    else:
-        generator = client.list_directories_and_files(share_name, directory_name, timeout=timeout)
+        file_list_args["snapshot"] = snapshot
+
+    generator = client.list_directories_and_files(**file_list_args)
 
     if exclude_dir:
         t_file_properties = cmd.get_models('file.models#FileProperties')
 
-        return list(f for f in generator if isinstance(f.properties, t_file_properties))
+        return {
+            'generator': (f for f in generator if isinstance(f.properties, t_file_properties)),
+            'next_marker': generator.next_marker
+        }
 
     return generator