Merge pull request #545 from xhochy/add-list-blobs

Add a BaseBlobService.list_blob_names method

azure-storage-blob/azure/storage/blob/_deserialization.py

@@ -351,6 +351,77 @@ def _convert_xml_to_blob_list(response):
     return blob_list
 
 
+def _convert_xml_to_blob_name_list(response):
+    '''
+    <?xml version="1.0" encoding="utf-8"?>
+    <EnumerationResults ServiceEndpoint="http://myaccount.blob.core.windows.net/" ContainerName="mycontainer">
+      <Prefix>string-value</Prefix>
+      <Marker>string-value</Marker>
+      <MaxResults>int-value</MaxResults>
+      <Delimiter>string-value</Delimiter>
+      <Blobs>
+        <Blob>
+          <Name>blob-name</name>
+          <Deleted>true</Deleted>
+          <Snapshot>date-time-value</Snapshot>
+          <Properties>
+            <Last-Modified>date-time-value</Last-Modified>
+            <Etag>etag</Etag>
+            <Content-Length>size-in-bytes</Content-Length>
+            <Content-Type>blob-content-type</Content-Type>
+            <Content-Encoding />
+            <Content-Language />
+            <Content-MD5 />
+            <Cache-Control />
+            <x-ms-blob-sequence-number>sequence-number</x-ms-blob-sequence-number>
+            <BlobType>BlockBlob|PageBlob|AppendBlob</BlobType>
+            <LeaseStatus>locked|unlocked</LeaseStatus>
+            <LeaseState>available | leased | expired | breaking | broken</LeaseState>
+            <LeaseDuration>infinite | fixed</LeaseDuration>
+            <CopyId>id</CopyId>
+            <CopyStatus>pending | success | aborted | failed </CopyStatus>
+            <CopySource>source url</CopySource>
+            <CopyProgress>bytes copied/bytes total</CopyProgress>
+            <CopyCompletionTime>datetime</CopyCompletionTime>
+            <CopyStatusDescription>error string</CopyStatusDescription>
+            <AccessTier>P4 | P6 | P10 | P20 | P30 | P40 | P50 | P60 | Archive | Cool | Hot</AccessTier>
+            <AccessTierChangeTime>date-time-value</AccessTierChangeTime>
+            <AccessTierInferred>true</AccessTierInferred>
+            <DeletedTime>datetime</DeletedTime>
+            <RemainingRetentionDays>int</RemainingRetentionDays>
+            <Creation-Time>date-time-value</Creation-Time>
+          </Properties>
+          <Metadata>   
+            <Name>value</Name>
+          </Metadata>
+        </Blob>
+        <BlobPrefix>
+          <Name>blob-prefix</Name>
+        </BlobPrefix>
+      </Blobs>
+      <NextMarker />
+    </EnumerationResults>
+    '''
+    if response is None or response.body is None:
+        return None
+
+    blob_list = _list()
+    list_element = ETree.fromstring(response.body)
+
+    setattr(blob_list, 'next_marker', list_element.findtext('NextMarker'))
+
+    blobs_element = list_element.find('Blobs')
+    blob_prefix_elements = blobs_element.findall('BlobPrefix')
+    if blob_prefix_elements is not None:
+        for blob_prefix_element in blob_prefix_elements:
+            blob_list.append(blob_prefix_element.findtext('Name'))
+
+    for blob_element in blobs_element.findall('Blob'):
+        blob_list.append(blob_element.findtext('Name'))
+
+    return blob_list
+
+
 def _convert_xml_to_block_list(response):
     '''
     <?xml version="1.0" encoding="utf-8"?>

azure-storage-blob/azure/storage/blob/baseblobservice.py

@@ -58,6 +58,7 @@
     _convert_xml_to_containers,
     _parse_blob,
     _convert_xml_to_blob_list,
+    _convert_xml_to_blob_name_list,
     _parse_container,
     _parse_snapshot_blob,
     _parse_lease,
@@ -1243,14 +1244,66 @@ def list_blobs(self, container_name, prefix=None, num_results=None, include=None
         args = (container_name,)
         kwargs = {'prefix': prefix, 'marker': marker, 'max_results': num_results,
                   'include': include, 'delimiter': delimiter, 'timeout': timeout,
-                  '_context': operation_context}
+                  '_context': operation_context,
+                  '_converter': _convert_xml_to_blob_list}
+        resp = self._list_blobs(*args, **kwargs)
+
+        return ListGenerator(resp, self._list_blobs, args, kwargs)
+
+    def list_blob_names(self, container_name, prefix=None, num_results=None,
+                        include=None, delimiter=None, marker=None,
+                        timeout=None):
+        '''
+        Returns a generator to list the blob names under the specified container.
+        The generator will lazily follow the continuation tokens returned by
+        the service and stop when all blobs have been returned or num_results is reached.
+
+        If num_results is specified and the account has more than that number of 
+        blobs, the generator will have a populated next_marker field once it 
+        finishes. This marker can be used to create a new generator if more 
+        results are desired.
+
+        :param str container_name:
+            Name of existing container.
+        :param str prefix:
+            Filters the results to return only blobs whose names
+            begin with the specified prefix.
+        :param int num_results:
+            Specifies the maximum number of blobs to return,
+            including all :class:`BlobPrefix` elements. If the request does not specify
+            num_results or specifies a value greater than 5,000, the server will
+            return up to 5,000 items. Setting num_results to a value less than
+            or equal to zero results in error response code 400 (Bad Request).
+        :param ~azure.storage.blob.models.Include include:
+            Specifies one or more additional datasets to include in the response.
+        :param str delimiter:
+            When the request includes this parameter, the operation
+            returns a :class:`~azure.storage.blob.models.BlobPrefix` element in the
+            result list that acts as a placeholder for all blobs whose names begin
+            with the same substring up to the appearance of the delimiter character.
+            The delimiter may be a single character or a string.
+        :param str marker:
+            An opaque continuation token. This value can be retrieved from the 
+            next_marker field of a previous generator object if num_results was 
+            specified and that generator has finished enumerating results. If 
+            specified, this generator will begin returning results from the point 
+            where the previous generator stopped.
+        :param int timeout:
+            The timeout parameter is expressed in seconds.
+        '''
+        operation_context = _OperationContext(location_lock=True)
+        args = (container_name,)
+        kwargs = {'prefix': prefix, 'marker': marker, 'max_results': num_results,
+                  'include': include, 'delimiter': delimiter, 'timeout': timeout,
+                  '_context': operation_context,
+                  '_converter': _convert_xml_to_blob_name_list}
         resp = self._list_blobs(*args, **kwargs)
 
         return ListGenerator(resp, self._list_blobs, args, kwargs)
 
     def _list_blobs(self, container_name, prefix=None, marker=None,
                     max_results=None, include=None, delimiter=None, timeout=None,
-                    _context=None):
+                    _context=None, _converter=None):
         '''
         Returns the list of blobs under the specified container.
 
@@ -1319,7 +1372,7 @@ def _list_blobs(self, container_name, prefix=None, marker=None,
             'timeout': _int_to_str(timeout),
         }
 
-        return self._perform_request(request, _convert_xml_to_blob_list, operation_context=_context)
+        return self._perform_request(request, _converter, operation_context=_context)
 
     def get_blob_account_information(self, container_name=None, blob_name=None, timeout=None):
         """

tests/blob/test_container.py

@@ -706,6 +706,20 @@ def test_delete_container_with_lease_id(self):
         exists = self.bs.exists(container_name)
         self.assertFalse(exists)
 
+    @record
+    def test_list_names(self):
+        # Arrange
+        container_name = self._create_container()
+        data = b'hello world'
+        self.bs.create_blob_from_bytes (container_name, 'blob1', data, )
+        self.bs.create_blob_from_bytes (container_name, 'blob2', data, )
+
+        # Act
+        blobs = list(self.bs.list_blob_names(container_name))
+
+        self.assertEqual(blobs, ['blob1', 'blob2'])
+
+
     @record
     def test_list_blobs(self):
         # Arrange

tests/recordings/test_container.test_list_names.yaml

@@ -0,0 +1,106 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Connection: [keep-alive]
+      Content-Length: ['0']
+      User-Agent: [Azure-Storage/1.4.0-1.4.0 (Python CPython 3.7.0; Darwin 18.2.0)]
+      x-ms-client-request-id: [54ed1e2a-e474-11e8-9ac5-acde48001122]
+      x-ms-date: ['Fri, 09 Nov 2018 23:08:12 GMT']
+      x-ms-version: ['2018-03-28']
+    method: PUT
+    uri: https://storagename.blob.core.windows.net/containerc02c0c5f?restype=container
+  response:
+    body: {string: ''}
+    headers:
+      Date: ['Fri, 09 Nov 2018 23:08:11 GMT']
+      ETag: ['"0x8D646983935CBA5"']
+      Last-Modified: ['Fri, 09 Nov 2018 23:08:12 GMT']
+      Server: [Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0]
+      Transfer-Encoding: [chunked]
+      x-ms-request-id: [13632f99-b01e-0015-7d81-780db2000000]
+      x-ms-version: ['2018-03-28']
+    status: {code: 201, message: Created}
+- request:
+    body: hello world
+    headers:
+      Connection: [keep-alive]
+      Content-Length: ['11']
+      User-Agent: [Azure-Storage/1.4.0-1.4.0 (Python CPython 3.7.0; Darwin 18.2.0)]
+      x-ms-blob-type: [BlockBlob]
+      x-ms-client-request-id: [5505f440-e474-11e8-9ac5-acde48001122]
+      x-ms-date: ['Fri, 09 Nov 2018 23:08:12 GMT']
+      x-ms-version: ['2018-03-28']
+    method: PUT
+    uri: https://storagename.blob.core.windows.net/containerc02c0c5f/blob1
+  response:
+    body: {string: ''}
+    headers:
+      Content-MD5: [XrY7u+Ae7tCTyyK7j1rNww==]
+      Date: ['Fri, 09 Nov 2018 23:08:12 GMT']
+      ETag: ['"0x8D646983937A651"']
+      Last-Modified: ['Fri, 09 Nov 2018 23:08:12 GMT']
+      Server: [Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0]
+      Transfer-Encoding: [chunked]
+      x-ms-request-id: [13632fb1-b01e-0015-1281-780db2000000]
+      x-ms-request-server-encrypted: ['true']
+      x-ms-version: ['2018-03-28']
+    status: {code: 201, message: Created}
+- request:
+    body: hello world
+    headers:
+      Connection: [keep-alive]
+      Content-Length: ['11']
+      User-Agent: [Azure-Storage/1.4.0-1.4.0 (Python CPython 3.7.0; Darwin 18.2.0)]
+      x-ms-blob-type: [BlockBlob]
+      x-ms-client-request-id: [550ceeda-e474-11e8-9ac5-acde48001122]
+      x-ms-date: ['Fri, 09 Nov 2018 23:08:12 GMT']
+      x-ms-version: ['2018-03-28']
+    method: PUT
+    uri: https://storagename.blob.core.windows.net/containerc02c0c5f/blob2
+  response:
+    body: {string: ''}
+    headers:
+      Content-MD5: [XrY7u+Ae7tCTyyK7j1rNww==]
+      Date: ['Fri, 09 Nov 2018 23:08:12 GMT']
+      ETag: ['"0x8D64698393EACDE"']
+      Last-Modified: ['Fri, 09 Nov 2018 23:08:12 GMT']
+      Server: [Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0]
+      Transfer-Encoding: [chunked]
+      x-ms-request-id: [13632fc6-b01e-0015-2481-780db2000000]
+      x-ms-request-server-encrypted: ['true']
+      x-ms-version: ['2018-03-28']
+    status: {code: 201, message: Created}
+- request:
+    body: null
+    headers:
+      Connection: [keep-alive]
+      User-Agent: [Azure-Storage/1.4.0-1.4.0 (Python CPython 3.7.0; Darwin 18.2.0)]
+      x-ms-client-request-id: [5513d470-e474-11e8-9ac5-acde48001122]
+      x-ms-date: ['Fri, 09 Nov 2018 23:08:12 GMT']
+      x-ms-version: ['2018-03-28']
+    method: GET
+    uri: https://storagename.blob.core.windows.net/containerc02c0c5f?restype=container&comp=list
+  response:
+    body: {string: "\uFEFF<?xml version=\"1.0\" encoding=\"utf-8\"?><EnumerationResults\
+        \ ServiceEndpoint=\"https://storagename.blob.core.windows.net/\" ContainerName=\"\
+        containerc02c0c5f\"><Blobs><Blob><Name>blob1</Name><Properties><Creation-Time>Fri,\
+        \ 09 Nov 2018 23:08:12 GMT</Creation-Time><Last-Modified>Fri, 09 Nov 2018\
+        \ 23:08:12 GMT</Last-Modified><Etag>0x8D646983937A651</Etag><Content-Length>11</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-Encoding\
+        \ /><Content-Language /><Content-MD5>XrY7u+Ae7tCTyyK7j1rNww==</Content-MD5><Cache-Control\
+        \ /><Content-Disposition /><BlobType>BlockBlob</BlobType><AccessTier>Hot</AccessTier><AccessTierInferred>true</AccessTierInferred><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><ServerEncrypted>true</ServerEncrypted></Properties></Blob><Blob><Name>blob2</Name><Properties><Creation-Time>Fri,\
+        \ 09 Nov 2018 23:08:12 GMT</Creation-Time><Last-Modified>Fri, 09 Nov 2018\
+        \ 23:08:12 GMT</Last-Modified><Etag>0x8D64698393EACDE</Etag><Content-Length>11</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-Encoding\
+        \ /><Content-Language /><Content-MD5>XrY7u+Ae7tCTyyK7j1rNww==</Content-MD5><Cache-Control\
+        \ /><Content-Disposition /><BlobType>BlockBlob</BlobType><AccessTier>Hot</AccessTier><AccessTierInferred>true</AccessTierInferred><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><ServerEncrypted>true</ServerEncrypted></Properties></Blob></Blobs><NextMarker\
+        \ /></EnumerationResults>"}
+    headers:
+      Content-Type: [application/xml]
+      Date: ['Fri, 09 Nov 2018 23:08:12 GMT']
+      Server: [Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0]
+      Transfer-Encoding: [chunked]
+      Vary: [Origin]
+      x-ms-request-id: [13632fe1-b01e-0015-3d81-780db2000000]
+      x-ms-version: ['2018-03-28']
+    status: {code: 200, message: OK}
+version: 1