[Storage] az storage blob download/download-batch: Add --overwrite

src/azure-cli/azure/cli/command_modules/storage/_params.py

@@ -1048,6 +1048,8 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
                      'if enabled, the memory-efficient algorithm will not be used because computing the MD5 hash '
                      'requires buffering entire blocks, and doing so defeats the purpose of the memory-efficient '
                      'algorithm.')
+        c.argument('overwrite', arg_type=get_three_state_flag(),
+                   help="Overwrite an existing file when specified. Default value is true.")
 
     with self.argument_context('storage blob download-batch') as c:
         c.ignore('source_container_name')
@@ -1056,6 +1058,8 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         c.extra('no_progress', progress_type)
         c.extra('max_concurrency', options_list=['--max-connections'], type=int, default=2,
                 help='The number of parallel connections with which to download.')
+        c.argument('overwrite', arg_type=get_three_state_flag(),
+                   help="Overwrite an existing file when specified. Default value is false.")
 
     with self.argument_context('storage blob delete') as c:
         from .sdkutil import get_delete_blob_snapshot_type_names

src/azure-cli/azure/cli/command_modules/storage/operations/blob.py

@@ -9,7 +9,7 @@
 
 from azure.cli.core.profiles import ResourceType, get_sdk
 from azure.cli.core.util import sdk_no_wait
-from azure.cli.core.azclierror import AzureResponseError
+from azure.cli.core.azclierror import AzureResponseError, FileOperationError
 from azure.cli.command_modules.storage.util import (filter_none, collect_blobs, collect_blob_objects,
                                                     collect_files_track2, mkdir_p, guess_content_type,
                                                     normalize_blob_file_path, check_precondition_success)
@@ -408,7 +408,7 @@ def action_file_copy(file_info):
 
 # pylint: disable=unused-argument
 def storage_blob_download_batch(client, source, destination, source_container_name, pattern=None, dryrun=False,
-                                progress_callback=None, **kwargs):
+                                progress_callback=None, overwrite=False, **kwargs):
     @check_precondition_success
     def _download_blob(*args, **kwargs):
         blob = download_blob(*args, **kwargs)
@@ -441,7 +441,6 @@ def _download_blob(*args, **kwargs):
 
     results = []
     for index, blob_normed in enumerate(blobs_to_download):
-        from azure.cli.core.azclierror import FileOperationError
         # add blob name and number to progress message
         if progress_callback:
             progress_callback.message = '{}/{}: "{}"'.format(
@@ -451,13 +450,13 @@ def _download_blob(*args, **kwargs):
         destination_path = os.path.join(destination, os.path.normpath(blob_normed))
         destination_folder = os.path.dirname(destination_path)
         # Failed when there is same name for file and folder
-        if os.path.isfile(destination_path) and os.path.exists(destination_folder):
+        if os.path.isfile(destination_path) and os.path.exists(destination_folder) and not overwrite:
             raise FileOperationError("%s already exists in %s. Please rename existing file or choose another "
-                                     "destination folder. ")
+                                     "destination folder. " % (blob_normed, destination))
         if not os.path.exists(destination_folder):
             mkdir_p(destination_folder)
         include, result = _download_blob(client=blob_client, file_path=destination_path,
-                                         progress_callback=progress_callback, **kwargs)
+                                         progress_callback=progress_callback, overwrite=overwrite, **kwargs)
         if include:
             results.append(result)
 
@@ -666,7 +665,7 @@ def upload_blob(cmd, client, file_path=None, container_name=None, blob_name=None
 
 
 def download_blob(client, file_path=None, open_mode='wb', start_range=None, end_range=None,
-                  progress_callback=None, **kwargs):
+                  progress_callback=None, overwrite=True, **kwargs):
     offset = None
     length = None
     if start_range is not None and end_range is not None:
@@ -678,6 +677,8 @@ def download_blob(client, file_path=None, open_mode='wb', start_range=None, end_
         kwargs['max_concurrency'] = 1
     download_stream = client.download_blob(offset=offset, length=length, **kwargs)
     if file_path:
+        if os.path.isfile(file_path) and not overwrite:
+            raise FileOperationError("%s already exists. Please rename existing file or use --overwrite" % (file_path))
         with open(file_path, open_mode) as stream:
             download_stream.readinto(stream)
         return download_stream.properties

src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_storage_blob_download_directory.yaml

@@ -15,12 +15,12 @@ interactions:
       ParameterSetName:
       - -n -g --query -o
       User-Agent:
-      - AZURECLI/2.35.0 azsdk-python-azure-mgmt-storage/20.0.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
+      - AZURECLI/2.38.0 azsdk-python-azure-mgmt-storage/20.0.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
     method: POST
     uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/clitest.rg000001/providers/Microsoft.Storage/storageAccounts/clitest000002/listKeys?api-version=2021-09-01&$expand=kerb
   response:
     body:
-      string: '{"keys":[{"creationTime":"2022-04-11T13:50:34.2610343Z","keyName":"key1","value":"veryFakedStorageAccountKey==","permissions":"FULL"},{"creationTime":"2022-04-11T13:50:34.2610343Z","keyName":"key2","value":"veryFakedStorageAccountKey==","permissions":"FULL"}]}'
+      string: '{"keys":[{"creationTime":"2022-07-26T06:44:50.9901204Z","keyName":"key1","value":"veryFakedStorageAccountKey==","permissions":"FULL"},{"creationTime":"2022-07-26T06:44:50.9901204Z","keyName":"key2","value":"veryFakedStorageAccountKey==","permissions":"FULL"}]}'
     headers:
       cache-control:
       - no-cache
@@ -29,7 +29,7 @@ interactions:
       content-type:
       - application/json
       date:
-      - Mon, 11 Apr 2022 13:50:55 GMT
+      - Tue, 26 Jul 2022 06:45:12 GMT
       expires:
       - '-1'
       pragma:
@@ -45,7 +45,7 @@ interactions:
       x-content-type-options:
       - nosniff
       x-ms-ratelimit-remaining-subscription-resource-requests:
-      - '11998'
+      - '11999'
     status:
       code: 200
       message: OK

src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_storage_blob_oauth.yaml

@@ -15,9 +15,9 @@ interactions:
       ParameterSetName:
       - -n --account-name --public-access --auth-mode
       User-Agent:
-      - AZURECLI/2.36.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
+      - AZURECLI/2.38.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
       x-ms-date:
-      - Wed, 18 May 2022 03:54:11 GMT
+      - Tue, 26 Jul 2022 06:45:06 GMT
       x-ms-version:
       - '2021-06-08'
     method: PUT
@@ -29,11 +29,11 @@ interactions:
       content-length:
       - '0'
       date:
-      - Wed, 18 May 2022 03:54:12 GMT
+      - Tue, 26 Jul 2022 06:45:12 GMT
       etag:
-      - '"0x8DA38821241AA78"'
+      - '"0x8DA6ED264473865"'
       last-modified:
-      - Wed, 18 May 2022 03:54:12 GMT
+      - Tue, 26 Jul 2022 06:45:13 GMT
       server:
       - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
       x-ms-version:
@@ -61,11 +61,11 @@ interactions:
       ParameterSetName:
       - -c -f -n --type --account-name --auth-mode
       User-Agent:
-      - AZURECLI/2.36.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
+      - AZURECLI/2.38.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
       x-ms-blob-type:
       - BlockBlob
       x-ms-date:
-      - Wed, 18 May 2022 03:54:13 GMT
+      - Tue, 26 Jul 2022 06:45:09 GMT
       x-ms-version:
       - '2021-06-08'
     method: PUT
@@ -79,11 +79,11 @@ interactions:
       content-md5:
       - DzQ7CTESaiDxM9Z8KwGKOw==
       date:
-      - Wed, 18 May 2022 03:54:13 GMT
+      - Tue, 26 Jul 2022 06:45:13 GMT
       etag:
-      - '"0x8DA388212E9F9B1"'
+      - '"0x8DA6ED264FDB391"'
       last-modified:
-      - Wed, 18 May 2022 03:54:14 GMT
+      - Tue, 26 Jul 2022 06:45:14 GMT
       server:
       - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
       x-ms-content-crc64:
@@ -109,9 +109,9 @@ interactions:
       ParameterSetName:
       - -n -c --account-name --auth-mode
       User-Agent:
-      - AZURECLI/2.36.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
+      - AZURECLI/2.38.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
       x-ms-date:
-      - Wed, 18 May 2022 03:54:14 GMT
+      - Tue, 26 Jul 2022 06:45:10 GMT
       x-ms-version:
       - '2021-06-08'
     method: HEAD
@@ -129,17 +129,17 @@ interactions:
       content-type:
       - application/octet-stream
       date:
-      - Wed, 18 May 2022 03:54:15 GMT
+      - Tue, 26 Jul 2022 06:45:14 GMT
       etag:
-      - '"0x8DA388212E9F9B1"'
+      - '"0x8DA6ED264FDB391"'
       last-modified:
-      - Wed, 18 May 2022 03:54:14 GMT
+      - Tue, 26 Jul 2022 06:45:14 GMT
       server:
       - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
       x-ms-blob-type:
       - BlockBlob
       x-ms-creation-time:
-      - Wed, 18 May 2022 03:54:14 GMT
+      - Tue, 26 Jul 2022 06:45:14 GMT
       x-ms-lease-state:
       - available
       x-ms-lease-status:
@@ -165,27 +165,27 @@ interactions:
       ParameterSetName:
       - -c --account-name --auth-mode
       User-Agent:
-      - AZURECLI/2.36.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
+      - AZURECLI/2.38.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
       x-ms-date:
-      - Wed, 18 May 2022 03:54:15 GMT
+      - Tue, 26 Jul 2022 06:45:12 GMT
       x-ms-version:
       - '2021-06-08'
     method: GET
     uri: https://clitest000002.blob.core.windows.net/container000003?restype=container&comp=list&maxresults=5000
   response:
     body:
       string: "\uFEFF<?xml version=\"1.0\" encoding=\"utf-8\"?><EnumerationResults
-        ServiceEndpoint=\"https://clitest000002.blob.core.windows.net/\" ContainerName=\"container000003\"><MaxResults>5000</MaxResults><Blobs><Blob><Name>blob000004</Name><Properties><Creation-Time>Wed,
-        18 May 2022 03:54:14 GMT</Creation-Time><Last-Modified>Wed, 18 May 2022 03:54:14
-        GMT</Last-Modified><Etag>0x8DA388212E9F9B1</Etag><Content-Length>1024</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-Encoding
+        ServiceEndpoint=\"https://clitest000002.blob.core.windows.net/\" ContainerName=\"container000003\"><MaxResults>5000</MaxResults><Blobs><Blob><Name>blob000004</Name><Properties><Creation-Time>Tue,
+        26 Jul 2022 06:45:14 GMT</Creation-Time><Last-Modified>Tue, 26 Jul 2022 06:45:14
+        GMT</Last-Modified><Etag>0x8DA6ED264FDB391</Etag><Content-Length>1024</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-Encoding
         /><Content-Language /><Content-CRC64 /><Content-MD5>DzQ7CTESaiDxM9Z8KwGKOw==</Content-MD5><Cache-Control
         /><Content-Disposition /><BlobType>BlockBlob</BlobType><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><ServerEncrypted>true</ServerEncrypted></Properties><OrMetadata
         /></Blob></Blobs><NextMarker /></EnumerationResults>"
     headers:
       content-type:
       - application/xml
       date:
-      - Wed, 18 May 2022 03:54:16 GMT
+      - Tue, 26 Jul 2022 06:45:15 GMT
       server:
       - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
       transfer-encoding:
@@ -209,9 +209,9 @@ interactions:
       ParameterSetName:
       - -c --account-name -n --auth-mode
       User-Agent:
-      - AZURECLI/2.36.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
+      - AZURECLI/2.38.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
       x-ms-date:
-      - Wed, 18 May 2022 03:54:16 GMT
+      - Tue, 26 Jul 2022 06:45:13 GMT
       x-ms-version:
       - '2021-06-08'
     method: HEAD
@@ -229,17 +229,17 @@ interactions:
       content-type:
       - application/octet-stream
       date:
-      - Wed, 18 May 2022 03:54:16 GMT
+      - Tue, 26 Jul 2022 06:45:16 GMT
       etag:
-      - '"0x8DA388212E9F9B1"'
+      - '"0x8DA6ED264FDB391"'
       last-modified:
-      - Wed, 18 May 2022 03:54:14 GMT
+      - Tue, 26 Jul 2022 06:45:14 GMT
       server:
       - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
       x-ms-blob-type:
       - BlockBlob
       x-ms-creation-time:
-      - Wed, 18 May 2022 03:54:14 GMT
+      - Tue, 26 Jul 2022 06:45:14 GMT
       x-ms-lease-state:
       - available
       x-ms-lease-status:
@@ -265,9 +265,9 @@ interactions:
       ParameterSetName:
       - -n -c --file --account-name --auth-mode
       User-Agent:
-      - AZURECLI/2.36.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
+      - AZURECLI/2.38.0 azsdk-python-storage-blob/12.12.0 Python/3.9.6 (Windows-10-10.0.19044-SP0)
       x-ms-date:
-      - Wed, 18 May 2022 03:54:17 GMT
+      - Tue, 26 Jul 2022 06:45:14 GMT
       x-ms-range:
       - bytes=0-33554431
       x-ms-version:
@@ -287,19 +287,19 @@ interactions:
       content-type:
       - application/octet-stream
       date:
-      - Wed, 18 May 2022 03:54:18 GMT
+      - Tue, 26 Jul 2022 06:45:18 GMT
       etag:
-      - '"0x8DA388212E9F9B1"'
+      - '"0x8DA6ED264FDB391"'
       last-modified:
-      - Wed, 18 May 2022 03:54:14 GMT
+      - Tue, 26 Jul 2022 06:45:14 GMT
       server:
       - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
       x-ms-blob-content-md5:
       - DzQ7CTESaiDxM9Z8KwGKOw==
       x-ms-blob-type:
       - BlockBlob
       x-ms-creation-time:
-      - Wed, 18 May 2022 03:54:14 GMT
+      - Tue, 26 Jul 2022 06:45:14 GMT
       x-ms-lease-state:
       - available
       x-ms-lease-status: