{storage-blob-preview} Storage blob features with api version 2019-12…

…-12 (#2136)

* update index

* create new extension

* add vendored SDK

* create new extension

* refine structure

* support basic list

* list with next marker

* list with include

* support delimiter

* add test for list

* support show

* add upload interface

* copy start interface

* validate source url

* refine

* support secondary mode

* add test

* download

* add sas token

* generate sas

* refine help

* add container sas

* sas pass

* set-tier

* tags  interface

* fix requires_sync in python sdk

* test filter

* add if_tags

* remove if tag in precondition

* refine sdk

* support upload with tier

* fix SDK in generate_sas

* add tag conditions for existing command

* add metadata

* support snaoshot

* support undelete

* add help for download

* test pass for tag

* add license

* fix style

* fix linter

* fix setup.py

* fix denpendency

* revert

* add test for append upload

* add test for append upload

* list test pass

* test pass

* test pass

* rearrange command

* add exist

.github/CODEOWNERS

@@ -100,6 +100,8 @@
 
 /src/blueprint/ @fengzhou-msft
 
+/src/storage-blob-preview/ @Juliehzl
+
 /src/storage-or-preview/ @Juliehzl
 
 /src/logic/ @bquantump

src/storage-blob-preview/HISTORY.rst

@@ -0,0 +1,8 @@
+.. :changelog:
+
+Release History
+===============
+
+0.1.0
+++++++
+* Initial release.

src/storage-blob-preview/README.md

@@ -0,0 +1,5 @@
+Microsoft Azure CLI 'storage-blob-preview' Extension
+==========================================
+
+This package is for the 'storage-blob-preview' extension.
+i.e. 'az storage-blob-preview'

src/storage-blob-preview/azext_storage_blob_preview/_client_factory.py

@@ -0,0 +1,119 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from azure.cli.core.profiles import ResourceType, get_sdk
+from .profiles import CUSTOM_DATA_STORAGE_BLOB
+
+MISSING_CREDENTIALS_ERROR_MESSAGE = """
+Missing credentials to access storage service. The following variations are accepted:
+    (1) account name and key (--account-name and --account-key options or
+        set AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY environment variables)
+    (2) account name and SAS token (--sas-token option used with either the --account-name
+        option or AZURE_STORAGE_ACCOUNT environment variable)
+    (3) account name (--account-name option or AZURE_STORAGE_ACCOUNT environment variable;
+        this will make calls to query for a storage account key using login credentials)
+    (4) connection string (--connection-string option or
+        set AZURE_STORAGE_CONNECTION_STRING environment variable); some shells will require
+        quoting to preserve literal character interpretation.
+"""
+
+
+def get_account_url(cli_ctx, account_name, service):
+    from knack.util import CLIError
+    if account_name is None:
+        raise CLIError("Please provide storage account name or connection string.")
+    storage_endpoint = cli_ctx.cloud.suffixes.storage_endpoint
+    return "https://{}.{}.{}".format(account_name, service, storage_endpoint)
+
+
+def cf_blob_service(cli_ctx, kwargs):
+    from knack.util import CLIError
+    client_args = {}
+    t_blob_service = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE_BLOB,
+                             '_blob_service_client#BlobServiceClient')
+    connection_string = kwargs.pop('connection_string', None)
+    account_name = kwargs.pop('account_name', None)
+    account_key = kwargs.pop('account_key', None)
+    token_credential = kwargs.pop('token_credential', None)
+    sas_token = kwargs.pop('sas_token', None)
+    location_mode = kwargs.pop('location_mode', None)
+    if location_mode:
+        client_args['_location_mode'] = location_mode
+
+    if connection_string:
+        return t_blob_service.from_connection_string(conn_str=connection_string)
+
+    account_url = get_account_url(cli_ctx, account_name=account_name, service='blob')
+    credential = account_key or sas_token or token_credential
+
+    if account_url and credential:
+        return t_blob_service(account_url=account_url, credential=credential, **client_args)
+    raise CLIError("Please provide valid connection string, or account name with account key, "
+                   "sas token or login auth mode.")
+
+
+def cf_blob_client(cli_ctx, kwargs):
+    return cf_blob_service(cli_ctx, kwargs).get_blob_client(container=kwargs.pop('container_name'),
+                                                            blob=kwargs.pop('blob_name'),
+                                                            snapshot=kwargs.pop('snapshot', None))
+
+
+def cf_blob_lease_client(cli_ctx, kwargs):
+    t_lease_service = get_sdk(cli_ctx, ResourceType.DATA_STORAGE_BLOB, '_lease#BlobLeaseClient')
+    blob_client = cf_blob_service(cli_ctx, kwargs).get_blob_client(container=kwargs.pop('container_name', None),
+                                                                   blob=kwargs.pop('blob_name', None))
+    return t_lease_service(client=blob_client, lease_id=kwargs.pop('lease_id', None))
+
+
+def cf_container_client(cli_ctx, kwargs):
+    return cf_blob_service(cli_ctx, kwargs).get_container_client(container=kwargs.pop('container_name', None))
+
+
+def cf_blob_sas(cli_ctx, kwargs):
+    t_blob_sas = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE_BLOB, '_shared_access_signature#BlobSharedAccessSignature')
+
+    if kwargs.pop('as_user', None):
+        from .operations.blob import _get_datetime_from_string
+        from datetime import datetime
+        service_client = cf_blob_service(cli_ctx, kwargs)
+        user_delegation_key = service_client.get_user_delegation_key(
+            _get_datetime_from_string(kwargs['start']) if kwargs['start'] else datetime.utcnow(),
+            _get_datetime_from_string(kwargs['expiry']))
+        return t_blob_sas(account_name=kwargs.pop('account_name', None),
+                          user_delegation_key=user_delegation_key)
+
+    return t_blob_sas(account_name=kwargs.pop('account_name', None),
+                      account_key=kwargs.pop('account_key', None))
+
+
+def cf_adls_service(cli_ctx, kwargs):
+    t_adls_service = get_sdk(cli_ctx, ResourceType.DATA_STORAGE_FILEDATALAKE,
+                             '_data_lake_service_client#DataLakeServiceClient')
+    connection_string = kwargs.pop('connection_string', None)
+    account_key = kwargs.pop('account_key', None)
+    token_credential = kwargs.pop('token_credential', None)
+    sas_token = kwargs.pop('sas_token', None)
+    if connection_string:
+        return t_adls_service.from_connection_string(connection_string=connection_string)
+
+    account_url = get_account_url(cli_ctx, account_name=kwargs.pop('account_name', None), service='dfs')
+    credential = account_key or sas_token or token_credential
+
+    if account_url and credential:
+        return t_adls_service(account_url=account_url, credential=credential)
+    return None
+
+
+def cf_adls_file_system(cli_ctx, kwargs):
+    return cf_adls_service(cli_ctx, kwargs).get_file_system_client(file_system=kwargs.pop('file_system_name'))
+
+
+def cf_adls_directory(cli_ctx, kwargs):
+    return cf_adls_file_system(cli_ctx, kwargs).get_directory_client(directory=kwargs.pop('directory_path'))
+
+
+def cf_adls_file(cli_ctx, kwargs):
+    return cf_adls_service(cli_ctx, kwargs).get_file_client(file_system=kwargs.pop('file_system_name', None),
+                                                            file_path=kwargs.pop('path', None))

src/storage-blob-preview/azext_storage_blob_preview/_help.py

@@ -0,0 +1,134 @@
+# coding=utf-8
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from knack.help_files import helps  # pylint: disable=unused-import
+
+helps['storage blob copy start'] = """
+type: command
+short-summary: List blobs in a given container.
+parameters:
+  - name: --source-uri -u
+    type: string
+    short-summary: >
+        A URL of up to 2 KB in length that specifies an Azure file or blob.
+        The value should be URL-encoded as it would appear in a request URI.
+        If the source is in another account, the source must either be public
+        or must be authenticated via a shared access signature. If the source
+        is public, no authentication is required.
+        Examples:
+        `https://myaccount.blob.core.windows.net/mycontainer/myblob`,
+        `https://myaccount.blob.core.windows.net/mycontainer/myblob?snapshot=<DateTime>`,
+        `https://otheraccount.blob.core.windows.net/mycontainer/myblob?sastoken`
+  - name: --destination-if-modified-since
+    type: string
+    short-summary: >
+        A DateTime value. Azure expects the date value passed in to be UTC.
+        If timezone is included, any non-UTC datetimes will be converted to UTC.
+        If a date is passed in without timezone info, it is assumed to be UTC.
+        Specify this conditional header to copy the blob only
+        if the destination blob has been modified since the specified date/time.
+        If the destination blob has not been modified, the Blob service returns
+        status code 412 (Precondition Failed).
+  - name: --destination-if-unmodified-since
+    type: string
+    short-summary: >
+        A DateTime value. Azure expects the date value passed in to be UTC.
+        If timezone is included, any non-UTC datetimes will be converted to UTC.
+        If a date is passed in without timezone info, it is assumed to be UTC.
+        Specify this conditional header to copy the blob only
+        if the destination blob has not been modified since the specified
+        date/time. If the destination blob has been modified, the Blob service
+        returns status code 412 (Precondition Failed).
+  - name: --source-if-modified-since
+    type: string
+    short-summary: >
+        A DateTime value. Azure expects the date value passed in to be UTC.
+        If timezone is included, any non-UTC datetimes will be converted to UTC.
+        If a date is passed in without timezone info, it is assumed to be UTC.
+        Specify this conditional header to copy the blob only if the source
+        blob has been modified since the specified date/time.
+  - name: --source-if-unmodified-since
+    type: string
+    short-summary: >
+        A DateTime value. Azure expects the date value passed in to be UTC.
+        If timezone is included, any non-UTC datetimes will be converted to UTC.
+        If a date is passed in without timezone info, it is assumed to be UTC.
+        Specify this conditional header to copy the blob only if the source blob
+        has not been modified since the specified date/time.
+examples:
+  - name: Copy a blob asynchronously. Use `az storage blob show` to check the status of the blobs.
+    text: |
+        az storage blob copy start --account-key 00000000 --account-name MyAccount --destination-blob MyDestinationBlob --destination-container MyDestinationContainer --source-uri https://storage.blob.core.windows.net/photos
+  - name: Copy a blob asynchronously. Use `az storage blob show` to check the status of the blobs.
+    text: |
+        az storage blob copy start --account-name MyAccount --destination-blob MyDestinationBlob --destination-container MyDestinationContainer --sas-token $sas --source-uri https://storage.blob.core.windows.net/photos
+"""
+
+helps['storage blob download'] = """
+type: command
+short-summary: Download a blob to a file path, with automatic chunking and progress notifications.
+"""
+
+helps['storage blob filter'] = """
+type: command
+short-summary: List blobs across all containers whose tags match a given search expression.
+long-summary: >
+    Filter blobs searches across all containers within a storage account but can be scoped within the expression to
+    a single container.
+parameters:
+  - name: --tag-filter
+    short-summary: >
+            The expression to find blobs whose tags matches the specified condition.
+            eg. ""yourtagname"='firsttag' and "yourtagname2"='secondtag'"
+            To specify a container, eg. "@container='containerName' and "Name"='C'"
+"""
+
+helps['storage blob list'] = """
+type: command
+short-summary: List blobs in a given container.
+examples:
+  - name: List all storage blobs in a container whose names start with 'foo'; will match names such as 'foo', 'foobar', and 'foo/bar'
+    text: az storage blob list -c MyContainer --prefix foo
+"""
+
+helps['storage blob metadata'] = """
+type: group
+short-summary: Manage blob metadata.
+"""
+
+helps['storage blob metadata show'] = """
+type: command
+short-summary: Return all user-defined metadata for the specified blob or snapshot.
+examples:
+  - name: Get all user-defined metadata for the specified blob.
+    text: az storage blob metadata show -n myblob -c mycontainer --account-name mystorageaccount --account-key 0000-0000
+"""
+
+helps['storage blob metadata update'] = """
+type: command
+short-summary: Set user-defined metadata for the specified blob as one or more name-value pairs.
+examples:
+  - name:  Set user-defined metadata for the specified blob as one or more name-value pairs.
+    text: az storage blob metadata update -n myblob -c mycontainer --metadata a=b c=d
+"""
+
+helps['storage blob tag'] = """
+type: group
+short-summary: Manage blob tags.
+"""
+
+helps['storage blob tag list'] = """
+type: command
+short-summary: Get tags on a blob or specific blob version, or snapshot.
+"""
+
+helps['storage blob tag set'] = """
+type: command
+short-summary: Set tags on a blob or specific blob version, but not snapshot.
+long-summary: >
+    Each call to this operation replaces all existing tags attached to the blob. To remove all
+    tags from the blob, call this operation with no tags set.
+"""