From da7764d335500be610e9f96adeee1d0ca101678b Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 23 May 2019 18:21:13 -0700 Subject: [PATCH 1/8] Mypy rebase --- sdk/core/azure-core/azure/__init__.py | 2 +- sdk/core/azure-core/azure/core/paging.py | 2 +- .../azure/core/pipeline/__init__.py | 6 +- .../azure-core/azure/core/pipeline/base.py | 8 +-- .../azure/core/pipeline/policies/base.py | 14 ++-- .../core/pipeline/policies/base_async.py | 4 +- .../core/pipeline/policies/credentials.py | 2 +- .../pipeline/policies/credentials_async.py | 6 +- .../core/pipeline/policies/custom_hook.py | 9 +-- .../azure/core/pipeline/policies/redirect.py | 2 +- .../core/pipeline/policies/redirect_async.py | 2 +- .../core/pipeline/policies/retry_async.py | 2 +- .../azure/core/pipeline/policies/universal.py | 68 ++++++++++--------- .../azure/core/pipeline/transport/aiohttp.py | 4 +- .../azure/core/pipeline/transport/base.py | 11 +-- .../pipeline/transport/requests_asyncio.py | 14 ++-- .../core/pipeline/transport/requests_basic.py | 16 +++-- .../core/pipeline/transport/requests_trio.py | 16 ++--- .../azure-core/azure/core/pipeline_client.py | 56 ++++++++++++--- .../azure/core/pipeline_client_async.py | 56 ++++++++++++--- .../azure-core/azure/core/polling/poller.py | 7 +- sdk/core/azure-core/dev_requirements.txt | 3 +- sdk/core/azure-core/setup.py | 6 +- 23 files changed, 198 insertions(+), 118 deletions(-) diff --git a/sdk/core/azure-core/azure/__init__.py b/sdk/core/azure-core/azure/__init__.py index 69e3be50dac4..0d1f7edf5dc6 100644 --- a/sdk/core/azure-core/azure/__init__.py +++ b/sdk/core/azure-core/azure/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/sdk/core/azure-core/azure/core/paging.py b/sdk/core/azure-core/azure/core/paging.py index aadb3348ea02..871730360e92 100644 --- a/sdk/core/azure-core/azure/core/paging.py +++ b/sdk/core/azure-core/azure/core/paging.py @@ -34,7 +34,7 @@ if TYPE_CHECKING: from .pipeline.transport.base import HttpResponse - from msrest.serialization import Deserializer, Model # pylint: disable=unused-import + from msrest.serialization import Deserializer, Model # type: ignore # pylint: disable=unused-import if sys.version_info >= (3, 5, 2): # Not executed on old Python, no syntax error diff --git a/sdk/core/azure-core/azure/core/pipeline/__init__.py b/sdk/core/azure-core/azure/core/pipeline/__init__.py index ac5d171e35cb..a302d2babc63 100644 --- a/sdk/core/azure-core/azure/core/pipeline/__init__.py +++ b/sdk/core/azure-core/azure/core/pipeline/__init__.py @@ -25,12 +25,16 @@ # -------------------------------------------------------------------------- import abc +from typing import (TypeVar, Any, Dict, Optional) try: ABC = abc.ABC except AttributeError: # Python 2.7, abc exists, but not ABC ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) # type: ignore +HTTPResponseType = TypeVar("HTTPResponseType") +HTTPRequestType = TypeVar("HTTPRequestType") + try: from contextlib import AbstractContextManager # type: ignore #pylint: disable=unused-import except ImportError: # Python <= 3.5 @@ -107,7 +111,7 @@ class PipelineResponse(object): However, nothing prevents a policy to actually sub-class this class a return it instead of the initial instance. """ def __init__(self, http_request, http_response, context): - # type: (HttpRequest[HTTPRequestType], HTTPResponseType, Optional[Dict[str, Any]]) -> None + # type: (HTTPRequestType, HTTPResponseType, Optional[Dict[str, Any]]) -> None self.http_request = http_request self.http_response = http_response self.context = context diff --git a/sdk/core/azure-core/azure/core/pipeline/base.py b/sdk/core/azure-core/azure/core/pipeline/base.py index 1c2d2676f676..6720e076dde3 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/base.py @@ -45,7 +45,7 @@ def __init__(self, policy): self._policy = policy def send(self, request): - # type: (PipelineRequest[HTTPRequestType], Any) -> PipelineResponse[HTTPRequestType, HTTPResponseType] + # type: (PipelineRequest) -> PipelineResponse self._policy.on_request(request) try: response = self.next.send(request) @@ -60,7 +60,7 @@ def send(self, request): class _TransportRunner(HTTPPolicy): def __init__(self, sender): - # type: (HttpTransport) -> None + # type: (Any) -> None super(_TransportRunner, self).__init__() self._sender = sender @@ -80,7 +80,7 @@ class Pipeline(AbstractContextManager, Generic[HTTPRequestType, HTTPResponseType """ def __init__(self, transport, policies=None): - # type: (HttpTransport, List[Union[HTTPPolicy, SansIOHTTPPolicy]]) -> None + # type: (Any, List[Union[HTTPPolicy, SansIOHTTPPolicy]]) -> None self._impl_policies = [] # type: List[HTTPPolicy] self._transport = transport # type: HTTPPolicy @@ -105,6 +105,6 @@ def __exit__(self, *exc_details): # pylint: disable=arguments-differ def run(self, request, **kwargs): # type: (HTTPRequestType, Any) -> PipelineResponse context = PipelineContext(self._transport, **kwargs) - pipeline_request = PipelineRequest(request, context) # type: PipelineRequest[HTTPRequestType] + pipeline_request = PipelineRequest(request, context) # type: PipelineRequest first_node = self._impl_policies[0] if self._impl_policies else _TransportRunner(self._transport) return first_node.send(pipeline_request) # type: ignore diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/base.py b/sdk/core/azure-core/azure/core/pipeline/policies/base.py index 75ceb41730ff..4f281c47fe61 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/base.py @@ -31,7 +31,7 @@ from typing import (TYPE_CHECKING, Generic, TypeVar, cast, IO, List, Union, Any, Mapping, Dict, Optional, # pylint: disable=unused-import Tuple, Callable, Iterator) -from azure.core.pipeline import ABC +from azure.core.pipeline import ABC, PipelineRequest, PipelineResponse HTTPResponseType = TypeVar("HTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") @@ -39,7 +39,7 @@ _LOGGER = logging.getLogger(__name__) -class HTTPPolicy(ABC, Generic[HTTPRequestType, HTTPResponseType]): +class HTTPPolicy(ABC, Generic[HTTPRequestType, HTTPResponseType]): # type: ignore """An HTTP policy ABC. """ def __init__(self): @@ -47,7 +47,7 @@ def __init__(self): @abc.abstractmethod def send(self, request): - # type: (PipelineRequest[HTTPRequestType]) -> PipelineResponse[HTTPRequestType, HTTPResponseType] + # type: (PipelineRequest) -> PipelineResponse """Mutate the request. Context content is dependent on the HttpTransport. """ @@ -64,18 +64,18 @@ class SansIOHTTPPolicy(Generic[HTTPRequestType, HTTPResponseType]): """ def on_request(self, request, **kwargs): - # type: (PipelineRequest[HTTPRequestType], Any) -> None + # type: (PipelineRequest, Any) -> None """Is executed before sending the request to next policy. """ def on_response(self, request, response, **kwargs): - # type: (PipelineRequest[HTTPRequestType], PipelineResponse[HTTPRequestType, HTTPResponseType], Any) -> None + # type: (PipelineRequest, PipelineResponse, Any) -> None """Is executed after the request comes back from the policy. """ #pylint: disable=no-self-use def on_exception(self, _request, **kwargs): #pylint: disable=unused-argument - # type: (PipelineRequest[HTTPRequestType], Any) -> bool + # type: (PipelineRequest, Any) -> bool """Is executed if an exception comes back from the following policy. Return True if the exception has been handled and should not @@ -98,7 +98,7 @@ class RequestHistory(object): """ def __init__(self, http_request, http_response=None, error=None, context=None): - # type: (PipelineRequest[HTTPRequestType], Exception, Optional[Dict[str, Any]]) -> None + # type: (PipelineRequest, PipelineResponse, Exception, Optional[Dict[str, Any]]) -> None self.http_request = copy.deepcopy(http_request) self.http_response = http_response self.error = error diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/base_async.py b/sdk/core/azure-core/azure/core/pipeline/policies/base_async.py index f7f8177c391b..4e35b36025cc 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/base_async.py @@ -26,7 +26,7 @@ # -------------------------------------------------------------------------- import abc -from typing import Generic, TypeVar +from typing import Generic, TypeVar, Optional, Any from azure.core.pipeline import PipelineRequest @@ -54,7 +54,7 @@ class AsyncHTTPPolicy(abc.ABC, Generic[HTTPRequestType, AsyncHTTPResponseType]): """ def __init__(self) -> None: # next will be set once in the pipeline - self.next = None # type: Optional[Union[AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType], AsyncHttpTransport[HTTPRequestType, AsyncHTTPResponseType]]] #pylint: disable=line-too-long + self.next = None # type: Optional[Any] @abc.abstractmethod async def send(self, request: PipelineRequest): diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/credentials.py b/sdk/core/azure-core/azure/core/pipeline/policies/credentials.py index 8cf1a02f46ca..9bb44f31c52d 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/credentials.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/credentials.py @@ -37,5 +37,5 @@ class BearerTokenCredentialPolicy(_BearerTokenCredentialPolicyBase, HTTPPolicy): def send(self, request): # type: (PipelineRequest) -> PipelineResponse token = self._credential.get_token(self._scopes) - self._update_headers(request.http_request.headers, token) + self._update_headers(request.http_request.headers, token) # type: ignore return self.next.send(request) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/credentials_async.py b/sdk/core/azure-core/azure/core/pipeline/policies/credentials_async.py index cfeae0ebded8..5251c744fc9c 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/credentials_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/credentials_async.py @@ -13,6 +13,6 @@ class AsyncBearerTokenCredentialPolicy(_BearerTokenCredentialPolicyBase, AsyncHT """Adds a bearer token Authorization header to requests.""" async def send(self, request: PipelineRequest) -> PipelineResponse: - token = await self._credential.get_token(self._scopes) - self._update_headers(request.http_request.headers, token) - return await self.next.send(request) + token = await self._credential.get_token(self._scopes) # type: ignore + self._update_headers(request.http_request.headers, token) # type: ignore + return await self.next.send(request) # type: ignore diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/custom_hook.py b/sdk/core/azure-core/azure/core/pipeline/policies/custom_hook.py index 737a8907dad7..0317019f89d3 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/custom_hook.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/custom_hook.py @@ -26,6 +26,7 @@ """ This module is the requests implementation of Pipeline ABC """ +from azure.core.pipeline import PipelineRequest, PipelineResponse from .base import SansIOHTTPPolicy class CustomHookPolicy(SansIOHTTPPolicy): @@ -35,12 +36,12 @@ class CustomHookPolicy(SansIOHTTPPolicy): def __init__(self, **kwargs): # pylint: disable=unused-argument self._callback = None - def on_request(self, request): # pylint: disable=arguments-differ + def on_request(self, request): # type: ignore # pylint: disable=arguments-differ # type: (PipelineRequest) -> None - self._callback = request.context.options.pop('raw_response_hook', None) + self._callback = request.context.options.pop('raw_response_hook', None) # type: ignore - def on_response(self, request, response): # pylint: disable=arguments-differ + def on_response(self, request, response): # type: ignore # pylint: disable=arguments-differ # type: (PipelineRequest, PipelineResponse) -> None if self._callback: self._callback(response) - request.context.options.update({'raw_response_hook': self._callback}) + request.context.options.update({'raw_response_hook': self._callback}) # type: ignore diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/redirect.py b/sdk/core/azure-core/azure/core/pipeline/policies/redirect.py index 7a0a9a16bddd..89d5f53b8d08 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/redirect.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/redirect.py @@ -30,7 +30,7 @@ import logging from typing import TYPE_CHECKING, List, Callable, Iterator, Any, Union, Dict, Optional # pylint: disable=unused-import try: - from urlparse import urlparse + from urlparse import urlparse # type: ignore except ImportError: from urllib.parse import urlparse diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/redirect_async.py b/sdk/core/azure-core/azure/core/pipeline/policies/redirect_async.py index c1573c4ef138..0bbfca945363 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/redirect_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/redirect_async.py @@ -30,7 +30,7 @@ from .redirect import RedirectPolicy -class AsyncRedirectPolicy(RedirectPolicy, AsyncHTTPPolicy): +class AsyncRedirectPolicy(RedirectPolicy, AsyncHTTPPolicy): # type: ignore """An async redirect policy.""" async def send(self, request): diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/retry_async.py b/sdk/core/azure-core/azure/core/pipeline/policies/retry_async.py index b54546730e42..273c42d2d0b9 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/retry_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/retry_async.py @@ -39,7 +39,7 @@ -class AsyncRetryPolicy(RetryPolicy, AsyncHTTPPolicy): +class AsyncRetryPolicy(RetryPolicy, AsyncHTTPPolicy): # type: ignore async def _sleep_for_retry(self, response, transport): diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/universal.py b/sdk/core/azure-core/azure/core/pipeline/policies/universal.py index 10a029139fc5..851ebe002382 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/universal.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/universal.py @@ -34,7 +34,8 @@ import xml.etree.ElementTree as ET import types import re -from typing import cast, IO +from typing import (Mapping, IO, TypeVar, TYPE_CHECKING, cast, List, Callable, Iterator, # pylint: disable=unused-import + Any, Union, Dict, Optional) from azure.core import __version__ as azcore_version from azure.core.exceptions import ( @@ -42,6 +43,7 @@ raise_with_traceback ) +from azure.core.pipeline import PipelineRequest, PipelineResponse from .base import SansIOHTTPPolicy @@ -54,9 +56,9 @@ class HeadersPolicy(SansIOHTTPPolicy): This will overwrite any headers already defined in the request. """ def __init__(self, base_headers=None, **kwargs): - # type: (Mapping[str, str]) -> None + # type: (Mapping[str, str], Any) -> None self._headers = base_headers or {} - self._headers.update(kwargs.pop('headers', {})) + self._headers.update(kwargs.pop('headers', {})) # type: ignore @property def headers(self): @@ -69,10 +71,10 @@ def add_header(self, key, value): def on_request(self, request, **kwargs): # type: (PipelineRequest, Any) -> None - request.http_request.headers.update(self.headers) - additional_headers = request.context.options.pop('headers', {}) + request.http_request.headers.update(self.headers) # type: ignore + additional_headers = request.context.options.pop('headers', {}) # type: ignore if additional_headers: - request.http_request.headers.update(additional_headers) + request.http_request.headers.update(additional_headers) # type: ignore class UserAgentPolicy(SansIOHTTPPolicy): @@ -113,17 +115,17 @@ def add_user_agent(self, value): def on_request(self, request, **kwargs): # type: (PipelineRequest, Any) -> None http_request = request.http_request - options = request.context.options + options = request.context.options # type: ignore if 'user_agent' in options: user_agent = options.pop('user_agent') if options.pop('user_agent_overwrite', self.overwrite): - http_request.headers[self._USERAGENT] = user_agent + http_request.headers[self._USERAGENT] = user_agent # type: ignore else: user_agent = "{} {}".format(self.user_agent, user_agent) - http_request.headers[self._USERAGENT] = user_agent + http_request.headers[self._USERAGENT] = user_agent # type: ignore - elif self.overwrite or self._USERAGENT not in http_request.headers: - http_request.headers[self._USERAGENT] = self.user_agent + elif self.overwrite or self._USERAGENT not in http_request.headers: # type: ignore + http_request.headers[self._USERAGENT] = self.user_agent # type: ignore class NetworkTraceLoggingPolicy(SansIOHTTPPolicy): @@ -136,59 +138,59 @@ def __init__(self, logging_enable=False, **kwargs): # pylint: disable=unused-arg def on_request(self, request, **kwargs): # type: (PipelineRequest, Any) -> None http_request = request.http_request - options = request.context.options + options = request.context.options # type: ignore if options.pop("logging_enable", self.enable_http_logger): - request.context["logging_enable"] = True + request.context["logging_enable"] = True # type: ignore if not _LOGGER.isEnabledFor(logging.DEBUG): return try: - _LOGGER.debug("Request URL: %r", http_request.url) - _LOGGER.debug("Request method: %r", http_request.method) + _LOGGER.debug("Request URL: %r", http_request.url) # type: ignore + _LOGGER.debug("Request method: %r", http_request.method) # type: ignore _LOGGER.debug("Request headers:") - for header, value in http_request.headers.items(): + for header, value in http_request.headers.items(): # type: ignore if header.lower() == 'authorization': value = '*****' _LOGGER.debug(" %r: %r", header, value) _LOGGER.debug("Request body:") # We don't want to log the binary data of a file upload. - if isinstance(http_request.body, types.GeneratorType): + if isinstance(http_request.body, types.GeneratorType): # type: ignore _LOGGER.debug("File upload") else: - _LOGGER.debug(str(http_request.body)) + _LOGGER.debug(str(http_request.body)) # type: ignore except Exception as err: # pylint: disable=broad-except _LOGGER.debug("Failed to log request: %r", err) def on_response(self, request, response, **kwargs): # type: (PipelineRequest, PipelineResponse, Any) -> None - if response.context.pop("logging_enable", self.enable_http_logger): + if response.context.pop("logging_enable", self.enable_http_logger): # type: ignore if not _LOGGER.isEnabledFor(logging.DEBUG): return try: - _LOGGER.debug("Response status: %r", response.http_response.status_code) + _LOGGER.debug("Response status: %r", response.http_response.status_code) # type: ignore _LOGGER.debug("Response headers:") - for res_header, value in response.http_response.headers.items(): + for res_header, value in response.http_response.headers.items(): # type: ignore _LOGGER.debug(" %r: %r", res_header, value) # We don't want to log binary data if the response is a file. _LOGGER.debug("Response content:") pattern = re.compile(r'attachment; ?filename=["\w.]+', re.IGNORECASE) - header = response.http_response.headers.get('content-disposition') + header = response.http_response.headers.get('content-disposition') # type: ignore if header and pattern.match(header): filename = header.partition('=')[2] _LOGGER.debug("File attachments: %s", filename) - elif response.http_response.headers.get("content-type", "").endswith("octet-stream"): + elif response.http_response.headers.get("content-type", "").endswith("octet-stream"): # type: ignore _LOGGER.debug("Body contains binary data.") - elif response.http_response.headers.get("content-type", "").startswith("image"): + elif response.http_response.headers.get("content-type", "").startswith("image"): # type: ignore _LOGGER.debug("Body contains image data.") else: - if response.context.options.get('stream', False): + if response.context.options.get('stream', False): # type: ignore _LOGGER.debug("Body is streamable") else: - _LOGGER.debug(response.http_response.text()) + _LOGGER.debug(response.http_response.text()) # type: ignore except Exception as err: # pylint: disable=broad-except _LOGGER.debug("Failed to log response: %s", repr(err)) @@ -204,7 +206,7 @@ class ContentDecodePolicy(SansIOHTTPPolicy): @classmethod def deserialize_from_text(cls, response, content_type=None): - # type: (Optional[Union[AnyStr, IO]], Optional[str]) -> Any + # type: (Optional[PipelineResponse], Optional[str]) -> Any """Decode response data according to content-type. Accept a stream of data as well, but will be load at once in memory for now. If no content-type, will return the string version (not bytes, not stream) @@ -212,7 +214,7 @@ def deserialize_from_text(cls, response, content_type=None): :type response: ~azure.core.pipeline.transport.HttpResponse :param str content_type: The content type. """ - data = response.text() + data = response.text() # type: ignore if not data: return None @@ -260,7 +262,7 @@ def _json_attemp(data): @classmethod def deserialize_from_http_generics(cls, response): - # type: (Optional[Union[AnyStr, IO]], Mapping) -> Any + # type: (PipelineResponse) -> Any """Deserialize from HTTP response. Use bytes and headers to NOT use any requests/aiohttp or whatever specific implementation. @@ -268,8 +270,8 @@ def deserialize_from_http_generics(cls, response): """ # Try to use content-type from headers if available content_type = None - if response.content_type: - content_type = response.content_type[0].strip().lower() + if response.content_type: # type: ignore + content_type = response.content_type[0].strip().lower() # type: ignore # Ouch, this server did not declare what it sent... # Let's guess it's JSON... @@ -293,10 +295,10 @@ def on_response(self, request, response, **kwargs): :raises xml.etree.ElementTree.ParseError: If bytes is not valid XML """ # If response was asked as stream, do NOT read anything and quit now - if response.context.options.get("stream", True): + if response.context.options.get("stream", True): # type: ignore return - response.context[self.CONTEXT_NAME] = self.deserialize_from_http_generics(response.http_response) + response.context[self.CONTEXT_NAME] = self.deserialize_from_http_generics(response.http_response) # type: ignore class ProxyPolicy(SansIOHTTPPolicy): diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/aiohttp.py index 0ef7ab00bc9b..09017983d6eb 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/aiohttp.py @@ -23,7 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import Any, AsyncIterator as AsyncIteratorType +from typing import Any, Optional, AsyncIterator as AsyncIteratorType from collections.abc import AsyncIterator import logging @@ -102,7 +102,7 @@ def _get_request_data(self, request): #pylint: disable=no-self-use return form_data return request.data - async def send(self, request: HttpRequest, **config: Any) -> AsyncHttpResponse: + async def send(self, request: HttpRequest, **config: Any) -> Optional[Any]: """Send the request using this HTTP sender. Will pre-load the body into memory to be available with a sync method. diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/base.py b/sdk/core/azure-core/azure/core/pipeline/transport/base.py index 8f8a9981ab0b..63a7ecd8184e 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/base.py @@ -30,7 +30,7 @@ import os import time try: - from urlparse import urlparse + from urlparse import urlparse # type: ignore except ImportError: from urllib.parse import urlparse import xml.etree.ElementTree as ET @@ -43,7 +43,8 @@ # If one day we reach the point where "requests" can be skip totally, # might provide our own implementation from requests.structures import CaseInsensitiveDict -from azure.core.pipeline import ABC, AbstractContextManager +from azure.core.pipeline import ABC, AbstractContextManager, PipelineRequest, PipelineResponse + HTTPResponseType = TypeVar("HTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") @@ -51,13 +52,13 @@ _LOGGER = logging.getLogger(__name__) -class HttpTransport(AbstractContextManager, ABC, Generic[HTTPRequestType, HTTPResponseType]): +class HttpTransport(AbstractContextManager, ABC, Generic[HTTPRequestType, HTTPResponseType]): # type: ignore """An http sender ABC. """ @abc.abstractmethod def send(self, request, **kwargs): - # type: (PipelineRequest[HTTPRequestType], Any) -> PipelineResponse[HTTPRequestType, HTTPResponseType] + # type: (PipelineRequest, Any) -> PipelineResponse """Send the request using this HTTP sender. """ @@ -205,7 +206,7 @@ class _HttpResponseBase(object): Full in-memory using "body" as bytes. """ def __init__(self, request, internal_response, block_size=None): - # type: (HttpRequest, Any) -> None + # type: (HttpRequest, Any, Optional[int]) -> None self.request = request self.internal_response = internal_response self.status_code = None # type: Optional[int] diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py index b7644fc0ebe4..c2b0525a667c 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py @@ -27,8 +27,8 @@ from collections.abc import AsyncIterator import functools import logging -from typing import Any, AsyncIterator as AsyncIteratorType -import urllib3 +from typing import Any, Union, Optional, AsyncIterator as AsyncIteratorType +import urllib3 # type: ignore import requests @@ -73,14 +73,14 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AsyncHttpResponse: self.open() loop = kwargs.get("loop", _get_running_loop()) response = None - error = None + error = None # type: Optional[Union[ServiceRequestError, ServiceResponseError]] if self.config.proxy_policy and 'proxies' not in kwargs: kwargs['proxies'] = self.config.proxy_policy.proxies try: response = await loop.run_in_executor( None, functools.partial( - self.session.request, + self.session.request, # type: ignore request.method, request.url, headers=request.headers, @@ -149,8 +149,8 @@ async def __anext__(self): raise -class AsyncioRequestsTransportResponse(AsyncHttpResponse, RequestsTransportResponse): +class AsyncioRequestsTransportResponse(AsyncHttpResponse, RequestsTransportResponse): # type: ignore - def stream_download(self) -> AsyncIteratorType[bytes]: + def stream_download(self) -> AsyncIteratorType[bytes]: # type: ignore """Generator for streaming request body data.""" - return AsyncioStreamDownloadGenerator(self.internal_response, self.block_size) + return AsyncioStreamDownloadGenerator(self.internal_response, self.block_size) # type: ignore diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py b/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py index 711cbe4aa7cd..d9cc81bfb35b 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py @@ -25,9 +25,10 @@ # -------------------------------------------------------------------------- from __future__ import absolute_import import logging +from typing import Iterator, Optional, Any, Union +import urllib3 # type: ignore +from urllib3.util.retry import Retry # type: ignore import requests -import urllib3 -from urllib3.util.retry import Retry from azure.core.configuration import Configuration @@ -35,6 +36,7 @@ ServiceRequestError, ServiceResponseError ) +from . import HttpRequest # pylint: disable=unused-import from .base import ( HttpTransport, @@ -107,7 +109,7 @@ def __next__(self): class RequestsTransportResponse(HttpResponse, _RequestsTransportResponseBase): def stream_download(self): - # type: (Optional[int], Optional[Callable]) -> Iterator[bytes] + # type: () -> Iterator[bytes] """Generator for streaming request body data.""" return StreamDownloadGenerator(self.internal_response, self.block_size) @@ -127,7 +129,7 @@ class RequestsTransport(HttpTransport): _protocols = ['http://', 'https://'] def __init__(self, configuration=None, session=None, session_owner=True): - # type: (Optional[requests.Session]) -> None + # type: (Optional[Any], Optional[requests.Session], Any) -> None self._session_owner = session_owner self.config = configuration or Configuration() self.session = session @@ -164,7 +166,7 @@ def close(self): self._session_owner = False self.session = None - def send(self, request, **kwargs): + def send(self, request, **kwargs): # type: ignore # type: (HttpRequest, Any) -> HttpResponse """Send request object according to configuration. @@ -176,12 +178,12 @@ def send(self, request, **kwargs): """ self.open() response = None - error = None + error = None # type: Optional[Union[ServiceRequestError, ServiceResponseError]] if self.config.proxy_policy and 'proxies' not in kwargs: kwargs['proxies'] = self.config.proxy_policy.proxies try: - response = self.session.request( + response = self.session.request( # type: ignore request.method, request.url, headers=request.headers, diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/requests_trio.py index c3225a96bcb7..656775a2f441 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/requests_trio.py @@ -26,9 +26,9 @@ from collections.abc import AsyncIterator import functools import logging -from typing import Any, Callable, Optional, AsyncIterator as AsyncIteratorType -import trio -import urllib3 +from typing import Any, Callable, Union, Optional, AsyncIterator as AsyncIteratorType +import trio # type: ignore +import urllib3 # type: ignore import requests @@ -84,15 +84,15 @@ async def __anext__(self): self.response.close() raise -class TrioRequestsTransportResponse(AsyncHttpResponse, RequestsTransportResponse): +class TrioRequestsTransportResponse(AsyncHttpResponse, RequestsTransportResponse): # type: ignore - def stream_download(self) -> AsyncIteratorType[bytes]: + def stream_download(self) -> AsyncIteratorType[bytes]: # type: ignore """Generator for streaming request body data. :param callback: Custom callback for monitoring progress. :param int chunk_size: """ - return TrioStreamDownloadGenerator(self.internal_response, self.block_size) + return TrioStreamDownloadGenerator(self.internal_response, self.block_size) # type: ignore class TrioRequestsTransport(RequestsTransport, AsyncHttpTransport): # type: ignore @@ -112,13 +112,13 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AsyncHttpResponse: self.open() trio_limiter = kwargs.get("trio_limiter", None) response = None - error = None + error = None # type: Optional[Union[ServiceRequestError, ServiceResponseError]] if self.config.proxy_policy and 'proxies' not in kwargs: kwargs['proxies'] = self.config.proxy_policy.proxies try: response = await trio.run_sync_in_worker_thread( functools.partial( - self.session.request, + self.session.request, # type: ignore request.method, request.url, headers=request.headers, diff --git a/sdk/core/azure-core/azure/core/pipeline_client.py b/sdk/core/azure-core/azure/core/pipeline_client.py index 6eb24e1838fe..11870cfe381a 100644 --- a/sdk/core/azure-core/azure/core/pipeline_client.py +++ b/sdk/core/azure-core/azure/core/pipeline_client.py @@ -26,7 +26,7 @@ import logging try: - from urlparse import urljoin, urlparse + from urlparse import urljoin, urlparse # type: ignore except ImportError: from urllib.parse import urljoin, urlparse import xml.etree.ElementTree as ET @@ -141,8 +141,14 @@ def format_url(self, url_template, **kwargs): url = urljoin(base + '/', url) return url - def get(self, url, params=None, headers=None, content=None, form_content=None): - # type: (str, Optional[Dict[str, str]], Optional[Dict[str, str]], Any, Optional[Dict[str, Any]]) -> HttpRequest + def get( + self, url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None # type: Optional[Dict[str, Any]] + ): + # type: (...) -> HttpRequest """Create a GET request object. :param str url: The request URL. @@ -154,8 +160,15 @@ def get(self, url, params=None, headers=None, content=None, form_content=None): request.method = 'GET' return request - def put(self, url, params=None, headers=None, content=None, form_content=None, stream_content=None): - # type: (str, Optional[Dict[str, str]], Optional[Dict[str, str]], Any, Optional[Dict[str, Any]]) -> HttpRequest + def put( + self, url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None # type: Any + ): + # type: (...) -> HttpRequest """Create a PUT request object. :param str url: The request URL. @@ -166,8 +179,15 @@ def put(self, url, params=None, headers=None, content=None, form_content=None, s request = self._request('PUT', url, params, headers, content, form_content, stream_content) return request - def post(self, url, params=None, headers=None, content=None, form_content=None, stream_content=None): - # type: (str, Optional[Dict[str, str]], Optional[Dict[str, str]], Any, Optional[Dict[str, Any]]) -> HttpRequest + def post( + self, url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None # type: Any + ): + # type: (...) -> HttpRequest """Create a POST request object. :param str url: The request URL. @@ -178,8 +198,15 @@ def post(self, url, params=None, headers=None, content=None, form_content=None, request = self._request('POST', url, params, headers, content, form_content, stream_content) return request - def head(self, url, params=None, headers=None, content=None, form_content=None, stream_content=None): - # type: (str, Optional[Dict[str, str]], Optional[Dict[str, str]], Any, Optional[Dict[str, Any]]) -> HttpRequest + def head( + self, url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None # type: Any + ): + # type: (...) -> HttpRequest """Create a HEAD request object. :param str url: The request URL. @@ -190,8 +217,15 @@ def head(self, url, params=None, headers=None, content=None, form_content=None, request = self._request('HEAD', url, params, headers, content, form_content, stream_content) return request - def patch(self, url, params=None, headers=None, content=None, form_content=None, stream_content=None): - # type: (str, Optional[Dict[str, str]], Optional[Dict[str, str]], Any, Optional[Dict[str, Any]]) -> HttpRequest + def patch( + self, url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None # type: Any + ): + # type: (...) -> HttpRequest """Create a PATCH request object. :param str url: The request URL. diff --git a/sdk/core/azure-core/azure/core/pipeline_client_async.py b/sdk/core/azure-core/azure/core/pipeline_client_async.py index 00772beafd55..8755b0191282 100644 --- a/sdk/core/azure-core/azure/core/pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/pipeline_client_async.py @@ -26,7 +26,7 @@ import logging try: - from urlparse import urljoin, urlparse + from urlparse import urljoin, urlparse # type: ignore except ImportError: from urllib.parse import urljoin, urlparse import xml.etree.ElementTree as ET @@ -127,8 +127,14 @@ def format_url(self, url_template, **kwargs): url = urljoin(base + '/', url) return url - def get(self, url, params=None, headers=None, content=None, form_content=None): - # type: (str, Optional[Dict[str, str]], Optional[Dict[str, str]], Any, Optional[Dict[str, Any]]) -> HttpRequest + def get( + self, url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + ): + # type: (...) -> HttpRequest """Create a GET request object. :param str url: The request URL. @@ -140,8 +146,15 @@ def get(self, url, params=None, headers=None, content=None, form_content=None): request.method = 'GET' return request - def put(self, url, params=None, headers=None, content=None, form_content=None, stream_content=None): - # type: (str, Optional[Dict[str, str]], Optional[Dict[str, str]], Any, Optional[Dict[str, Any]]) -> HttpRequest + def put( + self, url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None # type: Any + ): + # type: (...) -> HttpRequest """Create a PUT request object. :param str url: The request URL. @@ -152,8 +165,15 @@ def put(self, url, params=None, headers=None, content=None, form_content=None, s request = self._request('PUT', url, params, headers, content, form_content, stream_content) return request - def post(self, url, params=None, headers=None, content=None, form_content=None, stream_content=None): - # type: (str, Optional[Dict[str, str]], Optional[Dict[str, str]], Any, Optional[Dict[str, Any]]) -> HttpRequest + def post( + self, url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None # type: Any + ): + # type: (...) -> HttpRequest """Create a POST request object. :param str url: The request URL. @@ -164,8 +184,15 @@ def post(self, url, params=None, headers=None, content=None, form_content=None, request = self._request('POST', url, params, headers, content, form_content, stream_content) return request - def head(self, url, params=None, headers=None, content=None, form_content=None, stream_content=None): - # type: (str, Optional[Dict[str, str]], Optional[Dict[str, str]], Any, Optional[Dict[str, Any]]) -> HttpRequest + def head( + self, url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None # type: Any + ): + # type: (...) -> HttpRequest """Create a HEAD request object. :param str url: The request URL. @@ -176,8 +203,15 @@ def head(self, url, params=None, headers=None, content=None, form_content=None, request = self._request('HEAD', url, params, headers, content, form_content, stream_content) return request - def patch(self, url, params=None, headers=None, content=None, form_content=None, stream_content=None): - # type: (str, Optional[Dict[str, str]], Optional[Dict[str, str]], Any, Optional[Dict[str, Any]]) -> HttpRequest + def patch( + self, url, # type: str + params=None, # type: Optional[Dict[str, str]] + headers=None, # type: Optional[Dict[str, str]] + content=None, # type: Any + form_content=None, # type: Optional[Dict[str, Any]] + stream_content=None # type: Any + ): + # type: (...) -> HttpRequest """Create a PATCH request object. :param str url: The request URL. diff --git a/sdk/core/azure-core/azure/core/polling/poller.py b/sdk/core/azure-core/azure/core/polling/poller.py index bca868ca3723..15f390415135 100644 --- a/sdk/core/azure-core/azure/core/polling/poller.py +++ b/sdk/core/azure-core/azure/core/polling/poller.py @@ -26,7 +26,7 @@ import threading import uuid try: - from urlparse import urlparse # pylint: disable=unused-import + from urlparse import urlparse # type: ignore # pylint: disable=unused-import except ImportError: from urllib.parse import urlparse @@ -34,7 +34,8 @@ if TYPE_CHECKING: import requests - from msrest.serialization import Model # pylint: disable=unused-import + from azure.core.pipeline.transport.base import HttpResponse # type: ignore + from msrest.serialization import Model # type: ignore # pylint: disable=unused-import class PollingMethod(object): """ABC class for polling method. @@ -117,7 +118,7 @@ def __init__(self, client, initial_response, deserialization_callback, polling_m # This implicit test avoids bringing in an explicit dependency on Model directly try: - deserialization_callback = deserialization_callback.deserialize + deserialization_callback = deserialization_callback.deserialize # type: ignore except AttributeError: pass diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index 7962026b5974..d018223b285f 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -1,4 +1,5 @@ trio; python_version >= '3.5' aiohttp>=3.0; python_version >= '3.5' aiodns>=2.0; python_version >= '3.5' -typing_extensions>=3.7.2 \ No newline at end of file +typing_extensions>=3.7.2 +mypy>=0.7; python_version >= '3.5' \ No newline at end of file diff --git a/sdk/core/azure-core/setup.py b/sdk/core/azure-core/setup.py index 2647ae940d6b..ede4e01a2966 100644 --- a/sdk/core/azure-core/setup.py +++ b/sdk/core/azure-core/setup.py @@ -9,7 +9,7 @@ import re import os.path from io import open -from setuptools import find_packages, setup +from setuptools import find_packages, setup # type: ignore # Change the PACKAGE_NAME only to change folder and different name PACKAGE_NAME = "azure-core" @@ -25,7 +25,7 @@ try: import azure try: - ver = azure.__version__ + ver = azure.__version__ # type: ignore raise Exception( 'This package is incompatible with azure=={}. '.format(ver) + 'Uninstall it with "pip uninstall azure".' @@ -37,7 +37,7 @@ # Version extraction inspired from 'requests' with open(os.path.join(package_folder_path, 'version.py'), 'r') as fd: - version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', + version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', # type: ignore fd.read(), re.MULTILINE).group(1) if not version: From d8a4d34fcd5efa84c7e6333e18b86d65351c7dce Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 24 May 2019 10:34:37 -0700 Subject: [PATCH 2/8] use common type aliases --- sdk/core/azure-core/azure/core/common.py | 41 +++++++++++++++++++ .../azure/core/pipeline/base_async.py | 5 ++- .../azure-core/azure/core/polling/poller.py | 5 ++- 3 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 sdk/core/azure-core/azure/core/common.py diff --git a/sdk/core/azure-core/azure/core/common.py b/sdk/core/azure-core/azure/core/common.py new file mode 100644 index 000000000000..03b735451127 --- /dev/null +++ b/sdk/core/azure-core/azure/core/common.py @@ -0,0 +1,41 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from typing import List, Union, Optional, TypeVar, Callable +from azure.core.pipeline.policies import AsyncHTTPPolicy, SansIOHTTPPolicy +from azure.core.exceptions import ( + ServiceRequestError, + ServiceResponseError +) +import requests +from msrest.serialization import Model # type: ignore # pylint: disable=unused-import + +AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType") +HTTPRequestType = TypeVar("HTTPRequestType") + +ErrorType = Optional[Union[ServiceRequestError, ServiceResponseError]] +ImplPoliciesType = List[AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType]] #pylint: disable=unsubscriptable-object +PoliciesType = List[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] +DeserializationCallbackType = Union[Model, Callable[[requests.Response], Model]] diff --git a/sdk/core/azure-core/azure/core/pipeline/base_async.py b/sdk/core/azure-core/azure/core/pipeline/base_async.py index dd490198877d..d30d945f26fa 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/base_async.py @@ -29,6 +29,7 @@ from azure.core.pipeline import PipelineRequest, PipelineResponse, PipelineContext from azure.core.pipeline.policies import AsyncHTTPPolicy, SansIOHTTPPolicy +from azure.core.common import ImplPoliciesType, PoliciesType AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") @@ -88,8 +89,8 @@ class AsyncPipeline(AbstractAsyncContextManager, Generic[HTTPRequestType, AsyncH of the HTTP sender. """ - def __init__(self, transport, policies: List[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] = None) -> None: - self._impl_policies = [] # type: List[AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType]] + def __init__(self, transport, policies: PoliciesType = None) -> None: + self._impl_policies = [] # type: ImplPoliciesType self._transport = transport for policy in (policies or []): diff --git a/sdk/core/azure-core/azure/core/polling/poller.py b/sdk/core/azure-core/azure/core/polling/poller.py index 15f390415135..99d82c2eae49 100644 --- a/sdk/core/azure-core/azure/core/polling/poller.py +++ b/sdk/core/azure-core/azure/core/polling/poller.py @@ -31,10 +31,11 @@ from urllib.parse import urlparse from typing import Any, Callable, Union, List, Optional, TYPE_CHECKING +from azure.core.common import DeserializationCallbackType +from azure.core.pipeline.transport.base import HttpResponse # type: ignore if TYPE_CHECKING: import requests - from azure.core.pipeline.transport.base import HttpResponse # type: ignore from msrest.serialization import Model # type: ignore # pylint: disable=unused-import class PollingMethod(object): @@ -110,7 +111,7 @@ class LROPoller(object): """ def __init__(self, client, initial_response, deserialization_callback, polling_method): - # type: (Any, HttpResponse, Union[Model, Callable[[requests.Response], Model]], PollingMethod) -> None + # type: (Any, HttpResponse, DeserializationCallbackType, PollingMethod) -> None self._client = client self._response = initial_response self._callbacks = [] # type: List[Callable] From ddee45b3d0b2c02d49f737a7619baf6e748c47bf Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 24 May 2019 15:07:42 -0700 Subject: [PATCH 3/8] comments address --- sdk/core/azure-core/azure/core/common.py | 11 +---------- sdk/core/azure-core/azure/core/pipeline/base.py | 6 ++++-- sdk/core/azure-core/azure/core/pipeline/base_async.py | 8 ++++---- .../azure/core/pipeline/policies/retry_async.py | 1 - .../azure/core/pipeline/policies/universal.py | 7 ++++--- .../azure/core/pipeline/transport/requests_basic.py | 2 +- sdk/core/azure-core/azure/core/polling/poller.py | 5 ++++- 7 files changed, 18 insertions(+), 22 deletions(-) diff --git a/sdk/core/azure-core/azure/core/common.py b/sdk/core/azure-core/azure/core/common.py index 03b735451127..e3b990ff98d2 100644 --- a/sdk/core/azure-core/azure/core/common.py +++ b/sdk/core/azure-core/azure/core/common.py @@ -23,19 +23,10 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import List, Union, Optional, TypeVar, Callable -from azure.core.pipeline.policies import AsyncHTTPPolicy, SansIOHTTPPolicy +from typing import Union, Optional from azure.core.exceptions import ( ServiceRequestError, ServiceResponseError ) -import requests -from msrest.serialization import Model # type: ignore # pylint: disable=unused-import - -AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType") -HTTPRequestType = TypeVar("HTTPRequestType") ErrorType = Optional[Union[ServiceRequestError, ServiceResponseError]] -ImplPoliciesType = List[AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType]] #pylint: disable=unsubscriptable-object -PoliciesType = List[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] -DeserializationCallbackType = Union[Model, Callable[[requests.Response], Model]] diff --git a/sdk/core/azure-core/azure/core/pipeline/base.py b/sdk/core/azure-core/azure/core/pipeline/base.py index 6720e076dde3..5f8d997ac430 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/base.py @@ -29,10 +29,12 @@ Tuple, Callable, Iterator) from azure.core.pipeline import AbstractContextManager, PipelineRequest, PipelineResponse, PipelineContext from azure.core.pipeline.policies import HTTPPolicy, SansIOHTTPPolicy +from azure.core.pipeline.transport import HttpTransport HTTPResponseType = TypeVar("HTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") _LOGGER = logging.getLogger(__name__) +PoliciesType = List[Union[HTTPPolicy, SansIOHTTPPolicy]] class _SansIOHTTPPolicyRunner(HTTPPolicy, Generic[HTTPRequestType, HTTPResponseType]): @@ -60,7 +62,7 @@ def send(self, request): class _TransportRunner(HTTPPolicy): def __init__(self, sender): - # type: (Any) -> None + # type: (HttpTransport) -> None super(_TransportRunner, self).__init__() self._sender = sender @@ -80,7 +82,7 @@ class Pipeline(AbstractContextManager, Generic[HTTPRequestType, HTTPResponseType """ def __init__(self, transport, policies=None): - # type: (Any, List[Union[HTTPPolicy, SansIOHTTPPolicy]]) -> None + # type: (HttpTransport, PoliciesType) -> None self._impl_policies = [] # type: List[HTTPPolicy] self._transport = transport # type: HTTPPolicy diff --git a/sdk/core/azure-core/azure/core/pipeline/base_async.py b/sdk/core/azure-core/azure/core/pipeline/base_async.py index d30d945f26fa..e7f397aa10e0 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/base_async.py @@ -25,15 +25,15 @@ # -------------------------------------------------------------------------- import abc -from typing import Any, List, Union, Generic, TypeVar +from typing import Any, Union, List, Generic, TypeVar from azure.core.pipeline import PipelineRequest, PipelineResponse, PipelineContext from azure.core.pipeline.policies import AsyncHTTPPolicy, SansIOHTTPPolicy -from azure.core.common import ImplPoliciesType, PoliciesType AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") - +ImplPoliciesType = List[AsyncHTTPPolicy[HTTPRequestType, AsyncHTTPResponseType]] #pylint: disable=unsubscriptable-object +AsyncPoliciesType = List[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] try: from contextlib import AbstractAsyncContextManager # type: ignore @@ -89,7 +89,7 @@ class AsyncPipeline(AbstractAsyncContextManager, Generic[HTTPRequestType, AsyncH of the HTTP sender. """ - def __init__(self, transport, policies: PoliciesType = None) -> None: + def __init__(self, transport, policies: AsyncPoliciesType = None) -> None: self._impl_policies = [] # type: ImplPoliciesType self._transport = transport diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/retry_async.py b/sdk/core/azure-core/azure/core/pipeline/policies/retry_async.py index 673069e6bcb8..4ba2145adeb0 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/retry_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/retry_async.py @@ -92,4 +92,3 @@ async def send(self, request): self.update_context(response.context, retry_settings) return response - diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/universal.py b/sdk/core/azure-core/azure/core/pipeline/policies/universal.py index 851ebe002382..81c622a8a869 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/universal.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/universal.py @@ -34,7 +34,7 @@ import xml.etree.ElementTree as ET import types import re -from typing import (Mapping, IO, TypeVar, TYPE_CHECKING, cast, List, Callable, Iterator, # pylint: disable=unused-import +from typing import (Mapping, IO, TypeVar, TYPE_CHECKING, Type, cast, List, Callable, Iterator, # pylint: disable=unused-import Any, Union, Dict, Optional) from azure.core import __version__ as azcore_version @@ -48,6 +48,7 @@ _LOGGER = logging.getLogger(__name__) +ContentDecodePolicyType = TypeVar('ContentDecodePolicyType', bound='ContentDecodePolicy') class HeadersPolicy(SansIOHTTPPolicy): @@ -206,7 +207,7 @@ class ContentDecodePolicy(SansIOHTTPPolicy): @classmethod def deserialize_from_text(cls, response, content_type=None): - # type: (Optional[PipelineResponse], Optional[str]) -> Any + # type: (Type[ContentDecodePolicyType], PipelineResponse, Optional[str]) -> Any """Decode response data according to content-type. Accept a stream of data as well, but will be load at once in memory for now. If no content-type, will return the string version (not bytes, not stream) @@ -262,7 +263,7 @@ def _json_attemp(data): @classmethod def deserialize_from_http_generics(cls, response): - # type: (PipelineResponse) -> Any + # type: (Type[ContentDecodePolicyType], PipelineResponse) -> Any """Deserialize from HTTP response. Use bytes and headers to NOT use any requests/aiohttp or whatever specific implementation. diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py b/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py index d9cc81bfb35b..7531ced6b079 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py @@ -129,7 +129,7 @@ class RequestsTransport(HttpTransport): _protocols = ['http://', 'https://'] def __init__(self, configuration=None, session=None, session_owner=True): - # type: (Optional[Any], Optional[requests.Session], Any) -> None + # type: (Optional[Any], Optional[requests.Session], bool) -> None self._session_owner = session_owner self.config = configuration or Configuration() self.session = session diff --git a/sdk/core/azure-core/azure/core/polling/poller.py b/sdk/core/azure-core/azure/core/polling/poller.py index 99d82c2eae49..183a5a558a71 100644 --- a/sdk/core/azure-core/azure/core/polling/poller.py +++ b/sdk/core/azure-core/azure/core/polling/poller.py @@ -31,13 +31,16 @@ from urllib.parse import urlparse from typing import Any, Callable, Union, List, Optional, TYPE_CHECKING -from azure.core.common import DeserializationCallbackType from azure.core.pipeline.transport.base import HttpResponse # type: ignore if TYPE_CHECKING: import requests from msrest.serialization import Model # type: ignore # pylint: disable=unused-import + +DeserializationCallbackType = Union[Model, Callable[[requests.Response], Model]] + + class PollingMethod(object): """ABC class for polling method. """ From d454c4b86b5ceef22db7d590f8afc2f15adda6c6 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 24 May 2019 15:40:38 -0700 Subject: [PATCH 4/8] changes --- sdk/core/azure-core/azure/core/polling/poller.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/poller.py b/sdk/core/azure-core/azure/core/polling/poller.py index 183a5a558a71..91dc9747c430 100644 --- a/sdk/core/azure-core/azure/core/polling/poller.py +++ b/sdk/core/azure-core/azure/core/polling/poller.py @@ -36,9 +36,7 @@ if TYPE_CHECKING: import requests from msrest.serialization import Model # type: ignore # pylint: disable=unused-import - - -DeserializationCallbackType = Union[Model, Callable[[requests.Response], Model]] + DeserializationCallbackType = Union[Model, Callable[[requests.Response], Model]] class PollingMethod(object): From 16d66d0f1b28f61ca65ffcdd77ed7e73915c4cfa Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 24 May 2019 16:34:13 -0700 Subject: [PATCH 5/8] upgrade python for mypy --- sdk/core/azure-core/dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index d018223b285f..77df372c00dc 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -2,4 +2,4 @@ trio; python_version >= '3.5' aiohttp>=3.0; python_version >= '3.5' aiodns>=2.0; python_version >= '3.5' typing_extensions>=3.7.2 -mypy>=0.7; python_version >= '3.5' \ No newline at end of file +mypy>=0.7; python_version >= '3.6' \ No newline at end of file From b8fe814931f76191b362e67b08c5c6f7ba98d2db Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 24 May 2019 16:53:12 -0700 Subject: [PATCH 6/8] fix --- sdk/core/azure-core/azure/core/pipeline/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/base.py b/sdk/core/azure-core/azure/core/pipeline/base.py index 5f8d997ac430..3ee5ed5d53c1 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/base.py @@ -29,9 +29,9 @@ Tuple, Callable, Iterator) from azure.core.pipeline import AbstractContextManager, PipelineRequest, PipelineResponse, PipelineContext from azure.core.pipeline.policies import HTTPPolicy, SansIOHTTPPolicy -from azure.core.pipeline.transport import HttpTransport HTTPResponseType = TypeVar("HTTPResponseType") HTTPRequestType = TypeVar("HTTPRequestType") +HttpTransportType = TypeVar("HttpTransportType") _LOGGER = logging.getLogger(__name__) PoliciesType = List[Union[HTTPPolicy, SansIOHTTPPolicy]] @@ -62,7 +62,7 @@ def send(self, request): class _TransportRunner(HTTPPolicy): def __init__(self, sender): - # type: (HttpTransport) -> None + # type: (HttpTransportType) -> None super(_TransportRunner, self).__init__() self._sender = sender @@ -82,7 +82,7 @@ class Pipeline(AbstractContextManager, Generic[HTTPRequestType, HTTPResponseType """ def __init__(self, transport, policies=None): - # type: (HttpTransport, PoliciesType) -> None + # type: (HttpTransportType, PoliciesType) -> None self._impl_policies = [] # type: List[HTTPPolicy] self._transport = transport # type: HTTPPolicy From 4746204ac9984fc2ce490a5566890890ce796985 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 24 May 2019 17:03:51 -0700 Subject: [PATCH 7/8] small fix --- sdk/core/azure-core/azure/core/pipeline/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/base.py b/sdk/core/azure-core/azure/core/pipeline/base.py index 3ee5ed5d53c1..87b56f51abac 100644 --- a/sdk/core/azure-core/azure/core/pipeline/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/base.py @@ -84,7 +84,7 @@ class Pipeline(AbstractContextManager, Generic[HTTPRequestType, HTTPResponseType def __init__(self, transport, policies=None): # type: (HttpTransportType, PoliciesType) -> None self._impl_policies = [] # type: List[HTTPPolicy] - self._transport = transport # type: HTTPPolicy + self._transport = transport # type: ignore for policy in (policies or []): if isinstance(policy, SansIOHTTPPolicy): @@ -98,7 +98,7 @@ def __init__(self, transport, policies=None): def __enter__(self): # type: () -> Pipeline - self._transport.__enter__() + self._transport.__enter__() # type: ignore return self def __exit__(self, *exc_details): # pylint: disable=arguments-differ From 02761e0654e99ba9162ee023ecfdc8d7a4dcec78 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 24 May 2019 18:04:16 -0700 Subject: [PATCH 8/8] comments --- sdk/core/azure-core/azure/core/pipeline/policies/base.py | 2 +- sdk/core/azure-core/azure/core/pipeline/transport/aiohttp.py | 2 +- .../azure-core/azure/core/pipeline/transport/requests_basic.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/base.py b/sdk/core/azure-core/azure/core/pipeline/policies/base.py index 4f281c47fe61..d3893940a442 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/base.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/base.py @@ -98,7 +98,7 @@ class RequestHistory(object): """ def __init__(self, http_request, http_response=None, error=None, context=None): - # type: (PipelineRequest, PipelineResponse, Exception, Optional[Dict[str, Any]]) -> None + # type: (PipelineRequest, Optional[PipelineResponse], Exception, Optional[Dict[str, Any]]) -> None self.http_request = copy.deepcopy(http_request) self.http_response = http_response self.error = error diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/aiohttp.py index 09017983d6eb..f3468e24a989 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/aiohttp.py @@ -102,7 +102,7 @@ def _get_request_data(self, request): #pylint: disable=no-self-use return form_data return request.data - async def send(self, request: HttpRequest, **config: Any) -> Optional[Any]: + async def send(self, request: HttpRequest, **config: Any) -> Optional[AsyncHttpResponse]: """Send the request using this HTTP sender. Will pre-load the body into memory to be available with a sync method. diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py b/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py index 7531ced6b079..7881f7d5188c 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py @@ -129,7 +129,7 @@ class RequestsTransport(HttpTransport): _protocols = ['http://', 'https://'] def __init__(self, configuration=None, session=None, session_owner=True): - # type: (Optional[Any], Optional[requests.Session], bool) -> None + # type: (Optional[Configuration], Optional[requests.Session], bool) -> None self._session_owner = session_owner self.config = configuration or Configuration() self.session = session