From 5192810ff6a41e62e41d16fcf636663d177a1232 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 16 Mar 2017 16:50:41 +1300 Subject: [PATCH] Make mypy succeed with imports on master.py We get little benefit from our mypy QA checks at the moment, because we skip imports. This patch is what's needed to make mypy succeed with imports on a single file: master.py It also updates mypy to the current version, and enables a QA check. Mypy bugs I encountered: dict.update with kwargs not supported: https://github.com/python/mypy/issues/1031 property setters and getters must be adjacent: https://github.com/python/mypy/issues/1465 --- mitmproxy/ctx.py | 2 + mitmproxy/http.py | 12 ++--- mitmproxy/net/http/encoding.py | 12 +++-- mitmproxy/net/http/message.py | 7 ++- mitmproxy/net/http/request.py | 82 ++++++++++++++++---------------- mitmproxy/net/http/response.py | 26 +++++----- mitmproxy/net/http/url.py | 2 +- mitmproxy/options.py | 4 +- mitmproxy/proxy/config.py | 10 ++-- mitmproxy/proxy/protocol/base.py | 14 +++--- mitmproxy/proxy/server.py | 10 ++-- mitmproxy/stateobject.py | 3 +- mitmproxy/tcp.py | 4 +- mitmproxy/utils/strutils.py | 6 +-- mitmproxy/utils/typecheck.py | 12 ++--- mitmproxy/websocket.py | 11 +++-- setup.py | 2 +- tox.ini | 2 + 18 files changed, 116 insertions(+), 105 deletions(-) diff --git a/mitmproxy/ctx.py b/mitmproxy/ctx.py index adae8cf68c..7b5231e6ff 100644 --- a/mitmproxy/ctx.py +++ b/mitmproxy/ctx.py @@ -1,2 +1,4 @@ +import mitmproxy.master # noqa +import mitmproxy.log # noqa master = None # type: "mitmproxy.master.Master" log = None # type: "mitmproxy.log.Log" diff --git a/mitmproxy/http.py b/mitmproxy/http.py index c6b1753347..c09778fea7 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -52,9 +52,7 @@ def __init__( def get_state(self): state = super().get_state() - state.update( - is_replay=self.is_replay - ) + state["is_replay"] = self.is_replay return state def set_state(self, state): @@ -167,11 +165,12 @@ def __init__(self, client_conn, server_conn, live=None, mode="regular"): """ What mode was the proxy layer in when receiving this request? """ _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes.update( + # mypy doesn't support update with kwargs + _stateobject_attributes.update(dict( request=HTTPRequest, response=HTTPResponse, mode=str - ) + )) def __repr__(self): s = " Union[str, bytes]: +def decode( + encoded: Optional[bytes], encoding: str, errors: str='strict' +) -> Optional[AnyStr]: """ Decode the given input object @@ -62,7 +66,7 @@ def decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> U )) -def encode(decoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]: +def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optional[AnyStr]: """ Encode the given input object diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 506674d630..1040c6ce17 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -1,5 +1,5 @@ import re -from typing import Optional +from typing import Optional, Union # noqa from mitmproxy.utils import strutils from mitmproxy.net.http import encoding @@ -8,6 +8,8 @@ class MessageData(serializable.Serializable): + content = None # type: bytes + def __eq__(self, other): if isinstance(other, MessageData): return self.__dict__ == other.__dict__ @@ -31,6 +33,8 @@ def from_state(cls, state): class Message(serializable.Serializable): + data = None # type: MessageData + def __eq__(self, other): if isinstance(other, Message): return self.data == other.data @@ -159,6 +163,7 @@ def _get_content_type_charset(self) -> Optional[str]: ct = headers.parse_content_type(self.headers.get("content-type", "")) if ct: return ct[2].get("charset") + return None def _guess_encoding(self) -> str: enc = self._get_content_type_charset() diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 90a1f924c3..6f366a4fd4 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -82,8 +82,8 @@ def make( cls, method: str, url: str, - content: AnyStr = b"", - headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]] = () + content: Union[bytes, str] = "", + headers: Union[Dict[str, AnyStr], Iterable[Tuple[bytes, bytes]]] = () ): """ Simplified API for creating request objects. @@ -327,6 +327,15 @@ def pretty_url(self): return "%s:%d" % (self.pretty_host, self.port) return mitmproxy.net.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path) + def _get_query(self): + query = urllib.parse.urlparse(self.url).query + return tuple(mitmproxy.net.http.url.decode(query)) + + def _set_query(self, query_data): + query = mitmproxy.net.http.url.encode(query_data) + _, _, path, params, _, fragment = urllib.parse.urlparse(self.url) + self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment]) + @property def query(self) -> multidict.MultiDictView: """ @@ -337,19 +346,17 @@ def query(self) -> multidict.MultiDictView: self._set_query ) - def _get_query(self): - query = urllib.parse.urlparse(self.url).query - return tuple(mitmproxy.net.http.url.decode(query)) - - def _set_query(self, query_data): - query = mitmproxy.net.http.url.encode(query_data) - _, _, path, params, _, fragment = urllib.parse.urlparse(self.url) - self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment]) - @query.setter def query(self, value): self._set_query(value) + def _get_cookies(self): + h = self.headers.get_all("Cookie") + return tuple(cookies.parse_cookie_headers(h)) + + def _set_cookies(self, value): + self.headers["cookie"] = cookies.format_cookie_header(value) + @property def cookies(self) -> multidict.MultiDictView: """ @@ -362,13 +369,6 @@ def cookies(self) -> multidict.MultiDictView: self._set_cookies ) - def _get_cookies(self): - h = self.headers.get_all("Cookie") - return tuple(cookies.parse_cookie_headers(h)) - - def _set_cookies(self, value): - self.headers["cookie"] = cookies.format_cookie_header(value) - @cookies.setter def cookies(self, value): self._set_cookies(value) @@ -426,20 +426,6 @@ def constrain_encoding(self): ) ) - @property - def urlencoded_form(self): - """ - The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. - An empty multidict.MultiDictView if the content-type indicates non-form data - or the content could not be parsed. - - Starting with mitmproxy 1.0, key and value are strings. - """ - return multidict.MultiDictView( - self._get_urlencoded_form, - self._set_urlencoded_form - ) - def _get_urlencoded_form(self): is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() if is_valid_content_type: @@ -457,24 +443,24 @@ def _set_urlencoded_form(self, form_data): self.headers["content-type"] = "application/x-www-form-urlencoded" self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode() - @urlencoded_form.setter - def urlencoded_form(self, value): - self._set_urlencoded_form(value) - @property - def multipart_form(self): + def urlencoded_form(self): """ - The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. + The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. An empty multidict.MultiDictView if the content-type indicates non-form data or the content could not be parsed. - Key and value are bytes. + Starting with mitmproxy 1.0, key and value are strings. """ return multidict.MultiDictView( - self._get_multipart_form, - self._set_multipart_form + self._get_urlencoded_form, + self._set_urlencoded_form ) + @urlencoded_form.setter + def urlencoded_form(self, value): + self._set_urlencoded_form(value) + def _get_multipart_form(self): is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower() if is_valid_content_type: @@ -487,6 +473,20 @@ def _get_multipart_form(self): def _set_multipart_form(self, value): raise NotImplementedError() + @property + def multipart_form(self): + """ + The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. + An empty multidict.MultiDictView if the content-type indicates non-form data + or the content could not be parsed. + + Key and value are bytes. + """ + return multidict.MultiDictView( + self._get_multipart_form, + self._set_multipart_form + ) + @multipart_form.setter def multipart_form(self, value): self._set_multipart_form(value) diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index 53c9c1ca6a..8edd43b8bb 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -69,8 +69,8 @@ def __repr__(self): def make( cls, status_code: int=200, - content: AnyStr=b"", - headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]]=() + content: Union[bytes, str]=b"", + headers: Union[Dict[str, AnyStr], Iterable[Tuple[bytes, bytes]]]=() ): """ Simplified API for creating response objects. @@ -129,6 +129,17 @@ def reason(self): def reason(self, reason): self.data.reason = strutils.always_bytes(reason, "ISO-8859-1", "surrogateescape") + def _get_cookies(self): + h = self.headers.get_all("set-cookie") + return tuple(cookies.parse_set_cookie_headers(h)) + + def _set_cookies(self, value): + cookie_headers = [] + for k, v in value: + header = cookies.format_set_cookie_header([(k, v[0], v[1])]) + cookie_headers.append(header) + self.headers.set_all("set-cookie", cookie_headers) + @property def cookies(self) -> multidict.MultiDictView: """ @@ -146,17 +157,6 @@ def cookies(self) -> multidict.MultiDictView: self._set_cookies ) - def _get_cookies(self): - h = self.headers.get_all("set-cookie") - return tuple(cookies.parse_set_cookie_headers(h)) - - def _set_cookies(self, value): - cookie_headers = [] - for k, v in value: - header = cookies.format_set_cookie_header([(k, v[0], v[1])]) - cookie_headers.append(header) - self.headers.set_all("set-cookie", cookie_headers) - @cookies.setter def cookies(self, value): self._set_cookies(value) diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index 86ce976425..f2c8c4736c 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -1,4 +1,4 @@ -import urllib +import urllib.parse from typing import Sequence from typing import Tuple diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 5b84ac9302..703928032b 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -332,7 +332,7 @@ def __init__(self, **kwargs) -> None: Set supported SSL/TLS versions for client connections. SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. """, - choices=tcp.sslversion_choices.keys(), + choices=list(tcp.sslversion_choices.keys()), ) self.add_option( "ssl_version_server", str, "secure", @@ -340,7 +340,7 @@ def __init__(self, **kwargs) -> None: Set supported SSL/TLS versions for server connections. SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. """, - choices=tcp.sslversion_choices.keys(), + choices=list(tcp.sslversion_choices.keys()), ) self.add_option( "ssl_insecure", bool, False, diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 8417ebad7b..b809d89a56 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -37,11 +37,11 @@ class ProxyConfig: def __init__(self, options: moptions.Options) -> None: self.options = options - self.check_ignore = None - self.check_tcp = None - self.certstore = None - self.client_certs = None - self.openssl_verification_mode_server = None + self.check_ignore = None # type: HostMatcher + self.check_tcp = None # type: HostMatcher + self.certstore = None # type: certs.CertStore + self.client_certs = None # type: str + self.openssl_verification_mode_server = None # type: int self.configure(options, set(options.keys())) options.changed.connect(self.configure) diff --git a/mitmproxy/proxy/protocol/base.py b/mitmproxy/proxy/protocol/base.py index b10bb8f585..7c0f78ae5a 100644 --- a/mitmproxy/proxy/protocol/base.py +++ b/mitmproxy/proxy/protocol/base.py @@ -1,5 +1,7 @@ from mitmproxy import exceptions from mitmproxy import connections +from mitmproxy import controller # noqa +from mitmproxy.proxy import config # noqa class _LayerCodeCompletion: @@ -12,14 +14,10 @@ def __init__(self, **mixin_args): # pragma: no cover super().__init__(**mixin_args) if True: return - self.config = None - """@type: mitmproxy.proxy.ProxyConfig""" - self.client_conn = None - """@type: mitmproxy.connections.ClientConnection""" - self.server_conn = None - """@type: mitmproxy.connections.ServerConnection""" - self.channel = None - """@type: mitmproxy.controller.Channel""" + self.config = None # type: config.ProxyConfig + self.client_conn = None # type: connections.ClientConnection + self.server_conn = None # type: connections.ServerConnection + self.channel = None # type: controller.Channel self.ctx = None """@type: mitmproxy.proxy.protocol.Layer""" diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 166922340a..9f783bc323 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -3,10 +3,11 @@ from mitmproxy import exceptions from mitmproxy import connections +from mitmproxy import controller # noqa from mitmproxy import http from mitmproxy import log from mitmproxy import platform -from mitmproxy.proxy import ProxyConfig +from mitmproxy.proxy import config from mitmproxy.proxy import modes from mitmproxy.proxy import root_context from mitmproxy.net import tcp @@ -34,7 +35,7 @@ class ProxyServer(tcp.TCPServer): allow_reuse_address = True bound = True - def __init__(self, config: ProxyConfig): + def __init__(self, config: config.ProxyConfig) -> None: """ Raises ServerException if there's a startup problem. """ @@ -49,7 +50,7 @@ def __init__(self, config: ProxyConfig): raise exceptions.ServerException( 'Error starting proxy server: ' + repr(e) ) from e - self.channel = None + self.channel = None # type: controller.Channel def set_channel(self, channel): self.channel = channel @@ -67,8 +68,7 @@ def handle_client_connection(self, conn, client_address): class ConnectionHandler: def __init__(self, client_conn, client_address, config, channel): - self.config = config - """@type: mitmproxy.proxy.config.ProxyConfig""" + self.config = config # type: config.ProxyConfig self.client_conn = connections.ClientConnection( client_conn, client_address, diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 1415900171..a0deaec933 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -1,5 +1,6 @@ from typing import Any from typing import List +from typing import MutableMapping # noqa from mitmproxy.types import serializable @@ -19,7 +20,7 @@ class StateObject(serializable.Serializable): or StateObject instances themselves. """ - _stateobject_attributes = None + _stateobject_attributes = None # type: MutableMapping[str, Any] """ An attribute-name -> class-or-type dict containing all attributes that should be serialized. If the attribute is a class, it must implement the diff --git a/mitmproxy/tcp.py b/mitmproxy/tcp.py index 067fbfe307..fe9f217bb8 100644 --- a/mitmproxy/tcp.py +++ b/mitmproxy/tcp.py @@ -41,9 +41,7 @@ def __init__(self, client_conn, server_conn, live=None): self.messages = [] # type: List[TCPMessage] _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes.update( - messages=List[TCPMessage] - ) + _stateobject_attributes["messages"] = List[TCPMessage] def __repr__(self): return "".format(len(self.messages)) diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 2946561599..1b90c2e50b 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -1,11 +1,11 @@ import re import codecs -from typing import AnyStr, Optional +from typing import AnyStr, Optional, cast def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes]: if isinstance(str_or_bytes, bytes) or str_or_bytes is None: - return str_or_bytes + return cast(Optional[bytes], str_or_bytes) elif isinstance(str_or_bytes, str): return str_or_bytes.encode(*encode_args) else: @@ -18,7 +18,7 @@ def always_str(str_or_bytes: Optional[AnyStr], *decode_args) -> Optional[str]: str_or_bytes unmodified, if """ if isinstance(str_or_bytes, str) or str_or_bytes is None: - return str_or_bytes + return cast(Optional[str], str_or_bytes) elif isinstance(str_or_bytes, bytes): return str_or_bytes.decode(*decode_args) else: diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index bdd83ee6fc..e8e2121ead 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -25,10 +25,10 @@ def check_type(name: str, value: typing.Any, typeinfo: type) -> None: if typename.startswith("typing.Union"): try: - types = typeinfo.__args__ + types = typeinfo.__args__ # type: ignore except AttributeError: # Python 3.5.x - types = typeinfo.__union_params__ + types = typeinfo.__union_params__ # type: ignore for T in types: try: @@ -40,10 +40,10 @@ def check_type(name: str, value: typing.Any, typeinfo: type) -> None: raise e elif typename.startswith("typing.Tuple"): try: - types = typeinfo.__args__ + types = typeinfo.__args__ # type: ignore except AttributeError: # Python 3.5.x - types = typeinfo.__tuple_params__ + types = typeinfo.__tuple_params__ # type: ignore if not isinstance(value, (tuple, list)): raise e @@ -54,10 +54,10 @@ def check_type(name: str, value: typing.Any, typeinfo: type) -> None: return elif typename.startswith("typing.Sequence"): try: - T = typeinfo.__args__[0] + T = typeinfo.__args__[0] # type: ignore except AttributeError: # Python 3.5.0 - T = typeinfo.__parameters__[0] + T = typeinfo.__parameters__[0] # type: ignore if not isinstance(value, (tuple, list)): raise e diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py index 5d76aafc29..2efa7ad124 100644 --- a/mitmproxy/websocket.py +++ b/mitmproxy/websocket.py @@ -8,11 +8,13 @@ class WebSocketMessage(serializable.Serializable): - def __init__(self, type: int, from_client: bool, content: bytes, timestamp: Optional[int]=None): + def __init__( + self, type: int, from_client: bool, content: bytes, timestamp: Optional[int]=None + ) -> None: self.type = type self.from_client = from_client self.content = content - self.timestamp = timestamp or time.time() # type: int + self.timestamp = timestamp or int(time.time()) # type: int @classmethod def from_state(cls, state): @@ -62,7 +64,8 @@ def __init__(self, client_conn, server_conn, handshake_flow, live=None): self.handshake_flow = handshake_flow _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes.update( + # mypy doesn't support update with kwargs + _stateobject_attributes.update(dict( messages=List[WebSocketMessage], close_sender=str, close_code=str, @@ -77,7 +80,7 @@ def __init__(self, client_conn, server_conn, handshake_flow, live=None): # Do not include handshake_flow, to prevent recursive serialization! # Since mitmproxy-console currently only displays HTTPFlows, # dumping the handshake_flow will include the WebSocketFlow too. - ) + )) @classmethod def from_state(cls, state): diff --git a/setup.py b/setup.py index 9ea69f67a6..ec28eded75 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ 'dev': [ "Flask>=0.10.1, <0.13", "flake8>=3.2.1, <3.4", - "mypy>=0.471, <0.502", + "mypy>=0.501, <0.502", "rstcheck>=2.2, <4.0", "tox>=2.3, <3", "pytest>=3, <3.1", diff --git a/tox.ini b/tox.ini index a9904e8760..a1ed53f7a3 100644 --- a/tox.ini +++ b/tox.ini @@ -36,6 +36,8 @@ commands = mitmproxy/tools/dump.py \ mitmproxy/tools/web/ \ mitmproxy/contentviews/ + mypy --ignore-missing-imports \ + mitmproxy/master.py [testenv:individual_coverage] deps =