diff --git a/.travis.yml b/.travis.yml index 0206c845cc1..1f2de5b9652 100644 --- a/.travis.yml +++ b/.travis.yml @@ -180,6 +180,14 @@ jobs: script: - python setup.py check --metadata --restructuredtext --strict --verbose + - <<: *_lint_base + name: + - CONTRIBUTORS.txt sorting check + install: + - skip + script: + - LC_ALL=C sort -c CONTRIBUTORS.txt + - <<: *osx_python_base python: 3.5.3 env: diff --git a/CHANGES/3267.feature b/CHANGES/3267.feature new file mode 100644 index 00000000000..9696401193c --- /dev/null +++ b/CHANGES/3267.feature @@ -0,0 +1,2 @@ +Don't raise a warning if ``NETRC`` environment variable is not set and ``~/.netrc`` file +doesn't exist. diff --git a/CHANGES/3318.removal b/CHANGES/3318.removal new file mode 100644 index 00000000000..8c8236e7403 --- /dev/null +++ b/CHANGES/3318.removal @@ -0,0 +1 @@ +Deprecated use of boolean in ``resp.enable_compression()`` diff --git a/CHANGES/3337.bugfix b/CHANGES/3337.bugfix new file mode 100644 index 00000000000..b048eadc0a6 --- /dev/null +++ b/CHANGES/3337.bugfix @@ -0,0 +1 @@ +Fix tests/test_connector.py typo and tests/autobahn/server.py duplicate loop def. diff --git a/CHANGES/3341.misc b/CHANGES/3341.misc new file mode 100644 index 00000000000..2d8e3a61e30 --- /dev/null +++ b/CHANGES/3341.misc @@ -0,0 +1 @@ +Added a `lru_cache` to the `parse_mimetype` method diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index d3db40b4a57..2a149216923 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -1,5 +1,5 @@ -Contributors ------------- +- Contributors - +---------------- A. Jesse Jiryu Davis Adam Mills Adrián Chaves @@ -45,7 +45,6 @@ Brian Muller Bryce Drennan Carl George Cecile Tonglet -Colin Dunklau Chien-Wei Huang Chih-Yuan Chen Chris AtLee @@ -53,6 +52,7 @@ Chris Laws Chris Moore Christopher Schmitt Claudiu Popa +Colin Dunklau Damien Nadé Dan Xu Daniel García @@ -64,12 +64,12 @@ Denilson Amorim Denis Matiychuk Dima Veselov Dimitar Dimitrov +Dmitriy Safonov Dmitry Doroshev Dmitry Lukashin Dmitry Shamov Dmitry Trofimov Dmytro Kuznetsov -Dmitriy Safonov Dustin J. Mitchell Eduard Iskandarov Eli Ribble @@ -88,9 +88,9 @@ Gennady Andreyev Georges Dubus Greg Holt Gregory Haynes -Günther Jena Gus Goulart Gustavo Carneiro +Günther Jena Hu Bo Hugo Herter Hynek Schlawack @@ -126,13 +126,13 @@ Kirill Klenov Kirill Malovitsa Kyrylo Perevozchikov Lars P. Søndergaard -Loïc Lajeanne Louis-Philippe Huberdeau +Loïc Lajeanne Lu Gong Lubomir Gelo Ludovic Gasc -Lukasz Marcin Dobrzanski Luis Pedrosa +Lukasz Marcin Dobrzanski Makc Belousow Manuel Miranda Marat Sharafutdinov @@ -154,6 +154,7 @@ Mun Gwan-gyeong Nicolas Braem Nikolay Kim Nikolay Novik +Oisin Aylward Olaf Conradi Pahaz Blinov Panagiotis Kolokotronis @@ -175,9 +176,9 @@ Robert Lu Roman Podoliaka Samuel Colvin Sean Hunt -Sebastien Geffroy Sebastian Hanula Sebastian Hüther +Sebastien Geffroy SeongSoo Cho Sergey Ninua Sergey Skripnick @@ -198,8 +199,8 @@ Thanos Lefteris Thijs Vermeir Thomas Forbes Thomas Grainger -Tomasz Trebski Tolga Tezel +Tomasz Trebski Trinh Hoang Nhu Vadim Suharnikov Vaibhav Sagar @@ -218,12 +219,12 @@ Vladimir Shulyak Vladimir Zakharov Vladyslav Bondar W. Trevor King +Wei Lin +Weiwei Wang Will McGugan Willem de Groot William Grzybowski Wilson Ong -Wei Lin -Weiwei Wang Yang Zhou Yannick Koechlin Yannick Péroux diff --git a/Makefile b/Makefile index cea4463f595..b35c6e249a7 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,8 @@ all: test -install-cython: +.install-deps: $(shell find requirements -type f) @pip install -r requirements/cython.txt - -.install-deps: $(shell find requirements -type f) install-cython @pip install -U -r requirements/dev.txt @touch .install-deps @@ -26,6 +24,9 @@ flake: .flake isort --diff -rc aiohttp tests examples; \ false; \ fi + @if ! LC_ALL=C sort -c CONTRIBUTORS.txt; then \ + echo "CONTRIBUTORS.txt sort error"; \ + fi @touch .flake check_changes: diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index b93d7ce2250..e1d1f1037e8 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -10,9 +10,8 @@ class BaseProtocol(asyncio.Protocol): def __init__(self, loop: Optional[asyncio.AbstractEventLoop]=None) -> None: if loop is None: - self._loop = asyncio.get_event_loop() - else: - self._loop = loop + loop = asyncio.get_event_loop() + self._loop = loop # type: asyncio.AbstractEventLoop self._paused = False self._drain_waiter = None # type: Optional[asyncio.Future[None]] self._connection_lost = False diff --git a/aiohttp/client.py b/aiohttp/client.py index 540faad6b1e..e58d1377c8e 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -99,7 +99,7 @@ def __init__(self, *, connector=None, loop=None, cookies=None, read_timeout=sentinel, conn_timeout=None, timeout=sentinel, auto_decompress=True, trust_env=False, - trace_configs=None): + trace_configs=None) -> None: implicit_loop = False if loop is None: diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 9231b163feb..c8a70954abc 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -1,4 +1,6 @@ +import asyncio from contextlib import suppress +from typing import Optional from .base_protocol import BaseProtocol from .client_exceptions import (ClientOSError, ClientPayloadError, @@ -10,7 +12,10 @@ class ResponseHandler(BaseProtocol, DataQueue): """Helper class to adapt between Protocol and StreamReader.""" - def __init__(self, *, loop=None): + def __init__(self, *, + loop: Optional[asyncio.AbstractEventLoop]=None) -> None: + if loop is None: + loop = asyncio.get_event_loop() BaseProtocol.__init__(self, loop=loop) DataQueue.__init__(self, loop=loop) diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 995b3af0732..8e540c2b96c 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -4,12 +4,11 @@ import base64 import binascii import cgi -import datetime import functools import inspect -import logging import netrc import os +import platform import re import sys import time @@ -19,19 +18,18 @@ from math import ceil from pathlib import Path from types import TracebackType -from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterable, Iterator, - List, Mapping, Optional, Pattern, Tuple, Type, TypeVar, - Union, cast) +from typing import (Any, Callable, Dict, Iterable, Iterator, List, # noqa + Mapping, Optional, Pattern, Tuple, Type, TypeVar, Union, + cast) from urllib.parse import quote from urllib.request import getproxies import async_timeout import attr -from multidict import MultiDict +from multidict import MultiDict, MultiDictProxy from yarl import URL from . import hdrs -from .abc import AbstractAccessLogger from .log import client_logger from .typedefs import PathLike # noqa @@ -51,12 +49,6 @@ from typing_extensions import ContextManager -if TYPE_CHECKING: # pragma: no cover - # run in mypy mode only to prevent circular imports - from .web_request import BaseRequest # noqa - from .web_response import StreamResponse # noqa - - _T = TypeVar('_T') @@ -67,7 +59,7 @@ # for compatibility with older versions DEBUG = (getattr(sys.flags, 'dev_mode', False) or (not sys.flags.ignore_environment and - bool(os.environ.get('PYTHONASYNCIODEBUG')))) + bool(os.environ.get('PYTHONASYNCIODEBUG')))) # type: bool CHAR = set(chr(i) for i in range(0, 128)) @@ -164,30 +156,39 @@ def strip_auth_from_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]: def netrc_from_env() -> Optional[netrc.netrc]: - netrc_obj = None - netrc_path = os.environ.get('NETRC') # type: Optional[PathLike] - try: - if netrc_path is not None: - netrc_path = Path(netrc_path) - else: + """Attempt to load the netrc file from the path specified by the env-var + NETRC or in the default location in the user's home directory. + + Returns None if it couldn't be found or fails to parse. + """ + netrc_env = os.environ.get('NETRC') + + if netrc_env is not None: + netrc_path = Path(netrc_env) + else: + try: home_dir = Path.home() - if os.name == 'nt': # pragma: no cover - netrc_path = home_dir.joinpath('_netrc') - else: - netrc_path = home_dir.joinpath('.netrc') + except RuntimeError as e: # pragma: no cover + # if pathlib can't resolve home, it may raise a RuntimeError + client_logger.warning('Could not resolve home directory when ' + 'trying to look for .netrc file: %s', e) + return None - if netrc_path and netrc_path.is_file(): - try: - netrc_obj = netrc.netrc(str(netrc_path)) - except (netrc.NetrcParseError, OSError) as e: - client_logger.warning(".netrc file parses fail: %s", e) + netrc_path = home_dir / ( + '_netrc' if platform.system() == 'Windows' else '.netrc') - if netrc_obj is None: - client_logger.warning("could't find .netrc file") - except RuntimeError as e: # pragma: no cover - """ handle error raised by pathlib """ - client_logger.warning("could't find .netrc file: %s", e) - return netrc_obj + try: + return netrc.netrc(str(netrc_path)) + except netrc.NetrcParseError as e: + client_logger.warning('Could not parse .netrc file: %s', e) + except OSError as e: + # we couldn't read the file (doesn't exist, permissions, etc.) + if netrc_env or netrc_path.is_file(): + # only warn if the enviroment wanted us to load it, + # or it appears like the default file does actually exist + client_logger.warning('Could not read .netrc file: %s', e) + + return None @attr.s(frozen=True, slots=True) @@ -243,9 +244,10 @@ class MimeType: type = attr.ib(type=str) subtype = attr.ib(type=str) suffix = attr.ib(type=str) - parameters = attr.ib(type=MultiDict) # type: MultiDict[str] + parameters = attr.ib(type=MultiDictProxy) # type: MultiDictProxy[str] +@functools.lru_cache(maxsize=56) def parse_mimetype(mimetype: str) -> MimeType: """Parses a MIME type into its components. @@ -261,17 +263,17 @@ def parse_mimetype(mimetype: str) -> MimeType: """ if not mimetype: - return MimeType(type='', subtype='', suffix='', parameters=MultiDict()) + return MimeType(type='', subtype='', suffix='', + parameters=MultiDictProxy(MultiDict())) parts = mimetype.split(';') - params_lst = [] + params = MultiDict() # type: MultiDict[str] for item in parts[1:]: if not item: continue key, value = cast(Tuple[str, str], item.split('=', 1) if '=' in item else (item, '')) - params_lst.append((key.lower().strip(), value.strip(' "'))) - params = MultiDict(params_lst) + params.add(key.lower().strip(), value.strip(' "')) fulltype = parts[0].strip().lower() if fulltype == '*': @@ -283,7 +285,7 @@ def parse_mimetype(mimetype: str) -> MimeType: if '+' in stype else (stype, '')) return MimeType(type=mtype, subtype=stype, suffix=suffix, - parameters=params) + parameters=MultiDictProxy(params)) def guess_filename(obj: Any, default: Optional[str]=None) -> Optional[str]: @@ -323,231 +325,6 @@ def content_disposition_header(disptype: str, return value -KeyMethod = namedtuple('KeyMethod', 'key method') - - -class AccessLogger(AbstractAccessLogger): - """Helper object to log access. - - Usage: - log = logging.getLogger("spam") - log_format = "%a %{User-Agent}i" - access_logger = AccessLogger(log, log_format) - access_logger.log(request, response, time) - - Format: - %% The percent sign - %a Remote IP-address (IP-address of proxy if using reverse proxy) - %t Time when the request was started to process - %P The process ID of the child that serviced the request - %r First line of request - %s Response status code - %b Size of response in bytes, including HTTP headers - %T Time taken to serve the request, in seconds - %Tf Time taken to serve the request, in seconds with floating fraction - in .06f format - %D Time taken to serve the request, in microseconds - %{FOO}i request.headers['FOO'] - %{FOO}o response.headers['FOO'] - %{FOO}e os.environ['FOO'] - - """ - LOG_FORMAT_MAP = { - 'a': 'remote_address', - 't': 'request_start_time', - 'P': 'process_id', - 'r': 'first_request_line', - 's': 'response_status', - 'b': 'response_size', - 'T': 'request_time', - 'Tf': 'request_time_frac', - 'D': 'request_time_micro', - 'i': 'request_header', - 'o': 'response_header', - } - - LOG_FORMAT = '%a %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"' - FORMAT_RE = re.compile(r'%(\{([A-Za-z0-9\-_]+)\}([ioe])|[atPrsbOD]|Tf?)') - CLEANUP_RE = re.compile(r'(%[^s])') - _FORMAT_CACHE = {} # type: Dict[str, Tuple[str, List[KeyMethod]]] - - def __init__(self, logger: logging.Logger, - log_format: str=LOG_FORMAT) -> None: - """Initialise the logger. - - logger is a logger object to be used for logging. - log_format is an string with apache compatible log format description. - - """ - super().__init__(logger, log_format=log_format) - - _compiled_format = AccessLogger._FORMAT_CACHE.get(log_format) - if not _compiled_format: - _compiled_format = self.compile_format(log_format) - AccessLogger._FORMAT_CACHE[log_format] = _compiled_format - - self._log_format, self._methods = _compiled_format - - def compile_format(self, log_format: str) -> Tuple[str, List[KeyMethod]]: - """Translate log_format into form usable by modulo formatting - - All known atoms will be replaced with %s - Also methods for formatting of those atoms will be added to - _methods in appropriate order - - For example we have log_format = "%a %t" - This format will be translated to "%s %s" - Also contents of _methods will be - [self._format_a, self._format_t] - These method will be called and results will be passed - to translated string format. - - Each _format_* method receive 'args' which is list of arguments - given to self.log - - Exceptions are _format_e, _format_i and _format_o methods which - also receive key name (by functools.partial) - - """ - # list of (key, method) tuples, we don't use an OrderedDict as users - # can repeat the same key more than once - methods = list() - - for atom in self.FORMAT_RE.findall(log_format): - if atom[1] == '': - format_key1 = self.LOG_FORMAT_MAP[atom[0]] - m = getattr(AccessLogger, '_format_%s' % atom[0]) - key_method = KeyMethod(format_key1, m) - else: - format_key2 = (self.LOG_FORMAT_MAP[atom[2]], atom[1]) - m = getattr(AccessLogger, '_format_%s' % atom[2]) - key_method = KeyMethod(format_key2, - functools.partial(m, atom[1])) - - methods.append(key_method) - - log_format = self.FORMAT_RE.sub(r'%s', log_format) - log_format = self.CLEANUP_RE.sub(r'%\1', log_format) - return log_format, methods - - @staticmethod - def _format_i(key: str, - request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - if request is None: - return '(no headers)' - - # suboptimal, make istr(key) once - return request.headers.get(key, '-') - - @staticmethod - def _format_o(key: str, - request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - # suboptimal, make istr(key) once - return response.headers.get(key, '-') - - @staticmethod - def _format_a(request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - if request is None: - return '-' - ip = request.remote - return ip if ip is not None else '-' - - @staticmethod - def _format_t(request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - now = datetime.datetime.utcnow() - start_time = now - datetime.timedelta(seconds=time) - return start_time.strftime('[%d/%b/%Y:%H:%M:%S +0000]') - - @staticmethod - def _format_P(request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - return "<%s>" % os.getpid() - - @staticmethod - def _format_r(request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - if request is None: - return '-' - return '%s %s HTTP/%s.%s' % (request.method, request.path_qs, - request.version.major, - request.version.minor) - - @staticmethod - def _format_s(request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - return response.status - - @staticmethod - def _format_b(request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - return response.body_length - - @staticmethod - def _format_T(request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - return str(round(time)) - - @staticmethod - def _format_Tf(request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - return '%06f' % time - - @staticmethod - def _format_D(request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> str: - return str(round(time * 1000000)) - - def _format_line(self, - request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> Iterable[Tuple[str, - Callable[['BaseRequest', - 'StreamResponse', - float], - str]]]: - return [(key, method(request, response, time)) - for key, method in self._methods] - - def log(self, - request: 'BaseRequest', - response: 'StreamResponse', - time: float) -> None: - try: - fmt_info = self._format_line(request, response, time) - - values = list() - extra = dict() - for key, value in fmt_info: - values.append(value) - - if key.__class__ is str: - extra[key] = value - else: - k1, k2 = key - dct = extra.get(k1, {}) - dct[k2] = value # type: ignore - extra[k1] = dct # type: ignore - - self.logger.info(self._log_format % tuple(values), extra=extra) - except Exception: - self.logger.exception("Error in logging") - - class reify: """Use as a class method decorator. It operates almost exactly like the Python `@property` decorator, but it puts the result of the diff --git a/aiohttp/web.py b/aiohttp/web.py index 9a824819ceb..630585fc6d3 100644 --- a/aiohttp/web.py +++ b/aiohttp/web.py @@ -6,13 +6,14 @@ from collections.abc import Iterable from importlib import import_module -from . import (helpers, web_app, web_exceptions, web_fileresponse, - web_middlewares, web_protocol, web_request, web_response, - web_routedef, web_runner, web_server, web_urldispatcher, web_ws) +from . import (web_app, web_exceptions, web_fileresponse, web_middlewares, + web_protocol, web_request, web_response, web_routedef, + web_runner, web_server, web_urldispatcher, web_ws) from .log import access_logger from .web_app import * # noqa from .web_exceptions import * # noqa from .web_fileresponse import * # noqa +from .web_log import AccessLogger from .web_middlewares import * # noqa from .web_protocol import * # noqa from .web_request import * # noqa @@ -42,8 +43,8 @@ def run_app(app, *, host=None, port=None, path=None, sock=None, shutdown_timeout=60.0, ssl_context=None, - print=print, backlog=128, access_log_class=helpers.AccessLogger, - access_log_format=helpers.AccessLogger.LOG_FORMAT, + print=print, backlog=128, access_log_class=AccessLogger, + access_log_format=AccessLogger.LOG_FORMAT, access_log=access_logger, handle_signals=True, reuse_address=None, reuse_port=None): """Run an app locally""" diff --git a/aiohttp/web_app.py b/aiohttp/web_app.py index 2160a161509..3d1a71f2578 100644 --- a/aiohttp/web_app.py +++ b/aiohttp/web_app.py @@ -9,9 +9,10 @@ from . import hdrs from .abc import AbstractAccessLogger, AbstractMatchInfo, AbstractRouter from .frozenlist import FrozenList -from .helpers import DEBUG, AccessLogger +from .helpers import DEBUG from .log import web_logger from .signals import Signal +from .web_log import AccessLogger from .web_middlewares import _fix_request_current_app from .web_request import Request from .web_response import StreamResponse diff --git a/aiohttp/web_log.py b/aiohttp/web_log.py new file mode 100644 index 00000000000..ba6daa801ad --- /dev/null +++ b/aiohttp/web_log.py @@ -0,0 +1,236 @@ +import datetime +import functools +import logging +import os +import re +from collections import namedtuple +from typing import Callable, Dict, Iterable, List, Tuple # noqa + +from .abc import AbstractAccessLogger +from .web_request import BaseRequest +from .web_response import StreamResponse + + +KeyMethod = namedtuple('KeyMethod', 'key method') + + +class AccessLogger(AbstractAccessLogger): + """Helper object to log access. + + Usage: + log = logging.getLogger("spam") + log_format = "%a %{User-Agent}i" + access_logger = AccessLogger(log, log_format) + access_logger.log(request, response, time) + + Format: + %% The percent sign + %a Remote IP-address (IP-address of proxy if using reverse proxy) + %t Time when the request was started to process + %P The process ID of the child that serviced the request + %r First line of request + %s Response status code + %b Size of response in bytes, including HTTP headers + %T Time taken to serve the request, in seconds + %Tf Time taken to serve the request, in seconds with floating fraction + in .06f format + %D Time taken to serve the request, in microseconds + %{FOO}i request.headers['FOO'] + %{FOO}o response.headers['FOO'] + %{FOO}e os.environ['FOO'] + + """ + LOG_FORMAT_MAP = { + 'a': 'remote_address', + 't': 'request_start_time', + 'P': 'process_id', + 'r': 'first_request_line', + 's': 'response_status', + 'b': 'response_size', + 'T': 'request_time', + 'Tf': 'request_time_frac', + 'D': 'request_time_micro', + 'i': 'request_header', + 'o': 'response_header', + } + + LOG_FORMAT = '%a %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"' + FORMAT_RE = re.compile(r'%(\{([A-Za-z0-9\-_]+)\}([ioe])|[atPrsbOD]|Tf?)') + CLEANUP_RE = re.compile(r'(%[^s])') + _FORMAT_CACHE = {} # type: Dict[str, Tuple[str, List[KeyMethod]]] + + def __init__(self, logger: logging.Logger, + log_format: str=LOG_FORMAT) -> None: + """Initialise the logger. + + logger is a logger object to be used for logging. + log_format is an string with apache compatible log format description. + + """ + super().__init__(logger, log_format=log_format) + + _compiled_format = AccessLogger._FORMAT_CACHE.get(log_format) + if not _compiled_format: + _compiled_format = self.compile_format(log_format) + AccessLogger._FORMAT_CACHE[log_format] = _compiled_format + + self._log_format, self._methods = _compiled_format + + def compile_format(self, log_format: str) -> Tuple[str, List[KeyMethod]]: + """Translate log_format into form usable by modulo formatting + + All known atoms will be replaced with %s + Also methods for formatting of those atoms will be added to + _methods in appropriate order + + For example we have log_format = "%a %t" + This format will be translated to "%s %s" + Also contents of _methods will be + [self._format_a, self._format_t] + These method will be called and results will be passed + to translated string format. + + Each _format_* method receive 'args' which is list of arguments + given to self.log + + Exceptions are _format_e, _format_i and _format_o methods which + also receive key name (by functools.partial) + + """ + # list of (key, method) tuples, we don't use an OrderedDict as users + # can repeat the same key more than once + methods = list() + + for atom in self.FORMAT_RE.findall(log_format): + if atom[1] == '': + format_key1 = self.LOG_FORMAT_MAP[atom[0]] + m = getattr(AccessLogger, '_format_%s' % atom[0]) + key_method = KeyMethod(format_key1, m) + else: + format_key2 = (self.LOG_FORMAT_MAP[atom[2]], atom[1]) + m = getattr(AccessLogger, '_format_%s' % atom[2]) + key_method = KeyMethod(format_key2, + functools.partial(m, atom[1])) + + methods.append(key_method) + + log_format = self.FORMAT_RE.sub(r'%s', log_format) + log_format = self.CLEANUP_RE.sub(r'%\1', log_format) + return log_format, methods + + @staticmethod + def _format_i(key: str, + request: BaseRequest, + response: StreamResponse, + time: float) -> str: + if request is None: + return '(no headers)' + + # suboptimal, make istr(key) once + return request.headers.get(key, '-') + + @staticmethod + def _format_o(key: str, + request: BaseRequest, + response: StreamResponse, + time: float) -> str: + # suboptimal, make istr(key) once + return response.headers.get(key, '-') + + @staticmethod + def _format_a(request: BaseRequest, + response: StreamResponse, + time: float) -> str: + if request is None: + return '-' + ip = request.remote + return ip if ip is not None else '-' + + @staticmethod + def _format_t(request: BaseRequest, + response: StreamResponse, + time: float) -> str: + now = datetime.datetime.utcnow() + start_time = now - datetime.timedelta(seconds=time) + return start_time.strftime('[%d/%b/%Y:%H:%M:%S +0000]') + + @staticmethod + def _format_P(request: BaseRequest, + response: StreamResponse, + time: float) -> str: + return "<%s>" % os.getpid() + + @staticmethod + def _format_r(request: BaseRequest, + response: StreamResponse, + time: float) -> str: + if request is None: + return '-' + return '%s %s HTTP/%s.%s' % (request.method, request.path_qs, + request.version.major, + request.version.minor) + + @staticmethod + def _format_s(request: BaseRequest, + response: StreamResponse, + time: float) -> str: + return response.status + + @staticmethod + def _format_b(request: BaseRequest, + response: StreamResponse, + time: float) -> str: + return response.body_length + + @staticmethod + def _format_T(request: BaseRequest, + response: StreamResponse, + time: float) -> str: + return str(round(time)) + + @staticmethod + def _format_Tf(request: BaseRequest, + response: StreamResponse, + time: float) -> str: + return '%06f' % time + + @staticmethod + def _format_D(request: BaseRequest, + response: StreamResponse, + time: float) -> str: + return str(round(time * 1000000)) + + def _format_line(self, + request: BaseRequest, + response: StreamResponse, + time: float) -> Iterable[Tuple[str, + Callable[[BaseRequest, + StreamResponse, + float], + str]]]: + return [(key, method(request, response, time)) + for key, method in self._methods] + + def log(self, + request: BaseRequest, + response: StreamResponse, + time: float) -> None: + try: + fmt_info = self._format_line(request, response, time) + + values = list() + extra = dict() + for key, value in fmt_info: + values.append(value) + + if key.__class__ is str: + extra[key] = value + else: + k1, k2 = key + dct = extra.get(k1, {}) + dct[k2] = value # type: ignore + extra[k1] = dct # type: ignore + + self.logger.info(self._log_format % tuple(values), extra=extra) + except Exception: + self.logger.exception("Error in logging") diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index b9ee34746ad..f68750b4bf2 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -8,7 +8,6 @@ import yarl -from . import helpers from .base_protocol import BaseProtocol from .helpers import CeilTimeout from .http import (HttpProcessingError, HttpRequestParser, HttpVersion10, @@ -17,8 +16,9 @@ from .streams import EMPTY_PAYLOAD from .tcp_helpers import tcp_cork, tcp_keepalive, tcp_nodelay from .web_exceptions import HTTPException +from .web_log import AccessLogger from .web_request import BaseRequest -from .web_response import Response +from .web_response import Response, StreamResponse __all__ = ('RequestHandler', 'RequestPayloadError', 'PayloadAccessError') @@ -90,9 +90,9 @@ def __init__(self, manager, *, loop=None, keepalive_timeout=75, # NGINX default value is 75 secs tcp_keepalive=True, logger=server_logger, - access_log_class=helpers.AccessLogger, + access_log_class=AccessLogger, access_log=access_logger, - access_log_format=helpers.AccessLogger.LOG_FORMAT, + access_log_format=AccessLogger.LOG_FORMAT, debug=False, max_line_size=8190, max_headers=32768, @@ -346,6 +346,7 @@ async def start(self): handler = self._task_handler manager = self._manager keepalive_timeout = self._keepalive_timeout + resp = None while not self._force_close: if not self._messages: @@ -389,6 +390,13 @@ async def start(self): "please raise the exception instead", DeprecationWarning) + if self.debug: + if not isinstance(resp, StreamResponse): + self.log_debug("Possibly missing return " + "statement on request handler") + raise RuntimeError("Web-handler should return " + "a response instance, " + "got {!r}".format(resp)) await resp.prepare(request) await resp.write_eof() @@ -438,7 +446,7 @@ async def start(self): self.log_exception('Unhandled exception', exc_info=exc) self.force_close() finally: - if self.transport is None: + if self.transport is None and resp is not None: self.log_debug('Ignored premature client disconnection.') elif not self._force_close: if self._keepalive and not self._close: diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index e00add1c505..7e35b246547 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -75,7 +75,7 @@ class BaseRequest(collections.MutableMapping, HeadersMixin): POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT, hdrs.METH_TRACE, hdrs.METH_DELETE} - ATTRS = HeadersMixin.ATTRS | frozenset([ + ATTRS = HeadersMixin.ATTRS | frozenset([ # type: ignore '_message', '_protocol', '_payload_writer', '_payload', '_headers', '_method', '_version', '_rel_url', '_post', '_read_bytes', '_state', '_cache', '_task', '_client_max_size', '_loop', diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index c6e9afc71d0..f3fc59e43d6 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -131,6 +131,8 @@ def enable_compression(self, force=None): # Backwards compatibility for when force was a bool <0.17. if type(force) == bool: force = ContentCoding.deflate if force else ContentCoding.identity + warnings.warn("Using boolean for force is deprecated #3318", + DeprecationWarning) elif force is not None: assert isinstance(force, ContentCoding), ("force should one of " "None, bool or " diff --git a/aiohttp/worker.py b/aiohttp/worker.py index a4b32c48752..32d72b17094 100644 --- a/aiohttp/worker.py +++ b/aiohttp/worker.py @@ -12,7 +12,8 @@ from aiohttp import web -from .helpers import AccessLogger, set_result +from .helpers import set_result +from .web_log import AccessLogger try: diff --git a/docs/client_quickstart.rst b/docs/client_quickstart.rst index b19bec17cfc..93073c02bf3 100644 --- a/docs/client_quickstart.rst +++ b/docs/client_quickstart.rst @@ -61,6 +61,15 @@ Other HTTP methods are available as well:: A session contains a connection pool inside. Connection reusage and keep-alives (both are on by default) may speed up total performance. +A session context manager usage is not mandatory +but ``await session.close()`` method +should be called in this case, e.g.:: + + session = aiohttp.ClientSession() + async with session.get('...'): + # ... + await session.close() + Passing Parameters In URLs ========================== diff --git a/docs/contributing.rst b/docs/contributing.rst index 373d0b281dc..b908c35ffce 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -29,6 +29,15 @@ Workflow is pretty straightforward: 7. Optionally make backport Pull Request(s) for landing a bug fix into released aiohttp versions. +.. note:: + + The project uses *Squash-and-Merge* strategy for *GitHub Merge* button. + + Basically it means that there is **no need to rebase** a Pull Request against + *master* branch. Just ``git merge`` *master* into your working copy (a fork) if + needed. The Pull Request is automatically squashed into the single commit + once the PR is accepted. + Preconditions for running aiohttp test suite -------------------------------------------- diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index c88ba8f3b95..00a685ec51b 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -9,6 +9,7 @@ alives api api’s app +apps app’s arg Arsenic diff --git a/docs/third_party.rst b/docs/third_party.rst index 1d3dace0a82..608f23dbd35 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -231,3 +231,6 @@ period ask to raise the status. for `Azure Application Insights `_ implemented using ``aiohttp`` client, including a middleware for ``aiohttp`` servers to collect web apps telemetry. + +- `aiogmaps `_ + Asynchronous client for Google Maps API Web Services. Python 3.6+ required. diff --git a/requirements/ci-wheel.txt b/requirements/ci-wheel.txt index 5b1d37e5dc5..133f1adc31c 100644 --- a/requirements/ci-wheel.txt +++ b/requirements/ci-wheel.txt @@ -1,20 +1,20 @@ -r flake.txt attrs==18.2.0 async-generator==1.10 -async-timeout==3.0.0 +async-timeout==3.0.1 brotlipy==0.7.0 cchardet==2.1.4 chardet==3.0.4 coverage==4.5.1 -cython==0.28.5 -gunicorn==19.8.1 +cython==0.29 +gunicorn==19.9.0 pyflakes==2.0.0 multidict==4.4.2 -pytest==3.8.1 +pytest==3.8.2 pytest-cov==2.6.0 pytest-mock==1.10.0 pytest-xdist==1.23.2 -tox==3.4.0 +tox==3.5.2 twine==1.12.1 yarl==1.2.6 @@ -23,4 +23,4 @@ aiodns==1.1.1; platform_system!="Windows" # required c-ares will not build on w codecov==2.0.15; platform_system!="Windows" # We only use it in Travis CI uvloop==0.11.2; platform_system!="Windows" and implementation_name=="cpython" and python_version<"3.7" # MagicStack/uvloop#14 idna-ssl==1.1.0; python_version<"3.7" -typing_extensions==3.6.5; python_version<"3.7" +typing_extensions==3.6.6; python_version<"3.7" diff --git a/requirements/cython.txt b/requirements/cython.txt index 64c06c65b80..4d49466ac1e 100644 --- a/requirements/cython.txt +++ b/requirements/cython.txt @@ -1 +1 @@ -cython==0.28.5 +cython==0.29 diff --git a/requirements/wheel.txt b/requirements/wheel.txt index 8e8aca00515..79d8a057f92 100644 --- a/requirements/wheel.txt +++ b/requirements/wheel.txt @@ -1,3 +1,3 @@ -cython==0.28.5 -pytest==3.8.1 +cython==0.29 +pytest==3.8.2 twine==1.12.1 diff --git a/tests/autobahn/server.py b/tests/autobahn/server.py index 0a1a0e36505..693dcd017fe 100644 --- a/tests/autobahn/server.py +++ b/tests/autobahn/server.py @@ -50,8 +50,6 @@ async def finish(app, srv, handler): loop = asyncio.get_event_loop() logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s') - - loop = asyncio.get_event_loop() app, srv, handler = loop.run_until_complete(main(loop)) try: loop.run_forever() diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index c21b70e5c82..f9c870d35cc 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -338,7 +338,8 @@ async def handler(request): assert exc.got == fingerprint -async def test_format_task_get(aiohttp_server, loop) -> None: +async def test_format_task_get(aiohttp_server) -> None: + loop = asyncio.get_event_loop() async def handler(request): return web.Response(body=b'OK') @@ -592,7 +593,9 @@ async def handler(request): await client.get('/') -async def test_timeout_on_reading_data(loop, aiohttp_client, mocker) -> None: +async def test_timeout_on_reading_data(aiohttp_client, mocker) -> None: + loop = asyncio.get_event_loop() + mocker.patch('aiohttp.helpers.ceil').side_effect = ceil fut = loop.create_future() @@ -630,7 +633,8 @@ async def handler(request): assert resp.status == 200 -async def test_readline_error_on_conn_close(loop, aiohttp_client) -> None: +async def test_readline_error_on_conn_close(aiohttp_client) -> None: + loop = asyncio.get_event_loop() async def handler(request): resp_ = web.StreamResponse() @@ -2474,7 +2478,8 @@ async def handler(request): assert 1 == len(client.session.connector._conns) -async def test_server_close_keepalive_connection(loop) -> None: +async def test_server_close_keepalive_connection() -> None: + loop = asyncio.get_event_loop() class Proto(asyncio.Protocol): @@ -2515,7 +2520,8 @@ def connection_lost(self, exc): await server.wait_closed() -async def test_handle_keepalive_on_closed_connection(loop) -> None: +async def test_handle_keepalive_on_closed_connection() -> None: + loop = asyncio.get_event_loop() class Proto(asyncio.Protocol): @@ -2560,7 +2566,7 @@ def connection_lost(self, exc): await server.wait_closed() -async def test_error_in_performing_request(loop, ssl_ctx, +async def test_error_in_performing_request(ssl_ctx, aiohttp_client, aiohttp_server): async def handler(request): return web.Response() @@ -2569,6 +2575,7 @@ def exception_handler(loop, context): # skip log messages about destroyed but pending tasks pass + loop = asyncio.get_event_loop() loop.set_exception_handler(exception_handler) app = web.Application() @@ -2587,7 +2594,8 @@ def exception_handler(loop, context): await client.get('/') -async def test_await_after_cancelling(loop, aiohttp_client) -> None: +async def test_await_after_cancelling(aiohttp_client) -> None: + loop = asyncio.get_event_loop() async def handler(request): return web.Response() diff --git a/tests/test_client_ws_functional.py b/tests/test_client_ws_functional.py index 46c33264db1..9f0d2e2b20b 100644 --- a/tests/test_client_ws_functional.py +++ b/tests/test_client_ws_functional.py @@ -15,7 +15,7 @@ def ceil(val): mocker.patch('aiohttp.helpers.ceil').side_effect = ceil -async def test_send_recv_text(loop, aiohttp_client) -> None: +async def test_send_recv_text(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -41,7 +41,7 @@ async def handler(request): assert resp.get_extra_info('socket') is None -async def test_send_recv_bytes_bad_type(loop, aiohttp_client) -> None: +async def test_send_recv_bytes_bad_type(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -63,7 +63,7 @@ async def handler(request): await resp.close() -async def test_send_recv_bytes(loop, aiohttp_client) -> None: +async def test_send_recv_bytes(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -87,7 +87,7 @@ async def handler(request): await resp.close() -async def test_send_recv_text_bad_type(loop, aiohttp_client) -> None: +async def test_send_recv_text_bad_type(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -111,7 +111,7 @@ async def handler(request): await resp.close() -async def test_send_recv_json(loop, aiohttp_client) -> None: +async def test_send_recv_json(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -134,8 +134,8 @@ async def handler(request): await resp.close() -async def test_ping_pong(loop, aiohttp_client) -> None: - +async def test_ping_pong(aiohttp_client) -> None: + loop = asyncio.get_event_loop() closed = loop.create_future() async def handler(request): @@ -170,8 +170,8 @@ async def handler(request): await closed -async def test_ping_pong_manual(loop, aiohttp_client) -> None: - +async def test_ping_pong_manual(aiohttp_client) -> None: + loop = asyncio.get_event_loop() closed = loop.create_future() async def handler(request): @@ -211,7 +211,7 @@ async def handler(request): await closed -async def test_close(loop, aiohttp_client) -> None: +async def test_close(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -239,7 +239,7 @@ async def handler(request): assert msg.type == aiohttp.WSMsgType.CLOSED -async def test_concurrent_close(loop, aiohttp_client) -> None: +async def test_concurrent_close(aiohttp_client) -> None: client_ws = None async def handler(request): @@ -266,13 +266,13 @@ async def handler(request): msg = await ws.receive() assert msg.type == aiohttp.WSMsgType.CLOSING - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) msg = await ws.receive() assert msg.type == aiohttp.WSMsgType.CLOSED -async def test_close_from_server(loop, aiohttp_client) -> None: - +async def test_close_from_server(aiohttp_client) -> None: + loop = asyncio.get_event_loop() closed = loop.create_future() async def handler(request): @@ -303,8 +303,8 @@ async def handler(request): await closed -async def test_close_manual(loop, aiohttp_client) -> None: - +async def test_close_manual(aiohttp_client) -> None: + loop = asyncio.get_event_loop() closed = loop.create_future() async def handler(request): @@ -340,14 +340,14 @@ async def handler(request): assert resp.closed -async def test_close_timeout(loop, aiohttp_client) -> None: +async def test_close_timeout(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() await ws.prepare(request) await ws.receive_bytes() await ws.send_str('test') - await asyncio.sleep(1, loop=loop) + await asyncio.sleep(1) return ws app = web.Application() @@ -366,14 +366,15 @@ async def handler(request): assert isinstance(resp.exception(), asyncio.TimeoutError) -async def test_close_cancel(loop, aiohttp_client) -> None: +async def test_close_cancel(aiohttp_client) -> None: + loop = asyncio.get_event_loop() async def handler(request): ws = web.WebSocketResponse() await ws.prepare(request) await ws.receive_bytes() await ws.send_str('test') - await asyncio.sleep(10, loop=loop) + await asyncio.sleep(10) app = web.Application() app.router.add_route('GET', '/', handler) @@ -386,14 +387,14 @@ async def handler(request): assert text.data == 'test' t = loop.create_task(resp.close()) - await asyncio.sleep(0.1, loop=loop) + await asyncio.sleep(0.1) t.cancel() - await asyncio.sleep(0.1, loop=loop) + await asyncio.sleep(0.1) assert resp.closed assert resp.exception() is None -async def test_override_default_headers(loop, aiohttp_client) -> None: +async def test_override_default_headers(aiohttp_client) -> None: async def handler(request): assert request.headers[hdrs.SEC_WEBSOCKET_VERSION] == '8' @@ -413,7 +414,7 @@ async def handler(request): await resp.close() -async def test_additional_headers(loop, aiohttp_client) -> None: +async def test_additional_headers(aiohttp_client) -> None: async def handler(request): assert request.headers['x-hdr'] == 'xtra' @@ -433,7 +434,7 @@ async def handler(request): await resp.close() -async def test_recv_protocol_error(loop, aiohttp_client) -> None: +async def test_recv_protocol_error(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -458,7 +459,7 @@ async def handler(request): await resp.close() -async def test_recv_timeout(loop, aiohttp_client) -> None: +async def test_recv_timeout(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -466,7 +467,7 @@ async def handler(request): await ws.receive_str() - await asyncio.sleep(0.1, loop=request.app.loop) + await asyncio.sleep(0.1) await ws.close() return ws @@ -478,13 +479,13 @@ async def handler(request): await resp.send_str('ask') with pytest.raises(asyncio.TimeoutError): - with async_timeout.timeout(0.01, loop=app.loop): + with async_timeout.timeout(0.01): await resp.receive() await resp.close() -async def test_receive_timeout(loop, aiohttp_client) -> None: +async def test_receive_timeout(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -505,7 +506,7 @@ async def handler(request): await resp.close() -async def test_custom_receive_timeout(loop, aiohttp_client) -> None: +async def test_custom_receive_timeout(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -526,7 +527,7 @@ async def handler(request): await resp.close() -async def test_heartbeat(loop, aiohttp_client, ceil) -> None: +async def test_heartbeat(aiohttp_client, ceil) -> None: ping_received = False async def handler(request): @@ -551,7 +552,7 @@ async def handler(request): assert ping_received -async def test_heartbeat_no_pong(loop, aiohttp_client, ceil) -> None: +async def test_heartbeat_no_pong(aiohttp_client, ceil) -> None: ping_received = False async def handler(request): @@ -576,7 +577,7 @@ async def handler(request): assert ping_received -async def test_send_recv_compress(loop, aiohttp_client) -> None: +async def test_send_recv_compress(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -602,7 +603,7 @@ async def handler(request): assert resp.get_extra_info('socket') is None -async def test_send_recv_compress_wbits(loop, aiohttp_client) -> None: +async def test_send_recv_compress_wbits(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -630,7 +631,7 @@ async def handler(request): assert resp.get_extra_info('socket') is None -async def test_send_recv_compress_wbit_error(loop, aiohttp_client) -> None: +async def test_send_recv_compress_wbit_error(aiohttp_client) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -648,7 +649,7 @@ async def handler(request): await client.ws_connect('/', compress=1) -async def test_ws_client_async_for(loop, aiohttp_client) -> None: +async def test_ws_client_async_for(aiohttp_client) -> None: items = ['q1', 'q2', 'q3'] async def handler(request): @@ -674,7 +675,7 @@ async def handler(request): assert resp.closed -async def test_ws_async_with(loop, aiohttp_server) -> None: +async def test_ws_async_with(aiohttp_server) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -689,7 +690,7 @@ async def handler(request): server = await aiohttp_server(app) - async with aiohttp.ClientSession(loop=loop) as client: + async with aiohttp.ClientSession() as client: async with client.ws_connect(server.make_url('/')) as ws: await ws.send_str('request') msg = await ws.receive() @@ -698,7 +699,7 @@ async def handler(request): assert ws.closed -async def test_ws_async_with_send(loop, aiohttp_server) -> None: +async def test_ws_async_with_send(aiohttp_server) -> None: # send_xxx methods have to return awaitable objects async def handler(request): @@ -714,7 +715,7 @@ async def handler(request): server = await aiohttp_server(app) - async with aiohttp.ClientSession(loop=loop) as client: + async with aiohttp.ClientSession() as client: async with client.ws_connect(server.make_url('/')) as ws: await ws.send_str('request') msg = await ws.receive() @@ -723,7 +724,7 @@ async def handler(request): assert ws.closed -async def test_ws_async_with_shortcut(loop, aiohttp_server) -> None: +async def test_ws_async_with_shortcut(aiohttp_server) -> None: async def handler(request): ws = web.WebSocketResponse() @@ -737,7 +738,7 @@ async def handler(request): app.router.add_route('GET', '/', handler) server = await aiohttp_server(app) - async with aiohttp.ClientSession(loop=loop) as client: + async with aiohttp.ClientSession() as client: async with client.ws_connect(server.make_url('/')) as ws: await ws.send_str('request') msg = await ws.receive() @@ -746,8 +747,8 @@ async def handler(request): assert ws.closed -async def test_closed_async_for(loop, aiohttp_client) -> None: - +async def test_closed_async_for(aiohttp_client) -> None: + loop = asyncio.get_event_loop() closed = loop.create_future() async def handler(request): diff --git a/tests/test_connector.py b/tests/test_connector.py index 353240f2c33..33af56e02dd 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -209,7 +209,7 @@ def test_del_with_closed_loop(loop) -> None: assert exc_handler.called -def test_del_empty_conector(loop) -> None: +def test_del_empty_connector(loop) -> None: conn = aiohttp.BaseConnector(loop=loop) exc_handler = mock.Mock() diff --git a/tests/test_helpers.py b/tests/test_helpers.py index a3636d21bff..b03b4ffd718 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,6 +1,5 @@ import asyncio import base64 -import datetime import gc import os import platform @@ -12,7 +11,6 @@ from yarl import URL from aiohttp import helpers -from aiohttp.abc import AbstractAccessLogger IS_PYPY = platform.python_implementation() == 'PyPy' @@ -148,157 +146,6 @@ def test_basic_auth_from_not_url() -> None: helpers.BasicAuth.from_url('http://user:pass@example.com') -# ------------- access logger ------------------------- - - -def test_access_logger_format() -> None: - log_format = '%T "%{ETag}o" %X {X} %%P' - mock_logger = mock.Mock() - access_logger = helpers.AccessLogger(mock_logger, log_format) - expected = '%s "%s" %%X {X} %%%s' - assert expected == access_logger._log_format - - -@pytest.mark.skip( - IS_PYPY, - """ - Because of patching :py:class:`datetime.datetime`, under PyPy it - fails in :py:func:`isinstance` call in - :py:meth:`datetime.datetime.__sub__` (called from - :py:meth:`aiohttp.helpers.AccessLogger._format_t`): - - *** TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types - - (Pdb) from datetime import datetime - (Pdb) isinstance(now, datetime) - *** TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types - (Pdb) datetime.__class__ - - (Pdb) isinstance(now, datetime.__class__) - False - - Ref: https://bitbucket.org/pypy/pypy/issues/1187/call-to-isinstance-in-__sub__-self-other - Ref: https://github.com/celery/celery/issues/811 - Ref: https://stackoverflow.com/a/46102240/595220 - """, # noqa: E501 -) -def test_access_logger_atoms(mocker) -> None: - utcnow = datetime.datetime(1843, 1, 1, 0, 30) - mock_datetime = mocker.patch("aiohttp.helpers.datetime.datetime") - mock_getpid = mocker.patch("os.getpid") - mock_datetime.utcnow.return_value = utcnow - mock_getpid.return_value = 42 - log_format = '%a %t %P %r %s %b %T %Tf %D "%{H1}i" "%{H2}i"' - mock_logger = mock.Mock() - access_logger = helpers.AccessLogger(mock_logger, log_format) - request = mock.Mock(headers={'H1': 'a', 'H2': 'b'}, - method="GET", path_qs="/path", - version=(1, 1), - remote="127.0.0.2") - response = mock.Mock(headers={}, body_length=42, status=200) - access_logger.log(request, response, 3.1415926) - assert not mock_logger.exception.called - expected = ('127.0.0.2 [01/Jan/1843:00:29:56 +0000] <42> ' - 'GET /path HTTP/1.1 200 42 3 3.141593 3141593 "a" "b"') - extra = { - 'first_request_line': 'GET /path HTTP/1.1', - 'process_id': '<42>', - 'remote_address': '127.0.0.2', - 'request_start_time': '[01/Jan/1843:00:29:56 +0000]', - 'request_time': 3, - 'request_time_frac': '3.141593', - 'request_time_micro': 3141593, - 'response_size': 42, - 'response_status': 200, - 'request_header': {'H1': 'a', 'H2': 'b'}, - } - - mock_logger.info.assert_called_with(expected, extra=extra) - - -def test_access_logger_dicts() -> None: - log_format = '%{User-Agent}i %{Content-Length}o %{None}i' - mock_logger = mock.Mock() - access_logger = helpers.AccessLogger(mock_logger, log_format) - request = mock.Mock(headers={"User-Agent": "Mock/1.0"}, version=(1, 1), - remote="127.0.0.2") - response = mock.Mock(headers={"Content-Length": 123}) - access_logger.log(request, response, 0.0) - assert not mock_logger.error.called - expected = 'Mock/1.0 123 -' - extra = { - 'request_header': {"User-Agent": "Mock/1.0", 'None': '-'}, - 'response_header': {'Content-Length': 123} - } - - mock_logger.info.assert_called_with(expected, extra=extra) - - -def test_access_logger_unix_socket() -> None: - log_format = '|%a|' - mock_logger = mock.Mock() - access_logger = helpers.AccessLogger(mock_logger, log_format) - request = mock.Mock(headers={"User-Agent": "Mock/1.0"}, version=(1, 1), - remote="") - response = mock.Mock() - access_logger.log(request, response, 0.0) - assert not mock_logger.error.called - expected = '||' - mock_logger.info.assert_called_with(expected, extra={'remote_address': ''}) - - -def test_logger_no_message() -> None: - mock_logger = mock.Mock() - access_logger = helpers.AccessLogger(mock_logger, - "%r %{content-type}i") - extra_dict = { - 'first_request_line': '-', - 'request_header': {'content-type': '(no headers)'} - } - - access_logger.log(None, None, 0.0) - mock_logger.info.assert_called_with("- (no headers)", extra=extra_dict) - - -def test_logger_internal_error() -> None: - mock_logger = mock.Mock() - access_logger = helpers.AccessLogger(mock_logger, "%D") - access_logger.log(None, None, 'invalid') - mock_logger.exception.assert_called_with("Error in logging") - - -def test_logger_no_transport() -> None: - mock_logger = mock.Mock() - access_logger = helpers.AccessLogger(mock_logger, "%a") - access_logger.log(None, None, 0) - mock_logger.info.assert_called_with("-", extra={'remote_address': '-'}) - - -def test_logger_abc() -> None: - class Logger(AbstractAccessLogger): - def log(self, request, response, time): - 1 / 0 - - mock_logger = mock.Mock() - access_logger = Logger(mock_logger, None) - - with pytest.raises(ZeroDivisionError): - access_logger.log(None, None, None) - - class Logger(AbstractAccessLogger): - def log(self, request, response, time): - self.logger.info(self.log_format.format( - request=request, - response=response, - time=time - )) - - mock_logger = mock.Mock() - access_logger = Logger(mock_logger, '{request} {response} {time}') - access_logger.log('request', 'response', 1) - mock_logger.info.assert_called_with('request response 1') - - class ReifyMixin: reify = NotImplemented diff --git a/tests/test_payload.py b/tests/test_payload.py index af4ba705ae5..b75bf497bba 100644 --- a/tests/test_payload.py +++ b/tests/test_payload.py @@ -1,3 +1,4 @@ +import asyncio from io import StringIO from unittest import mock @@ -114,7 +115,8 @@ def test_async_iterable_payload_not_async_iterable() -> None: payload.AsyncIterablePayload(object()) -async def test_stream_reader_long_lines(loop) -> None: +async def test_stream_reader_long_lines() -> None: + loop = asyncio.get_event_loop() DATA = b'0' * 1024 ** 3 stream = streams.StreamReader(mock.Mock(), loop=loop) diff --git a/tests/test_web_log.py b/tests/test_web_log.py new file mode 100644 index 00000000000..8261eb84311 --- /dev/null +++ b/tests/test_web_log.py @@ -0,0 +1,159 @@ +import datetime +import platform +from unittest import mock + +import pytest + +from aiohttp.abc import AbstractAccessLogger +from aiohttp.web_log import AccessLogger + + +IS_PYPY = platform.python_implementation() == 'PyPy' + + +def test_access_logger_format() -> None: + log_format = '%T "%{ETag}o" %X {X} %%P' + mock_logger = mock.Mock() + access_logger = AccessLogger(mock_logger, log_format) + expected = '%s "%s" %%X {X} %%%s' + assert expected == access_logger._log_format + + +@pytest.mark.skip( + IS_PYPY, + """ + Because of patching :py:class:`datetime.datetime`, under PyPy it + fails in :py:func:`isinstance` call in + :py:meth:`datetime.datetime.__sub__` (called from + :py:meth:`aiohttp.AccessLogger._format_t`): + + *** TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types + + (Pdb) from datetime import datetime + (Pdb) isinstance(now, datetime) + *** TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types + (Pdb) datetime.__class__ + + (Pdb) isinstance(now, datetime.__class__) + False + + Ref: https://bitbucket.org/pypy/pypy/issues/1187/call-to-isinstance-in-__sub__-self-other + Ref: https://github.com/celery/celery/issues/811 + Ref: https://stackoverflow.com/a/46102240/595220 + """, # noqa: E501 +) +def test_access_logger_atoms(mocker) -> None: + utcnow = datetime.datetime(1843, 1, 1, 0, 30) + mock_datetime = mocker.patch("aiohttp.datetime.datetime") + mock_getpid = mocker.patch("os.getpid") + mock_datetime.utcnow.return_value = utcnow + mock_getpid.return_value = 42 + log_format = '%a %t %P %r %s %b %T %Tf %D "%{H1}i" "%{H2}i"' + mock_logger = mock.Mock() + access_logger = AccessLogger(mock_logger, log_format) + request = mock.Mock(headers={'H1': 'a', 'H2': 'b'}, + method="GET", path_qs="/path", + version=(1, 1), + remote="127.0.0.2") + response = mock.Mock(headers={}, body_length=42, status=200) + access_logger.log(request, response, 3.1415926) + assert not mock_logger.exception.called + expected = ('127.0.0.2 [01/Jan/1843:00:29:56 +0000] <42> ' + 'GET /path HTTP/1.1 200 42 3 3.141593 3141593 "a" "b"') + extra = { + 'first_request_line': 'GET /path HTTP/1.1', + 'process_id': '<42>', + 'remote_address': '127.0.0.2', + 'request_start_time': '[01/Jan/1843:00:29:56 +0000]', + 'request_time': 3, + 'request_time_frac': '3.141593', + 'request_time_micro': 3141593, + 'response_size': 42, + 'response_status': 200, + 'request_header': {'H1': 'a', 'H2': 'b'}, + } + + mock_logger.info.assert_called_with(expected, extra=extra) + + +def test_access_logger_dicts() -> None: + log_format = '%{User-Agent}i %{Content-Length}o %{None}i' + mock_logger = mock.Mock() + access_logger = AccessLogger(mock_logger, log_format) + request = mock.Mock(headers={"User-Agent": "Mock/1.0"}, version=(1, 1), + remote="127.0.0.2") + response = mock.Mock(headers={"Content-Length": 123}) + access_logger.log(request, response, 0.0) + assert not mock_logger.error.called + expected = 'Mock/1.0 123 -' + extra = { + 'request_header': {"User-Agent": "Mock/1.0", 'None': '-'}, + 'response_header': {'Content-Length': 123} + } + + mock_logger.info.assert_called_with(expected, extra=extra) + + +def test_access_logger_unix_socket() -> None: + log_format = '|%a|' + mock_logger = mock.Mock() + access_logger = AccessLogger(mock_logger, log_format) + request = mock.Mock(headers={"User-Agent": "Mock/1.0"}, version=(1, 1), + remote="") + response = mock.Mock() + access_logger.log(request, response, 0.0) + assert not mock_logger.error.called + expected = '||' + mock_logger.info.assert_called_with(expected, extra={'remote_address': ''}) + + +def test_logger_no_message() -> None: + mock_logger = mock.Mock() + access_logger = AccessLogger(mock_logger, + "%r %{content-type}i") + extra_dict = { + 'first_request_line': '-', + 'request_header': {'content-type': '(no headers)'} + } + + access_logger.log(None, None, 0.0) + mock_logger.info.assert_called_with("- (no headers)", extra=extra_dict) + + +def test_logger_internal_error() -> None: + mock_logger = mock.Mock() + access_logger = AccessLogger(mock_logger, "%D") + access_logger.log(None, None, 'invalid') + mock_logger.exception.assert_called_with("Error in logging") + + +def test_logger_no_transport() -> None: + mock_logger = mock.Mock() + access_logger = AccessLogger(mock_logger, "%a") + access_logger.log(None, None, 0) + mock_logger.info.assert_called_with("-", extra={'remote_address': '-'}) + + +def test_logger_abc() -> None: + class Logger(AbstractAccessLogger): + def log(self, request, response, time): + 1 / 0 + + mock_logger = mock.Mock() + access_logger = Logger(mock_logger, None) + + with pytest.raises(ZeroDivisionError): + access_logger.log(None, None, None) + + class Logger(AbstractAccessLogger): + def log(self, request, response, time): + self.logger.info(self.log_format.format( + request=request, + response=response, + time=time + )) + + mock_logger = mock.Mock() + access_logger = Logger(mock_logger, '{request} {response} {time}') + access_logger.log('request', 'response', 1) + mock_logger.info.assert_called_with('request response 1') diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index 9ee0a242401..47536862177 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -98,7 +98,8 @@ def ceil(val): mocker.patch('aiohttp.helpers.ceil').side_effect = ceil -async def test_shutdown(srv, loop, transport) -> None: +async def test_shutdown(srv, transport) -> None: + loop = asyncio.get_event_loop() assert transport is srv.transport srv._keepalive = True @@ -117,7 +118,7 @@ async def test_shutdown(srv, loop, transport) -> None: assert srv.transport is None assert not srv._task_handler - await asyncio.sleep(0.1, loop=loop) + await asyncio.sleep(0.1) assert task_handler.done() @@ -132,7 +133,8 @@ async def test_double_shutdown(srv, transport) -> None: assert srv.transport is None -async def test_shutdown_wait_error_handler(loop, srv, transport) -> None: +async def test_shutdown_wait_error_handler(srv, transport) -> None: + loop = asyncio.get_event_loop() async def _error_handle(): pass @@ -142,14 +144,14 @@ async def _error_handle(): assert srv._error_handler.done() -async def test_close_after_response(srv, loop, transport) -> None: +async def test_close_after_response(srv, transport) -> None: srv.data_received( b'GET / HTTP/1.0\r\n' b'Host: example.com\r\n' b'Content-Length: 0\r\n\r\n') h = srv._task_handler - await asyncio.sleep(0.1, loop=loop) + await asyncio.sleep(0.1) assert srv._waiter is None assert srv._task_handler is None @@ -192,7 +194,7 @@ def test_eof_received(make_srv) -> None: # assert srv.reader._eof -async def test_connection_lost(srv, loop) -> None: +async def test_connection_lost(srv) -> None: srv.data_received( b'GET / HTTP/1.1\r\n' b'Host: example.com\r\n' @@ -200,7 +202,7 @@ async def test_connection_lost(srv, loop) -> None: srv._keepalive = True handle = srv._task_handler - await asyncio.sleep(0, loop=loop) # wait for .start() starting + await asyncio.sleep(0) # wait for .start() starting srv.connection_lost(None) assert srv._force_close @@ -229,24 +231,24 @@ def test_srv_keep_alive_disable(srv) -> None: handle.cancel.assert_called_with() -async def test_simple(srv, loop, buf) -> None: +async def test_simple(srv, buf) -> None: srv.data_received( b'GET / HTTP/1.1\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert buf.startswith(b'HTTP/1.1 200 OK\r\n') -async def test_bad_method(srv, loop, buf) -> None: +async def test_bad_method(srv, buf) -> None: srv.data_received( b':BAD; / HTTP/1.0\r\n' b'Host: example.com\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert buf.startswith(b'HTTP/1.0 400 Bad Request\r\n') -async def test_data_received_error(srv, loop, buf) -> None: +async def test_data_received_error(srv, buf) -> None: transport = srv.transport srv._request_parser = mock.Mock() srv._request_parser.feed_data.side_effect = TypeError @@ -255,31 +257,31 @@ async def test_data_received_error(srv, loop, buf) -> None: b'!@#$ / HTTP/1.0\r\n' b'Host: example.com\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert buf.startswith(b'HTTP/1.0 500 Internal Server Error\r\n') assert transport.close.called assert srv._error_handler is None -async def test_line_too_long(srv, loop, buf) -> None: +async def test_line_too_long(srv, buf) -> None: srv.data_received(b''.join([b'a' for _ in range(10000)]) + b'\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert buf.startswith(b'HTTP/1.0 400 Bad Request\r\n') -async def test_invalid_content_length(srv, loop, buf) -> None: +async def test_invalid_content_length(srv, buf) -> None: srv.data_received( b'GET / HTTP/1.0\r\n' b'Host: example.com\r\n' b'Content-Length: sdgg\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert buf.startswith(b'HTTP/1.0 400 Bad Request\r\n') async def test_handle_error__utf( - make_srv, buf, transport, loop, request_handler + make_srv, buf, transport, request_handler ): request_handler.side_effect = RuntimeError('что-то пошло не так') @@ -292,7 +294,7 @@ async def test_handle_error__utf( b'GET / HTTP/1.0\r\n' b'Host: example.com\r\n' b'Content-Length: 0\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert b'HTTP/1.0 500 Internal Server Error' in buf assert b'Content-Type: text/html; charset=utf-8' in buf @@ -305,7 +307,7 @@ async def test_handle_error__utf( async def test_unhandled_runtime_error( - make_srv, loop, transport, request_handler + make_srv, transport, request_handler ): async def handle(request): @@ -332,7 +334,7 @@ async def handle(request): async def test_handle_uncompleted( - make_srv, loop, transport, handle_with_error, request_handler): + make_srv, transport, handle_with_error, request_handler): closed = False def close(): @@ -359,7 +361,7 @@ def close(): async def test_handle_uncompleted_pipe( - make_srv, loop, transport, request_handler, handle_with_error): + make_srv, transport, request_handler, handle_with_error): closed = False normal_completed = False @@ -376,7 +378,7 @@ def close(): async def handle(request): nonlocal normal_completed normal_completed = True - await asyncio.sleep(0.05, loop=loop) + await asyncio.sleep(0.05) return web.Response() # normal @@ -385,7 +387,7 @@ async def handle(request): b'GET / HTTP/1.1\r\n' b'Host: example.com\r\n' b'Content-Length: 0\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) # with exception request_handler.side_effect = handle_with_error() @@ -396,7 +398,7 @@ async def handle(request): assert srv._task_handler - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) await srv._task_handler assert normal_completed @@ -406,7 +408,7 @@ async def handle(request): "Error handling request", exc_info=mock.ANY) -async def test_lingering(srv, loop, transport) -> None: +async def test_lingering(srv, transport) -> None: assert not transport.close.called async def handle(message, request, writer): @@ -418,65 +420,65 @@ async def handle(message, request, writer): b'Host: example.com\r\n' b'Content-Length: 3\r\n\r\n') - await asyncio.sleep(0.05, loop=loop) + await asyncio.sleep(0.05) assert not transport.close.called srv.data_received(b'123') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) transport.close.assert_called_with() -async def test_lingering_disabled(make_srv, loop, +async def test_lingering_disabled(make_srv, transport, request_handler) -> None: async def handle_request(request): - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) srv = make_srv(lingering_time=0) srv.connection_made(transport) request_handler.side_effect = handle_request - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert not transport.close.called srv.data_received( b'GET / HTTP/1.0\r\n' b'Host: example.com\r\n' b'Content-Length: 50\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert not transport.close.called - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) transport.close.assert_called_with() async def test_lingering_timeout( - make_srv, loop, transport, ceil, request_handler + make_srv, transport, ceil, request_handler ): async def handle_request(request): - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) srv = make_srv(lingering_time=1e-30) srv.connection_made(transport) request_handler.side_effect = handle_request - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert not transport.close.called srv.data_received( b'GET / HTTP/1.0\r\n' b'Host: example.com\r\n' b'Content-Length: 50\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert not transport.close.called - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) transport.close.assert_called_with() async def test_handle_payload_access_error( - make_srv, loop, transport, request_handler + make_srv, transport, request_handler ): srv = make_srv(lingering_time=0) srv.connection_made(transport) @@ -486,20 +488,20 @@ async def test_handle_payload_access_error( b'some data' ) # start request_handler task - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) with pytest.raises(web.PayloadAccessError): await request_handler.call_args[0][0].content.read() -def test_handle_cancel(make_srv, loop, transport) -> None: +async def test_handle_cancel(make_srv, transport) -> None: log = mock.Mock() srv = make_srv(logger=log, debug=True) srv.connection_made(transport) async def handle_request(message, payload, writer): - await asyncio.sleep(10, loop=loop) + await asyncio.sleep(10) srv.handle_request = handle_request @@ -511,12 +513,37 @@ async def cancel(): b'Content-Length: 10\r\n' b'Host: example.com\r\n\r\n') - loop.run_until_complete( - asyncio.gather(srv._task_handler, cancel(), loop=loop)) + await asyncio.gather(srv._task_handler, cancel()) assert log.debug.called -def test_handle_cancelled(make_srv, loop, transport) -> None: +async def test_handle_none_response(make_srv, transport, request_handler): + loop = asyncio.get_event_loop() + log = mock.Mock() + + srv = make_srv(logger=log, debug=True) + srv.connection_made(transport) + + handle = mock.Mock() + handle.return_value = loop.create_future() + handle.return_value.set_result(None) + request_handler.side_effect = handle + + srv.data_received( + b'GET / HTTP/1.0\r\n' + b'Content-Length: 10\r\n' + b'Host: example.com\r\n\r\n') + + assert srv._task_handler + + await asyncio.sleep(0, loop=loop) + await srv._task_handler + assert request_handler.called + log.debug.assert_called_with("Possibly missing return " + "statement on request handler") + + +async def test_handle_cancelled(make_srv, transport) -> None: log = mock.Mock() srv = make_srv(logger=log, debug=True) @@ -524,35 +551,36 @@ def test_handle_cancelled(make_srv, loop, transport) -> None: srv.handle_request = mock.Mock() # start request_handler task - loop.run_until_complete(asyncio.sleep(0, loop=loop)) + await asyncio.sleep(0) srv.data_received( b'GET / HTTP/1.0\r\n' b'Host: example.com\r\n\r\n') r_handler = srv._task_handler - assert loop.run_until_complete(r_handler) is None + assert (await r_handler) is None -async def test_handle_400(srv, loop, buf, transport) -> None: +async def test_handle_400(srv, buf, transport) -> None: srv.data_received(b'GET / HT/asd\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert b'400 Bad Request' in buf -def test_handle_500(srv, loop, buf, transport, request_handler) -> None: +async def test_handle_500(srv, buf, transport, request_handler) -> None: request_handler.side_effect = ValueError srv.data_received( b'GET / HTTP/1.0\r\n' b'Host: example.com\r\n\r\n') - loop.run_until_complete(srv._task_handler) + await srv._task_handler assert b'500 Internal Server Error' in buf -async def test_keep_alive(make_srv, loop, transport, ceil) -> None: +async def test_keep_alive(make_srv, transport, ceil) -> None: + loop = asyncio.get_event_loop() srv = make_srv(keepalive_timeout=0.05) srv.KEEPALIVE_RESCHEDULE_DELAY = 0.1 srv.connection_made(transport) @@ -567,19 +595,19 @@ async def test_keep_alive(make_srv, loop, transport, ceil) -> None: b'Host: example.com\r\n' b'Content-Length: 0\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) waiter = srv._waiter assert waiter assert srv._keepalive_handle is not None assert not transport.close.called - await asyncio.sleep(0.2, loop=loop) + await asyncio.sleep(0.2) assert transport.close.called assert waiter.cancelled -def test_srv_process_request_without_timeout(make_srv, - loop, transport) -> None: +async def test_srv_process_request_without_timeout(make_srv, + transport) -> None: srv = make_srv() srv.connection_made(transport) @@ -587,7 +615,7 @@ def test_srv_process_request_without_timeout(make_srv, b'GET / HTTP/1.0\r\n' b'Host: example.com\r\n\r\n') - loop.run_until_complete(srv._task_handler) + await srv._task_handler assert transport.close.called @@ -600,12 +628,12 @@ def test_keep_alive_timeout_nondefault(make_srv) -> None: assert 10 == srv.keepalive_timeout -async def test_supports_connect_method(srv, loop, +async def test_supports_connect_method(srv, transport, request_handler) -> None: srv.data_received( b'CONNECT aiohttp.readthedocs.org:80 HTTP/1.0\r\n' b'Content-Length: 0\r\n\r\n') - await asyncio.sleep(0.1, loop=loop) + await asyncio.sleep(0.1) assert request_handler.called assert isinstance( @@ -613,18 +641,18 @@ async def test_supports_connect_method(srv, loop, streams.StreamReader) -async def test_content_length_0(srv, loop, request_handler) -> None: +async def test_content_length_0(srv, request_handler) -> None: srv.data_received( b'GET / HTTP/1.1\r\n' b'Host: example.org\r\n' b'Content-Length: 0\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert request_handler.called assert request_handler.call_args[0][0].content == streams.EMPTY_PAYLOAD -def test_rudimentary_transport(srv, loop) -> None: +def test_rudimentary_transport(srv) -> None: transport = mock.Mock() srv.connection_made(transport) @@ -647,10 +675,10 @@ def test_rudimentary_transport(srv, loop) -> None: assert not srv._reading_paused -async def test_close(srv, loop, transport) -> None: +async def test_close(srv, transport) -> None: transport.close.side_effect = partial(srv.connection_lost, None) srv.connection_made(transport) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) srv.handle_request = mock.Mock() srv.handle_request.side_effect = helpers.noop @@ -666,19 +694,19 @@ async def test_close(srv, loop, transport) -> None: b'Host: example.com\r\n' b'Content-Length: 0\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert srv._task_handler assert srv._waiter srv.close() - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert srv._task_handler is None assert srv.transport is None assert transport.close.called async def test_pipeline_multiple_messages( - srv, loop, transport, request_handler + srv, transport, request_handler ): transport.close.side_effect = partial(srv.connection_lost, None) @@ -706,14 +734,14 @@ async def handle(request): assert len(srv._messages) == 2 assert srv._waiter is not None - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert srv._task_handler is not None assert srv._waiter is not None assert processed == 2 async def test_pipeline_response_order( - srv, loop, buf, transport, request_handler + srv, buf, transport, request_handler ): transport.close.side_effect = partial(srv.connection_lost, None) srv._keepalive = True @@ -722,7 +750,7 @@ async def test_pipeline_response_order( async def handle1(request): nonlocal processed - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) resp = web.StreamResponse() await resp.prepare(request) await resp.write(b'test1') @@ -735,7 +763,7 @@ async def handle1(request): b'GET / HTTP/1.1\r\n' b'Host: example.com\r\n' b'Content-Length: 0\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) # second @@ -753,11 +781,11 @@ async def handle2(request): b'GET / HTTP/1.1\r\n' b'Host: example.com\r\n' b'Content-Length: 0\r\n\r\n') - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert srv._task_handler is not None - await asyncio.sleep(0.1, loop=loop) + await asyncio.sleep(0.1) assert processed == [1, 2] @@ -781,7 +809,8 @@ def test_data_received_force_close(srv) -> None: assert not srv._messages -async def test__process_keepalive(loop, srv) -> None: +async def test__process_keepalive(srv) -> None: + loop = asyncio.get_event_loop() # wait till the waiter is waiting await asyncio.sleep(0) @@ -796,7 +825,8 @@ async def test__process_keepalive(loop, srv) -> None: assert srv._force_close -async def test__process_keepalive_schedule_next(loop, srv) -> None: +async def test__process_keepalive_schedule_next(srv) -> None: + loop = asyncio.get_event_loop() # wait till the waiter is waiting await asyncio.sleep(0) @@ -813,16 +843,17 @@ async def test__process_keepalive_schedule_next(loop, srv) -> None: ) -def test__process_keepalive_force_close(loop, srv) -> None: +async def test__process_keepalive_force_close(srv) -> None: + loop = asyncio.get_event_loop() srv._force_close = True with mock.patch.object(loop, "call_at") as call_at_patched: srv._process_keepalive() assert not call_at_patched.called -def test_two_data_received_without_waking_up_start_task(srv, loop) -> None: +async def test_two_data_received_without_waking_up_start_task(srv) -> None: # make a chance to srv.start() method start waiting for srv._waiter - loop.run_until_complete(asyncio.sleep(0.01)) + await asyncio.sleep(0.01) assert srv._waiter is not None srv.data_received( diff --git a/tests/test_web_request.py b/tests/test_web_request.py index b5e1d639380..cf4f6801d15 100644 --- a/tests/test_web_request.py +++ b/tests/test_web_request.py @@ -511,8 +511,8 @@ def test_clone_headers_dict() -> None: assert req2.raw_headers == ((b'B', b'C'),) -async def test_cannot_clone_after_read(loop, protocol) -> None: - payload = StreamReader(protocol, loop=loop) +async def test_cannot_clone_after_read(protocol) -> None: + payload = StreamReader(protocol) payload.feed_data(b'data') payload.feed_eof() req = make_mocked_request('GET', '/path', payload=payload) @@ -521,8 +521,8 @@ async def test_cannot_clone_after_read(loop, protocol) -> None: req.clone() -async def test_make_too_big_request(loop, protocol) -> None: - payload = StreamReader(protocol, loop=loop) +async def test_make_too_big_request(protocol) -> None: + payload = StreamReader(protocol) large_file = 1024 ** 2 * b'x' too_large_file = large_file + b'x' payload.feed_data(too_large_file) @@ -534,8 +534,8 @@ async def test_make_too_big_request(loop, protocol) -> None: assert err.value.status_code == 413 -async def test_make_too_big_request_adjust_limit(loop, protocol) -> None: - payload = StreamReader(protocol, loop=loop) +async def test_make_too_big_request_adjust_limit(protocol) -> None: + payload = StreamReader(protocol) large_file = 1024 ** 2 * b'x' too_large_file = large_file + b'x' payload.feed_data(too_large_file) @@ -547,8 +547,8 @@ async def test_make_too_big_request_adjust_limit(loop, protocol) -> None: assert len(txt) == 1024**2 + 1 -async def test_multipart_formdata(loop, protocol) -> None: - payload = StreamReader(protocol, loop=loop) +async def test_multipart_formdata(protocol) -> None: + payload = StreamReader(protocol) payload.feed_data(b"""-----------------------------326931944431359\r Content-Disposition: form-data; name="a"\r \r @@ -568,8 +568,8 @@ async def test_multipart_formdata(loop, protocol) -> None: assert dict(result) == {'a': 'b', 'c': 'd'} -async def test_make_too_big_request_limit_None(loop, protocol) -> None: - payload = StreamReader(protocol, loop=loop) +async def test_make_too_big_request_limit_None(protocol) -> None: + payload = StreamReader(protocol) large_file = 1024 ** 2 * b'x' too_large_file = large_file + b'x' payload.feed_data(too_large_file) diff --git a/tests/test_web_response.py b/tests/test_web_response.py index 5ae43ebb782..8e970fe1a66 100644 --- a/tests/test_web_response.py +++ b/tests/test_web_response.py @@ -326,7 +326,8 @@ async def test_force_compression_no_accept_backwards_compat() -> None: assert not resp.chunked assert not resp.compression - resp.enable_compression(force=True) + with pytest.warns(DeprecationWarning): + resp.enable_compression(force=True) assert resp.compression msg = await resp.prepare(req) @@ -339,7 +340,8 @@ async def test_force_compression_false_backwards_compat() -> None: resp = StreamResponse() assert not resp.compression - resp.enable_compression(force=False) + with pytest.warns(DeprecationWarning): + resp.enable_compression(force=False) assert resp.compression msg = await resp.prepare(req)