From 9cfa4fe39e2a3b8fe4e7052f8dda02a7adfe2123 Mon Sep 17 00:00:00 2001 From: Nikolay Kim Date: Sun, 26 Feb 2017 22:31:15 -0800 Subject: [PATCH] unify ServerHttpProtocol and RequestHandler; replace FileSender with FileResponse --- aiohttp/__init__.py | 4 +- aiohttp/http.py | 4 +- aiohttp/http_message.py | 26 +- aiohttp/test_utils.py | 21 +- aiohttp/web.py | 16 +- .../{file_sender.py => web_fileresponse.py} | 69 ++-- aiohttp/{server.py => web_protocol.py} | 261 +++++++-------- aiohttp/web_response.py | 12 +- aiohttp/web_server.py | 134 +------- aiohttp/web_urldispatcher.py | 9 +- tests/test_client_functional_oldstyle.py | 106 +++--- tests/test_http_message.py | 137 +++----- tests/test_web_exceptions.py | 16 +- .../{test_server.py => test_web_protocol.py} | 314 ++++++------------ tests/test_web_request_handler.py | 3 +- tests/test_web_response.py | 69 ++-- tests/test_web_sendfile.py | 26 +- tests/test_web_sendfile_functional.py | 36 +- tests/test_web_server.py | 4 +- tests/test_web_websocket.py | 4 +- 20 files changed, 517 insertions(+), 754 deletions(-) rename aiohttp/{file_sender.py => web_fileresponse.py} (77%) rename aiohttp/{server.py => web_protocol.py} (73%) rename tests/{test_server.py => test_web_protocol.py} (68%) diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 7c25157230d..5007b99f96f 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -10,7 +10,6 @@ from .http_websocket import WSMsgType, WSCloseCode, WSMessage, WebSocketError # noqa from .streams import * # noqa from .multipart import * # noqa -from .file_sender import FileSender # noqa from .cookiejar import CookieJar # noqa from .payload import * # noqa from .payload_streamer import * # noqa @@ -32,8 +31,7 @@ payload.__all__ + # noqa payload_streamer.__all__ + # noqa streams.__all__ + # noqa - ('hdrs', 'FileSender', - 'HttpVersion', 'HttpVersion10', 'HttpVersion11', + ('hdrs', 'HttpVersion', 'HttpVersion10', 'HttpVersion11', 'WSMsgType', 'MsgType', 'WSCloseCode', 'WebSocketError', 'WSMessage', 'CookieJar', diff --git a/aiohttp/http.py b/aiohttp/http.py index 41957627172..7f908fe620c 100644 --- a/aiohttp/http.py +++ b/aiohttp/http.py @@ -1,3 +1,5 @@ +from yarl import URL # noqa + from .http_exceptions import HttpProcessingError from .http_message import (RESPONSES, SERVER_SOFTWARE, HttpMessage, HttpVersion, HttpVersion10, HttpVersion11, @@ -13,7 +15,7 @@ # .http_message 'RESPONSES', 'SERVER_SOFTWARE', - 'HttpMessage', 'Request', 'Response', 'PayloadWriter', + 'HttpMessage', 'Request', 'PayloadWriter', 'HttpVersion', 'HttpVersion10', 'HttpVersion11', # .http_parser diff --git a/aiohttp/http_message.py b/aiohttp/http_message.py index e7bd1f0fa94..d0b5bf392aa 100644 --- a/aiohttp/http_message.py +++ b/aiohttp/http_message.py @@ -7,7 +7,6 @@ import sys import zlib from urllib.parse import SplitResult -from wsgiref.handlers import format_date_time import yarl from multidict import CIMultiDict, istr @@ -36,7 +35,7 @@ class PayloadWriter(AbstractPayloadWriter): - def __init__(self, stream, loop): + def __init__(self, stream, loop, acquire=True): if loop is None: loop = asyncio.get_event_loop() @@ -53,13 +52,29 @@ def __init__(self, stream, loop): self._compress = None self._drain_waiter = None + self._replacement = None + if self._stream.available: self._transport = self._stream.transport self._stream.available = False - else: + elif acquire: self._stream.acquire(self.set_transport) + def replace(self, factory): + """Hack: for internal use only """ + if self._transport is not None: + self._transport = None + self._stream.available = True + return factory(self._stream, self.loop) + else: + self._replacement = factory(self._stream, self.loop, False) + return self._replacement + def set_transport(self, transport): + if self._replacement is not None: + self._replacement.set_transport(transport) + return + self._transport = transport chunk = b''.join(self._buffer) @@ -196,7 +211,7 @@ def drain(self, last=False): class HttpMessage(PayloadWriter): """HttpMessage allows to write headers and payload to a stream.""" - HOP_HEADERS = None # Must be set by subclass. + HOP_HEADERS = () # Must be set by subclass. SERVER_SOFTWARE = 'Python/{0[0]}.{0[1]} aiohttp/{1}'.format( sys.version_info, aiohttp.__version__) @@ -205,7 +220,8 @@ class HttpMessage(PayloadWriter): websocket = False # Upgrade: WEBSOCKET has_chunked_hdr = False # Transfer-encoding: chunked - def __init__(self, transport, version, close, loop=None): + def __init__(self, transport, + version=HttpVersion11, close=False, loop=None): super().__init__(transport, loop) self.version = version diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index a47ae4ca48e..b2e52f5ead5 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -17,8 +17,8 @@ from aiohttp.client import _RequestContextManager from . import ClientSession, hdrs -from .helpers import PY_35, sentinel -from .http import HttpVersion, PayloadWriter, RawRequestMessage +from .helpers import PY_35, noop, sentinel +from .http import HttpVersion, RawRequestMessage from .signals import Signal from .web import Application, Request, Server, UrlMappingMatchInfo @@ -484,6 +484,7 @@ def make_mocked_request(method, path, headers=None, *, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, + payload_writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, @@ -497,6 +498,10 @@ def make_mocked_request(method, path, headers=None, *, """ + task = mock.Mock() + loop = mock.Mock() + loop.create_future.return_value = () + if version < HttpVersion(1, 1): closing = True @@ -526,6 +531,10 @@ def make_mocked_request(method, path, headers=None, *, writer = mock.Mock() writer.transport = transport + if payload_writer is sentinel: + payload_writer = mock.Mock() + payload_writer.write_eof.side_effect = noop + protocol.transport = transport protocol.writer = writer @@ -543,14 +552,8 @@ def timeout(*args, **kw): time_service.timeout = mock.Mock() time_service.timeout.side_effect = timeout - task = mock.Mock() - loop = mock.Mock() - loop.create_future.return_value = () - - w = PayloadWriter(writer, loop=loop) - req = Request(message, payload, - protocol, w, time_service, task, + protocol, payload_writer, time_service, task, secure_proxy_ssl_header=secure_proxy_ssl_header, client_max_size=client_max_size) diff --git a/aiohttp/web.py b/aiohttp/web.py index b658757c4f6..e4072d6124e 100644 --- a/aiohttp/web.py +++ b/aiohttp/web.py @@ -10,15 +10,18 @@ from yarl import URL -from . import (hdrs, web_exceptions, web_middlewares, web_request, - web_response, web_server, web_urldispatcher, web_ws) +from . import (hdrs, web_exceptions, web_fileresponse, web_middlewares, + web_protocol, web_request, web_response, web_server, + web_urldispatcher, web_ws) from .abc import AbstractMatchInfo, AbstractRouter from .helpers import FrozenList from .http import HttpVersion # noqa from .log import access_logger, web_logger from .signals import PostSignal, PreSignal, Signal from .web_exceptions import * # noqa +from .web_fileresponse import * # noqa from .web_middlewares import * # noqa +from .web_protocol import * # noqa from .web_request import * # noqa from .web_response import * # noqa from .web_server import Server @@ -26,7 +29,9 @@ from .web_urldispatcher import PrefixedSubAppResource from .web_ws import * # noqa -__all__ = (web_request.__all__ + +__all__ = (web_protocol.__all__ + + web_fileresponse.__all__ + + web_request.__all__ + web_response.__all__ + web_exceptions.__all__ + web_urldispatcher.__all__ + @@ -222,10 +227,10 @@ def cleanup(self): """ yield from self.on_cleanup.send(self) - def _make_request(self, message, payload, protocol, writer, + def _make_request(self, message, payload, protocol, writer, task, _cls=web_request.Request): return _cls( - message, payload, protocol, writer, protocol._time_service, None, + message, payload, protocol, writer, protocol._time_service, task, secure_proxy_ssl_header=self._secure_proxy_ssl_header, client_max_size=self._client_max_size) @@ -250,6 +255,7 @@ def _handle(self, request): for app in match_info.apps: for factory in app._middlewares: handler = yield from factory(app, handler) + resp = yield from handler(request) assert isinstance(resp, web_response.StreamResponse), \ diff --git a/aiohttp/file_sender.py b/aiohttp/web_fileresponse.py similarity index 77% rename from aiohttp/file_sender.py rename to aiohttp/web_fileresponse.py index c368d812517..b03881783df 100644 --- a/aiohttp/file_sender.py +++ b/aiohttp/web_fileresponse.py @@ -1,6 +1,7 @@ import asyncio import mimetypes import os +import pathlib from . import hdrs from .helpers import create_future @@ -9,6 +10,8 @@ HTTPRequestRangeNotSatisfiable) from .web_response import StreamResponse +__all__ = ('FileResponse',) + NOSENDFILE = bool(os.environ.get("AIOHTTP_NOSENDFILE")) @@ -81,15 +84,20 @@ def write_eof(self, chunk=b''): pass -class FileSender: - """A helper that can be used to send files.""" +class FileResponse(StreamResponse): + """A response object can be used to send files.""" + + def __init__(self, path, chunk_size=256*1024, *args, **kwargs): + super().__init__(*args, **kwargs) + + if isinstance(path, str): + path = pathlib.Path(path) - def __init__(self, *, resp_factory=StreamResponse, chunk_size=256*1024): - self._response_factory = resp_factory + self._path = path self._chunk_size = chunk_size @asyncio.coroutine - def _sendfile_system(self, request, resp, fobj, count): + def _sendfile_system(self, request, fobj, count): # Write count bytes of fobj to resp using # the os.sendfile system call. # @@ -103,14 +111,17 @@ def _sendfile_system(self, request, resp, fobj, count): transport = request.transport if transport.get_extra_info("sslcontext"): - yield from self._sendfile_fallback(request, resp, fobj, count) + writer = yield from self._sendfile_fallback(request, fobj, count) else: - writer = yield from resp.prepare( - request, PayloadWriterFactory=SendfilePayloadWriter) + writer = request._writer.replace(SendfilePayloadWriter) + request._writer = writer + yield from super().prepare(request) yield from writer.sendfile(fobj, count) + return writer + @asyncio.coroutine - def _sendfile_fallback(self, request, resp, fobj, count): + def _sendfile_fallback(self, request, fobj, count): # Mimic the _sendfile_system() method, but without using the # os.sendfile() system call. This should be used on systems # that don't support the os.sendfile(). @@ -119,21 +130,23 @@ def _sendfile_fallback(self, request, resp, fobj, count): # fobj is transferred in chunks controlled by the # constructor's chunk_size argument. - yield from resp.prepare(request) + writer = (yield from super().prepare(request)) - resp.set_tcp_cork(True) + self.set_tcp_cork(True) try: chunk_size = self._chunk_size chunk = fobj.read(chunk_size) while True: - yield from resp.write(chunk) + yield from writer.write(chunk) count = count - chunk_size if count <= 0: break chunk = fobj.read(min(chunk_size, count)) finally: - resp.set_tcp_nodelay(True) + self.set_tcp_nodelay(True) + + yield from writer.drain() if hasattr(os, "sendfile") and not NOSENDFILE: # pragma: no cover _sendfile = _sendfile_system @@ -141,8 +154,9 @@ def _sendfile_fallback(self, request, resp, fobj, count): _sendfile = _sendfile_fallback @asyncio.coroutine - def send(self, request, filepath): - """Send filepath to client using request.""" + def prepare(self, request): + filepath = self._path + gzip = False if 'gzip' in request.headers.get(hdrs.ACCEPT_ENCODING, ''): gzip_path = filepath.with_name(filepath.name + '.gz') @@ -155,7 +169,8 @@ def send(self, request, filepath): modsince = request.if_modified_since if modsince is not None and st.st_mtime <= modsince.timestamp(): - raise HTTPNotModified() + self.set_status(HTTPNotModified.status_code) + return (yield from super().prepare(request)) ct, encoding = mimetypes.guess_type(str(filepath)) if not ct: @@ -170,7 +185,8 @@ def send(self, request, filepath): start = rng.start end = rng.stop except ValueError: - raise HTTPRequestRangeNotSatisfiable + self.set_status(HTTPRequestRangeNotSatisfiable.status_code) + return (yield from super().prepare(request)) # If a range request has been made, convert start, end slice notation # into file pointer offset and count @@ -192,18 +208,17 @@ def send(self, request, filepath): # the current length of the selected representation). count = file_size - start - resp = self._response_factory(status=status) - resp.content_type = ct + self.set_status(status) + self.content_type = ct if encoding: - resp.headers[hdrs.CONTENT_ENCODING] = encoding + self.headers[hdrs.CONTENT_ENCODING] = encoding if gzip: - resp.headers[hdrs.VARY] = hdrs.ACCEPT_ENCODING - resp.last_modified = st.st_mtime + self.headers[hdrs.VARY] = hdrs.ACCEPT_ENCODING + self.last_modified = st.st_mtime + self.content_length = count - resp.content_length = count - with filepath.open('rb') as f: + with filepath.open('rb') as fobj: if start: - f.seek(start) - yield from self._sendfile(request, resp, f, count) + fobj.seek(start) - return resp + return (yield from self._sendfile(request, fobj, count)) diff --git a/aiohttp/server.py b/aiohttp/web_protocol.py similarity index 73% rename from aiohttp/server.py rename to aiohttp/web_protocol.py index e3133f23bad..0392d157e73 100644 --- a/aiohttp/server.py +++ b/aiohttp/web_protocol.py @@ -1,5 +1,3 @@ -"""simple HTTP server.""" - import asyncio import asyncio.streams import http.server @@ -10,27 +8,20 @@ from contextlib import suppress from html import escape as html_escape -from . import hdrs, helpers -from .helpers import CeilTimeout, TimeService, create_future, ensure_future +from . import helpers, http +from .helpers import CeilTimeout, create_future, ensure_future from .http import HttpProcessingError, HttpRequestParser, PayloadWriter from .log import access_logger, server_logger -from .streams import StreamWriter - -__all__ = ('ServerHttpProtocol',) +from .streams import EMPTY_PAYLOAD, StreamWriter +from .web_exceptions import HTTPException +from .web_request import BaseRequest +from .web_response import Response +__all__ = ('RequestHandler',) -RESPONSES = http.server.BaseHTTPRequestHandler.responses -DEFAULT_ERROR_MESSAGE = """ - - - {status} {reason} - - -

{status} {reason}

- {message} - -""" - +ERROR = http.RawRequestMessage( + 'UNKNOWN', '/', http.HttpVersion10, {}, + {}, True, False, False, False, http.URL('/')) if hasattr(socket, 'SO_KEEPALIVE'): def tcp_keepalive(server, transport): @@ -41,14 +32,14 @@ def tcp_keepalive(server, transport): # pragma: no cover pass -class ServerHttpProtocol(asyncio.streams.FlowControlMixin, asyncio.Protocol): - """Simple HTTP protocol implementation. +class RequestHandler(asyncio.streams.FlowControlMixin, asyncio.Protocol): + """HTTP protocol implementation. - ServerHttpProtocol handles incoming HTTP request. It reads request line, + RequestHandler handles incoming HTTP request. It reads request line, request headers and request payload and calls handle_request() method. By default it always returns with 404 response. - ServerHttpProtocol handles errors in incoming request, like bad + RequestHandler handles errors in incoming request, like bad status line, bad headers or incomplete payload. If any error occurs, connection gets closed. @@ -82,8 +73,7 @@ class ServerHttpProtocol(asyncio.streams.FlowControlMixin, asyncio.Protocol): _request_count = 0 _keepalive = False # keep transport open - def __init__(self, *, loop=None, - time_service=None, + def __init__(self, manager, *, loop=None, keepalive_timeout=75, # NGINX default value is 75 secs tcp_keepalive=True, slow_request_timeout=None, @@ -94,8 +84,7 @@ def __init__(self, *, loop=None, max_line_size=8190, max_headers=32768, max_field_size=8190, - lingering_time=30.0, - lingering_timeout=5.0, + lingering_time=10.0, max_concurrent_handlers=2, **kwargs): @@ -109,19 +98,17 @@ def __init__(self, *, loop=None, super().__init__(loop=loop) self._loop = loop if loop is not None else asyncio.get_event_loop() - if time_service is not None: - self._time_service_owner = False - self._time_service = time_service - else: - self._time_service_owner = True - self._time_service = TimeService(self._loop) + + self._manager = manager + self._time_service = manager.time_service + self._request_handler = manager.request_handler + self._request_factory = manager.request_factory self._tcp_keepalive = tcp_keepalive self._keepalive_time = None self._keepalive_handle = None self._keepalive_timeout = keepalive_timeout self._lingering_time = float(lingering_time) - self._lingering_timeout = float(lingering_timeout) self._messages = deque() self._message_tail = b'' @@ -154,6 +141,20 @@ def __init__(self, *, loop=None, self._close = False self._force_close = False + def __repr__(self): + self._request = None + if self._request is None: + meth = 'none' + path = 'none' + else: + meth = 'none' + path = 'none' + # meth = self._request.method + # path = self._request.rel_url.raw_path + return "<{} {}:{} {}>".format( + self.__class__.__name__, meth, path, + 'connected' if self.transport is not None else 'disconnected') + @property def time_service(self): return self._time_service @@ -216,11 +217,17 @@ def connection_made(self, transport): tcp_keepalive(self, transport) self.writer.set_tcp_nodelay(True) + self._manager.connection_made(self, transport) def connection_lost(self, exc): + self._manager.connection_lost(self, exc) + super().connection_lost(exc) + self._manager = None self._force_close = True + self._request_factory = None + self._request_handler = None self._request_parser = None self.transport = self.writer = None @@ -241,9 +248,6 @@ def connection_lost(self, exc): self._request_handlers = () - if self._time_service_owner: - self._time_service.close() - def set_parser(self, parser): assert self._payload_parser is None @@ -268,17 +272,17 @@ def data_received(self, data): # something happened during parsing self.close() self._error_handler = ensure_future( - self.handle_error( + self.handle_parse_error( PayloadWriter(self.writer, self._loop), - 400, None, exc, exc.message), + 400, exc, exc.message), loop=self._loop) except Exception as exc: # 500: internal error self.close() self._error_handler = ensure_future( - self.handle_error( + self.handle_parse_error( PayloadWriter(self.writer, self._loop), - 500, None, exc), loop=self._loop) + 500, exc), loop=self._loop) else: for (msg, payload) in messages: self._request_count += 1 @@ -391,13 +395,47 @@ def start(self, message, payload, handler): """ loop = self._loop handler = handler[0] + manager = self._manager keepalive_timeout = self._keepalive_timeout while not self._force_close: - try: - writer = PayloadWriter(self.writer, loop) - yield from self.handle_request(message, payload, writer) + if self.access_log: + now = loop.time() + manager.requests_count += 1 + writer = PayloadWriter(self.writer, loop) + request = self._request_factory( + message, payload, self, writer, handler) + try: + try: + resp = yield from self._request_handler(request) + except HTTPException as exc: + resp = exc + except asyncio.CancelledError: + self.log_debug('Ignored premature client disconnection') + break + except asyncio.TimeoutError: + self.log_debug('Request handler timed out.') + resp = self.handle_error(request, 504) + except Exception as exc: + resp = self.handle_error(request, 500, exc) + + yield from resp.prepare(request) + yield from resp.write_eof() + + # notify server about keep-alive + self._keepalive = resp.keep_alive + + # Restore default state. + # Should be no-op if server code didn't touch these attributes. + writer.set_tcp_cork(False) + writer.set_tcp_nodelay(True) + + # log access + if self.access_log: + self.log_access(message, None, resp, loop.time() - now) + + # check payload if not payload.is_eof(): lingering_time = self._lingering_time if not self._force_close and lingering_time: @@ -421,16 +459,9 @@ def start(self, message, payload, handler): self.log_debug('Uncompleted request.') self.close() - except asyncio.CancelledError: - self.log_debug('Ignored premature client disconnection') - break - except asyncio.TimeoutError: - self.log_debug('Request handler timed out.') - yield from self.handle_error(writer, 504, message) - break except Exception as exc: - yield from self.handle_error(writer, 500, message, exc) - break + self.log_exception('Unhandled exception', exc_info=exc) + self.force_close() finally: if self.transport is None: self.log_debug('Ignored premature client disconnection.') @@ -466,107 +497,51 @@ def start(self, message, payload, handler): if self.transport is not None: self.transport.close() - @asyncio.coroutine - def handle_error(self, writer, status=500, message=None, - exc=None, reason=None, SEP=': ', END='\r\n'): + def handle_error(self, request, status=500, exc=None, message=None): """Handle errors. Returns HTTP response with specific status code. Logs additional information. It always closes current connection.""" - if self.access_log: - now = self._loop.time() + self.log_exception("Error handling request", exc_info=exc) if status == 500: - self.log_exception("Error handling request") - - try: - # some data already got sent, connection is broken - if writer.output_size > 0 or self.transport is None: - self.force_close() - return - - try: - if not reason: - reason, msg = RESPONSES[status] - else: - msg = reason - reason, _ = RESPONSES[status] - except KeyError: - status = 500 - reason, msg = RESPONSES[500] - - writer.status = status - - if self.debug and exc is not None: + msg = "

500 Internal Server Error

" + if self.debug: try: tb = traceback.format_exc() tb = html_escape(tb) - msg += '

Traceback:

\n
{}
'.format(tb) - except: + msg += '

Traceback:

\n
'
+                    msg += tb
+                    msg += '
' + except: # pragma: no cover pass + else: + msg += "Server got itself in trouble" + msg = ("500 Internal Server Error" + "" + msg + "") + else: + msg = message - html = DEFAULT_ERROR_MESSAGE.format( - status=status, reason=reason, message=msg).encode('utf-8') - - headers = { - hdrs.CONNECTION: 'close', - hdrs.CONTENT_TYPE: 'text/html; charset=utf-8', - hdrs.CONTENT_LENGTH: str(len(html)), - hdrs.DATE: self._time_service.strtime()} - writer.headers = headers - - # status line - status_line = 'HTTP/1.1 {} {}\r\n'.format(status, reason) - - # status + headers - headers = status_line + ''.join( - [k + SEP + v + END for k, v in headers.items()]) - headers = headers.encode('utf-8') + b'\r\n' - writer.buffer_data(headers + html) - - # disable CORK, enable NODELAY if needed - writer.set_tcp_nodelay(True) - yield from writer.write_eof() - finally: - self.keep_alive(False) - if self.access_log: - self.log_access(message, None, writer, self._loop.time() - now) + resp = Response(status=status, text=msg, content_type='text/html') + resp.force_close() - @asyncio.coroutine - def handle_request(self, message, payload, writer, SEP=': ', END='\r\n'): - """Handle a single HTTP request. + # some data already got sent, connection is broken + if request.writer.output_size > 0 or self.transport is None: + self.force_close() - Subclass should override this method. By default it always - returns 404 response. + return resp - :param message: Request headers - :type message: aiohttp.protocol.HttpRequestParser - :param payload: Request payload - :type payload: aiohttp.streams.FlowControlStreamReader - """ - if self.access_log: - now = self._loop.time() - - body = b'Page Not Found!' - headers = { - hdrs.CONNECTION: 'close', - hdrs.CONTENT_TYPE: 'text/plain', - hdrs.CONTENT_LENGTH: str(len(body)), - hdrs.DATE: self._time_service.strtime()} - writer.status = 404 - writer.headers = headers - - # status line - status_line = 'HTTP/{}.{} {} {}\r\n'.format( - message.version[0], message.version[1], 404, 'Not Found') - - # status + headers - headers = status_line + ''.join( - [k + SEP + v + END for k, v in headers.items()]) - headers = headers.encode('utf-8') + b'\r\n' - writer.buffer_data(headers + body) - yield from writer.write_eof() - - self.keep_alive(False) - if self.access_log: - self.log_access(message, None, response, self._loop.time() - now) + @asyncio.coroutine + def handle_parse_error(self, writer, status, exc=None, message=None): + request = BaseRequest( + ERROR, EMPTY_PAYLOAD, + self, writer, self._time_service, None) + + resp = self.handle_error(request, status, exc, message) + yield from resp.prepare(request) + yield from resp.write_eof() + + # Restore default state. + # Should be no-op if server code didn't touch these attributes. + self.writer.set_tcp_cork(False) + self.writer.set_tcp_nodelay(True) diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index 33609c9c061..ed844305ad8 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -11,8 +11,7 @@ from . import hdrs, payload from .helpers import HeadersMixin, SimpleCookie, sentinel -from .http import (RESPONSES, SERVER_SOFTWARE, HttpVersion10, - HttpVersion11, PayloadWriter) +from .http import RESPONSES, SERVER_SOFTWARE, HttpVersion10, HttpVersion11 __all__ = ('ContentCoding', 'StreamResponse', 'Response', 'json_response') @@ -305,15 +304,14 @@ def _start_compression(self, request): return @asyncio.coroutine - def prepare(self, request, PayloadWriterFactory=PayloadWriter): + def prepare(self, request): if self._payload_writer is not None: return self._payload_writer yield from request._prepare_hook(self) - return self._start(request, PayloadWriterFactory=PayloadWriterFactory) + return self._start(request) def _start(self, request, - PayloadWriterFactory=PayloadWriter, HttpVersion10=HttpVersion10, HttpVersion11=HttpVersion11, CONNECTION=hdrs.CONNECTION, @@ -575,14 +573,14 @@ def write_eof(self): else: yield from super().write_eof() - def _start(self, request, PayloadWriterFactory=PayloadWriter): + def _start(self, request): if not self._chunked and hdrs.CONTENT_LENGTH not in self._headers: if self._body is not None: self._headers[hdrs.CONTENT_LENGTH] = str(len(self._body)) else: self._headers[hdrs.CONTENT_LENGTH] = '0' - return super()._start(request, PayloadWriterFactory) + return super()._start(request) def json_response(data=sentinel, *, text=None, body=None, status=200, diff --git a/aiohttp/web_server.py b/aiohttp/web_server.py index 6cffcaa78ce..8e240e2e0c4 100644 --- a/aiohttp/web_server.py +++ b/aiohttp/web_server.py @@ -1,106 +1,11 @@ """Low level HTTP server.""" - import asyncio -import traceback -from html import escape as html_escape from .helpers import TimeService -from .server import ServerHttpProtocol -from .web_exceptions import HTTPException, HTTPInternalServerError +from .web_protocol import RequestHandler from .web_request import BaseRequest -__all__ = ('RequestHandler', 'Server') - - -class RequestHandler(ServerHttpProtocol): - _request = None - - def __init__(self, manager, **kwargs): - kwargs['time_service'] = manager.time_service - - super().__init__(**kwargs) - - self._manager = manager - self._request_factory = manager.request_factory - self._handler = manager.handler - - def __repr__(self): - if self._request is None: - meth = 'none' - path = 'none' - else: - meth = self._request.method - path = self._request.rel_url.raw_path - return "<{} {}:{} {}>".format( - self.__class__.__name__, meth, path, - 'connected' if self.transport is not None else 'disconnected') - - def connection_made(self, transport): - super().connection_made(transport) - - self._manager.connection_made(self, transport) - - def connection_lost(self, exc): - self._manager.connection_lost(self, exc) - - super().connection_lost(exc) - self._request_factory = None - self._manager = None - self._handler = None - - @asyncio.coroutine - def handle_request(self, message, payload, writer): - self._manager._requests_count += 1 - if self.access_log: - now = self._loop.time() - - request = self._request_factory(message, payload, self, writer) - self._request = request - - try: - resp = yield from self._handler(request) - except (asyncio.CancelledError, asyncio.TimeoutError): - raise - except HTTPException as exc: - resp = exc - except Exception as exc: - msg = "

500 Internal Server Error

" - if self.debug: - try: - tb = traceback.format_exc() - tb = html_escape(tb) - msg += '

Traceback:

\n
'
-                    msg += tb
-                    msg += '
' - except: # pragma: no cover - pass - else: - msg += "Server got itself in trouble" - msg = ("500 Internal Server Error" - "" + msg + "") - resp = HTTPInternalServerError( - text=msg, content_type='text/html') - self.logger.exception( - "Error handling request", exc_info=exc) - - yield from resp.prepare(request) - yield from resp.write_eof() - - # notify server about keep-alive - # assign to parent class attr - self._keepalive = resp.keep_alive - - # Restore default state. - # Should be no-op if server code didn't touch these attributes. - self.writer.set_tcp_cork(False) - self.writer.set_tcp_nodelay(True) - - # log access - if self.access_log: - self.log_access(message, None, resp, self._loop.time() - now) - - # for repr - self._request = None +__all__ = ('Server',) class Server: @@ -108,30 +13,13 @@ class Server: def __init__(self, handler, *, request_factory=None, loop=None, **kwargs): if loop is None: loop = asyncio.get_event_loop() - self._handler = handler - self._request_factory = request_factory or self._make_request self._loop = loop self._connections = {} self._kwargs = kwargs - self._requests_count = 0 - self._time_service = TimeService(self._loop) - - @property - def requests_count(self): - """Number of processed requests.""" - return self._requests_count - - @property - def handler(self): - return self._handler - - @property - def request_factory(self): - return self._request_factory - - @property - def time_service(self): - return self._time_service + self.time_service = TimeService(self._loop) + self.requests_count = 0 + self.request_handler = handler + self.request_factory = request_factory or self._make_request @property def connections(self): @@ -144,21 +32,19 @@ def connection_lost(self, handler, exc=None): if handler in self._connections: del self._connections[handler] - def _make_request(self, message, payload, protocol, writer): + def _make_request(self, message, payload, protocol, writer, task): return BaseRequest( message, payload, protocol, writer, - protocol._time_service, None) + protocol.time_service, task) @asyncio.coroutine def shutdown(self, timeout=None): coros = [conn.shutdown(timeout) for conn in self._connections] yield from asyncio.gather(*coros, loop=self._loop) self._connections.clear() - self._time_service.close() + self.time_service.close() finish_connections = shutdown def __call__(self): - return RequestHandler( - self, loop=self._loop, - **self._kwargs) + return RequestHandler(self, loop=self._loop, **self._kwargs) diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index c401edba924..9f1bda539b2 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -17,10 +17,10 @@ from . import hdrs, helpers from .abc import AbstractMatchInfo, AbstractRouter, AbstractView -from .file_sender import FileSender -from .http import HttpVersion11, PayloadWriter +from .http import HttpVersion11 from .web_exceptions import (HTTPExpectationFailed, HTTPForbidden, HTTPMethodNotAllowed, HTTPNotFound) +from .web_fileresponse import FileResponse from .web_response import Response, StreamResponse __all__ = ('UrlDispatcher', 'UrlMappingMatchInfo', @@ -399,9 +399,8 @@ def __init__(self, prefix, directory, *, name=None, raise ValueError( "No directory exists at '{}'".format(directory)) from error self._directory = directory - self._file_sender = FileSender(resp_factory=response_factory, - chunk_size=chunk_size) self._show_index = show_index + self._chunk_size = chunk_size self._follow_symlinks = follow_symlinks self._expect_handler = expect_handler @@ -482,7 +481,7 @@ def _handle(self, request): else: raise HTTPForbidden() elif filepath.is_file(): - ret = yield from self._file_sender.send(request, filepath) + ret = FileResponse(filepath, chunk_size=self._chunk_size) else: raise HTTPNotFound diff --git a/tests/test_client_functional_oldstyle.py b/tests/test_client_functional_oldstyle.py index 96928db9a7a..1e83491ef72 100644 --- a/tests/test_client_functional_oldstyle.py +++ b/tests/test_client_functional_oldstyle.py @@ -25,7 +25,7 @@ import aiohttp import aiohttp.http -from aiohttp import client, helpers, server, test_utils +from aiohttp import client, helpers, test_utils, web from aiohttp.multipart import MultipartWriter from aiohttp.test_utils import run_briefly, unused_port @@ -56,28 +56,24 @@ def url(self, *suffix): return urllib.parse.urljoin( self._url, '/'.join(str(s) for s in suffix)) - class TestHttpServer(server.ServerHttpProtocol): - - def connection_made(self, transport): - transports.append(transport) - - super().connection_made(transport) - - def handle_request(self, message, payload): + @asyncio.coroutine + def handler(request): + if properties.get('close', False): + return - if properties.get('close', False): - return + for hdr, val in request.message.headers.items(): + if (hdr.upper() == 'EXPECT') and (val == '100-continue'): + request.writer.write(b'HTTP/1.0 100 Continue\r\n\r\n') + break - for hdr, val in message.headers.items(): - if (hdr.upper() == 'EXPECT') and (val == '100-continue'): - self.transport.write(b'HTTP/1.0 100 Continue\r\n\r\n') - break + rob = router(properties, request) + return (yield from rob.dispatch()) - body = yield from payload.read() + class TestHttpServer(web.RequestHandler): - rob = router( - self, properties, self.transport, message, body) - yield from rob.dispatch() + def connection_made(self, transport): + transports.append(transport) + super().connection_made(transport) if use_ssl: here = os.path.join(os.path.dirname(__file__), '..', 'tests') @@ -94,7 +90,8 @@ def run(loop, fut): host, port = listen_addr server_coroutine = thread_loop.create_server( - lambda: TestHttpServer(keepalive_timeout=0.5), + lambda: TestHttpServer( + web.Server(handler, loop=loop), keepalive_timeout=0.5), host, port, ssl=sslcontext) server = thread_loop.run_until_complete(server_coroutine) @@ -137,20 +134,19 @@ class Router: _response_version = "1.1" _responses = http.server.BaseHTTPRequestHandler.responses - def __init__(self, srv, props, transport, message, payload): + def __init__(self, props, request): # headers self._headers = http.client.HTTPMessage() - for hdr, val in message.headers.items(): + for hdr, val in request.message.headers.items(): self._headers.add_header(hdr, val) - self._srv = srv self._props = props - self._transport = transport - self._method = message.method - self._uri = message.path - self._version = message.version - self._compression = message.compression - self._body = payload + self._request = request + self._method = request.message.method + self._uri = request.message.path + self._version = request.message.version + self._compression = request.message.compression + self._body = request.content url = urllib.parse.urlsplit(self._uri) self._path = url.path @@ -171,18 +167,18 @@ def dispatch(self): # pragma: no cover match = route.match(self._path) if match is not None: try: - return getattr(self, fn)(match) + return (yield from getattr(self, fn)(match)) except Exception: out = io.StringIO() traceback.print_exc(file=out) - self._response(500, out.getvalue()) + return (yield from self._response(500, out.getvalue())) return () - return self._response(self._start_response(404)) + return (yield from self._response(self._start_response(404))) def _start_response(self, code): - return aiohttp.http.Response(self._srv.writer, code) + return web.Response(status=code) @asyncio.coroutine def _response(self, response, body=None, @@ -205,7 +201,7 @@ def _response(self, response, body=None, 'version': '%s.%s' % self._version, 'path': self._uri, 'headers': r_headers, - 'origin': self._transport.get_extra_info('addr', ' ')[0], + 'origin': self._request.transport.get_extra_info('addr', ' ')[0], 'query': self._query, 'form': {}, 'compression': cmod, @@ -214,7 +210,8 @@ def _response(self, response, body=None, if body: # pragma: no cover resp['content'] = body else: - resp['content'] = self._body.decode('utf-8', 'ignore') + resp['content'] = ( + yield from self._request.read()).decode('utf-8', 'ignore') ct = self._headers.get('content-type', '').lower() @@ -228,8 +225,9 @@ def _response(self, response, body=None, for key, val in self._headers.items(): out.write(bytes('{}: {}\r\n'.format(key, val), 'latin1')) + b = yield from self._request.read() out.write(b'\r\n') - out.write(self._body) + out.write(b) out.write(b'\r\n') out.seek(0) @@ -261,12 +259,14 @@ def _response(self, response, body=None, if headers: hdrs.extend(headers.items()) + # headers + for key, val in hdrs: + response.headers[key] = val + if chunked: - response.enable_chunked_encoding() + self._request.writer.enable_chunking() - # headers - response.add_headers(*hdrs) - response.send_headers() + yield from response.prepare(self._request) # write payload if write_body: @@ -277,11 +277,7 @@ def _response(self, response, body=None, else: response.write(body.encode('utf8')) - yield from response.write_eof() - - # keep-alive - if response.keep_alive(): - self._srv.keep_alive(True) + return response class Functional(Router): @@ -292,15 +288,16 @@ def method(self, match): @Router.define('/keepalive$') def keepalive(self, match): - self._transport._requests = getattr( - self._transport, '_requests', 0) + 1 + transport = self._request.transport + + transport._requests = getattr(transport, '_requests', 0) + 1 resp = self._start_response(200) if 'close=' in self._query: return self._response( - resp, 'requests={}'.format(self._transport._requests)) + resp, 'requests={}'.format(transport._requests)) else: return self._response( - resp, 'requests={}'.format(self._transport._requests), + resp, 'requests={}'.format(transport._requests), headers={'CONNECTION': 'keep-alive'}) @Router.define('/cookies$') @@ -311,12 +308,13 @@ def cookies(self, match): resp = self._start_response(200) for cookie in cookies.output(header='').split('\n'): - resp.add_header('Set-Cookie', cookie.strip()) + resp.headers.extend({'Set-Cookie': cookie.strip()}) + + resp.headers.extend( + {'Set-Cookie': + 'ISAWPLB{A7F52349-3531-4DA9-8776-F74BC6F4F1BB}=' + '{925EC0B8-CB17-4BEB-8A35-1033813B0523}; HttpOnly; Path=/'}) - resp.add_header( - 'Set-Cookie', - 'ISAWPLB{A7F52349-3531-4DA9-8776-F74BC6F4F1BB}=' - '{925EC0B8-CB17-4BEB-8A35-1033813B0523}; HttpOnly; Path=/') return self._response(resp) @Router.define('/cookies_partial$') diff --git a/tests/test_http_message.py b/tests/test_http_message.py index 7148a61fc04..cfaff11d795 100644 --- a/tests/test_http_message.py +++ b/tests/test_http_message.py @@ -34,38 +34,17 @@ def test_start_request(stream, loop): assert msg.status_line == 'GET /index.html HTTP/1.1\r\n' -def test_start_response_with_reason(stream, loop): - msg = http.Response(stream, 333, close=True, reason="My Reason", loop=loop) - - assert msg.status == 333 - assert msg.reason == "My Reason" - assert msg.status_line == 'HTTP/1.1 333 My Reason\r\n' - - -def test_start_response_with_unknown_reason(stream, loop): - msg = http.Response(stream, 777, close=True, loop=loop) - - assert msg.status == 777 - assert msg.reason == "" - assert msg.status_line == 'HTTP/1.1 777 \r\n' - - -def test_force_close(stream, loop): - msg = http.Response(stream, 200, loop=loop) - assert not msg.closing - msg.force_close() - assert msg.closing - - def test_force_chunked(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.Request( + stream, 'GET', '/index.html', close=True, loop=loop) assert not msg.chunked msg.enable_chunking() assert msg.chunked def test_keep_alive(stream, loop): - msg = http.Response(stream, 200, close=True, loop=loop) + msg = http.Request( + stream, 'GET', '/index.html', close=True, loop=loop) assert not msg.keep_alive() msg.keepalive = True assert msg.keep_alive() @@ -75,39 +54,37 @@ def test_keep_alive(stream, loop): def test_keep_alive_http10(stream, loop): - msg = http.Response(stream, 200, http_version=(1, 0), loop=loop) + msg = http.HttpMessage(stream, version=(1, 0), close=True, loop=loop) assert not msg.keepalive assert not msg.keep_alive() - msg = http.Response(stream, 200, http_version=(1, 1), loop=loop) + msg = http.HttpMessage(stream, version=(1, 1), loop=loop) assert msg.keepalive is None def test_http_message_keepsalive(stream, loop): - msg = http.Response(stream, 200, http_version=(0, 9), loop=loop) + msg = http.HttpMessage(stream, version=(0, 9), loop=loop) assert not msg.keep_alive() - msg = http.Response(stream, 200, http_version=(1, 0), loop=loop) + msg = http.HttpMessage(stream, version=(1, 0), loop=loop) assert not msg.keep_alive() - msg = http.Response(stream, 200, http_version=(1, 0), loop=loop) + msg = http.HttpMessage(stream, version=(1, 0), loop=loop) msg.headers[hdrs.CONNECTION] = 'keep-alive' assert msg.keep_alive() - msg = http.Response( - stream, 200, http_version=(1, 1), close=False, loop=loop) + msg = http.HttpMessage(stream, version=(1, 1), close=False, loop=loop) assert msg.keep_alive() - msg = http.Response( - stream, 200, http_version=(1, 1), close=True, loop=loop) + msg = http.HttpMessage(stream, version=(1, 1), close=True, loop=loop) assert not msg.keep_alive() - msg = http.Response(stream, 200, http_version=(0, 9), loop=loop) + msg = http.HttpMessage(stream, version=(0, 9), loop=loop) msg.keepalive = True assert msg.keep_alive() def test_add_header(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, version=(1, 1), loop=loop) assert [] == list(msg.headers) msg.add_header('content-type', 'plain/html') @@ -115,7 +92,7 @@ def test_add_header(stream, loop): def test_add_header_with_spaces(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, version=(1, 1), loop=loop) assert [] == list(msg.headers) msg.add_header('content-type', ' plain/html ') @@ -123,7 +100,7 @@ def test_add_header_with_spaces(stream, loop): def test_add_header_non_ascii(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, version=(1, 1), loop=loop) assert [] == list(msg.headers) with pytest.raises(AssertionError): @@ -131,7 +108,7 @@ def test_add_header_non_ascii(stream, loop): def test_add_header_invalid_value_type(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, version=(1, 1), loop=loop) assert [] == list(msg.headers) with pytest.raises(AssertionError): @@ -142,7 +119,7 @@ def test_add_header_invalid_value_type(stream, loop): def test_add_headers(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, version=(1, 1), loop=loop) assert [] == list(msg.headers) msg.add_headers(('content-type', 'plain/html')) @@ -150,7 +127,7 @@ def test_add_headers(stream, loop): def test_add_headers_length(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, loop=loop) assert msg.length is None msg.add_headers(('content-length', '42')) @@ -158,7 +135,7 @@ def test_add_headers_length(stream, loop): def test_add_headers_upgrade(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, loop=loop) assert not msg.upgrade msg.add_headers(('connection', 'upgrade')) @@ -166,19 +143,19 @@ def test_add_headers_upgrade(stream, loop): def test_add_headers_upgrade_websocket(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, loop=loop) msg.add_headers(('upgrade', 'test')) assert not msg.websocket assert [('Upgrade', 'test')] == list(msg.headers.items()) - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, loop=loop) msg.add_headers(('upgrade', 'websocket')) assert msg.websocket assert [('Upgrade', 'websocket')] == list(msg.headers.items()) def test_add_headers_connection_keepalive(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, loop=loop) msg.add_headers(('connection', 'keep-alive')) assert [] == list(msg.headers) @@ -189,7 +166,7 @@ def test_add_headers_connection_keepalive(stream, loop): def test_add_headers_hop_headers(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, loop=loop) msg.HOP_HEADERS = (hdrs.TRANSFER_ENCODING,) msg.add_headers(('connection', 'test'), ('transfer-encoding', 't')) @@ -197,36 +174,26 @@ def test_add_headers_hop_headers(stream, loop): def test_default_headers_http_10(stream, loop): - msg = http.Response(stream, 200, - http_version=http.HttpVersion10, loop=loop) + msg = http.HttpMessage(stream, version=http.HttpVersion10, loop=loop) msg._add_default_headers() - assert 'DATE' in msg.headers assert 'keep-alive' == msg.headers['CONNECTION'] def test_default_headers_http_11(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, loop=loop) msg._add_default_headers() - assert 'DATE' in msg.headers assert 'CONNECTION' not in msg.headers -def test_default_headers_server(stream, loop): - msg = http.Response(stream, 200, loop=loop) - msg._add_default_headers() - - assert 'SERVER' in msg.headers - - def test_default_headers_chunked(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg._add_default_headers() assert 'TRANSFER-ENCODING' not in msg.headers - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.enable_chunking() msg.send_headers() @@ -234,7 +201,7 @@ def test_default_headers_chunked(stream, loop): def test_default_headers_connection_upgrade(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, loop=loop) msg.upgrade = True msg._add_default_headers() @@ -242,7 +209,7 @@ def test_default_headers_connection_upgrade(stream, loop): def test_default_headers_connection_close(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.HttpMessage(stream, loop=loop) msg.force_close() msg._add_default_headers() @@ -250,8 +217,7 @@ def test_default_headers_connection_close(stream, loop): def test_default_headers_connection_keep_alive_http_10(stream, loop): - msg = http.Response(stream, 200, - http_version=http.HttpVersion10, loop=loop) + msg = http.HttpMessage(stream, version=http.HttpVersion10, loop=loop) msg.keepalive = True msg._add_default_headers() @@ -259,8 +225,7 @@ def test_default_headers_connection_keep_alive_http_10(stream, loop): def test_default_headers_connection_keep_alive_11(stream, loop): - msg = http.Response(stream, 200, - http_version=http.HttpVersion11, loop=loop) + msg = http.HttpMessage(stream, version=http.HttpVersion11, loop=loop) msg.keepalive = True msg._add_default_headers() @@ -268,21 +233,21 @@ def test_default_headers_connection_keep_alive_11(stream, loop): def test_send_headers(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.add_headers(('content-type', 'plain/html')) assert not msg.is_headers_sent() msg.send_headers() content = b''.join(msg._buffer) - assert content.startswith(b'HTTP/1.1 200 OK\r\n') + assert content.startswith(b'GET / HTTP/1.1\r\n') assert b'Content-Type: plain/html' in content assert msg.headers_sent assert msg.is_headers_sent() def test_send_headers_non_ascii(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.add_headers(('x-header', 'текст')) assert not msg.is_headers_sent() @@ -290,14 +255,14 @@ def test_send_headers_non_ascii(stream, loop): content = b''.join(msg._buffer) - assert content.startswith(b'HTTP/1.1 200 OK\r\n') + assert content.startswith(b'GET / HTTP/1.1\r\n') assert b'X-Header: \xd1\x82\xd0\xb5\xd0\xba\xd1\x81\xd1\x82' in content assert msg.headers_sent assert msg.is_headers_sent() def test_send_headers_nomore_add(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.add_headers(('content-type', 'plain/html')) msg.send_headers() @@ -306,7 +271,7 @@ def test_send_headers_nomore_add(stream, loop): def test_prepare_length(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.add_headers(('content-length', '42')) msg.send_headers() @@ -314,7 +279,7 @@ def test_prepare_length(stream, loop): def test_prepare_chunked_force(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.enable_chunking() msg.add_headers(('content-length', '42')) msg.send_headers() @@ -322,19 +287,19 @@ def test_prepare_chunked_force(stream, loop): def test_prepare_chunked_no_length(stream, loop): - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.send_headers() assert msg.chunked def test_prepare_eof(stream, loop): - msg = http.Response(stream, 200, http_version=(1, 0), loop=loop) + msg = http.Request(stream, 'GET', '/', http_version=(1, 0), loop=loop) msg.send_headers() assert msg.length is None def test_write_auto_send_headers(stream, loop): - msg = http.Response(stream, 200, http_version=(1, 0), loop=loop) + msg = http.Request(stream, 'GET', '/', http_version=(1, 0), loop=loop) msg.send_headers() msg.write(b'data1') assert msg.headers_sent @@ -342,7 +307,7 @@ def test_write_auto_send_headers(stream, loop): def test_write_payload_eof(stream, loop): write = stream.transport.write = mock.Mock() - msg = http.Response(stream, 200, http_version=(1, 0), loop=loop) + msg = http.Request(stream, 'GET', '/', http_version=(1, 0), loop=loop) msg.send_headers() msg.write(b'data1') @@ -359,7 +324,7 @@ def test_write_payload_eof(stream, loop): def test_write_payload_chunked(stream, loop): write = stream.transport.write = mock.Mock() - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.enable_chunking() msg.send_headers() @@ -374,7 +339,7 @@ def test_write_payload_chunked(stream, loop): def test_write_payload_chunked_multiple(stream, loop): write = stream.transport.write = mock.Mock() - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.enable_chunking() msg.send_headers() @@ -391,7 +356,7 @@ def test_write_payload_chunked_multiple(stream, loop): def test_write_payload_length(stream, loop): write = stream.transport.write = mock.Mock() - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.add_headers(('content-length', '2')) msg.send_headers() @@ -407,7 +372,7 @@ def test_write_payload_length(stream, loop): def test_write_payload_chunked_filter(stream, loop): write = stream.transport.write = mock.Mock() - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.send_headers() msg.enable_chunking() @@ -422,7 +387,7 @@ def test_write_payload_chunked_filter(stream, loop): @asyncio.coroutine def test_write_payload_chunked_filter_mutiple_chunks(stream, loop): write = stream.transport.write = mock.Mock() - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.send_headers() msg.enable_chunking() @@ -441,7 +406,7 @@ def test_write_payload_chunked_filter_mutiple_chunks(stream, loop): @asyncio.coroutine def test_write_payload_deflate_compression(stream, loop): write = stream.transport.write = mock.Mock() - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.add_headers(('content-length', '{}'.format(len(COMPRESSED)))) msg.send_headers() @@ -458,7 +423,7 @@ def test_write_payload_deflate_compression(stream, loop): @asyncio.coroutine def test_write_payload_deflate_and_chunked(stream, loop): write = stream.transport.write = mock.Mock() - msg = http.Response(stream, 200, loop=loop) + msg = http.Request(stream, 'GET', '/', loop=loop) msg.send_headers() msg.enable_compression('deflate') @@ -476,7 +441,7 @@ def test_write_payload_deflate_and_chunked(stream, loop): def test_write_drain(stream, loop): - msg = http.Response(stream, 200, http_version=(1, 0), loop=loop) + msg = http.Request(stream, 'GET', '/', http_version=(1, 0), loop=loop) msg.drain = mock.Mock() msg.send_headers() msg.write(b'1' * (64 * 1024 * 2), drain=False) @@ -496,7 +461,7 @@ def test_dont_override_request_headers_with_default_values(stream, loop): def test_dont_override_response_headers_with_default_values(stream, loop): - msg = http.Response(stream, 200, http_version=(1, 0), loop=loop) + msg = http.Request(stream, 'GET', '/', http_version=(1, 0), loop=loop) msg.add_header('DATE', 'now') msg.add_header('SERVER', 'custom') msg._add_default_headers() diff --git a/tests/test_web_exceptions.py b/tests/test_web_exceptions.py index fb6a1ea87ce..3029396cb55 100644 --- a/tests/test_web_exceptions.py +++ b/tests/test_web_exceptions.py @@ -5,7 +5,7 @@ import pytest -from aiohttp import signals, web +from aiohttp import helpers, signals, web from aiohttp.test_utils import make_mocked_request @@ -21,18 +21,18 @@ def request(buf): writer = mock.Mock() writer.drain.return_value = () - def acquire(cb): - cb(writer) + def append(data=b''): + buf.extend(data) + return helpers.noop() - writer.acquire.side_effect = acquire + writer.buffer_data.side_effect = append + writer.write.side_effect = append + writer.write_eof.side_effect = append - def append(data): - buf.extend(data) - writer.transport.write.side_effect = append app = mock.Mock() app._debug = False app.on_response_prepare = signals.Signal(app) - req = make_mocked_request(method, path, app=app, writer=writer) + req = make_mocked_request(method, path, app=app, payload_writer=writer) return req diff --git a/tests/test_server.py b/tests/test_web_protocol.py similarity index 68% rename from tests/test_server.py rename to tests/test_web_protocol.py index c6cdd686068..9b4d53f0384 100644 --- a/tests/test_server.py +++ b/tests/test_web_protocol.py @@ -8,21 +8,29 @@ import pytest -from aiohttp import helpers, http, server, streams +from aiohttp import helpers, http, streams, web @pytest.yield_fixture -def make_srv(loop): +def make_srv(loop, manager): srv = None - def maker(cls=server.ServerHttpProtocol, **kwargs): + def maker(*, cls=web.RequestHandler, **kwargs): nonlocal srv - srv = cls(loop=loop, access_log=None, **kwargs) + m = kwargs.pop('manager', manager) + srv = cls(m, loop=loop, access_log=None, **kwargs) return srv yield maker + if srv is not None: - srv.connection_lost(None) + if srv.transport is not None: + srv.connection_lost(None) + + +@pytest.fixture +def manager(request_handler, loop): + return web.Server(request_handler, loop=loop) @pytest.fixture @@ -38,12 +46,24 @@ def buf(): return bytearray() +@pytest.fixture +def request_handler(): + + @asyncio.coroutine + def handler(request): + return web.Response() + + m = mock.Mock() + m.side_effect = handler + return m + + @pytest.fixture def handle_with_error(): def wrapper(exc=ValueError): @asyncio.coroutine - def handle(message, payload, writer): + def handle(request): raise exc h = mock.Mock() @@ -78,22 +98,8 @@ def ceil(val): mocker.patch('aiohttp.helpers.ceil').side_effect = ceil -@asyncio.coroutine -def test_handle_request(srv, buf, writer): - message = mock.Mock() - message.headers = [] - message.version = (1, 1) - yield from srv.handle_request(message, mock.Mock(), writer) - - content = bytes(buf) - assert content.startswith(b'HTTP/1.1 404 Not Found\r\n') - - @asyncio.coroutine def test_shutdown(srv, loop, transport): - srv.handle_request = mock.Mock() - srv.handle_request.side_effect = helpers.noop - assert transport is srv.transport srv._keepalive = True @@ -171,12 +177,8 @@ def test_double_shutdown(srv, transport): @asyncio.coroutine def test_close_after_response(srv, loop, transport): - srv.handle_request = mock.Mock() - srv.handle_request.side_effect = helpers.noop - srv._keepalive = False - srv.data_received( - b'GET / HTTP/1.1\r\n' + b'GET / HTTP/1.0\r\n' b'Host: example.com\r\n' b'Content-Length: 0\r\n\r\n') h, = srv._request_handlers @@ -269,7 +271,7 @@ def test_bad_method(srv, loop, buf): b'Host: example.com\r\n\r\n') yield from asyncio.sleep(0, loop=loop) - assert buf.startswith(b'HTTP/1.1 400 Bad Request\r\n') + assert buf.startswith(b'HTTP/1.0 400 Bad Request\r\n') @asyncio.coroutine @@ -282,7 +284,7 @@ def test_internal_error(srv, loop, buf): b'Host: example.com\r\n\r\n') yield from asyncio.sleep(0, loop=loop) - assert buf.startswith(b'HTTP/1.1 500 Internal Server Error\r\n') + assert buf.startswith(b'HTTP/1.0 500 Internal Server Error\r\n') @asyncio.coroutine @@ -290,7 +292,7 @@ def test_line_too_long(srv, loop, buf): srv.data_received(b''.join([b'a' for _ in range(10000)]) + b'\r\n\r\n') yield from asyncio.sleep(0, loop=loop) - assert buf.startswith(b'HTTP/1.1 400 Bad Request\r\n') + assert buf.startswith(b'HTTP/1.0 400 Bad Request\r\n') @asyncio.coroutine @@ -301,104 +303,37 @@ def test_invalid_content_length(srv, loop, buf): b'Content-Length: sdgg\r\n\r\n') yield from asyncio.sleep(0, loop=loop) - assert buf.startswith(b'HTTP/1.1 400 Bad Request\r\n') + assert buf.startswith(b'HTTP/1.0 400 Bad Request\r\n') @asyncio.coroutine -def test_handle_error(srv, buf, writer): - srv.keep_alive(True) - - yield from srv.handle_error(writer, 404) - assert b'HTTP/1.1 404 Not Found' in buf - assert not srv._keepalive - +def test_handle_error__utf(make_srv, buf, transport, loop, request_handler): + request_handler.side_effect = RuntimeError('что-то пошло не так') -@asyncio.coroutine -def test_handle_error__utf(make_srv, buf, transport, writer): srv = make_srv(debug=True) srv.connection_made(transport) srv.keep_alive(True) srv.logger = mock.Mock() - try: - raise RuntimeError('что-то пошло не так') - except RuntimeError as exc: - yield from srv.handle_error(writer, exc=exc) + srv.data_received( + b'GET / HTTP/1.0\r\n' + b'Host: example.com\r\n' + b'Content-Length: 0\r\n\r\n') + yield from asyncio.sleep(0, loop=loop) - assert b'HTTP/1.1 500 Internal Server Error' in buf + assert b'HTTP/1.0 500 Internal Server Error' in buf assert b'Content-Type: text/html; charset=utf-8' in buf - pattern = escape("raise RuntimeError('что-то пошло не так')") + pattern = escape("RuntimeError: что-то пошло не так") assert pattern.encode('utf-8') in buf assert not srv._keepalive - srv.logger.exception.assert_called_with("Error handling request") - - -@asyncio.coroutine -def test_handle_error_traceback_exc(make_srv, buf, transport, writer): - log = mock.Mock() - srv = make_srv(debug=True, logger=log) - srv.connection_made(transport) - srv.transport.get_extra_info.return_value = '127.0.0.1' - srv._request_handlers.append(mock.Mock()) - - with mock.patch('aiohttp.server.traceback') as m_trace: - m_trace.format_exc.side_effect = ValueError - - yield from srv.handle_error(writer, 500, exc=object()) - - assert buf.startswith(b'HTTP/1.1 500 Internal Server Error') - assert log.exception.called + srv.logger.exception.assert_called_with( + "Error handling request", exc_info=mock.ANY) @asyncio.coroutine -def test_handle_error_debug(srv, buf, writer): - srv.debug = True - - try: - raise ValueError() - except Exception as exc: - yield from srv.handle_error(writer, 999, exc=exc) - - assert b'HTTP/1.1 500 Internal' in buf - assert b'Traceback (most recent call last):' in buf - - -@asyncio.coroutine -def test_handle_error_500(make_srv, loop, buf, transport, writer): - log = mock.Mock() - - srv = make_srv(logger=log) - srv.connection_made(transport) - - yield from srv.handle_error(writer, 500) - assert log.exception.called - - -@asyncio.coroutine -def test_handle(srv, loop, transport): - - def get_mock_coro(return_value): - @asyncio.coroutine - def mock_coro(*args, **kwargs): - return return_value - return mock.Mock(wraps=mock_coro) - - srv.connection_made(transport) - - handle = srv.handle_request = get_mock_coro(return_value=None) - - srv.data_received( - b'GET / HTTP/1.0\r\n' - b'Host: example.com\r\n\r\n') - - yield from srv._request_handlers[0] - assert handle.called - assert transport.close.called - - -@asyncio.coroutine -def test_handle_uncompleted(make_srv, loop, transport, handle_with_error): +def test_handle_uncompleted( + make_srv, loop, transport, handle_with_error, request_handler): closed = False def close(): @@ -407,10 +342,10 @@ def close(): transport.close.side_effect = close - srv = make_srv(lingering_timeout=0) + srv = make_srv(lingering_time=0) srv.connection_made(transport) srv.logger.exception = mock.Mock() - handle = srv.handle_request = handle_with_error() + request_handler.side_effect = handle_with_error() srv.data_received( b'GET / HTTP/1.0\r\n' @@ -418,13 +353,15 @@ def close(): b'Content-Length: 50000\r\n\r\n') yield from srv._request_handlers[0] - assert handle.called + assert request_handler.called assert closed - srv.logger.exception.assert_called_with("Error handling request") + srv.logger.exception.assert_called_with( + "Error handling request", exc_info=mock.ANY) @asyncio.coroutine -def test_handle_uncompleted_pipe(make_srv, loop, transport, handle_with_error): +def test_handle_uncompleted_pipe( + make_srv, loop, transport, request_handler, handle_with_error): closed = False normal_completed = False @@ -434,19 +371,19 @@ def close(): transport.close.side_effect = close - srv = make_srv(lingering_timeout=0) + srv = make_srv(lingering_time=0) srv.connection_made(transport) srv.logger.exception = mock.Mock() @asyncio.coroutine - def handle(message, request, writer): + def handle(request): nonlocal normal_completed normal_completed = True yield from asyncio.sleep(0.05, loop=loop) - yield from writer.write_eof() + return web.Response() # normal - srv.handle_request = handle + request_handler.side_effect = handle srv.data_received( b'GET / HTTP/1.1\r\n' b'Host: example.com\r\n' @@ -454,7 +391,7 @@ def handle(message, request, writer): yield from asyncio.sleep(0, loop=loop) # with exception - handle = srv.handle_request = handle_with_error() + request_handler.side_effect = handle_with_error() srv.data_received( b'GET / HTTP/1.1\r\n' b'Host: example.com\r\n' @@ -466,9 +403,10 @@ def handle(message, request, writer): yield from srv._request_handlers[0] assert normal_completed - assert handle.called + assert request_handler.called assert closed - srv.logger.exception.assert_called_with("Error handling request") + srv.logger.exception.assert_called_with( + "Error handling request", exc_info=mock.ANY) @asyncio.coroutine @@ -495,16 +433,15 @@ def handle(message, request, writer): @asyncio.coroutine -def test_lingering_disabled(make_srv, loop, transport): - - class Server(server.ServerHttpProtocol): +def test_lingering_disabled(make_srv, loop, transport, request_handler): - @asyncio.coroutine - def handle_request(self, message, payload, writer): - yield from asyncio.sleep(0, loop=loop) + @asyncio.coroutine + def handle_request(request): + yield from asyncio.sleep(0, loop=loop) - srv = make_srv(Server, lingering_time=0) + srv = make_srv(lingering_time=0) srv.connection_made(transport) + request_handler.side_effect = handle_request yield from asyncio.sleep(0, loop=loop) assert not transport.close.called @@ -520,15 +457,15 @@ def handle_request(self, message, payload, writer): @asyncio.coroutine -def test_lingering_timeout(make_srv, loop, transport, ceil): +def test_lingering_timeout(make_srv, loop, transport, ceil, request_handler): - class Server(server.ServerHttpProtocol): - - def handle_request(self, message, payload, writer): - yield from asyncio.sleep(0, loop=loop) + @asyncio.coroutine + def handle_request(request): + yield from asyncio.sleep(0, loop=loop) - srv = make_srv(Server, lingering_time=1e-30) + srv = make_srv(lingering_time=1e-30) srv.connection_made(transport) + request_handler.side_effect = handle_request yield from asyncio.sleep(0, loop=loop) assert not transport.close.called @@ -544,23 +481,6 @@ def handle_request(self, message, payload, writer): transport.close.assert_called_with() -def test_handle_coro(srv, loop, transport): - called = False - - @asyncio.coroutine - def coro(message, payload, writer): - nonlocal called - called = True - srv.eof_received() - - srv.handle_request = coro - srv.data_received( - b'GET / HTTP/1.0\r\n' - b'Host: example.com\r\n\r\n') - loop.run_until_complete(srv._request_handlers[0]) - assert called - - def test_handle_cancel(make_srv, loop, transport): log = mock.Mock() @@ -612,9 +532,8 @@ def test_handle_400(srv, loop, buf, transport): assert b'400 Bad Request' in buf -def test_handle_500(srv, loop, buf, transport): - handle = srv.handle_request = mock.Mock() - handle.side_effect = ValueError +def test_handle_500(srv, loop, buf, transport, request_handler): + request_handler.side_effect = ValueError srv.data_received( b'GET / HTTP/1.0\r\n' @@ -624,15 +543,6 @@ def test_handle_500(srv, loop, buf, transport): assert b'500 Internal Server Error' in buf -@asyncio.coroutine -def test_handle_error_no_handle_task(srv, transport, writer): - srv.keep_alive(True) - srv.connection_lost(None) - - yield from srv.handle_error(writer, 300) - assert not srv._keepalive - - @asyncio.coroutine def test_keep_alive(make_srv, loop, transport, ceil): srv = make_srv(keepalive_timeout=0.05) @@ -680,35 +590,28 @@ def test_keep_alive_timeout_nondefault(make_srv): @asyncio.coroutine -def test_supports_connect_method(srv, loop, transport): - srv.connection_made(transport) - - with mock.patch.object(srv, 'handle_request') as m_handle_request: - srv.data_received( - b'CONNECT aiohttp.readthedocs.org:80 HTTP/1.0\r\n' - b'Content-Length: 0\r\n\r\n') - yield from asyncio.sleep(0.1, loop=loop) - - srv.connection_lost(None) - yield from asyncio.sleep(0.05, loop=loop) - - assert m_handle_request.called - assert isinstance( - m_handle_request.call_args[0][1], streams.FlowControlStreamReader) +def test_supports_connect_method(srv, loop, transport, request_handler): + srv.data_received( + b'CONNECT aiohttp.readthedocs.org:80 HTTP/1.0\r\n' + b'Content-Length: 0\r\n\r\n') + yield from asyncio.sleep(0.1, loop=loop) + assert request_handler.called + assert isinstance( + request_handler.call_args[0][0].content, + streams.FlowControlStreamReader) -def test_content_length_0(srv, loop, transport): - with mock.patch.object(srv, 'handle_request') as m_handle_request: - srv.data_received( - b'GET / HTTP/1.1\r\n' - b'Host: example.org\r\n' - b'Content-Length: 0\r\n\r\n') - loop.run_until_complete(srv._request_handlers[0]) +@asyncio.coroutine +def test_content_length_0(srv, loop, request_handler): + srv.data_received( + b'GET / HTTP/1.1\r\n' + b'Host: example.org\r\n' + b'Content-Length: 0\r\n\r\n') + yield from asyncio.sleep(0, loop=loop) - assert m_handle_request.called - assert m_handle_request.call_args[0] == ( - mock.ANY, streams.EMPTY_PAYLOAD, mock.ANY) + assert request_handler.called + assert request_handler.call_args[0][0].content == streams.EMPTY_PAYLOAD def test_rudimentary_transport(srv, loop): @@ -766,20 +669,19 @@ def test_close(srv, loop, transport): @asyncio.coroutine -def test_pipeline_multiple_messages(srv, loop, transport): +def test_pipeline_multiple_messages(srv, loop, transport, request_handler): transport.close.side_effect = partial(srv.connection_lost, None) srv._max_concurrent_handlers = 1 processed = 0 @asyncio.coroutine - def handle(message, request, writer): + def handle(request): nonlocal processed processed += 1 - yield from writer.write_eof() + return web.Response() - srv.handle_request = mock.Mock() - srv.handle_request.side_effect = handle + request_handler.side_effect = handle assert transport is srv.transport @@ -803,23 +705,24 @@ def handle(message, request, writer): @asyncio.coroutine -def test_pipeline_response_order(srv, loop, buf, transport): +def test_pipeline_response_order(srv, loop, buf, transport, request_handler): transport.close.side_effect = partial(srv.connection_lost, None) - srv.connection_made(transport) srv._keepalive = True - srv.handle_request = mock.Mock() processed = [] @asyncio.coroutine - def handle1(message, payload, writer): + def handle1(request): nonlocal processed yield from asyncio.sleep(0.01, loop=loop) - writer.write(b'test1') - yield from writer.write_eof() + resp = web.StreamResponse() + yield from resp.prepare(request) + yield from resp.write(b'test1') + yield from resp.write_eof() processed.append(1) + return resp - srv.handle_request.side_effect = handle1 + request_handler.side_effect = handle1 srv.data_received( b'GET / HTTP/1.1\r\n' b'Host: example.com\r\n' @@ -828,13 +731,16 @@ def handle1(message, payload, writer): # second @asyncio.coroutine - def handle2(message, request, writer): + def handle2(request): nonlocal processed - writer.write(b'test2') - yield from writer.write_eof() + resp = web.StreamResponse() + yield from resp.prepare(request) + resp.write(b'test2') + yield from resp.write_eof() processed.append(2) + return resp - srv.handle_request.side_effect = handle2 + request_handler.side_effect = handle2 srv.data_received( b'GET / HTTP/1.1\r\n' b'Host: example.com\r\n' diff --git a/tests/test_web_request_handler.py b/tests/test_web_request_handler.py index b724e474568..734df681aa3 100644 --- a/tests/test_web_request_handler.py +++ b/tests/test_web_request_handler.py @@ -15,7 +15,8 @@ def test_repr(loop): handler.transport = object() request = make_mocked_request('GET', '/index.html') handler._request = request - assert '' == repr(handler) + # assert '' == repr(handler) + assert '' == repr(handler) def test_connections(loop): diff --git a/tests/test_web_response.py b/tests/test_web_response.py index 7604cbfd22e..4a49b7d2aab 100644 --- a/tests/test_web_response.py +++ b/tests/test_web_response.py @@ -35,11 +35,21 @@ def writer(buf): def acquire(cb): cb(writer.transport) + def buffer_data(chunk): + buf.extend(chunk) + def write(chunk): buf.extend(chunk) + @asyncio.coroutine + def write_eof(chunk=b''): + buf.extend(chunk) + writer.acquire.side_effect = acquire writer.transport.write.side_effect = write + writer.write.side_effect = write + writer.write_eof.side_effect = write_eof + writer.buffer_data.side_effect = buffer_data writer.drain.return_value = () return writer @@ -169,11 +179,11 @@ def test_last_modified_reset(): @asyncio.coroutine def test_start(): - req = make_request('GET', '/') + req = make_request('GET', '/', payload_writer=mock.Mock()) resp = StreamResponse() assert resp.keep_alive is None - msg = yield from resp.prepare(req, PayloadWriterFactory=mock.Mock()) + msg = yield from resp.prepare(req) assert msg.buffer_data.called msg2 = yield from resp.prepare(req) @@ -196,20 +206,20 @@ def test_chunked_encoding(): resp.enable_chunked_encoding() assert resp.chunked - msg = yield from resp.prepare(req, mock.Mock()) + msg = yield from resp.prepare(req) assert msg.chunked @asyncio.coroutine def test_chunk_size(): - req = make_request('GET', '/') + req = make_request('GET', '/', payload_writer=mock.Mock()) resp = StreamResponse() assert not resp.chunked resp.enable_chunked_encoding(chunk_size=8192) assert resp.chunked - msg = yield from resp.prepare(req, mock.Mock()) + msg = yield from resp.prepare(req) assert msg.chunked assert msg.enable_chunking.called assert msg.filter is not None @@ -229,7 +239,7 @@ def test_chunked_encoding_forbidden_for_http_10(): @asyncio.coroutine def test_compression_no_accept(): - req = make_request('GET', '/') + req = make_request('GET', '/', payload_writer=mock.Mock()) resp = StreamResponse() assert not resp.chunked @@ -237,13 +247,13 @@ def test_compression_no_accept(): resp.enable_compression() assert resp.compression - msg = yield from resp.prepare(req, mock.Mock()) + msg = yield from resp.prepare(req) assert not msg.enable_compression.called @asyncio.coroutine def test_force_compression_no_accept_backwards_compat(): - req = make_request('GET', '/') + req = make_request('GET', '/', payload_writer=mock.Mock()) resp = StreamResponse() assert not resp.chunked @@ -251,21 +261,21 @@ def test_force_compression_no_accept_backwards_compat(): resp.enable_compression(force=True) assert resp.compression - msg = yield from resp.prepare(req, mock.Mock()) + msg = yield from resp.prepare(req) assert msg.enable_compression.called assert msg.filter is not None @asyncio.coroutine def test_force_compression_false_backwards_compat(): - req = make_request('GET', '/') + req = make_request('GET', '/', payload_writer=mock.Mock()) resp = StreamResponse() assert not resp.compression resp.enable_compression(force=False) assert resp.compression - msg = yield from resp.prepare(req, mock.Mock()) + msg = yield from resp.prepare(req) assert not msg.enable_compression.called @@ -281,7 +291,7 @@ def test_compression_default_coding(): resp.enable_compression() assert resp.compression - msg = yield from resp.prepare(req, mock.Mock()) + msg = yield from resp.prepare(req) msg.enable_compression.assert_called_with('deflate') assert 'deflate' == resp.headers.get(hdrs.CONTENT_ENCODING) @@ -298,7 +308,7 @@ def test_force_compression_deflate(): resp.enable_compression(ContentCoding.deflate) assert resp.compression - msg = yield from resp.prepare(req, mock.Mock()) + msg = yield from resp.prepare(req) msg.enable_compression.assert_called_with('deflate') assert 'deflate' == resp.headers.get(hdrs.CONTENT_ENCODING) @@ -311,7 +321,7 @@ def test_force_compression_no_accept_deflate(): resp.enable_compression(ContentCoding.deflate) assert resp.compression - msg = yield from resp.prepare(req, mock.Mock()) + msg = yield from resp.prepare(req) msg.enable_compression.assert_called_with('deflate') assert 'deflate' == resp.headers.get(hdrs.CONTENT_ENCODING) @@ -326,7 +336,7 @@ def test_force_compression_gzip(): resp.enable_compression(ContentCoding.gzip) assert resp.compression - msg = yield from resp.prepare(req, mock.Mock()) + msg = yield from resp.prepare(req) msg.enable_compression.assert_called_with('gzip') assert 'gzip' == resp.headers.get(hdrs.CONTENT_ENCODING) @@ -339,7 +349,7 @@ def test_force_compression_no_accept_gzip(): resp.enable_compression(ContentCoding.gzip) assert resp.compression - msg = yield from resp.prepare(req, mock.Mock()) + msg = yield from resp.prepare(req) msg.enable_compression.assert_called_with('gzip') assert 'gzip' == resp.headers.get(hdrs.CONTENT_ENCODING) @@ -391,13 +401,11 @@ def test_cannot_write_after_eof(): @asyncio.coroutine def test_repr_after_eof(): resp = StreamResponse() - writer = mock.Mock() - yield from resp.prepare(make_request('GET', '/', writer=writer)) + yield from resp.prepare(make_request('GET', '/')) assert resp.prepared resp.write(b'data') - writer.drain.return_value = () yield from resp.write_eof() assert not resp.prepared resp_repr = repr(resp) @@ -416,8 +424,7 @@ def test_cannot_write_eof_before_headers(): def test_cannot_write_eof_twice(): resp = StreamResponse() writer = mock.Mock() - resp_impl = yield from resp.prepare( - make_request('GET', '/', writer=writer)) + resp_impl = yield from resp.prepare(make_request('GET', '/')) resp_impl.write = mock.Mock() resp_impl.write_eof = mock.Mock() resp_impl.write_eof.return_value = () @@ -433,16 +440,16 @@ def test_cannot_write_eof_twice(): @asyncio.coroutine -def test_write_returns_drain(): +def _test_write_returns_drain(): resp = StreamResponse() yield from resp.prepare(make_request('GET', '/')) with mock.patch('aiohttp.http_message.noop') as noop: - assert noop.return_value == resp.write(b'data') + assert noop == resp.write(b'data') @asyncio.coroutine -def test_write_returns_empty_tuple_on_empty_data(): +def _test_write_returns_empty_tuple_on_empty_data(): resp = StreamResponse() yield from resp.prepare(make_request('GET', '/')) @@ -635,7 +642,7 @@ def test_get_nodelay_prepared(): resp = StreamResponse() writer = mock.Mock() writer.tcp_nodelay = False - req = make_request('GET', '/', writer=writer) + req = make_request('GET', '/', payload_writer=writer) yield from resp.prepare(req) assert not resp.tcp_nodelay @@ -644,7 +651,7 @@ def test_get_nodelay_prepared(): def test_set_nodelay_prepared(): resp = StreamResponse() writer = mock.Mock() - req = make_request('GET', '/', writer=writer) + req = make_request('GET', '/', payload_writer=writer) yield from resp.prepare(req) resp.set_tcp_nodelay(True) @@ -668,7 +675,7 @@ def test_get_cork_prepared(): resp = StreamResponse() writer = mock.Mock() writer.tcp_cork = False - req = make_request('GET', '/', writer=writer) + req = make_request('GET', '/', payload_writer=writer) yield from resp.prepare(req) assert not resp.tcp_cork @@ -677,7 +684,7 @@ def test_get_cork_prepared(): def test_set_cork_prepared(): resp = StreamResponse() writer = mock.Mock() - req = make_request('GET', '/', writer=writer) + req = make_request('GET', '/', payload_writer=writer) yield from resp.prepare(req) resp.set_tcp_cork(True) @@ -815,7 +822,7 @@ def test_assign_nonstr_text(): @asyncio.coroutine def test_send_headers_for_empty_body(buf, writer): - req = make_request('GET', '/', writer=writer) + req = make_request('GET', '/', payload_writer=writer) resp = Response() yield from resp.prepare(req) @@ -830,7 +837,7 @@ def test_send_headers_for_empty_body(buf, writer): @asyncio.coroutine def test_render_with_body(buf, writer): - req = make_request('GET', '/', writer=writer) + req = make_request('GET', '/', payload_writer=writer) resp = Response(body=b'data') yield from resp.prepare(req) @@ -849,7 +856,7 @@ def test_render_with_body(buf, writer): def test_send_set_cookie_header(buf, writer): resp = Response() resp.cookies['name'] = 'value' - req = make_request('GET', '/', writer=writer) + req = make_request('GET', '/', payload_writer=writer) yield from resp.prepare(req) yield from resp.write_eof() diff --git a/tests/test_web_sendfile.py b/tests/test_web_sendfile.py index 29e774c45e3..b3e43456f1c 100644 --- a/tests/test_web_sendfile.py +++ b/tests/test_web_sendfile.py @@ -1,13 +1,13 @@ from unittest import mock from aiohttp import hdrs, helpers -from aiohttp.file_sender import FileSender, SendfilePayloadWriter from aiohttp.test_utils import make_mocked_coro, make_mocked_request +from aiohttp.web_fileresponse import FileResponse, SendfilePayloadWriter def test_static_handle_eof(loop): fake_loop = mock.Mock() - with mock.patch('aiohttp.file_sender.os') as m_os: + with mock.patch('aiohttp.web_fileresponse.os') as m_os: out_fd = 30 in_fd = 31 fut = helpers.create_future(loop) @@ -23,7 +23,7 @@ def test_static_handle_eof(loop): def test_static_handle_again(loop): fake_loop = mock.Mock() - with mock.patch('aiohttp.file_sender.os') as m_os: + with mock.patch('aiohttp.web_fileresponse.os') as m_os: out_fd = 30 in_fd = 31 fut = helpers.create_future(loop) @@ -41,7 +41,7 @@ def test_static_handle_again(loop): def test_static_handle_exception(loop): fake_loop = mock.Mock() - with mock.patch('aiohttp.file_sender.os') as m_os: + with mock.patch('aiohttp.web_fileresponse.os') as m_os: out_fd = 30 in_fd = 31 fut = helpers.create_future(loop) @@ -58,7 +58,7 @@ def test_static_handle_exception(loop): def test__sendfile_cb_return_on_cancelling(loop): fake_loop = mock.Mock() - with mock.patch('aiohttp.file_sender.os') as m_os: + with mock.patch('aiohttp.web_fileresponse.os') as m_os: out_fd = 30 in_fd = 31 fut = helpers.create_future(loop) @@ -89,10 +89,10 @@ def test_using_gzip_if_header_present_and_file_available(loop): filepath.open = mock.mock_open() filepath.with_name.return_value = gz_filepath - file_sender = FileSender() + file_sender = FileResponse(filepath) file_sender._sendfile = make_mocked_coro(None) - loop.run_until_complete(file_sender.send(request, filepath)) + loop.run_until_complete(file_sender.prepare(request)) assert not filepath.open.called assert gz_filepath.open.called @@ -115,10 +115,10 @@ def test_gzip_if_header_not_present_and_file_available(loop): filepath.stat.return_value = mock.MagicMock() filepath.stat.st_size = 1024 - file_sender = FileSender() + file_sender = FileResponse(filepath) file_sender._sendfile = make_mocked_coro(None) - loop.run_until_complete(file_sender.send(request, filepath)) + loop.run_until_complete(file_sender.prepare(request)) assert filepath.open.called assert not gz_filepath.open.called @@ -141,10 +141,10 @@ def test_gzip_if_header_not_present_and_file_not_available(loop): filepath.stat.return_value = mock.MagicMock() filepath.stat.st_size = 1024 - file_sender = FileSender() + file_sender = FileResponse(filepath) file_sender._sendfile = make_mocked_coro(None) - loop.run_until_complete(file_sender.send(request, filepath)) + loop.run_until_complete(file_sender.prepare(request)) assert filepath.open.called assert not gz_filepath.open.called @@ -168,10 +168,10 @@ def test_gzip_if_header_present_and_file_not_available(loop): filepath.stat.return_value = mock.MagicMock() filepath.stat.st_size = 1024 - file_sender = FileSender() + file_sender = FileResponse(filepath) file_sender._sendfile = make_mocked_coro(None) - loop.run_until_complete(file_sender.send(request, filepath)) + loop.run_until_complete(file_sender.prepare(request)) assert filepath.open.called assert not gz_filepath.open.called diff --git a/tests/test_web_sendfile_functional.py b/tests/test_web_sendfile_functional.py index 779b58558bf..7abb0921775 100644 --- a/tests/test_web_sendfile_functional.py +++ b/tests/test_web_sendfile_functional.py @@ -6,7 +6,6 @@ import aiohttp from aiohttp import web -from aiohttp.file_sender import FileSender try: import ssl @@ -17,7 +16,7 @@ @pytest.fixture(params=['sendfile', 'fallback'], ids=['sendfile', 'fallback']) def sender(request): def maker(*args, **kwargs): - ret = FileSender(*args, **kwargs) + ret = web.FileResponse(*args, **kwargs) if request.param == 'fallback': ret._sendfile = ret._sendfile_fallback return ret @@ -30,8 +29,7 @@ def test_static_file_ok(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender().send(request, filepath) - return resp + return sender(filepath) app = web.Application(loop=loop) app.router.add_get('/', handler) @@ -85,8 +83,7 @@ def test_static_file_with_content_type(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender(chunk_size=16).send(request, filepath) - return resp + return sender(filepath, chunk_size=16) app = web.Application(loop=loop) app.router.add_get('/', handler) @@ -109,8 +106,7 @@ def test_static_file_with_content_encoding(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender().send(request, filepath) - return resp + return sender(filepath) app = web.Application(loop=loop) app.router.add_get('/', handler) @@ -134,8 +130,7 @@ def test_static_file_if_modified_since(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender().send(request, filepath) - return resp + return sender(filepath) app = web.Application(loop=loop) app.router.add_get('/', handler) @@ -159,8 +154,7 @@ def test_static_file_if_modified_since_past_date(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender().send(request, filepath) - return resp + return sender(filepath) app = web.Application(loop=loop) app.router.add_get('/', handler) @@ -180,8 +174,7 @@ def test_static_file_if_modified_since_invalid_date(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender().send(request, filepath) - return resp + return sender(filepath) app = web.Application(loop=loop) app.router.add_get('/', handler) @@ -201,8 +194,7 @@ def test_static_file_if_modified_since_future_date(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender().send(request, filepath) - return resp + return sender(filepath) app = web.Application(loop=loop) app.router.add_get('/', handler) @@ -312,8 +304,7 @@ def test_static_file_range(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender(chunk_size=16).send(request, filepath) - return resp + return sender(filepath, chunk_size=16) app = web.Application(loop=loop) app.router.add_get('/', handler) @@ -359,8 +350,7 @@ def test_static_file_range_end_bigger_than_size(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender(chunk_size=16).send(request, filepath) - return resp + return sender(filepath, chunk_size=16) app = web.Application(loop=loop) app.router.add_get('/', handler) @@ -389,8 +379,7 @@ def test_static_file_range_tail(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender(chunk_size=16).send(request, filepath) - return resp + return sender(filepath, chunk_size=16) app = web.Application(loop=loop) app.router.add_get('/', handler) @@ -413,8 +402,7 @@ def test_static_file_invalid_range(loop, test_client, sender): @asyncio.coroutine def handler(request): - resp = yield from sender(chunk_size=16).send(request, filepath) - return resp + return sender(filepath, chunk_size=16) app = web.Application(loop=loop) app.router.add_get('/', handler) diff --git a/tests/test_web_server.py b/tests/test_web_server.py index 4fb36f86f97..710d2b05162 100644 --- a/tests/test_web_server.py +++ b/tests/test_web_server.py @@ -56,9 +56,7 @@ def handler(request): resp = yield from cli.get('/path/to') assert resp.status == 504 - txt = yield from resp.text() - assert "

504 Gateway Timeout

" in txt - + yield from resp.text() logger.debug.assert_called_with("Request handler timed out.") diff --git a/tests/test_web_websocket.py b/tests/test_web_websocket.py index 7cf4fdca6ab..0f290588463 100644 --- a/tests/test_web_websocket.py +++ b/tests/test_web_websocket.py @@ -24,6 +24,7 @@ def app(loop): def writer(): writer = mock.Mock() writer.drain.return_value = () + writer.write_eof.return_value = () return writer @@ -49,7 +50,8 @@ def maker(method, path, headers=None, protocols=False): headers['SEC-WEBSOCKET-PROTOCOL'] = 'chat, superchat' return make_mocked_request( - method, path, headers, app=app, protocol=protocol, writer=writer) + method, path, headers, + app=app, protocol=protocol, payload_writer=writer) return maker