From d7f41bdb9bc627edf424513cb91858707aba78fc Mon Sep 17 00:00:00 2001 From: Anton Kasyanov Date: Sun, 24 Jul 2016 13:47:10 +0200 Subject: [PATCH] Refactored ProxyConnector into TCPConnector --- aiohttp/client.py | 23 +- aiohttp/client_reqrep.py | 12 +- aiohttp/connector.py | 106 ++++---- docs/client.rst | 33 +-- docs/client_reference.rst | 19 +- tests/test_connector.py | 67 ++--- ...{test_proxy_connector.py => test_proxy.py} | 249 ++++++++++++------ 7 files changed, 323 insertions(+), 186 deletions(-) rename tests/{test_proxy_connector.py => test_proxy.py} (73%) diff --git a/aiohttp/client.py b/aiohttp/client.py index 3af9cda26c4..51099b0a464 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -104,7 +104,9 @@ def request(self, method, url, *, compress=None, chunked=None, expect100=False, - read_until_eof=True): + read_until_eof=True, + proxy=None, + proxy_auth=None): """Perform HTTP request.""" return _RequestContextManager( @@ -123,7 +125,9 @@ def request(self, method, url, *, compress=compress, chunked=chunked, expect100=expect100, - read_until_eof=read_until_eof)) + read_until_eof=read_until_eof, + proxy=proxy, + proxy_auth=proxy_auth,)) @asyncio.coroutine def _request(self, method, url, *, @@ -139,7 +143,9 @@ def _request(self, method, url, *, compress=None, chunked=None, expect100=False, - read_until_eof=True): + read_until_eof=True, + proxy=None, + proxy_auth=None): if version is not None: warnings.warn("HTTP version should be specified " @@ -181,7 +187,8 @@ def _request(self, method, url, *, cookies=cookies, encoding=encoding, auth=auth, version=version, compress=compress, chunked=chunked, expect100=expect100, - loop=self._loop, response_class=self._response_class) + loop=self._loop, response_class=self._response_class, + proxy=proxy, proxy_auth=proxy_auth,) conn = yield from self._connector.connect(req) try: @@ -621,7 +628,9 @@ def request(method, url, *, loop=None, read_until_eof=True, request_class=None, - response_class=None): + response_class=None, + proxy=None, + proxy_auth=None): """Constructs and sends a request. Returns response object. :param str method: HTTP method @@ -692,7 +701,9 @@ def request(method, url, *, compress=compress, chunked=chunked, expect100=expect100, - read_until_eof=read_until_eof), + read_until_eof=read_until_eof, + proxy=proxy, + proxy_auth=proxy_auth,), session=session) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 1cf091684be..f1ec39df7f4 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -67,7 +67,8 @@ def __init__(self, method, url, *, auth=None, encoding='utf-8', version=aiohttp.HttpVersion11, compress=None, chunked=None, expect100=False, - loop=None, response_class=None): + loop=None, response_class=None, + proxy=None, proxy_auth=None): if loop is None: loop = asyncio.get_event_loop() @@ -91,6 +92,7 @@ def __init__(self, method, url, *, self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) + self.update_proxy(proxy, proxy_auth) self.update_body_from_data(data, skip_auto_headers) self.update_transfer_encoding() @@ -366,6 +368,14 @@ def update_expect_continue(self, expect=False): if expect: self._continue = helpers.create_future(self.loop) + def update_proxy(self, proxy, proxy_auth): + if proxy and not proxy.startswith('http://'): + raise ValueError("Only http proxies are supported") + if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): + raise ValueError("proxy_auth must be None or BasicAuth() tuple") + self.proxy = proxy + self.proxy_auth = proxy_auth + @asyncio.coroutine def write_bytes(self, request, reader): """Support coroutines that yields bytes objects.""" diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 9260599ef2a..be6ba62e7c5 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -19,7 +19,7 @@ from .errors import HttpProxyError, ProxyConnectionError from .errors import ClientOSError, ClientTimeoutError from .errors import FingerprintMismatch -from .helpers import BasicAuth, is_ip_address +from .helpers import is_ip_address from .resolver import DefaultResolver @@ -587,6 +587,15 @@ def _create_connection(self, req): Has same keyword arguments as BaseEventLoop.create_connection. """ + if req.proxy: + transport, proto = yield from self._create_proxy_connection(req) + else: + transport, proto = yield from self._create_direct_connection(req) + + return transport, proto + + @asyncio.coroutine + def _create_direct_connection(self, req): if req.ssl: sslcontext = self.ssl_context else: @@ -629,56 +638,17 @@ def _create_connection(self, req): 'Can not connect to %s:%s [%s]' % (req.host, req.port, exc.strerror)) from exc - -class ProxyConnector(TCPConnector): - """Http Proxy connector. - - :param str proxy: Proxy URL address. Only HTTP proxy supported. - :param proxy_auth: (optional) Proxy HTTP Basic Auth - :type proxy_auth: aiohttp.helpers.BasicAuth - :param args: see :class:`TCPConnector` - :param kwargs: see :class:`TCPConnector` - - Usage: - - >>> conn = ProxyConnector(proxy="http://some.proxy.com") - >>> session = ClientSession(connector=conn) - >>> resp = yield from session.get('http://python.org') - - """ - - def __init__(self, proxy, *, proxy_auth=None, force_close=True, - **kwargs): - super().__init__(force_close=force_close, **kwargs) - self._proxy = proxy - self._proxy_auth = proxy_auth - assert proxy.startswith('http://'), ( - "Only http proxy supported", proxy) - assert proxy_auth is None or isinstance(proxy_auth, BasicAuth), ( - "proxy_auth must be None or BasicAuth() tuple", proxy_auth) - - @property - def proxy(self): - """Proxy URL.""" - return self._proxy - - @property - def proxy_auth(self): - """Proxy auth info. - - Should be BasicAuth instance. - """ - return self._proxy_auth - @asyncio.coroutine - def _create_connection(self, req): + def _create_proxy_connection(self, req): proxy_req = ClientRequest( - hdrs.METH_GET, self._proxy, + hdrs.METH_GET, req.proxy, headers={hdrs.HOST: req.host}, - auth=self._proxy_auth, + auth=req.proxy_auth, loop=self._loop) try: - transport, proto = yield from super()._create_connection(proxy_req) + # create connection to proxy server + transport, proto = yield from self._create_direct_connection( + proxy_req) except OSError as exc: raise ProxyConnectionError(*exc.args) from exc @@ -735,6 +705,50 @@ def _create_connection(self, req): return transport, proto +class ProxyConnector(TCPConnector): + """Http Proxy connector. + Deprecated, use ClientSession.request with proxy parameters. + Is still here for backward compatibility. + + :param str proxy: Proxy URL address. Only HTTP proxy supported. + :param proxy_auth: (optional) Proxy HTTP Basic Auth + :type proxy_auth: aiohttp.helpers.BasicAuth + :param args: see :class:`TCPConnector` + :param kwargs: see :class:`TCPConnector` + + Usage: + + >>> conn = ProxyConnector(proxy="http://some.proxy.com") + >>> session = ClientSession(connector=conn) + >>> resp = yield from session.get('http://python.org') + + """ + + def __init__(self, proxy, *, proxy_auth=None, force_close=True, + **kwargs): + super().__init__(force_close=force_close, **kwargs) + self._proxy = proxy + self._proxy_auth = proxy_auth + + @property + def proxy(self): + return self._proxy + + @property + def proxy_auth(self): + return self._proxy_auth + + @asyncio.coroutine + def _create_connection(self, req): + """ + Use TCPConnector _create_connection, to emulate old ProxyConnector. + """ + req.update_proxy(self._proxy, self._proxy_auth) + transport, proto = yield from super()._create_connection(req) + + return transport, proto + + class UnixConnector(BaseConnector): """Unix socket connector. diff --git a/docs/client.rst b/docs/client.rst index 7361d2f80d8..def106cc8ef 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -421,8 +421,8 @@ In order to specify the nameservers to when resolving the hostnames, aiodns is required. from aiohttp.resolver import AsyncResolver - - + + resolver = AsyncResolver(nameservers=["8.8.8.8", "8.8.4.4"]) conn = aiohttp.TCPConnector(resolver=resolver) @@ -493,26 +493,27 @@ Proxy support ------------- aiohttp supports proxy. You have to use -:class:`~aiohttp.ProxyConnector`:: +:attr:`proxy`:: + + async with aiohttp.ClientSession() as session: + async with session.get("http://python.org", + proxy="http://some.proxy.com") as resp: + print(resp.status) - conn = aiohttp.ProxyConnector(proxy="http://some.proxy.com") - session = aiohttp.ClientSession(connector=conn) - async with session.get('http://python.org') as resp: - print(resp.status) +it also supports proxy authorization:: -:class:`~aiohttp.ProxyConnector` also supports proxy authorization:: - conn = aiohttp.ProxyConnector( - proxy="http://some.proxy.com", - proxy_auth=aiohttp.BasicAuth('user', 'pass')) - session = aiohttp.ClientSession(connector=conn) - async with session.get('http://python.org') as r: - assert r.status == 200 + async with aiohttp.ClientSession() as session: + proxy_auth = aiohttp.BasicAuth('user', 'pass') + async with session.get("http://python.org", + proxy="http://some.proxy.com", + proxy_auth=proxy_auth) as resp: + print(resp.status) Authentication credentials can be passed in proxy URL:: - conn = aiohttp.ProxyConnector( - proxy="http://user:pass@some.proxy.com") + session.get("http://python.org", + proxy="http://user:pass@some.proxy.com") Response Status Codes diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 3c0e59da6ba..c8525e7e854 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -143,7 +143,8 @@ The client session supports the context manager protocol for self closing. max_redirects=10, encoding='utf-8',\ version=HttpVersion(major=1, minor=1),\ compress=None, chunked=None, expect100=False,\ - read_until_eof=True) + read_until_eof=True,\ + proxy=None, proxy_auth=None) :async-with: :coroutine: @@ -209,9 +210,18 @@ The client session supports the context manager protocol for self closing. does not have Content-Length header. ``True`` by default (optional). + :param str proxy: Proxy URL (optional) + + :param aiohttp.BasicAuth proxy_auth: an object that represents proxy HTTP + Basic Authorization (optional) + :return ClientResponse: a :class:`client response ` object. + .. versionadded:: 0.23 + + Added :attr:`proxy` and :attr:`proxy_auth` parameters. + .. comethod:: get(url, *, allow_redirects=True, **kwargs) :async-with: :coroutine: @@ -674,7 +684,7 @@ There are standard connectors: 1. :class:`TCPConnector` for regular *TCP sockets* (both *HTTP* and *HTTPS* schemes supported). -2. :class:`ProxyConnector` for connecting via HTTP proxy. +2. :class:`ProxyConnector` for connecting via HTTP proxy (deprecated). 3. :class:`UnixConnector` for connecting via UNIX socket (it's used mostly for testing purposes). @@ -941,6 +951,11 @@ ProxyConnector :class:`ProxyConnector` is inherited from :class:`TCPConnector`. + .. deprecated:: 0.23 + + Use :meth:`ClientSession.request` with :attr:`proxy` and :attr:`proxy_auth` + parameters. + Usage:: conn == ProxyConnector(proxy="http://some.proxy.com") diff --git a/tests/test_connector.py b/tests/test_connector.py index c8cffea960e..1ae823dd279 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -14,7 +14,7 @@ from aiohttp import web from aiohttp import client from aiohttp import helpers -from aiohttp.client import ClientResponse +from aiohttp.client import ClientResponse, ClientRequest from aiohttp.connector import Connection @@ -257,11 +257,9 @@ def test_connect(self): tr, proto = unittest.mock.Mock(), unittest.mock.Mock() proto.is_connected.return_value = True - class Req: - host = 'host' - port = 80 - ssl = False - response = unittest.mock.Mock() + req = ClientRequest('GET', 'http://host:80', + loop=self.loop, + response_class=unittest.mock.Mock()) conn = aiohttp.BaseConnector(loop=self.loop) key = ('host', 80, False) @@ -270,7 +268,7 @@ class Req: conn._create_connection.return_value = helpers.create_future(self.loop) conn._create_connection.return_value.set_result((tr, proto)) - connection = self.loop.run_until_complete(conn.connect(Req())) + connection = self.loop.run_until_complete(conn.connect(req)) self.assertFalse(conn._create_connection.called) self.assertEqual(connection._transport, tr) self.assertEqual(connection._protocol, proto) @@ -483,11 +481,9 @@ def go(): tr, proto = unittest.mock.Mock(), unittest.mock.Mock() proto.is_connected.return_value = True - class Req: - host = 'host' - port = 80 - ssl = False - response = unittest.mock.Mock() + req = ClientRequest('GET', 'http://host:80', + loop=self.loop, + response_class=unittest.mock.Mock()) conn = aiohttp.BaseConnector(loop=self.loop, limit=1) key = ('host', 80, False) @@ -497,7 +493,7 @@ class Req: self.loop) conn._create_connection.return_value.set_result((tr, proto)) - connection1 = yield from conn.connect(Req()) + connection1 = yield from conn.connect(req) self.assertEqual(connection1._transport, tr) self.assertEqual(1, len(conn._acquired[key])) @@ -507,7 +503,7 @@ class Req: @asyncio.coroutine def f(): nonlocal acquired - connection2 = yield from conn.connect(Req()) + connection2 = yield from conn.connect(req) acquired = True self.assertEqual(1, len(conn._acquired[key])) connection2.release() @@ -531,11 +527,9 @@ def go(): tr, proto = unittest.mock.Mock(), unittest.mock.Mock() proto.is_connected.return_value = True - class Req: - host = 'host' - port = 80 - ssl = False - response = unittest.mock.Mock() + req = ClientRequest('GET', 'http://host:80', + loop=self.loop, + response_class=unittest.mock.Mock()) conn = aiohttp.BaseConnector(loop=self.loop, limit=1) key = ('host', 80, False) @@ -545,14 +539,14 @@ class Req: self.loop) conn._create_connection.return_value.set_result((tr, proto)) - connection = yield from conn.connect(Req()) + connection = yield from conn.connect(req) self.assertEqual(connection._transport, tr) self.assertEqual(1, len(conn._acquired[key])) with self.assertRaises(asyncio.TimeoutError): # limit exhausted - yield from asyncio.wait_for(conn.connect(Req), 0.01, + yield from asyncio.wait_for(conn.connect(req), 0.01, loop=self.loop) connection.close() self.loop.run_until_complete(go()) @@ -583,11 +577,10 @@ def go(): proto = unittest.mock.Mock() proto.is_connected.return_value = True - class Req: - host = 'host' - port = 80 - ssl = False - response = unittest.mock.Mock(_should_close=False) + req = ClientRequest('GET', 'http://host:80', + loop=self.loop, + response_class=unittest.mock.Mock( + _should_close=False)) max_connections = 2 num_connections = 0 @@ -629,7 +622,7 @@ def f(start=True): return num_requests += 1 if not start: - connection = yield from conn.connect(Req()) + connection = yield from conn.connect(req) yield from asyncio.sleep(0, loop=self.loop) connection.release() tasks = [ @@ -652,11 +645,9 @@ def go(): tr, proto = unittest.mock.Mock(), unittest.mock.Mock() proto.is_connected.return_value = True - class Req: - host = 'host' - port = 80 - ssl = False - response = unittest.mock.Mock() + req = ClientRequest('GET', 'http://host:80', + loop=self.loop, + response_class=unittest.mock.Mock()) conn = aiohttp.BaseConnector(loop=self.loop, limit=1) key = ('host', 80, False) @@ -666,7 +657,7 @@ class Req: self.loop) conn._create_connection.return_value.set_result((tr, proto)) - connection = yield from conn.connect(Req()) + connection = yield from conn.connect(req) self.assertEqual(1, len(conn._acquired)) conn.close() @@ -845,14 +836,12 @@ def test_resolver_not_called_with_address_is_ip(self): resolver = unittest.mock.MagicMock() connector = aiohttp.TCPConnector(resolver=resolver, loop=self.loop) - class Req: - host = '127.0.0.1' - port = 80 - ssl = False - response = unittest.mock.Mock() + req = ClientRequest('GET', 'http://127.0.0.1:80', + loop=self.loop, + response_class=unittest.mock.Mock()) with self.assertRaises(OSError): - self.loop.run_until_complete(connector.connect(Req())) + self.loop.run_until_complete(connector.connect(req)) resolver.resolve.assert_not_called() diff --git a/tests/test_proxy_connector.py b/tests/test_proxy.py similarity index 73% rename from tests/test_proxy_connector.py rename to tests/test_proxy.py index dfebbebbbc8..1418a78f9d6 100644 --- a/tests/test_proxy_connector.py +++ b/tests/test_proxy.py @@ -6,7 +6,7 @@ from unittest import mock -class TestProxyConnector(unittest.TestCase): +class TestProxy(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() @@ -28,27 +28,19 @@ def coro(*args, **kw): yield # pragma: no cover mock.side_effect = coro - def test_ctor(self): - with self.assertRaises(AssertionError): - aiohttp.ProxyConnector('https://localhost:8118', loop=self.loop) - - def test_ctor2(self): - connector = aiohttp.ProxyConnector('http://localhost:8118', - loop=self.loop) - - self.assertEqual('http://localhost:8118', connector.proxy) - self.assertTrue(connector.force_close) - @unittest.mock.patch('aiohttp.connector.ClientRequest') def test_connect(self, ClientRequestMock): - req = ClientRequest('GET', 'http://www.python.org', loop=self.loop) - self.assertEqual(req.path, '/') - + req = ClientRequest( + 'GET', 'http://www.python.org', + proxy='http://proxy.example.com', + loop=self.loop + ) + self.assertEqual(req.proxy, 'http://proxy.example.com') + + # mock all the things! loop_mock = unittest.mock.Mock() - connector = aiohttp.ProxyConnector('http://proxy.example.com', - loop=loop_mock) + connector = aiohttp.TCPConnector(loop=loop_mock) self.assertIs(loop_mock, connector._loop) - resolve_host = unittest.mock.Mock() self._fake_coroutine(resolve_host, [unittest.mock.MagicMock()]) connector._resolve_host = resolve_host @@ -60,32 +52,34 @@ def test_connect(self, ClientRequestMock): self.assertIs(conn._transport, tr) self.assertIs(conn._protocol, proto) - # resolve_host.assert_called_once_with('proxy.example.com', 80) - tr.get_extra_info.assert_called_once_with('sslcontext') - ClientRequestMock.assert_called_with( 'GET', 'http://proxy.example.com', auth=None, headers={'HOST': 'www.python.org'}, loop=loop_mock) - conn.close() def test_proxy_auth(self): - with self.assertRaises(AssertionError) as ctx: - aiohttp.ProxyConnector('http://proxy.example.com', - proxy_auth=('user', 'pass'), - loop=unittest.mock.Mock()) - self.assertEqual(ctx.exception.args[0], - ("proxy_auth must be None or BasicAuth() tuple", - ('user', 'pass'))) + with self.assertRaises(ValueError) as ctx: + ClientRequest( + 'GET', 'http://python.org', + proxy='http://proxy.example.com', + proxy_auth=('user', 'pass'), + loop=unittest.mock.Mock()) + self.assertEqual( + ctx.exception.args[0], + "proxy_auth must be None or BasicAuth() tuple", + ) def test_proxy_connection_error(self): - connector = aiohttp.ProxyConnector('http://proxy.example.com', - loop=self.loop) + connector = aiohttp.TCPConnector(loop=self.loop) connector._resolve_host = resolve_mock = unittest.mock.Mock() self._fake_coroutine(resolve_mock, OSError('dont take it serious')) - req = ClientRequest('GET', 'http://www.python.org', loop=self.loop) + req = ClientRequest( + 'GET', 'http://www.python.org', + proxy='http://proxy.example.com', + loop=self.loop, + ) expected_headers = dict(req.headers) with self.assertRaises(aiohttp.ProxyConnectionError): self.loop.run_until_complete(connector.connect(req)) @@ -94,25 +88,29 @@ def test_proxy_connection_error(self): @unittest.mock.patch('aiohttp.connector.ClientRequest') def test_auth(self, ClientRequestMock): - proxy_req = ClientRequest('GET', 'http://proxy.example.com', - auth=aiohttp.helpers.BasicAuth('user', - 'pass'), - loop=self.loop) + proxy_req = ClientRequest( + 'GET', 'http://proxy.example.com', + auth=aiohttp.helpers.BasicAuth('user', 'pass'), + loop=self.loop + ) ClientRequestMock.return_value = proxy_req self.assertIn('AUTHORIZATION', proxy_req.headers) self.assertNotIn('PROXY-AUTHORIZATION', proxy_req.headers) loop_mock = unittest.mock.Mock() - connector = aiohttp.ProxyConnector( - 'http://proxy.example.com', loop=loop_mock, - proxy_auth=aiohttp.helpers.BasicAuth('user', 'pass')) + connector = aiohttp.TCPConnector(loop=loop_mock) connector._resolve_host = resolve_mock = unittest.mock.Mock() self._fake_coroutine(resolve_mock, [unittest.mock.MagicMock()]) tr, proto = unittest.mock.Mock(), unittest.mock.Mock() self._fake_coroutine(loop_mock.create_connection, (tr, proto)) - req = ClientRequest('GET', 'http://www.python.org', loop=self.loop) + req = ClientRequest( + 'GET', 'http://www.python.org', + proxy='http://proxy.example.com', + proxy_auth=aiohttp.helpers.BasicAuth('user', 'pass'), + loop=self.loop, + ) self.assertNotIn('AUTHORIZATION', req.headers) self.assertNotIn('PROXY-AUTHORIZATION', req.headers) conn = self.loop.run_until_complete(connector.connect(req)) @@ -145,15 +143,18 @@ def test_auth_from_url(self, ClientRequestMock): self.assertNotIn('PROXY-AUTHORIZATION', proxy_req.headers) loop_mock = unittest.mock.Mock() - connector = aiohttp.ProxyConnector( - 'http://user:pass@proxy.example.com', loop=loop_mock) + connector = aiohttp.TCPConnector(loop=loop_mock) connector._resolve_host = resolve_mock = unittest.mock.Mock() self._fake_coroutine(resolve_mock, [unittest.mock.MagicMock()]) tr, proto = unittest.mock.Mock(), unittest.mock.Mock() self._fake_coroutine(loop_mock.create_connection, (tr, proto)) - req = ClientRequest('GET', 'http://www.python.org', loop=self.loop) + req = ClientRequest( + 'GET', 'http://www.python.org', + proxy='http://user:pass@proxy.example.com', + loop=self.loop, + ) self.assertNotIn('AUTHORIZATION', req.headers) self.assertNotIn('PROXY-AUTHORIZATION', req.headers) conn = self.loop.run_until_complete(connector.connect(req)) @@ -177,12 +178,15 @@ def test_auth__not_modifying_request(self, ClientRequestMock): proxy_req_headers = dict(proxy_req.headers) loop_mock = unittest.mock.Mock() - connector = aiohttp.ProxyConnector( - 'http://user:pass@proxy.example.com', loop=loop_mock) + connector = aiohttp.TCPConnector(loop=loop_mock) connector._resolve_host = resolve_mock = unittest.mock.Mock() self._fake_coroutine(resolve_mock, OSError('nothing personal')) - req = ClientRequest('GET', 'http://www.python.org', loop=self.loop) + req = ClientRequest( + 'GET', 'http://www.python.org', + proxy='http://user:pass@proxy.example.com', + loop=self.loop, + ) req_headers = dict(req.headers) with self.assertRaises(aiohttp.ProxyConnectionError): self.loop.run_until_complete(connector.connect(req)) @@ -204,13 +208,16 @@ def test_https_connect(self, ClientRequestMock): proxy_resp.start = start_mock = unittest.mock.Mock() self._fake_coroutine(start_mock, unittest.mock.Mock(status=200)) - connector = aiohttp.ProxyConnector( - 'http://proxy.example.com', loop=loop_mock) + connector = aiohttp.TCPConnector(loop=loop_mock) tr, proto = unittest.mock.Mock(), unittest.mock.Mock() self._fake_coroutine(loop_mock.create_connection, (tr, proto)) - req = ClientRequest('GET', 'https://www.python.org', loop=self.loop) + req = ClientRequest( + 'GET', 'https://www.python.org', + proxy='http://proxy.example.com', + loop=self.loop, + ) self.loop.run_until_complete(connector._create_connection(req)) self.assertEqual(req.path, '/') @@ -237,14 +244,17 @@ def test_https_connect_runtime_error(self, ClientRequestMock): proxy_resp.start = start_mock = unittest.mock.Mock() self._fake_coroutine(start_mock, unittest.mock.Mock(status=200)) - connector = aiohttp.ProxyConnector( - 'http://proxy.example.com', loop=loop_mock) + connector = aiohttp.TCPConnector(loop=loop_mock) tr, proto = unittest.mock.Mock(), unittest.mock.Mock() tr.get_extra_info.return_value = None self._fake_coroutine(loop_mock.create_connection, (tr, proto)) - req = ClientRequest('GET', 'https://www.python.org', loop=self.loop) + req = ClientRequest( + 'GET', 'https://www.python.org', + proxy='http://proxy.example.com', + loop=self.loop, + ) with self.assertRaisesRegex( RuntimeError, "Transport does not expose socket instance"): self.loop.run_until_complete(connector._create_connection(req)) @@ -268,14 +278,17 @@ def test_https_connect_http_proxy_error(self, ClientRequestMock): self._fake_coroutine( start_mock, unittest.mock.Mock(status=400, reason='bad request')) - connector = aiohttp.ProxyConnector( - 'http://proxy.example.com', loop=loop_mock) + connector = aiohttp.TCPConnector(loop=loop_mock) tr, proto = unittest.mock.Mock(), unittest.mock.Mock() tr.get_extra_info.return_value = None self._fake_coroutine(loop_mock.create_connection, (tr, proto)) - req = ClientRequest('GET', 'https://www.python.org', loop=self.loop) + req = ClientRequest( + 'GET', 'https://www.python.org', + proxy='http://proxy.example.com', + loop=self.loop, + ) with self.assertRaisesRegex( aiohttp.HttpProxyError, "400, message='bad request'"): self.loop.run_until_complete(connector._create_connection(req)) @@ -298,14 +311,17 @@ def test_https_connect_resp_start_error(self, ClientRequestMock): proxy_resp.start = start_mock = unittest.mock.Mock() self._fake_coroutine(start_mock, OSError("error message")) - connector = aiohttp.ProxyConnector( - 'http://proxy.example.com', loop=loop_mock) + connector = aiohttp.TCPConnector(loop=loop_mock) tr, proto = unittest.mock.Mock(), unittest.mock.Mock() tr.get_extra_info.return_value = None self._fake_coroutine(loop_mock.create_connection, (tr, proto)) - req = ClientRequest('GET', 'https://www.python.org', loop=self.loop) + req = ClientRequest( + 'GET', 'https://www.python.org', + proxy='http://proxy.example.com', + loop=self.loop, + ) with self.assertRaisesRegex(OSError, "error message"): self.loop.run_until_complete(connector._create_connection(req)) @@ -316,31 +332,34 @@ def test_request_port(self, ClientRequestMock): ClientRequestMock.return_value = proxy_req loop_mock = unittest.mock.Mock() - connector = aiohttp.ProxyConnector('http://proxy.example.com', - loop=loop_mock) + connector = aiohttp.TCPConnector(loop=loop_mock) tr, proto = unittest.mock.Mock(), unittest.mock.Mock() tr.get_extra_info.return_value = None self._fake_coroutine(loop_mock.create_connection, (tr, proto)) - req = ClientRequest('GET', 'http://localhost:1234/path', - loop=self.loop) + req = ClientRequest( + 'GET', 'http://localhost:1234/path', + proxy='http://proxy.example.com', + loop=self.loop, + ) self.loop.run_until_complete(connector._create_connection(req)) self.assertEqual(req.path, 'http://localhost:1234/path') def test_proxy_auth_property(self): - connector = aiohttp.ProxyConnector( - 'http://proxy.example.com', + req = aiohttp.ClientRequest( + 'GET', 'http://localhost:1234/path', + proxy='http://proxy.example.com', proxy_auth=aiohttp.helpers.BasicAuth('user', 'pass'), loop=self.loop) - self.assertEqual(('user', 'pass', 'latin1'), connector.proxy_auth) - connector.close() + self.assertEqual(('user', 'pass', 'latin1'), req.proxy_auth) def test_proxy_auth_property_default(self): - connector = aiohttp.ProxyConnector('http://proxy.example.com', - loop=self.loop) - self.assertIsNone(connector.proxy_auth) - connector.close() + req = aiohttp.ClientRequest( + 'GET', 'http://localhost:1234/path', + proxy='http://proxy.example.com', + loop=self.loop) + self.assertIsNone(req.proxy_auth) @unittest.mock.patch('aiohttp.connector.ClientRequest') def test_https_connect_pass_ssl_context(self, ClientRequestMock): @@ -356,13 +375,16 @@ def test_https_connect_pass_ssl_context(self, ClientRequestMock): proxy_resp.start = start_mock = unittest.mock.Mock() self._fake_coroutine(start_mock, unittest.mock.Mock(status=200)) - connector = aiohttp.ProxyConnector( - 'http://proxy.example.com', loop=loop_mock) + connector = aiohttp.TCPConnector(loop=loop_mock) tr, proto = unittest.mock.Mock(), unittest.mock.Mock() self._fake_coroutine(loop_mock.create_connection, (tr, proto)) - req = ClientRequest('GET', 'https://www.python.org', loop=self.loop) + req = ClientRequest( + 'GET', 'https://www.python.org', + proxy='http://proxy.example.com', + loop=self.loop, + ) self.loop.run_until_complete(connector._create_connection(req)) loop_mock.create_connection.assert_called_with( @@ -397,8 +419,7 @@ def test_https_auth(self, ClientRequestMock): proxy_resp.start = start_mock = unittest.mock.Mock() self._fake_coroutine(start_mock, unittest.mock.Mock(status=200)) - connector = aiohttp.ProxyConnector( - 'http://proxy.example.com', loop=loop_mock) + connector = aiohttp.TCPConnector(loop=loop_mock) tr, proto = unittest.mock.Mock(), unittest.mock.Mock() self._fake_coroutine(loop_mock.create_connection, (tr, proto)) @@ -406,7 +427,11 @@ def test_https_auth(self, ClientRequestMock): self.assertIn('AUTHORIZATION', proxy_req.headers) self.assertNotIn('PROXY-AUTHORIZATION', proxy_req.headers) - req = ClientRequest('GET', 'https://www.python.org', loop=self.loop) + req = ClientRequest( + 'GET', 'https://www.python.org', + proxy='http://proxy.example.com', + loop=self.loop + ) self.assertNotIn('AUTHORIZATION', req.headers) self.assertNotIn('PROXY-AUTHORIZATION', req.headers) self.loop.run_until_complete(connector._create_connection(req)) @@ -420,3 +445,75 @@ def test_https_auth(self, ClientRequestMock): self.loop.run_until_complete(proxy_req.close()) proxy_resp.close() self.loop.run_until_complete(req.close()) + + +class TestProxyConnector(unittest.TestCase): + + """ + Backward compatibility simple test. Most testing happens in TextProxy. + """ + + def setUp(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(None) + + def tearDown(self): + # just in case if we have transport close callbacks + self.loop.stop() + self.loop.run_forever() + self.loop.close() + gc.collect() + + def _fake_coroutine(self, mock, return_value): + + def coro(*args, **kw): + if isinstance(return_value, Exception): + raise return_value + return return_value + yield # pragma: no cover + mock.side_effect = coro + + def test_ctor(self): + connector = aiohttp.ProxyConnector( + 'http://localhost:8118', + proxy_auth=aiohttp.helpers.BasicAuth('user', 'pass'), + loop=self.loop, + ) + + self.assertEqual('http://localhost:8118', connector.proxy) + self.assertEqual( + aiohttp.helpers.BasicAuth('user', 'pass'), + connector.proxy_auth + ) + self.assertTrue(connector.force_close) + + @unittest.mock.patch('aiohttp.connector.ClientRequest') + def test_connect(self, ClientRequestMock): + req = ClientRequest('GET', 'http://www.python.org', loop=self.loop) + self.assertEqual(req.path, '/') + + loop_mock = unittest.mock.Mock() + connector = aiohttp.ProxyConnector('http://proxy.example.com', + loop=loop_mock) + self.assertIs(loop_mock, connector._loop) + + resolve_host = unittest.mock.Mock() + self._fake_coroutine(resolve_host, [unittest.mock.MagicMock()]) + connector._resolve_host = resolve_host + + tr, proto = unittest.mock.Mock(), unittest.mock.Mock() + self._fake_coroutine(loop_mock.create_connection, (tr, proto)) + conn = self.loop.run_until_complete(connector.connect(req)) + self.assertEqual(req.path, 'http://www.python.org/') + self.assertIs(conn._transport, tr) + self.assertIs(conn._protocol, proto) + + # resolve_host.assert_called_once_with('proxy.example.com', 80) + tr.get_extra_info.assert_called_once_with('sslcontext') + + ClientRequestMock.assert_called_with( + 'GET', 'http://proxy.example.com', + auth=None, + headers={'HOST': 'www.python.org'}, + loop=loop_mock) + conn.close()