diff --git a/CHANGES.rst b/CHANGES.rst index 9da7f855392..73ce2316c3b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,7 +6,8 @@ CHANGES - Drop deprecated `WSClientDisconnectedError` (BACKWARD INCOMPATIBLE) -- +- Use `yarl.URL` in client API. The change is 99% backward compatible + but `ClientResponse.url` is an `yarl.URL` instance now. - diff --git a/README.rst b/README.rst index 18f19abfdf3..808be0a58c8 100644 --- a/README.rst +++ b/README.rst @@ -116,8 +116,10 @@ Requirements ------------ - Python >= 3.4.2 +- asyncio-timeout_ - chardet_ - multidict_ +- yarl_ Optionally you may install the cChardet_ and aiodns_ libraries (highly recommended for sake of speed). @@ -125,6 +127,8 @@ recommended for sake of speed). .. _chardet: https://pypi.python.org/pypi/chardet .. _aiodns: https://pypi.python.org/pypi/aiodns .. _multidict: https://pypi.python.org/pypi/multidict +.. _yarl: https://pypi.python.org/pypi/yarl +.. _asyncio-timeout: https://pypi.python.org/pypi/asyncio_timeout .. _cChardet: https://pypi.python.org/pypi/cchardet License diff --git a/aiohttp/client.py b/aiohttp/client.py index 4c616ef898d..c4b3f3a4d3b 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -6,10 +6,10 @@ import os import sys import traceback -import urllib.parse import warnings from multidict import CIMultiDict, MultiDict, MultiDictProxy, istr +from yarl import URL import aiohttp @@ -180,8 +180,11 @@ def _request(self, method, url, *, for i in skip_auto_headers: skip_headers.add(istr(i)) + if isinstance(proxy, str): + proxy = URL(proxy) + while True: - url, _ = urllib.parse.urldefrag(url) + url = URL(url).with_fragment(None) cookies = self._cookie_jar.filter_cookies(url) @@ -237,15 +240,15 @@ def _request(self, method, url, *, if headers.get(hdrs.CONTENT_LENGTH): headers.pop(hdrs.CONTENT_LENGTH) - r_url = (resp.headers.get(hdrs.LOCATION) or - resp.headers.get(hdrs.URI)) + r_url = URL(resp.headers.get(hdrs.LOCATION) or + resp.headers.get(hdrs.URI)) - scheme = urllib.parse.urlsplit(r_url)[0] + scheme = r_url.scheme if scheme not in ('http', 'https', ''): resp.close() raise ValueError('Can redirect only to http or https') elif not scheme: - r_url = urllib.parse.urljoin(url, r_url) + r_url = url.join(r_url) url = r_url params = None diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 86310074f72..5ab83564647 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -1,5 +1,4 @@ import asyncio -import collections import http.cookies import io import json @@ -7,10 +6,10 @@ import os import sys import traceback -import urllib.parse import warnings from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy +from yarl import URL import aiohttp @@ -31,9 +30,6 @@ PY_35 = sys.version_info >= (3, 5) -HTTP_PORT = 80 -HTTPS_PORT = 443 - class ClientRequest: @@ -75,7 +71,15 @@ def __init__(self, method, url, *, if loop is None: loop = asyncio.get_event_loop() - self.url = url + assert isinstance(url, URL), url + assert isinstance(proxy, (URL, type(None))), proxy + + if params: + q = MultiDict(url.query) + url2 = url.with_query(params) + q.extend(url2.query) + url = url.with_query(q) + self.url = url.with_fragment(None) self.method = method.upper() self.encoding = encoding self.chunked = chunked @@ -89,7 +93,6 @@ def __init__(self, method, url, *, self.update_version(version) self.update_host(url) - self.update_path(params) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) @@ -101,59 +104,30 @@ def __init__(self, method, url, *, self.update_transfer_encoding() self.update_expect_continue(expect100) - def update_host(self, url): - """Update destination host, port and connection type (ssl).""" - url_parsed = urllib.parse.urlsplit(url) + @property + def host(self): + return self.url.host - # check for network location part - netloc = url_parsed.netloc - if not netloc: - raise ValueError('Host could not be detected.') + @property + def port(self): + return self.url.port + def update_host(self, url): + """Update destination host, port and connection type (ssl).""" # get host/port - host = url_parsed.hostname - if not host: + if not url.host: raise ValueError('Host could not be detected.') - try: - port = url_parsed.port - except ValueError: - raise ValueError( - 'Port number could not be converted.') from None - - # check domain idna encoding - try: - host = host.encode('idna').decode('utf-8') - netloc = self.make_netloc(host, url_parsed.port) - except UnicodeError: - raise ValueError('URL has an invalid label.') - # basic auth info - username, password = url_parsed.username, url_parsed.password + username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') # Record entire netloc for usage in host header - self.netloc = netloc - scheme = url_parsed.scheme + scheme = url.scheme self.ssl = scheme in ('https', 'wss') - # set port number if it isn't already set - if not port: - if self.ssl: - port = HTTPS_PORT - else: - port = HTTP_PORT - - self.host, self.port, self.scheme = host, port, scheme - - def make_netloc(self, host, port): - ret = host - if port: - ret = ret + ':' + str(port) - return ret - def update_version(self, version): """Convert request version to two elements tuple. @@ -172,25 +146,8 @@ def update_version(self, version): def update_path(self, params): """Build path.""" # extract path - scheme, netloc, path, query, fragment = urllib.parse.urlsplit(self.url) - if not path: - path = '/' - - if isinstance(params, collections.Mapping): - params = list(params.items()) - - if params: - if not isinstance(params, str): - params = urllib.parse.urlencode(params) - if query: - query = '%s&%s' % (query, params) - else: - query = params - self.path = urllib.parse.urlunsplit(('', '', helpers.requote_uri(path), - query, '')) - self.url = urllib.parse.urlunsplit( - (scheme, netloc, self.path, '', fragment)) + self.url = self.url.with_query(params) def update_headers(self, headers): """Update request headers.""" @@ -214,7 +171,10 @@ def update_auto_headers(self, skip_auto_headers): # add host if hdrs.HOST not in used_headers: - self.headers[hdrs.HOST] = self.netloc + netloc = self.url.host + if not self.url.is_default_port(): + netloc += ':' + str(self.url.port) + self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = self.SERVER_SOFTWARE @@ -378,7 +338,7 @@ def update_expect_continue(self, expect=False): self._continue = helpers.create_future(self.loop) def update_proxy(self, proxy, proxy_auth): - if proxy and not proxy.startswith('http://'): + if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") @@ -488,7 +448,11 @@ def write_bytes(self, request, reader): def send(self, writer, reader): writer.set_tcp_cork(True) - request = aiohttp.Request(writer, self.method, self.path, self.version) + path = self.url.raw_path + if self.url.raw_query_string: + path += '?' + self.url.raw_query_string + request = aiohttp.Request(writer, self.method, path, + self.version) if self.compress: request.add_compression_filter(self.compress) @@ -511,7 +475,7 @@ def send(self, writer, reader): self.write_bytes(request, reader), loop=self.loop) self.response = self.response_class( - self.method, self.url, self.host, + self.method, self.url, self.url.host, writer=self._writer, continue100=self._continue, timeout=self._timeout) self.response._post_init(self.loop) @@ -556,7 +520,7 @@ class ClientResponse: def __init__(self, method, url, host='', *, writer=None, continue100=None, timeout=5*60): - super().__init__() + assert isinstance(url, URL) self.method = method self.url = url @@ -591,8 +555,7 @@ def __del__(self, _warnings=warnings): def __repr__(self): out = io.StringIO() - ascii_encodable_url = self.url.encode('ascii', 'backslashreplace') \ - .decode('ascii') + ascii_encodable_url = str(self.url) if self.reason: ascii_encodable_reason = self.reason.encode('ascii', 'backslashreplace') \ diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 8ada87a55d6..cdfdca709bb 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -11,6 +11,8 @@ from math import ceil from types import MappingProxyType +from yarl import URL + import aiohttp from . import hdrs, helpers @@ -638,9 +640,7 @@ def _create_proxy_connection(self, req): raise ProxyConnectionError(*exc.args) from exc if not req.ssl: - req.path = '{scheme}://{host}{path}'.format(scheme=req.scheme, - host=req.netloc, - path=req.path) + req.path = str(req.url) if hdrs.AUTHORIZATION in proxy_req.headers: auth = proxy_req.headers[hdrs.AUTHORIZATION] del proxy_req.headers[hdrs.AUTHORIZATION] @@ -723,6 +723,7 @@ def __init__(self, proxy, *, proxy_auth=None, force_close=True, conn_timeout=conn_timeout, keepalive_timeout=keepalive_timeout, limit=limit, loop=loop) + assert isinstance(proxy, URL) self._proxy = proxy self._proxy_auth = proxy_auth diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 907b0ca52fc..facd1e7b66c 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -4,7 +4,8 @@ from collections.abc import Mapping from http.cookies import Morsel, SimpleCookie from math import ceil -from urllib.parse import urlsplit + +from yarl import URL from .abc import AbstractCookieJar from .helpers import is_ip_address @@ -76,10 +77,9 @@ def _expire_cookie(self, when, domain, name): self._next_expiration = min(self._next_expiration, when) self._expirations[(domain, name)] = when - def update_cookies(self, cookies, response_url=None): + def update_cookies(self, cookies, response_url=URL()): """Update cookies.""" - url_parsed = urlsplit(response_url or "") - hostname = url_parsed.hostname + hostname = response_url.host if not self._unsafe and is_ip_address(hostname): # Don't accept cookies from IPs @@ -119,7 +119,7 @@ def update_cookies(self, cookies, response_url=None): path = cookie["path"] if not path or not path.startswith("/"): # Set the cookie's path to the response path - path = url_parsed.path + path = response_url.path if not path.startswith("/"): path = "/" else: @@ -152,13 +152,12 @@ def update_cookies(self, cookies, response_url=None): self._do_expiration() - def filter_cookies(self, request_url): + def filter_cookies(self, request_url=URL()): """Returns this jar's cookies filtered by their attributes.""" self._do_expiration() - url_parsed = urlsplit(request_url) filtered = SimpleCookie() - hostname = url_parsed.hostname or "" - is_not_secure = url_parsed.scheme not in ("https", "wss") + hostname = request_url.host or "" + is_not_secure = request_url.scheme not in ("https", "wss") for cookie in self: name = cookie.key @@ -178,7 +177,7 @@ def filter_cookies(self, request_url): elif not self._is_domain_match(domain, hostname): continue - if not self._is_path_match(url_parsed.path, cookie["path"]): + if not self._is_path_match(request_url.path, cookie["path"]): continue if is_not_secure and cookie["secure"]: diff --git a/aiohttp/errors.py b/aiohttp/errors.py index c7286a27568..85c6eed7ba9 100644 --- a/aiohttp/errors.py +++ b/aiohttp/errors.py @@ -176,7 +176,3 @@ def __repr__(self): return '<{} expected={} got={} host={} port={}>'.format( self.__class__.__name__, self.expected, self.got, self.host, self.port) - - -class InvalidURL(Exception): - """Invalid URL.""" diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 5b3f524efe3..37329d58426 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -11,13 +11,12 @@ import warnings from collections import namedtuple from pathlib import Path -from urllib.parse import quote, urlencode +from urllib.parse import urlencode from async_timeout import timeout from multidict import MultiDict, MultiDictProxy from . import hdrs -from .errors import InvalidURL try: from asyncio import ensure_future @@ -440,54 +439,6 @@ def __set__(self, inst, value): raise AttributeError("reified property is read-only") -# The unreserved URI characters (RFC 3986) -UNRESERVED_SET = frozenset( - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + - "0123456789-._~") - - -def unquote_unreserved(uri): - """Un-escape any percent-escape sequences in a URI that are unreserved - characters. This leaves all reserved, illegal and non-ASCII bytes encoded. - """ - parts = uri.split('%') - for i in range(1, len(parts)): - h = parts[i][0:2] - if len(h) == 2 and h.isalnum(): - try: - c = chr(int(h, 16)) - except ValueError: - raise InvalidURL("Invalid percent-escape sequence: '%s'" % h) - - if c in UNRESERVED_SET: - parts[i] = c + parts[i][2:] - else: - parts[i] = '%' + parts[i] - else: - parts[i] = '%' + parts[i] - return ''.join(parts) - - -def requote_uri(uri): - """Re-quote the given URI. - - This function passes the given URI through an unquote/quote cycle to - ensure that it is fully and consistently quoted. - """ - safe_with_percent = "!#$%&'()*+,/:;=?@[]~" - safe_without_percent = "!#$&'()*+,/:;=?@[]~" - try: - # Unquote only the unreserved characters - # Then quote only illegal characters (do not quote reserved, - # unreserved, or '%') - return quote(unquote_unreserved(uri), safe=safe_with_percent) - except InvalidURL: - # We couldn't unquote the given URI, so let's try quoting it, but - # there may be unquoted '%'s in the URI. We need to make sure they're - # properly quoted so they do not cause issues elsewhere. - return quote(uri, safe=safe_without_percent) - - _ipv4_pattern = ('^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}' '(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$') _ipv6_pattern = ( diff --git a/aiohttp/web.py b/aiohttp/web.py index f06535c6aa2..434a5643586 100644 --- a/aiohttp/web.py +++ b/aiohttp/web.py @@ -16,7 +16,6 @@ from .web_urldispatcher import * # noqa from .web_ws import * # noqa - __all__ = (web_reqrep.__all__ + web_exceptions.__all__ + web_urldispatcher.__all__ + diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 70ecea57184..ccb2a496251 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1135,7 +1135,13 @@ Response object .. attribute:: url - URL of request (:class:`str`). + URL of request (:class:`~yarl.URL`). + + .. versionchanged:: 1.1 + + The attribute is :class:`~yarl.URL` now instead of :class:`str`. + + For giving a string use ``str(resp.url)``. .. attribute:: connection diff --git a/docs/index.rst b/docs/index.rst index 09e092aae52..01aefbfb525 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -122,10 +122,11 @@ Dependencies ------------ - Python 3.4.2+ -- *chardet* library -- *multidict* library -- *async_timeout* library -- *Optional* :term:`cchardet` library as faster replacement for +- *chardet* +- *multidict* +- *async_timeout* +- *yarl* +- *Optional* :term:`cchardet` as faster replacement for :term:`chardet`. Install it explicitly via: @@ -134,7 +135,7 @@ Dependencies $ pip install cchardet -- *Optional* :term:`aiodns` library for fast DNS resolving. The +- *Optional* :term:`aiodns` for fast DNS resolving. The library is highly recommended. .. code-block:: bash diff --git a/requirements-ci.txt b/requirements-ci.txt index aef1916adb4..9d6bdbfc61e 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -20,4 +20,5 @@ gunicorn pygments>=2.1 #aiodns # Enable from .travis.yml as required c-ares will not build on windows twine +yarl -e . diff --git a/setup.py b/setup.py index 181ce680657..e703d49d72b 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ def build_extension(self, ext): raise RuntimeError('Unable to determine version.') -install_requires = ['chardet', 'multidict>=2.0', 'async_timeout'] +install_requires = ['chardet', 'multidict>=2.0', 'async_timeout', 'yarl'] if sys.version_info < (3, 4, 2): raise RuntimeError("aiohttp requires Python 3.4.2+") diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 8c162122da5..5b0fa8aa577 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -391,7 +391,7 @@ def handler_ok(request): resp = yield from client.get('/redirect') try: assert resp.status == 200 - assert resp.url.endswith('/ok') + assert resp.url.path == '/ok' finally: yield from resp.release() @@ -408,7 +408,7 @@ def handler_ok(request): resp = yield from client.get('/ok#fragment') try: assert resp.status == 200 - assert resp.url.endswith('/ok') + assert resp.url.path == '/ok' finally: yield from resp.release() diff --git a/tests/test_client_request.py b/tests/test_client_request.py index c68d3b0ba21..08daf1e2215 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -5,7 +5,6 @@ import inspect import io import os.path -import re import unittest import urllib.parse import zlib @@ -14,6 +13,7 @@ import pytest from multidict import CIMultiDict, CIMultiDictProxy, upstr +from yarl import URL import aiohttp from aiohttp import BaseConnector, helpers @@ -24,9 +24,9 @@ def make_request(loop): request = None - def maker(*args, **kwargs): + def maker(method, url, *args, **kwargs): nonlocal request - request = ClientRequest(*args, loop=loop, **kwargs) + request = ClientRequest(method, URL(url), *args, loop=loop, **kwargs) return request yield maker @@ -137,7 +137,7 @@ def test_host_header_host_without_port(make_request): def test_host_header_host_with_default_port(make_request): req = make_request('get', 'http://python.org:80/') - assert req.headers['HOST'] == 'python.org:80' + assert req.headers['HOST'] == 'python.org' def test_host_header_host_with_nondefault_port(make_request): @@ -159,7 +159,7 @@ def test_host_header_explicit_host_with_port(make_request): def test_default_loop(loop): asyncio.set_event_loop(loop) - req = ClientRequest('get', 'http://python.org/') + req = ClientRequest('get', URL('http://python.org/')) assert req.loop is loop @@ -219,7 +219,7 @@ def test_invalid_idna(make_request): def test_no_path(make_request): req = make_request('get', 'http://python.org') - assert '/' == req.path + assert '/' == req.url.path def test_ipv6_default_http_port(make_request): @@ -275,7 +275,7 @@ def test_basic_auth_from_url(make_request): req = make_request('get', 'http://nkim:1234@python.org') assert 'AUTHORIZATION' in req.headers assert 'Basic bmtpbToxMjM0' == req.headers['AUTHORIZATION'] - assert 'python.org' == req.netloc + assert 'python.org' == req.host def test_basic_auth_from_url_overriden(make_request): @@ -283,49 +283,49 @@ def test_basic_auth_from_url_overriden(make_request): auth=aiohttp.BasicAuth('nkim', '1234')) assert 'AUTHORIZATION' in req.headers assert 'Basic bmtpbToxMjM0' == req.headers['AUTHORIZATION'] - assert 'python.org' == req.netloc + assert 'python.org' == req.host def test_path_is_not_double_encoded1(make_request): req = make_request('get', "http://0.0.0.0/get/test case") - assert req.path == "/get/test%20case" + assert req.url.raw_path == "/get/test%20case" def test_path_is_not_double_encoded2(make_request): req = make_request('get', "http://0.0.0.0/get/test%2fcase") - assert req.path == "/get/test%2fcase" + assert req.url.raw_path == "/get/test%2Fcase" def test_path_is_not_double_encoded3(make_request): req = make_request('get', "http://0.0.0.0/get/test%20case") - assert req.path == "/get/test%20case" + assert req.url.raw_path == "/get/test%20case" def test_path_safe_chars_preserved(make_request): - req = make_request('get', "http://0.0.0.0/get/%:=") - assert req.path == "/get/%:=" + req = make_request('get', "http://0.0.0.0/get/:=") + assert req.url.path == "/get/:=" def test_params_are_added_before_fragment1(make_request): req = make_request('GET', "http://example.com/path#fragment", params={"a": "b"}) - assert req.url == "http://example.com/path?a=b#fragment" + assert str(req.url) == "http://example.com/path?a=b" def test_params_are_added_before_fragment2(make_request): req = make_request('GET', "http://example.com/path?key=value#fragment", params={"a": "b"}) - assert req.url == "http://example.com/path?key=value&a=b#fragment" + assert str(req.url) == "http://example.com/path?key=value&a=b" def test_path_not_contain_fragment1(make_request): req = make_request('GET', "http://example.com/path#fragment") - assert req.path == "/path" + assert req.url.path == "/path" def test_path_not_contain_fragment2(make_request): req = make_request('GET', "http://example.com/path?key=value#fragment") - assert req.path == "/path?key=value" + assert str(req.url) == "http://example.com/path?key=value" def test_cookies(make_request): @@ -347,19 +347,19 @@ def test_cookies_merge_with_headers(make_request): def test_unicode_get1(make_request): req = make_request('get', 'http://python.org', params={'foo': 'f\xf8\xf8'}) - assert '/?foo=f%C3%B8%C3%B8' == req.path + assert 'http://python.org/?foo=f%C3%B8%C3%B8' == str(req.url) def test_unicode_get2(make_request): req = make_request('', 'http://python.org', params={'f\xf8\xf8': 'f\xf8\xf8'}) - assert '/?f%C3%B8%C3%B8=f%C3%B8%C3%B8' == req.path + assert 'http://python.org/?f%C3%B8%C3%B8=f%C3%B8%C3%B8' == str(req.url) def test_unicode_get3(make_request): req = make_request('', 'http://python.org', params={'foo': 'foo'}) - assert '/?foo=foo' == req.path + assert 'http://python.org/?foo=foo' == str(req.url) def test_unicode_get4(make_request): @@ -367,7 +367,7 @@ def join(*suffix): return urllib.parse.urljoin('http://python.org/', '/'.join(suffix)) req = make_request('', join('\xf8'), params={'foo': 'foo'}) - assert '/%C3%B8?foo=foo' == req.path + assert 'http://python.org/%C3%B8?foo=foo' == str(req.url) def test_query_multivalued_param(make_request): @@ -376,42 +376,38 @@ def test_query_multivalued_param(make_request): meth, 'http://python.org', params=(('test', 'foo'), ('test', 'baz'))) - assert req.path == '/?test=foo&test=baz' + assert str(req.url) == 'http://python.org/?test=foo&test=baz' def test_query_str_param(make_request): for meth in ClientRequest.ALL_METHODS: req = make_request(meth, 'http://python.org', params='test=foo') - assert req.path == '/?test=foo' + assert str(req.url) == 'http://python.org/?test=foo' def test_query_bytes_param_raises(make_request): for meth in ClientRequest.ALL_METHODS: - with pytest.raises(TypeError) as ctx: + with pytest.raises(TypeError): make_request(meth, 'http://python.org', params=b'test=foo') - assert re.match('not a valid non-string.*or mapping', str(ctx.value)) def test_query_str_param_is_not_encoded(make_request): for meth in ClientRequest.ALL_METHODS: req = make_request(meth, 'http://python.org', params='test=f+oo') - assert req.path == '/?test=f+oo' + assert str(req.url) == 'http://python.org/?test=f+oo' def test_params_update_path_and_url(make_request): req = make_request('get', 'http://python.org', params=(('test', 'foo'), ('test', 'baz'))) - assert req.path == '/?test=foo&test=baz' - assert req.url == 'http://python.org/?test=foo&test=baz' + assert str(req.url) == 'http://python.org/?test=foo&test=baz' def test_params_empty_path_and_url(make_request): req_empty = make_request('get', 'http://python.org', params={}) - assert req_empty.path == '/' - assert req_empty.url == 'http://python.org/' + assert str(req_empty.url) == 'http://python.org' req_none = make_request('get', 'http://python.org') - assert req_none.path == '/' - assert req_none.url == 'http://python.org/' + assert str(req_none.url) == 'http://python.org' def test_gen_netloc_all(make_request): @@ -419,8 +415,8 @@ def test_gen_netloc_all(make_request): 'https://aiohttp:pwpwpw@' + '12345678901234567890123456789' + '012345678901234567890:8080') - assert req.netloc == '12345678901234567890123456789' +\ - '012345678901234567890:8080' + assert req.headers['HOST'] == '12345678901234567890123456789' +\ + '012345678901234567890:8080' def test_gen_netloc_no_port(make_request): @@ -428,17 +424,8 @@ def test_gen_netloc_no_port(make_request): 'https://aiohttp:pwpwpw@' + '12345678901234567890123456789' + '012345678901234567890/') - assert req.netloc == '12345678901234567890123456789' +\ - '012345678901234567890' - - -def test_gen_notloc_failed(make_request): - with pytest.raises(ValueError) as excinfo: - make_request('get', - 'https://aiohttp:pwpwpw@' + - '123456789012345678901234567890123456789' + - '01234567890123456789012345/') - assert excinfo.value.message == "URL has an invalid label." + assert req.headers['HOST'] == '12345678901234567890123456789' +\ + '012345678901234567890' class TestClientRequest(unittest.TestCase): @@ -465,35 +452,35 @@ def tearDown(self): gc.collect() def test_no_content_length(self): - req = ClientRequest('get', 'http://python.org', loop=self.loop) + req = ClientRequest('get', URL('http://python.org'), loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('0', req.headers.get('CONTENT-LENGTH')) self.loop.run_until_complete(req.close()) resp.close() def test_no_content_length2(self): - req = ClientRequest('head', 'http://python.org', loop=self.loop) + req = ClientRequest('head', URL('http://python.org'), loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('0', req.headers.get('CONTENT-LENGTH')) self.loop.run_until_complete(req.close()) resp.close() def test_content_type_auto_header_get(self): - req = ClientRequest('get', 'http://python.org', loop=self.loop) + req = ClientRequest('get', URL('http://python.org'), loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertNotIn('CONTENT-TYPE', req.headers) resp.close() def test_content_type_auto_header_form(self): - req = ClientRequest('post', 'http://python.org', data={'hey': 'you'}, - loop=self.loop) + req = ClientRequest('post', URL('http://python.org'), + data={'hey': 'you'}, loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('application/x-www-form-urlencoded', req.headers.get('CONTENT-TYPE')) resp.close() def test_content_type_auto_header_bytes(self): - req = ClientRequest('post', 'http://python.org', data=b'hey you', + req = ClientRequest('post', URL('http://python.org'), data=b'hey you', loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('application/octet-stream', @@ -501,7 +488,7 @@ def test_content_type_auto_header_bytes(self): resp.close() def test_content_type_skip_auto_header_bytes(self): - req = ClientRequest('post', 'http://python.org', data=b'hey you', + req = ClientRequest('post', URL('http://python.org'), data=b'hey you', skip_auto_headers={'Content-Type'}, loop=self.loop) resp = req.send(self.transport, self.protocol) @@ -509,14 +496,15 @@ def test_content_type_skip_auto_header_bytes(self): resp.close() def test_content_type_skip_auto_header_form(self): - req = ClientRequest('post', 'http://python.org', data={'hey': 'you'}, - loop=self.loop, skip_auto_headers={'Content-Type'}) + req = ClientRequest('post', URL('http://python.org'), + data={'hey': 'you'}, loop=self.loop, + skip_auto_headers={'Content-Type'}) resp = req.send(self.transport, self.protocol) self.assertNotIn('CONTENT-TYPE', req.headers) resp.close() def test_content_type_auto_header_content_length_no_skip(self): - req = ClientRequest('get', 'http://python.org', + req = ClientRequest('get', URL('http://python.org'), data=io.BytesIO(b'hey'), skip_auto_headers={'Content-Length'}, loop=self.loop) @@ -527,10 +515,10 @@ def test_content_type_auto_header_content_length_no_skip(self): def test_post_data(self): for meth in ClientRequest.POST_METHODS: req = ClientRequest( - meth, 'http://python.org/', + meth, URL('http://python.org/'), data={'life': '42'}, loop=self.loop) resp = req.send(self.transport, self.protocol) - self.assertEqual('/', req.path) + self.assertEqual('/', req.url.path) self.assertEqual(b'life=42', req.body) self.assertEqual('application/x-www-form-urlencoded', req.headers['CONTENT-TYPE']) @@ -541,7 +529,7 @@ def test_post_data(self): 'aiohttp.client_reqrep.ClientRequest.update_body_from_data') def test_pass_falsy_data(self, _): req = ClientRequest( - 'post', 'http://python.org/', + 'post', URL('http://python.org/'), data={}, loop=self.loop) req.update_body_from_data.assert_called_once_with({}, frozenset()) self.loop.run_until_complete(req.close()) @@ -549,19 +537,19 @@ def test_pass_falsy_data(self, _): def test_get_with_data(self): for meth in ClientRequest.GET_METHODS: req = ClientRequest( - meth, 'http://python.org/', data={'life': '42'}, + meth, URL('http://python.org/'), data={'life': '42'}, loop=self.loop) - self.assertEqual('/', req.path) + self.assertEqual('/', req.url.path) self.assertEqual(b'life=42', req.body) self.loop.run_until_complete(req.close()) def test_bytes_data(self): for meth in ClientRequest.POST_METHODS: req = ClientRequest( - meth, 'http://python.org/', + meth, URL('http://python.org/'), data=b'binary data', loop=self.loop) resp = req.send(self.transport, self.protocol) - self.assertEqual('/', req.path) + self.assertEqual('/', req.url.path) self.assertEqual(b'binary data', req.body) self.assertEqual('application/octet-stream', req.headers['CONTENT-TYPE']) @@ -570,7 +558,7 @@ def test_bytes_data(self): @mock.patch('aiohttp.client_reqrep.aiohttp') def test_content_encoding(self, m_http): - req = ClientRequest('get', 'http://python.org/', data='foo', + req = ClientRequest('get', URL('http://python.org/'), data='foo', compress='deflate', loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked') @@ -582,7 +570,7 @@ def test_content_encoding(self, m_http): @mock.patch('aiohttp.client_reqrep.aiohttp') def test_content_encoding_dont_set_headers_if_no_body(self, m_http): - req = ClientRequest('get', 'http://python.org/', + req = ClientRequest('get', URL('http://python.org/'), compress='deflate', loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertNotIn('TRANSFER-ENCODING', req.headers) @@ -593,7 +581,7 @@ def test_content_encoding_dont_set_headers_if_no_body(self, m_http): @mock.patch('aiohttp.client_reqrep.aiohttp') def test_content_encoding_header(self, m_http): req = ClientRequest( - 'get', 'http://python.org/', data='foo', + 'get', URL('http://python.org/'), data='foo', headers={'Content-Encoding': 'deflate'}, loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked') @@ -608,7 +596,7 @@ def test_content_encoding_header(self, m_http): def test_chunked(self): req = ClientRequest( - 'get', 'http://python.org/', + 'get', URL('http://python.org/'), headers={'TRANSFER-ENCODING': 'gzip'}, loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('gzip', req.headers['TRANSFER-ENCODING']) @@ -616,7 +604,7 @@ def test_chunked(self): resp.close() req = ClientRequest( - 'get', 'http://python.org/', + 'get', URL('http://python.org/'), headers={'Transfer-encoding': 'chunked'}, loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('chunked', req.headers['TRANSFER-ENCODING']) @@ -626,7 +614,7 @@ def test_chunked(self): @mock.patch('aiohttp.client_reqrep.aiohttp') def test_chunked_explicit(self, m_http): req = ClientRequest( - 'get', 'http://python.org/', chunked=True, loop=self.loop) + 'get', URL('http://python.org/'), chunked=True, loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('chunked', req.headers['TRANSFER-ENCODING']) @@ -638,7 +626,7 @@ def test_chunked_explicit(self, m_http): @mock.patch('aiohttp.client_reqrep.aiohttp') def test_chunked_explicit_size(self, m_http): req = ClientRequest( - 'get', 'http://python.org/', chunked=1024, loop=self.loop) + 'get', URL('http://python.org/'), chunked=1024, loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('chunked', req.headers['TRANSFER-ENCODING']) m_http.Request.return_value\ @@ -648,7 +636,7 @@ def test_chunked_explicit_size(self, m_http): def test_chunked_length(self): req = ClientRequest( - 'get', 'http://python.org/', + 'get', URL('http://python.org/'), headers={'CONTENT-LENGTH': '1000'}, chunked=1024, loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked') @@ -661,7 +649,7 @@ def test_file_upload_not_chunked(self): fname = os.path.join(here, 'sample.key') with open(fname, 'rb') as f: req = ClientRequest( - 'post', 'http://python.org/', + 'post', URL('http://python.org/'), data=f, loop=self.loop) self.assertFalse(req.chunked) @@ -672,7 +660,7 @@ def test_file_upload_not_chunked(self): def test_precompressed_data_stays_intact(self): data = zlib.compress(b'foobar') req = ClientRequest( - 'post', 'http://python.org/', + 'post', URL('http://python.org/'), data=data, headers={'CONTENT-ENCODING': 'deflate'}, compress=False, @@ -689,7 +677,7 @@ def test_file_upload_not_chunked_seek(self): with open(fname, 'rb') as f: f.seek(100) req = ClientRequest( - 'post', 'http://python.org/', + 'post', URL('http://python.org/'), data=f, loop=self.loop) self.assertEqual(req.headers['CONTENT-LENGTH'], @@ -701,7 +689,7 @@ def test_file_upload_force_chunked(self): fname = os.path.join(here, 'sample.key') with open(fname, 'rb') as f: req = ClientRequest( - 'post', 'http://python.org/', + 'post', URL('http://python.org/'), data=f, chunked=True, loop=self.loop) @@ -710,7 +698,7 @@ def test_file_upload_force_chunked(self): self.loop.run_until_complete(req.close()) def test_expect100(self): - req = ClientRequest('get', 'http://python.org/', + req = ClientRequest('get', URL('http://python.org/'), expect100=True, loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('100-continue', req.headers['EXPECT']) @@ -719,7 +707,7 @@ def test_expect100(self): resp.close() def test_expect_100_continue_header(self): - req = ClientRequest('get', 'http://python.org/', + req = ClientRequest('get', URL('http://python.org/'), headers={'expect': '100-continue'}, loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('100-continue', req.headers['EXPECT']) @@ -733,7 +721,7 @@ def gen(): return b' result' req = ClientRequest( - 'POST', 'http://python.org/', data=gen(), loop=self.loop) + 'POST', URL('http://python.org/'), data=gen(), loop=self.loop) self.assertTrue(req.chunked) self.assertTrue(inspect.isgenerator(req.body)) self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked') @@ -750,7 +738,7 @@ def gen(): def test_data_file(self): req = ClientRequest( - 'POST', 'http://python.org/', + 'POST', URL('http://python.org/'), data=io.BufferedReader(io.BytesIO(b'*' * 2)), loop=self.loop) self.assertTrue(req.chunked) @@ -775,7 +763,7 @@ def gen(): yield from fut req = ClientRequest( - 'POST', 'http://python.org/', data=gen(), loop=self.loop) + 'POST', URL('http://python.org/'), data=gen(), loop=self.loop) self.assertTrue(req.chunked) self.assertTrue(inspect.isgenerator(req.body)) self.assertEqual(req.headers['TRANSFER-ENCODING'], 'chunked') @@ -800,7 +788,7 @@ def gen(): yield object() req = ClientRequest( - 'POST', 'http://python.org/', data=gen(), loop=self.loop) + 'POST', URL('http://python.org/'), data=gen(), loop=self.loop) resp = req.send(self.transport, self.protocol) self.loop.run_until_complete(req._writer) self.assertTrue(self.protocol.set_exception.called) @@ -814,7 +802,7 @@ def gen(): yield from fut req = ClientRequest( - 'POST', 'http://python.org/', data=gen(), loop=self.loop) + 'POST', URL('http://python.org/'), data=gen(), loop=self.loop) inner_exc = ValueError() @@ -842,7 +830,7 @@ def gen(): return b' result' req = ClientRequest( - 'POST', 'http://python.org/', data=gen(), + 'POST', URL('http://python.org/'), data=gen(), expect100=True, loop=self.loop) self.assertTrue(req.chunked) self.assertTrue(inspect.isgenerator(req.body)) @@ -864,7 +852,7 @@ def coro(): def test_data_continue(self): req = ClientRequest( - 'POST', 'http://python.org/', data=b'data', + 'POST', URL('http://python.org/'), data=b'data', expect100=True, loop=self.loop) def coro(): @@ -890,7 +878,7 @@ def gen(): return b'result' req = ClientRequest( - 'POST', 'http://python.org/', data=gen(), loop=self.loop) + 'POST', URL('http://python.org/'), data=gen(), loop=self.loop) resp = req.send(self.transport, self.protocol) self.loop.run_until_complete(req.close()) self.assertEqual( @@ -906,7 +894,7 @@ def read(self, decode=False): return 'customized!' req = ClientRequest( - 'GET', 'http://python.org/', response_class=CustomResponse, + 'GET', URL('http://python.org/'), response_class=CustomResponse, loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertEqual('customized!', resp.read()) @@ -914,7 +902,7 @@ def read(self, decode=False): resp.close() def test_terminate(self): - req = ClientRequest('get', 'http://python.org', loop=self.loop) + req = ClientRequest('get', URL('http://python.org'), loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertIsNotNone(req._writer) writer = req._writer = mock.Mock() @@ -925,7 +913,7 @@ def test_terminate(self): resp.close() def test_terminate_with_closed_loop(self): - req = ClientRequest('get', 'http://python.org', loop=self.loop) + req = ClientRequest('get', URL('http://python.org'), loop=self.loop) resp = req.send(self.transport, self.protocol) self.assertIsNotNone(req._writer) writer = req._writer = mock.Mock() @@ -937,7 +925,7 @@ def test_terminate_with_closed_loop(self): resp.close() def test_terminate_without_writer(self): - req = ClientRequest('get', 'http://python.org', loop=self.loop) + req = ClientRequest('get', URL('http://python.org'), loop=self.loop) self.assertIsNone(req._writer) req.terminate() @@ -981,12 +969,13 @@ def create_connection(req): return self.transport, self.protocol self.connector._create_connection = create_connection - resp = yield from aiohttp.request('get', - 'http://example.com/path/to', - request_class=CustomRequest, - response_class=CustomResponse, - connector=self.connector, - loop=self.loop) + resp = yield from aiohttp.request( + 'get', + URL('http://example.com/path/to'), + request_class=CustomRequest, + response_class=CustomResponse, + connector=self.connector, + loop=self.loop) self.assertIsInstance(resp, CustomResponse) self.assertTrue(called) resp.close() diff --git a/tests/test_client_response.py b/tests/test_client_response.py index ce46478f037..7ed6e426ee5 100644 --- a/tests/test_client_response.py +++ b/tests/test_client_response.py @@ -6,6 +6,8 @@ import unittest from unittest import mock +from yarl import URL + import aiohttp from aiohttp import helpers from aiohttp.client_reqrep import ClientResponse @@ -19,7 +21,7 @@ def setUp(self): self.connection = mock.Mock() self.stream = aiohttp.StreamParser(loop=self.loop) - self.response = ClientResponse('get', 'http://def-cl-resp.org') + self.response = ClientResponse('get', URL('http://def-cl-resp.org')) self.response._post_init(self.loop) self.response._setup_connection(self.connection) @@ -29,7 +31,7 @@ def tearDown(self): gc.collect() def test_del(self): - response = ClientResponse('get', 'http://del-cl-resp.org') + response = ClientResponse('get', URL('http://del-cl-resp.org')) response._post_init(self.loop) connection = mock.Mock() @@ -51,14 +53,14 @@ def test_close(self): def test_wait_for_100_1(self): response = ClientResponse( - 'get', 'http://python.org', continue100=object()) + 'get', URL('http://python.org'), continue100=object()) response._post_init(self.loop) self.assertTrue(response.waiting_for_continue()) response.close() def test_wait_for_100_2(self): response = ClientResponse( - 'get', 'http://python.org') + 'get', URL('http://python.org')) response._post_init(self.loop) self.assertFalse(response.waiting_for_continue()) response.close() @@ -71,13 +73,13 @@ def test_repr(self): repr(self.response)) def test_repr_non_ascii_url(self): - response = ClientResponse('get', 'http://fake-host.org/\u03bb') + response = ClientResponse('get', URL('http://fake-host.org/\u03bb')) self.assertIn( - "", + "", repr(response)) def test_repr_non_ascii_reason(self): - response = ClientResponse('get', 'http://fake-host.org/path') + response = ClientResponse('get', URL('http://fake-host.org/path')) response.reason = '\u03bb' self.assertIn( "", @@ -243,7 +245,7 @@ def side_effect(*args, **kwargs): def test_override_flow_control(self): class MyResponse(ClientResponse): flow_control_class = aiohttp.StreamReader - response = MyResponse('get', 'http://my-cl-resp.org') + response = MyResponse('get', URL('http://my-cl-resp.org')) response._post_init(self.loop) response._setup_connection(self.connection) self.assertIsInstance(response.content, aiohttp.StreamReader) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index c9455611e0f..efd34faf5c8 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -8,6 +8,7 @@ import pytest from multidict import CIMultiDict, MultiDict +from yarl import URL import aiohttp from aiohttp import web @@ -94,7 +95,7 @@ def test_init_headers_list_of_tuples_with_duplicates(create_session): def test_init_cookies_with_simple_dict(create_session): session = create_session(cookies={"c1": "cookie1", "c2": "cookie2"}) - cookies = session.cookie_jar.filter_cookies('') + cookies = session.cookie_jar.filter_cookies() assert set(cookies) == {'c1', 'c2'} assert cookies['c1'].value == 'cookie1' assert cookies['c2'].value == 'cookie2' @@ -104,7 +105,7 @@ def test_init_cookies_with_list_of_tuples(create_session): session = create_session(cookies=[("c1", "cookie1"), ("c2", "cookie2")]) - cookies = session.cookie_jar.filter_cookies('') + cookies = session.cookie_jar.filter_cookies() assert set(cookies) == {'c1', 'c2'} assert cookies['c1'].value == 'cookie1' assert cookies['c2'].value == 'cookie2' @@ -402,7 +403,7 @@ def handler(request): # Filtering the cookie jar before sending the request, # getting the request URL as only parameter - jar.filter_cookies.assert_called_with(req_url) + jar.filter_cookies.assert_called_with(URL(req_url)) # Updating the cookie jar with the response cookies assert jar.update_cookies.called @@ -420,3 +421,16 @@ def test_session_default_version(loop): def test_session_loop(loop): session = aiohttp.ClientSession(loop=loop) assert session.loop is loop + + +def test_proxy_str(session, params): + with mock.patch("aiohttp.client.ClientSession._request") as patched: + session.get("http://test.example.com", + proxy='http://proxy.com', + **params) + assert patched.called, "`ClientSession._request` not called" + assert list(patched.call_args) == [("GET", "http://test.example.com",), + dict( + allow_redirects=True, + proxy='http://proxy.com', + **params)] diff --git a/tests/test_connector.py b/tests/test_connector.py index 6ed6f4aa694..4a9039dc0f9 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -12,6 +12,8 @@ import pytest +from yarl import URL + import aiohttp from aiohttp import client, helpers, web from aiohttp.client import ClientRequest @@ -287,7 +289,7 @@ def test_connect(loop): tr, proto = unittest.mock.Mock(), unittest.mock.Mock() proto.is_connected.return_value = True - req = ClientRequest('GET', 'http://host:80', + req = ClientRequest('GET', URL('http://host:80'), loop=loop, response_class=unittest.mock.Mock()) @@ -526,7 +528,7 @@ def test_connect_with_limit(loop): tr, proto = unittest.mock.Mock(), unittest.mock.Mock() proto.is_connected.return_value = True - req = ClientRequest('GET', 'http://host:80', + req = ClientRequest('GET', URL('http://host:80'), loop=loop, response_class=unittest.mock.Mock()) @@ -569,7 +571,7 @@ def test_connect_with_limit_cancelled(loop): tr, proto = unittest.mock.Mock(), unittest.mock.Mock() proto.is_connected.return_value = True - req = ClientRequest('GET', 'http://host:80', + req = ClientRequest('GET', URL('http://host:80'), loop=loop, response_class=unittest.mock.Mock()) @@ -618,7 +620,7 @@ def test_connect_with_limit_concurrent(loop): proto = unittest.mock.Mock() proto.is_connected.return_value = True - req = ClientRequest('GET', 'http://host:80', + req = ClientRequest('GET', URL('http://host:80'), loop=loop, response_class=unittest.mock.Mock( _should_close=False)) @@ -684,7 +686,7 @@ def test_close_with_acquired_connection(loop): tr, proto = unittest.mock.Mock(), unittest.mock.Mock() proto.is_connected.return_value = True - req = ClientRequest('GET', 'http://host:80', + req = ClientRequest('GET', URL('http://host:80'), loop=loop, response_class=unittest.mock.Mock()) @@ -878,7 +880,8 @@ def test_resolver_not_called_with_address_is_ip(self): resolver = unittest.mock.MagicMock() connector = aiohttp.TCPConnector(resolver=resolver, loop=self.loop) - req = ClientRequest('GET', 'http://127.0.0.1:{}'.format(unused_port()), + req = ClientRequest('GET', + URL('http://127.0.0.1:{}'.format(unused_port())), loop=self.loop, response_class=unittest.mock.Mock()) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index 359fb63d2f7..4d86e86383a 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -6,6 +6,8 @@ import pytest +from yarl import URL + from aiohttp import CookieJar @@ -172,7 +174,7 @@ def test_domain_filter_ip_cookie_send(loop): ) jar.update_cookies(cookies) - cookies_sent = jar.filter_cookies("http://1.2.3.4/").output( + cookies_sent = jar.filter_cookies(URL("http://1.2.3.4/")).output( header='Cookie:') assert cookies_sent == 'Cookie: shared-cookie=first' @@ -180,7 +182,7 @@ def test_domain_filter_ip_cookie_send(loop): def test_domain_filter_ip_cookie_receive(loop, cookies_to_receive): jar = CookieJar(loop=loop) - jar.update_cookies(cookies_to_receive, "http://1.2.3.4/") + jar.update_cookies(cookies_to_receive, URL("http://1.2.3.4/")) assert len(jar) == 0 @@ -190,7 +192,7 @@ def test_preserving_ip_domain_cookies(loop): "shared-cookie=first; " "ip-cookie=second; Domain=127.0.0.1;" )) - cookies_sent = jar.filter_cookies("http://127.0.0.1/").output( + cookies_sent = jar.filter_cookies(URL("http://127.0.0.1/")).output( header='Cookie:') assert cookies_sent == ('Cookie: ip-cookie=second\r\n' 'Cookie: shared-cookie=first') @@ -199,10 +201,10 @@ def test_preserving_ip_domain_cookies(loop): def test_ignore_domain_ending_with_dot(loop): jar = CookieJar(loop=loop, unsafe=True) jar.update_cookies(SimpleCookie("cookie=val; Domain=example.com.;"), - "http://www.example.com") - cookies_sent = jar.filter_cookies("http://www.example.com/") + URL("http://www.example.com")) + cookies_sent = jar.filter_cookies(URL("http://www.example.com/")) assert cookies_sent.output(header='Cookie:') == "Cookie: cookie=val" - cookies_sent = jar.filter_cookies("http://example.com/") + cookies_sent = jar.filter_cookies(URL("http://example.com/")) assert cookies_sent.output(header='Cookie:') == "" @@ -220,11 +222,11 @@ def tearDown(self): def request_reply_with_same_url(self, url): self.jar.update_cookies(self.cookies_to_send) - cookies_sent = self.jar.filter_cookies(url) + cookies_sent = self.jar.filter_cookies(URL(url)) self.jar.clear() - self.jar.update_cookies(self.cookies_to_receive, url) + self.jar.update_cookies(self.cookies_to_receive, URL(url)) cookies_received = SimpleCookie() for cookie in self.jar: dict.__setitem__(cookies_received, cookie.key, cookie) @@ -281,7 +283,7 @@ def timed_request(self, url, update_time, send_time): self.jar.update_cookies(self.cookies_to_send) with mock.patch.object(self.loop, 'time', return_value=send_time): - cookies_sent = self.jar.filter_cookies(url) + cookies_sent = self.jar.filter_cookies(URL(url)) self.jar.clear() @@ -352,12 +354,13 @@ def test_domain_filter_diff_host(self): }) def test_domain_filter_host_only(self): - self.jar.update_cookies(self.cookies_to_receive, "http://example.com/") + self.jar.update_cookies(self.cookies_to_receive, + URL("http://example.com/")) - cookies_sent = self.jar.filter_cookies("http://example.com/") + cookies_sent = self.jar.filter_cookies(URL("http://example.com/")) self.assertIn("unconstrained-cookie", set(cookies_sent.keys())) - cookies_sent = self.jar.filter_cookies("http://different.org/") + cookies_sent = self.jar.filter_cookies(URL("http://different.org/")) self.assertNotIn("unconstrained-cookie", set(cookies_sent.keys())) def test_secure_filter(self): diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 202ff1e7d58..582c7d56389 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -233,19 +233,6 @@ def prop(self): a.prop = 123 -def test_requote_uri_with_unquoted_percents(): - # Ensure we handle unquoted percent signs in redirects. - bad_uri = 'http://example.com/fiz?buz=%ppicture' - quoted = 'http://example.com/fiz?buz=%25ppicture' - assert quoted == helpers.requote_uri(bad_uri) - - -def test_requote_uri_properly_requotes(): - # Ensure requoting doesn't break expectations. - quoted = 'http://example.com/fiz?buz=%25ppicture' - assert quoted == helpers.requote_uri(quoted) - - def test_create_future_with_new_loop(): # We should use the new create_future() if it's available. mock_loop = mock.Mock() diff --git a/tests/test_proxy.py b/tests/test_proxy.py index deff66e0ee3..7b27d42a368 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -4,6 +4,8 @@ import unittest from unittest import mock +from yarl import URL + import aiohttp from aiohttp.client_reqrep import ClientRequest, ClientResponse from aiohttp.test_utils import make_mocked_coro @@ -25,11 +27,11 @@ def tearDown(self): @mock.patch('aiohttp.connector.ClientRequest') def test_connect(self, ClientRequestMock): req = ClientRequest( - 'GET', 'http://www.python.org', - proxy='http://proxy.example.com', + 'GET', URL('http://www.python.org'), + proxy=URL('http://proxy.example.com'), loop=self.loop ) - self.assertEqual(req.proxy, 'http://proxy.example.com') + self.assertEqual(str(req.proxy), 'http://proxy.example.com') # mock all the things! connector = aiohttp.TCPConnector(loop=self.loop) @@ -38,12 +40,12 @@ def test_connect(self, ClientRequestMock): tr, proto = mock.Mock(), mock.Mock() self.loop.create_connection = make_mocked_coro((tr, proto)) conn = self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.path, 'http://www.python.org/') + self.assertEqual(req.path, 'http://www.python.org') self.assertIs(conn._transport, tr) self.assertIs(conn._protocol, proto) ClientRequestMock.assert_called_with( - 'GET', 'http://proxy.example.com', + 'GET', URL('http://proxy.example.com'), auth=None, headers={'Host': 'www.python.org'}, loop=self.loop) @@ -51,8 +53,8 @@ def test_connect(self, ClientRequestMock): def test_proxy_auth(self): with self.assertRaises(ValueError) as ctx: ClientRequest( - 'GET', 'http://python.org', - proxy='http://proxy.example.com', + 'GET', URL('http://python.org'), + proxy=URL('http://proxy.example.com'), proxy_auth=('user', 'pass'), loop=mock.Mock()) self.assertEqual( @@ -66,20 +68,20 @@ def test_proxy_connection_error(self): raise_exception=OSError('dont take it serious')) req = ClientRequest( - 'GET', 'http://www.python.org', - proxy='http://proxy.example.com', + 'GET', URL('http://www.python.org'), + proxy=URL('http://proxy.example.com'), loop=self.loop, ) expected_headers = dict(req.headers) with self.assertRaises(aiohttp.ProxyConnectionError): self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.path, '/') + self.assertEqual(req.url.path, '/') self.assertEqual(dict(req.headers), expected_headers) @mock.patch('aiohttp.connector.ClientRequest') def test_auth(self, ClientRequestMock): proxy_req = ClientRequest( - 'GET', 'http://proxy.example.com', + 'GET', URL('http://proxy.example.com'), auth=aiohttp.helpers.BasicAuth('user', 'pass'), loop=self.loop ) @@ -94,8 +96,8 @@ def test_auth(self, ClientRequestMock): self.loop.create_connection = make_mocked_coro((tr, proto)) req = ClientRequest( - 'GET', 'http://www.python.org', - proxy='http://proxy.example.com', + 'GET', URL('http://www.python.org'), + proxy=URL('http://proxy.example.com'), proxy_auth=aiohttp.helpers.BasicAuth('user', 'pass'), loop=self.loop, ) @@ -103,28 +105,29 @@ def test_auth(self, ClientRequestMock): self.assertNotIn('PROXY-AUTHORIZATION', req.headers) conn = self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.path, 'http://www.python.org/') + self.assertEqual(req.path, 'http://www.python.org') self.assertNotIn('AUTHORIZATION', req.headers) self.assertIn('PROXY-AUTHORIZATION', req.headers) self.assertNotIn('AUTHORIZATION', proxy_req.headers) self.assertNotIn('PROXY-AUTHORIZATION', proxy_req.headers) ClientRequestMock.assert_called_with( - 'GET', 'http://proxy.example.com', + 'GET', URL('http://proxy.example.com'), auth=aiohttp.helpers.BasicAuth('user', 'pass'), loop=mock.ANY, headers=mock.ANY) conn.close() def test_auth_utf8(self): proxy_req = ClientRequest( - 'GET', 'http://proxy.example.com', + 'GET', URL('http://proxy.example.com'), auth=aiohttp.helpers.BasicAuth('юзер', 'пасс', 'utf-8'), loop=self.loop) self.assertIn('AUTHORIZATION', proxy_req.headers) @mock.patch('aiohttp.connector.ClientRequest') def test_auth_from_url(self, ClientRequestMock): - proxy_req = ClientRequest('GET', 'http://user:pass@proxy.example.com', + proxy_req = ClientRequest('GET', + URL('http://user:pass@proxy.example.com'), loop=self.loop) ClientRequestMock.return_value = proxy_req self.assertIn('AUTHORIZATION', proxy_req.headers) @@ -137,28 +140,29 @@ def test_auth_from_url(self, ClientRequestMock): self.loop.create_connection = make_mocked_coro((tr, proto)) req = ClientRequest( - 'GET', 'http://www.python.org', - proxy='http://user:pass@proxy.example.com', + 'GET', URL('http://www.python.org'), + proxy=URL('http://user:pass@proxy.example.com'), loop=self.loop, ) self.assertNotIn('AUTHORIZATION', req.headers) self.assertNotIn('PROXY-AUTHORIZATION', req.headers) conn = self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.path, 'http://www.python.org/') + self.assertEqual(req.path, 'http://www.python.org') self.assertNotIn('AUTHORIZATION', req.headers) self.assertIn('PROXY-AUTHORIZATION', req.headers) self.assertNotIn('AUTHORIZATION', proxy_req.headers) self.assertNotIn('PROXY-AUTHORIZATION', proxy_req.headers) ClientRequestMock.assert_called_with( - 'GET', 'http://user:pass@proxy.example.com', + 'GET', URL('http://user:pass@proxy.example.com'), auth=None, loop=mock.ANY, headers=mock.ANY) conn.close() @mock.patch('aiohttp.connector.ClientRequest') def test_auth__not_modifying_request(self, ClientRequestMock): - proxy_req = ClientRequest('GET', 'http://user:pass@proxy.example.com', + proxy_req = ClientRequest('GET', + URL('http://user:pass@proxy.example.com'), loop=self.loop) ClientRequestMock.return_value = proxy_req proxy_req_headers = dict(proxy_req.headers) @@ -168,24 +172,24 @@ def test_auth__not_modifying_request(self, ClientRequestMock): raise_exception=OSError('nothing personal')) req = ClientRequest( - 'GET', 'http://www.python.org', - proxy='http://user:pass@proxy.example.com', + 'GET', URL('http://www.python.org'), + proxy=URL('http://user:pass@proxy.example.com'), loop=self.loop, ) req_headers = dict(req.headers) with self.assertRaises(aiohttp.ProxyConnectionError): self.loop.run_until_complete(connector.connect(req)) self.assertEqual(req.headers, req_headers) - self.assertEqual(req.path, '/') + self.assertEqual(req.url.path, '/') self.assertEqual(proxy_req.headers, proxy_req_headers) @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect(self, ClientRequestMock): - proxy_req = ClientRequest('GET', 'http://proxy.example.com', + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), loop=self.loop) ClientRequestMock.return_value = proxy_req - proxy_resp = ClientResponse('get', 'http://proxy.example.com') + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop proxy_req.send = send_mock = mock.Mock() send_mock.return_value = proxy_resp @@ -200,13 +204,13 @@ def test_https_connect(self, ClientRequestMock): self.loop.create_connection = make_mocked_coro((tr, proto)) req = ClientRequest( - 'GET', 'https://www.python.org', - proxy='http://proxy.example.com', + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), loop=self.loop, ) self.loop.run_until_complete(connector._create_connection(req)) - self.assertEqual(req.path, '/') + self.assertEqual(req.url.path, '/') self.assertEqual(proxy_req.method, 'CONNECT') self.assertEqual(proxy_req.path, 'www.python.org:443') tr.pause_reading.assert_called_once_with() @@ -218,11 +222,11 @@ def test_https_connect(self, ClientRequestMock): @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect_runtime_error(self, ClientRequestMock): - proxy_req = ClientRequest('GET', 'http://proxy.example.com', + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), loop=self.loop) ClientRequestMock.return_value = proxy_req - proxy_resp = ClientResponse('get', 'http://proxy.example.com') + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop proxy_req.send = send_mock = mock.Mock() send_mock.return_value = proxy_resp @@ -238,8 +242,8 @@ def test_https_connect_runtime_error(self, ClientRequestMock): self.loop.create_connection = make_mocked_coro((tr, proto)) req = ClientRequest( - 'GET', 'https://www.python.org', - proxy='http://proxy.example.com', + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), loop=self.loop, ) with self.assertRaisesRegex( @@ -252,11 +256,11 @@ def test_https_connect_runtime_error(self, ClientRequestMock): @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect_http_proxy_error(self, ClientRequestMock): - proxy_req = ClientRequest('GET', 'http://proxy.example.com', + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), loop=self.loop) ClientRequestMock.return_value = proxy_req - proxy_resp = ClientResponse('get', 'http://proxy.example.com') + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop proxy_req.send = send_mock = mock.Mock() send_mock.return_value = proxy_resp @@ -274,8 +278,8 @@ def test_https_connect_http_proxy_error(self, ClientRequestMock): self.loop.create_connection = make_mocked_coro((tr, proto)) req = ClientRequest( - 'GET', 'https://www.python.org', - proxy='http://proxy.example.com', + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), loop=self.loop, ) with self.assertRaisesRegex( @@ -288,11 +292,11 @@ def test_https_connect_http_proxy_error(self, ClientRequestMock): @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect_resp_start_error(self, ClientRequestMock): - proxy_req = ClientRequest('GET', 'http://proxy.example.com', + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), loop=self.loop) ClientRequestMock.return_value = proxy_req - proxy_resp = ClientResponse('get', 'http://proxy.example.com') + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop proxy_req.send = send_mock = mock.Mock() send_mock.return_value = proxy_resp @@ -309,8 +313,8 @@ def test_https_connect_resp_start_error(self, ClientRequestMock): self.loop.create_connection = make_mocked_coro((tr, proto)) req = ClientRequest( - 'GET', 'https://www.python.org', - proxy='http://proxy.example.com', + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), loop=self.loop, ) with self.assertRaisesRegex(OSError, "error message"): @@ -318,7 +322,7 @@ def test_https_connect_resp_start_error(self, ClientRequestMock): @mock.patch('aiohttp.connector.ClientRequest') def test_request_port(self, ClientRequestMock): - proxy_req = ClientRequest('GET', 'http://proxy.example.com', + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), loop=self.loop) ClientRequestMock.return_value = proxy_req @@ -332,8 +336,8 @@ def test_request_port(self, ClientRequestMock): self.loop.create_connection = make_mocked_coro((tr, proto)) req = ClientRequest( - 'GET', 'http://localhost:1234/path', - proxy='http://proxy.example.com', + 'GET', URL('http://localhost:1234/path'), + proxy=URL('http://proxy.example.com'), loop=self.loop, ) self.loop.run_until_complete(connector._create_connection(req)) @@ -341,26 +345,26 @@ def test_request_port(self, ClientRequestMock): def test_proxy_auth_property(self): req = aiohttp.ClientRequest( - 'GET', 'http://localhost:1234/path', - proxy='http://proxy.example.com', + 'GET', URL('http://localhost:1234/path'), + proxy=URL('http://proxy.example.com'), proxy_auth=aiohttp.helpers.BasicAuth('user', 'pass'), loop=self.loop) self.assertEqual(('user', 'pass', 'latin1'), req.proxy_auth) def test_proxy_auth_property_default(self): req = aiohttp.ClientRequest( - 'GET', 'http://localhost:1234/path', - proxy='http://proxy.example.com', + 'GET', URL('http://localhost:1234/path'), + proxy=URL('http://proxy.example.com'), loop=self.loop) self.assertIsNone(req.proxy_auth) @mock.patch('aiohttp.connector.ClientRequest') def test_https_connect_pass_ssl_context(self, ClientRequestMock): - proxy_req = ClientRequest('GET', 'http://proxy.example.com', + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), loop=self.loop) ClientRequestMock.return_value = proxy_req - proxy_resp = ClientResponse('get', 'http://proxy.example.com') + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop proxy_req.send = send_mock = mock.Mock() send_mock.return_value = proxy_resp @@ -375,8 +379,8 @@ def test_https_connect_pass_ssl_context(self, ClientRequestMock): self.loop.create_connection = make_mocked_coro((tr, proto)) req = ClientRequest( - 'GET', 'https://www.python.org', - proxy='http://proxy.example.com', + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), loop=self.loop, ) self.loop.run_until_complete(connector._create_connection(req)) @@ -387,7 +391,7 @@ def test_https_connect_pass_ssl_context(self, ClientRequestMock): sock=mock.ANY, server_hostname='www.python.org') - self.assertEqual(req.path, '/') + self.assertEqual(req.url.path, '/') self.assertEqual(proxy_req.method, 'CONNECT') self.assertEqual(proxy_req.path, 'www.python.org:443') tr.pause_reading.assert_called_once_with() @@ -399,13 +403,13 @@ def test_https_connect_pass_ssl_context(self, ClientRequestMock): @mock.patch('aiohttp.connector.ClientRequest') def test_https_auth(self, ClientRequestMock): - proxy_req = ClientRequest('GET', 'http://proxy.example.com', + proxy_req = ClientRequest('GET', URL('http://proxy.example.com'), auth=aiohttp.helpers.BasicAuth('user', 'pass'), loop=self.loop) ClientRequestMock.return_value = proxy_req - proxy_resp = ClientResponse('get', 'http://proxy.example.com') + proxy_resp = ClientResponse('get', URL('http://proxy.example.com')) proxy_resp._loop = self.loop proxy_req.send = send_mock = mock.Mock() send_mock.return_value = proxy_resp @@ -423,15 +427,15 @@ def test_https_auth(self, ClientRequestMock): self.assertNotIn('PROXY-AUTHORIZATION', proxy_req.headers) req = ClientRequest( - 'GET', 'https://www.python.org', - proxy='http://proxy.example.com', + 'GET', URL('https://www.python.org'), + proxy=URL('http://proxy.example.com'), loop=self.loop ) self.assertNotIn('AUTHORIZATION', req.headers) self.assertNotIn('PROXY-AUTHORIZATION', req.headers) self.loop.run_until_complete(connector._create_connection(req)) - self.assertEqual(req.path, '/') + self.assertEqual(req.url.path, '/') self.assertNotIn('AUTHORIZATION', req.headers) self.assertNotIn('PROXY-AUTHORIZATION', req.headers) self.assertNotIn('AUTHORIZATION', proxy_req.headers) @@ -463,12 +467,12 @@ def tearDown(self): def test_ctor(self): connector = aiohttp.ProxyConnector( - 'http://localhost:8118', + URL('http://localhost:8118'), proxy_auth=aiohttp.helpers.BasicAuth('user', 'pass'), loop=self.loop, ) - self.assertEqual('http://localhost:8118', connector.proxy) + self.assertEqual('http://localhost:8118', str(connector.proxy)) self.assertEqual( aiohttp.helpers.BasicAuth('user', 'pass'), connector.proxy_auth @@ -477,10 +481,11 @@ def test_ctor(self): @mock.patch('aiohttp.connector.ClientRequest') def test_connect(self, ClientRequestMock): - req = ClientRequest('GET', 'http://www.python.org', loop=self.loop) - self.assertEqual(req.path, '/') + req = ClientRequest('GET', URL('http://www.python.org'), + loop=self.loop) + self.assertEqual(req.url.path, '/') - connector = aiohttp.ProxyConnector('http://proxy.example.com', + connector = aiohttp.ProxyConnector(URL('http://proxy.example.com'), loop=self.loop) self.assertIs(self.loop, connector._loop) @@ -489,7 +494,7 @@ def test_connect(self, ClientRequestMock): tr, proto = mock.Mock(), mock.Mock() self.loop.create_connection = make_mocked_coro((tr, proto)) conn = self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.path, 'http://www.python.org/') + self.assertEqual(req.path, 'http://www.python.org') self.assertIs(conn._transport, tr) self.assertIs(conn._protocol, proto) @@ -497,7 +502,7 @@ def test_connect(self, ClientRequestMock): tr.get_extra_info.assert_called_once_with('sslcontext') ClientRequestMock.assert_called_with( - 'GET', 'http://proxy.example.com', + 'GET', URL('http://proxy.example.com'), auth=None, headers={'Host': 'www.python.org'}, loop=self.loop) diff --git a/tests/test_wsgi.py b/tests/test_wsgi.py index 5ebce0b4ade..9c86cb80aa1 100644 --- a/tests/test_wsgi.py +++ b/tests/test_wsgi.py @@ -7,6 +7,7 @@ from unittest import mock import multidict + import aiohttp from aiohttp import helpers, protocol, wsgi