From 3fd0afbaf168efb40be3c21511f81958f8e44beb Mon Sep 17 00:00:00 2001 From: hdk5 Date: Tue, 10 Apr 2018 11:35:59 +0300 Subject: [PATCH 1/5] Fixed ClientResponse URL fragment --- CHANGES/PR_ID.bugfix | 1 + aiohttp/client.py | 1 - aiohttp/client_reqrep.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 CHANGES/PR_ID.bugfix diff --git a/CHANGES/PR_ID.bugfix b/CHANGES/PR_ID.bugfix new file mode 100644 index 00000000000..aa5fb0ac55e --- /dev/null +++ b/CHANGES/PR_ID.bugfix @@ -0,0 +1 @@ +URL fragment is stripped in ClientRequest only diff --git a/aiohttp/client.py b/aiohttp/client.py index d0e0bb6b39a..2e5d4ee38cd 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -290,7 +290,6 @@ async def _request(self, method, url, *, "with AUTH argument or credentials " "encoded in URL") - url = url.with_fragment(None) cookies = self._cookie_jar.filter_cookies(url) if proxy is not None: diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 0cf5e49502a..0834a485bd3 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -191,8 +191,8 @@ def __init__(self, method, url, *, url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) - self.url = url.with_fragment(None) self.original_url = url + self.url = url.with_fragment(None) self.method = method.upper() self.chunked = chunked self.compress = compress From 45622df88b76b067a6ea9ae0c32b72334d4fc4b6 Mon Sep 17 00:00:00 2001 From: hdk5 Date: Tue, 10 Apr 2018 12:31:05 +0300 Subject: [PATCH 2/5] Added news entry --- CHANGES/2925.bugfix | 1 + CHANGES/PR_ID.bugfix | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 CHANGES/2925.bugfix delete mode 100644 CHANGES/PR_ID.bugfix diff --git a/CHANGES/2925.bugfix b/CHANGES/2925.bugfix new file mode 100644 index 00000000000..6e9ba661666 --- /dev/null +++ b/CHANGES/2925.bugfix @@ -0,0 +1 @@ +URL fragment is stripped only from ClientRequest diff --git a/CHANGES/PR_ID.bugfix b/CHANGES/PR_ID.bugfix deleted file mode 100644 index aa5fb0ac55e..00000000000 --- a/CHANGES/PR_ID.bugfix +++ /dev/null @@ -1 +0,0 @@ -URL fragment is stripped in ClientRequest only From 2f819d90daa5901e4601f09868ad220d320e5608 Mon Sep 17 00:00:00 2001 From: hdk5 Date: Wed, 11 Apr 2018 15:54:31 +0300 Subject: [PATCH 3/5] ClientResponse real_url property + doc + test --- CHANGES/2925.bugfix | 1 - CHANGES/2925.feature | 1 + aiohttp/client_reqrep.py | 13 +++++++++++-- docs/client_reference.rst | 8 ++++++++ tests/test_client_request.py | 8 ++++++++ tests/test_client_response.py | 15 +++++++++++++++ 6 files changed, 43 insertions(+), 3 deletions(-) delete mode 100644 CHANGES/2925.bugfix create mode 100644 CHANGES/2925.feature diff --git a/CHANGES/2925.bugfix b/CHANGES/2925.bugfix deleted file mode 100644 index 6e9ba661666..00000000000 --- a/CHANGES/2925.bugfix +++ /dev/null @@ -1 +0,0 @@ -URL fragment is stripped only from ClientRequest diff --git a/CHANGES/2925.feature b/CHANGES/2925.feature new file mode 100644 index 00000000000..7c9562efb88 --- /dev/null +++ b/CHANGES/2925.feature @@ -0,0 +1 @@ +ClientResponse and RequestInfo now have real_url property, which is request url without fragment part being stripped diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 0834a485bd3..201eef40601 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -54,7 +54,11 @@ class RequestInfo: url = attr.ib(type=URL) method = attr.ib(type=str) headers = attr.ib(type=CIMultiDictProxy) + real_url = attr.ib(type=URL) + @real_url.default + def real_url_default(self): + return self.url class Fingerprint: HASHFUNC_BY_DIGESTLEN = { @@ -244,7 +248,7 @@ def port(self): @property def request_info(self): - return RequestInfo(self.url, self.method, self.headers) + return RequestInfo(self.url, self.method, self.headers, self.original_url) def update_host(self, url): """Update destination host, port and connection type (ssl).""" @@ -582,7 +586,8 @@ def __init__(self, method, url, *, self.headers = None self.cookies = SimpleCookie() - self._url = url + self._real_url = url + self._url = url.with_fragment(None) self._body = None self._writer = writer self._continue = continue100 # None by default @@ -608,6 +613,10 @@ def url_obj(self): "Deprecated, use .url #1654", DeprecationWarning, stacklevel=2) return self._url + @property + def real_url(self): + return self._real_url + @property def host(self): return self._url.host diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 669359a92b5..f28e4ca715f 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1059,6 +1059,10 @@ Response object URL of request (:class:`~yarl.URL`). + .. attribute:: real_url + + Unmodified URL of request (:class:`~yarl.URL`). + .. attribute:: connection :class:`Connection` used for handling response. @@ -1453,6 +1457,10 @@ RequestInfo HTTP headers for request, :class:`multidict.CIMultiDict` instance. + .. attribute:: real_url + + Requested *url* with URL fragment unstripped, :class:`yarl.URL` instance. + BasicAuth ^^^^^^^^^ diff --git a/tests/test_client_request.py b/tests/test_client_request.py index b862434da4d..aceb675e26d 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -105,6 +105,14 @@ def test_request_info(make_request): req.headers) +def test_request_info_with_fragment(make_request): + req = make_request('get', 'http://python.org/#urlfragment') + assert req.request_info == aiohttp.RequestInfo(URL('http://python.org/'), + 'GET', + req.headers, + URL('http://python.org/#urlfragment')) + + def test_version_err(make_request): with pytest.raises(ValueError): make_request('get', 'http://python.org/', version='1.c') diff --git a/tests/test_client_response.py b/tests/test_client_response.py index 820baf88a17..2ab2d5bf7ff 100644 --- a/tests/test_client_response.py +++ b/tests/test_client_response.py @@ -1018,3 +1018,18 @@ def side_effect(*args, **kwargs): trace.send_response_chunk_received.call_args == mock.call(response_body) ) + + +def test_response_real_url(loop, session): + url = URL('http://def-cl-resp.org/#urlfragment') + response = ClientResponse('get', url, + request_info=mock.Mock(), + writer=mock.Mock(), + continue100=None, + timer=TimerNoop(), + auto_decompress=True, + traces=[], + loop=loop, + session=session) + assert response.url == url.with_fragment(None) + assert response.real_url == url From f005304b430d3c71d0b2427f0bbd50c600810a8b Mon Sep 17 00:00:00 2001 From: hdk5 Date: Wed, 11 Apr 2018 16:13:38 +0300 Subject: [PATCH 4/5] Fixed flake8 and doc-spelling errors --- aiohttp/client_reqrep.py | 4 +++- docs/spelling_wordlist.txt | 1 + tests/test_client_request.py | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 201eef40601..b0a1f5e342f 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -60,6 +60,7 @@ class RequestInfo: def real_url_default(self): return self.url + class Fingerprint: HASHFUNC_BY_DIGESTLEN = { 16: md5, @@ -248,7 +249,8 @@ def port(self): @property def request_info(self): - return RequestInfo(self.url, self.method, self.headers, self.original_url) + return RequestInfo(self.url, self.method, + self.headers, self.original_url) def update_host(self, url): """Update destination host, port and connection type (ssl).""" diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index ab0166c1397..bb90d8c724c 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -273,6 +273,7 @@ unittest Unittest unix unsets +unstripped upstr url urldispatcher diff --git a/tests/test_client_request.py b/tests/test_client_request.py index aceb675e26d..c95419bbcbd 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -107,10 +107,10 @@ def test_request_info(make_request): def test_request_info_with_fragment(make_request): req = make_request('get', 'http://python.org/#urlfragment') - assert req.request_info == aiohttp.RequestInfo(URL('http://python.org/'), - 'GET', - req.headers, - URL('http://python.org/#urlfragment')) + assert req.request_info == aiohttp.RequestInfo( + URL('http://python.org/'), + 'GET', req.headers, + URL('http://python.org/#urlfragment')) def test_version_err(make_request): From aac2dc40200cb335329eb568080de2998ea8843a Mon Sep 17 00:00:00 2001 From: hdk5 Date: Tue, 17 Apr 2018 20:52:03 +0300 Subject: [PATCH 5/5] Updated docs --- docs/client_reference.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index f28e4ca715f..0a9db4ac851 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1063,6 +1063,8 @@ Response object Unmodified URL of request (:class:`~yarl.URL`). + .. versionadded:: 3.2 + .. attribute:: connection :class:`Connection` used for handling response. @@ -1461,6 +1463,8 @@ RequestInfo Requested *url* with URL fragment unstripped, :class:`yarl.URL` instance. + .. versionadded:: 3.2 + BasicAuth ^^^^^^^^^