From e91ec9d8f5ac36c361840303b6e7a6acead50932 Mon Sep 17 00:00:00 2001 From: xiang_xli Date: Mon, 4 Dec 2023 21:02:59 +0800 Subject: [PATCH 01/30] #7583 --- CHANGES/7583.feature | 1 + CONTRIBUTORS.txt | 1 + aiohttp/cookiejar.py | 37 ++++++++++++- tests/test_cookiejar.py | 115 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 CHANGES/7583.feature diff --git a/CHANGES/7583.feature b/CHANGES/7583.feature new file mode 100644 index 00000000000..a700c5eb5fb --- /dev/null +++ b/CHANGES/7583.feature @@ -0,0 +1 @@ +Implement filter_cookies() with domain-matching and path-matching on the keys, instead of testing every single cookie. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 539a8807689..921ab0d3529 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -355,6 +355,7 @@ William Grzybowski William S. Wilson Ong wouter bolsterlee +Xiang Li Yang Zhou Yannick Koechlin Yannick Péroux diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 6e26b029026..68b042cf3af 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -261,8 +261,43 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": request_origin = request_url.origin() is_not_secure = request_origin not in self._treat_as_secure_origin + # d: domain, d could be subdomain + # p: path + d = hostname + + pairs = set() + while d: + p = request_url.path + while p: + pairs.add((d, p)) + path_leftovers = p.rsplit("/", maxsplit=1) + # handle last element for rsplit + if len(path_leftovers) > 1: + p = path_leftovers[0] + else: + p = None + + # handle last element for split + leftovers = d.split(".", maxsplit=1) + if len(leftovers) > 1: + d = leftovers[-1] + else: + d = None + + # shared cookie, it should have max of 1 entry + pairs.add(("", "/")) + print(f'pairs: {pairs}') + + import itertools + + cookies = itertools.chain.from_iterable( + self._cookies[p].values() for p in pairs + ) + + # try push + # Point 2: https://www.rfc-editor.org/rfc/rfc6265.html#section-5.4 - for cookie in sorted(self, key=lambda c: len(c["path"])): + for cookie in list(cookies): name = cookie.key domain = cookie["domain"] diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index 0937206e259..9e40bd6f622 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -253,6 +253,121 @@ async def test_filter_cookies_str_deprecated(loop: Any) -> None: jar.filter_cookies("http://éé.com") +async def test_filter_cookies_with_domain_path_lookup(loop: Any) -> None: + jar = CookieJar() + cookies = SimpleCookie( + "shared-cookie=first; " + "domain-cookie=second; Domain=example.com; " + "subdomain1-cookie=third; Domain=test1.example.com; " + "subdomain2-cookie=fourth; Domain=test2.example.com; " + "dotted-domain-cookie=fifth; Domain=.example.com; " + "different-domain-cookie=sixth; Domain=different.org; " + "secure-cookie=seventh; Domain=secure.com; Secure; " + "no-path-cookie=eighth; Domain=pathtest.com; " + "path1-cookie=ninth; Domain=pathtest.com; Path=/; " + "path2-cookie=tenth; Domain=pathtest.com; Path=/one; " + "path3-cookie=eleventh; Domain=pathtest.com; Path=/one/two; " + "path4-cookie=twelfth; Domain=pathtest.com; Path=/one/two/; " + "expires-cookie=thirteenth; Domain=expirestest.com; Path=/;" + " Expires=Tue, 1 Jan 1980 12:00:00 GMT; " + "max-age-cookie=fourteenth; Domain=maxagetest.com; Path=/;" + " Max-Age=60; " + "invalid-max-age-cookie=fifteenth; Domain=invalid-values.com; " + " Max-Age=string; " + "invalid-expires-cookie=sixteenth; Domain=invalid-values.com; " + " Expires=string;" + ) + jar.update_cookies(cookies) + cookies = jar.filter_cookies(URL("http://pathtest.com/")) + + expected_cookies = [ + "shared-cookie", + "no-path-cookie", + "path1-cookie", + ] + assert len(cookies) == 3 + for c in cookies: + assert c in expected_cookies + + +async def test_filter_cookies_with_domain_path_lookup_subdomain(loop: Any) -> None: + jar = CookieJar() + cookies = SimpleCookie( + "shared-cookie=first; " + "domain-cookie=second; Domain=example.com; " + "subdomain1-cookie=third; Domain=test1.example.com; " + "subdomain2-cookie=fourth; Domain=test2.example.com; " + "dotted-domain-cookie=fifth; Domain=.example.com; " + "different-domain-cookie=sixth; Domain=different.org; " + "secure-cookie=seventh; Domain=secure.com; Secure; " + "no-path-cookie=eighth; Domain=pathtest.com; " + "path1-cookie=ninth; Domain=pathtest.com; Path=/; " + "path2-cookie=tenth; Domain=pathtest.com; Path=/one; " + "path3-cookie=eleventh; Domain=pathtest.com; Path=/one/two; " + "path4-cookie=twelfth; Domain=pathtest.com; Path=/one/two/; " + "expires-cookie=thirteenth; Domain=expirestest.com; Path=/;" + " Expires=Tue, 1 Jan 1980 12:00:00 GMT; " + "max-age-cookie=fourteenth; Domain=maxagetest.com; Path=/;" + " Max-Age=60; " + "invalid-max-age-cookie=fifteenth; Domain=invalid-values.com; " + " Max-Age=string; " + "invalid-expires-cookie=sixteenth; Domain=invalid-values.com; " + " Expires=string;" + ) + jar.update_cookies(cookies) + cookies = jar.filter_cookies(URL("http://test1.example.com/")) + + expected_cookies = [ + "shared-cookie", + "domain-cookie", + "subdomain1-cookie", + "dotted-domain-cookie", + ] + # assert len(cookies) == 4 + for c in cookies: + print(f"c: {c}") + assert c in expected_cookies + + +async def test_filter_cookies_with_domain_path_lookup_multilevelpath(loop: Any) -> None: + jar = CookieJar() + cookies = SimpleCookie( + "shared-cookie=first; " + "domain-cookie=second; Domain=example.com; " + "subdomain1-cookie=third; Domain=test1.example.com; " + "subdomain2-cookie=fourth; Domain=test2.example.com; " + "dotted-domain-cookie=fifth; Domain=.example.com; " + "different-domain-cookie=sixth; Domain=different.org; " + "secure-cookie=seventh; Domain=secure.com; Secure; " + "no-path-cookie=eighth; Domain=pathtest.com; " + "path1-cookie=ninth; Domain=pathtest.com; Path=/; " + "path2-cookie=tenth; Domain=pathtest.com; Path=/one; " + "path3-cookie=eleventh; Domain=pathtest.com; Path=/one/two; " + "path4-cookie=twelfth; Domain=pathtest.com; Path=/one/two/; " + "expires-cookie=thirteenth; Domain=expirestest.com; Path=/;" + " Expires=Tue, 1 Jan 1980 12:00:00 GMT; " + "max-age-cookie=fourteenth; Domain=maxagetest.com; Path=/;" + " Max-Age=60; " + "invalid-max-age-cookie=fifteenth; Domain=invalid-values.com; " + " Max-Age=string; " + "invalid-expires-cookie=sixteenth; Domain=invalid-values.com; " + " Expires=string;" + ) + jar.update_cookies(cookies) + cookies = jar.filter_cookies(URL("http://pathtest.com/one/two/")) + + expected_cookies = [ + "path2-cookie", + "shared-cookie", + "path3-cookie", + "path4-cookie", + ] + assert len(cookies) == 4 + for c in cookies: + # print(f"c: {c}") + assert c in expected_cookies + + async def test_domain_filter_ip_cookie_send(loop: Any) -> None: jar = CookieJar() cookies = SimpleCookie( From d901fcff0c012cea3aa892a9307d745c5a68187c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:10:46 +0000 Subject: [PATCH 02/30] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/cookiejar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 68b042cf3af..0f90dab5639 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -78,7 +78,7 @@ def __init__( *, unsafe: bool = False, quote_cookie: bool = True, - treat_as_secure_origin: Union[StrOrURL, List[StrOrURL], None] = None + treat_as_secure_origin: Union[StrOrURL, List[StrOrURL], None] = None, ) -> None: self._cookies: DefaultDict[Tuple[str, str], SimpleCookie] = defaultdict( SimpleCookie @@ -286,7 +286,7 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": # shared cookie, it should have max of 1 entry pairs.add(("", "/")) - print(f'pairs: {pairs}') + print(f"pairs: {pairs}") import itertools From 4bbf230c7e688c9eec951e5264cdee4fbf0daaeb Mon Sep 17 00:00:00 2001 From: xiangxli Date: Mon, 4 Dec 2023 22:51:56 +0800 Subject: [PATCH 03/30] Update aiohttp/cookiejar.py Co-authored-by: Sam Bull --- aiohttp/cookiejar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 0f90dab5639..97597d73689 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -297,7 +297,7 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": # try push # Point 2: https://www.rfc-editor.org/rfc/rfc6265.html#section-5.4 - for cookie in list(cookies): + for cookie in cookies: name = cookie.key domain = cookie["domain"] From 9a8a869b022e9fba309a1e6523edb69bf1f1e594 Mon Sep 17 00:00:00 2001 From: xiangxli Date: Mon, 4 Dec 2023 22:52:08 +0800 Subject: [PATCH 04/30] Update aiohttp/cookiejar.py Co-authored-by: Sam Bull --- aiohttp/cookiejar.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 97597d73689..1e23900ffda 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -286,7 +286,6 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": # shared cookie, it should have max of 1 entry pairs.add(("", "/")) - print(f"pairs: {pairs}") import itertools From 93250eb318db458dce362ba51e0c04cb667ddd5b Mon Sep 17 00:00:00 2001 From: xiangxli Date: Mon, 4 Dec 2023 22:52:19 +0800 Subject: [PATCH 05/30] Update aiohttp/cookiejar.py Co-authored-by: Sam Bull --- aiohttp/cookiejar.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 1e23900ffda..75a095acb51 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -278,11 +278,10 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": p = None # handle last element for split - leftovers = d.split(".", maxsplit=1) - if len(leftovers) > 1: - d = leftovers[-1] - else: - d = None + try: + d = d.split(".", maxsplit=1)[1] + except IndexError: + d = "" # shared cookie, it should have max of 1 entry pairs.add(("", "/")) From 26deaeeb6bfd597067dc03f2f7107734562d0811 Mon Sep 17 00:00:00 2001 From: xiang_xli Date: Mon, 4 Dec 2023 22:02:06 +0800 Subject: [PATCH 06/30] remove print --- tests/test_cookiejar.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index 9e40bd6f622..4ff9ad0d5cb 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -323,9 +323,8 @@ async def test_filter_cookies_with_domain_path_lookup_subdomain(loop: Any) -> No "subdomain1-cookie", "dotted-domain-cookie", ] - # assert len(cookies) == 4 + assert len(cookies) == 4 for c in cookies: - print(f"c: {c}") assert c in expected_cookies @@ -364,7 +363,6 @@ async def test_filter_cookies_with_domain_path_lookup_multilevelpath(loop: Any) ] assert len(cookies) == 4 for c in cookies: - # print(f"c: {c}") assert c in expected_cookies From 48f4efc843888f8f054294450c32c9c90befa746 Mon Sep 17 00:00:00 2001 From: xiang_xli Date: Mon, 4 Dec 2023 23:04:46 +0800 Subject: [PATCH 07/30] modify to suggetions --- aiohttp/cookiejar.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 75a095acb51..3ea6fc86dfd 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -7,6 +7,7 @@ import re import time import warnings +import itertools from collections import defaultdict from http.cookies import BaseCookie, Morsel, SimpleCookie from math import ceil @@ -265,28 +266,21 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": # p: path d = hostname - pairs = set() + pairs = [] while d: p = request_url.path while p: - pairs.add((d, p)) - path_leftovers = p.rsplit("/", maxsplit=1) - # handle last element for rsplit - if len(path_leftovers) > 1: - p = path_leftovers[0] - else: - p = None + pairs.append((d, p)) + p = p.rsplit("/", maxsplit=1)[0] - # handle last element for split try: d = d.split(".", maxsplit=1)[1] except IndexError: + # handle last element for split d = "" # shared cookie, it should have max of 1 entry - pairs.add(("", "/")) - - import itertools + pairs.append(("", "/")) cookies = itertools.chain.from_iterable( self._cookies[p].values() for p in pairs From 53057958c543611b70c4d0910723b65a2ee189dd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 15:05:28 +0000 Subject: [PATCH 08/30] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/cookiejar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 3ea6fc86dfd..3dbaf87d875 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -1,13 +1,13 @@ import calendar import contextlib import datetime +import itertools import os # noqa import pathlib import pickle import re import time import warnings -import itertools from collections import defaultdict from http.cookies import BaseCookie, Morsel, SimpleCookie from math import ceil From 4fcbd0840bc79bfe6ad7560fd8ba734e82e841ce Mon Sep 17 00:00:00 2001 From: xiang_xli Date: Tue, 5 Dec 2023 00:23:52 +0800 Subject: [PATCH 09/30] modify tests to suggestions --- tests/test_cookiejar.py | 78 +++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 46 deletions(-) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index 4ff9ad0d5cb..ae127c9abb7 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -290,45 +290,37 @@ async def test_filter_cookies_with_domain_path_lookup(loop: Any) -> None: assert c in expected_cookies -async def test_filter_cookies_with_domain_path_lookup_subdomain(loop: Any) -> None: - jar = CookieJar() - cookies = SimpleCookie( - "shared-cookie=first; " - "domain-cookie=second; Domain=example.com; " - "subdomain1-cookie=third; Domain=test1.example.com; " - "subdomain2-cookie=fourth; Domain=test2.example.com; " - "dotted-domain-cookie=fifth; Domain=.example.com; " - "different-domain-cookie=sixth; Domain=different.org; " - "secure-cookie=seventh; Domain=secure.com; Secure; " - "no-path-cookie=eighth; Domain=pathtest.com; " - "path1-cookie=ninth; Domain=pathtest.com; Path=/; " - "path2-cookie=tenth; Domain=pathtest.com; Path=/one; " - "path3-cookie=eleventh; Domain=pathtest.com; Path=/one/two; " - "path4-cookie=twelfth; Domain=pathtest.com; Path=/one/two/; " - "expires-cookie=thirteenth; Domain=expirestest.com; Path=/;" - " Expires=Tue, 1 Jan 1980 12:00:00 GMT; " - "max-age-cookie=fourteenth; Domain=maxagetest.com; Path=/;" - " Max-Age=60; " - "invalid-max-age-cookie=fifteenth; Domain=invalid-values.com; " - " Max-Age=string; " - "invalid-expires-cookie=sixteenth; Domain=invalid-values.com; " - " Expires=string;" - ) - jar.update_cookies(cookies) - cookies = jar.filter_cookies(URL("http://test1.example.com/")) - - expected_cookies = [ - "shared-cookie", - "domain-cookie", - "subdomain1-cookie", - "dotted-domain-cookie", - ] - assert len(cookies) == 4 - for c in cookies: - assert c in expected_cookies - +@pytest.mark.parametrize( + ("url", "expected_cookies"), + [ + ( + "http://pathtest.com/one/two/", + [ + "path2-cookie", + "shared-cookie", + "path3-cookie", + "path4-cookie", + ] + ), + ( + "http://test1.example.com/", + [ + "shared-cookie", + "domain-cookie", + "subdomain1-cookie", + "dotted-domain-cookie", + ] + ) -async def test_filter_cookies_with_domain_path_lookup_multilevelpath(loop: Any) -> None: + ], + ids=( + "/one/two/ path", + "test1.example.com subdomain", + ) +) +async def test_filter_cookies_with_domain_path_lookup_multilevelpath( + loop: Any, url: Any, expected_cookies: Any, +) -> None: jar = CookieJar() cookies = SimpleCookie( "shared-cookie=first; " @@ -353,15 +345,9 @@ async def test_filter_cookies_with_domain_path_lookup_multilevelpath(loop: Any) " Expires=string;" ) jar.update_cookies(cookies) - cookies = jar.filter_cookies(URL("http://pathtest.com/one/two/")) + cookies = jar.filter_cookies(URL(url)) - expected_cookies = [ - "path2-cookie", - "shared-cookie", - "path3-cookie", - "path4-cookie", - ] - assert len(cookies) == 4 + assert len(cookies) == len(expected_cookies) for c in cookies: assert c in expected_cookies From b176af3ae21bf1fc9d30e0664e77c2a0ae6d94bd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 16:27:56 +0000 Subject: [PATCH 10/30] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_cookiejar.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index ae127c9abb7..ddb4fc2b642 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -300,7 +300,7 @@ async def test_filter_cookies_with_domain_path_lookup(loop: Any) -> None: "shared-cookie", "path3-cookie", "path4-cookie", - ] + ], ), ( "http://test1.example.com/", @@ -309,17 +309,18 @@ async def test_filter_cookies_with_domain_path_lookup(loop: Any) -> None: "domain-cookie", "subdomain1-cookie", "dotted-domain-cookie", - ] - ) - + ], + ), ], ids=( "/one/two/ path", "test1.example.com subdomain", - ) + ), ) async def test_filter_cookies_with_domain_path_lookup_multilevelpath( - loop: Any, url: Any, expected_cookies: Any, + loop: Any, + url: Any, + expected_cookies: Any, ) -> None: jar = CookieJar() cookies = SimpleCookie( From 02b6baaa06109e2f76b3fba0b7f3818670521443 Mon Sep 17 00:00:00 2001 From: xiangxli Date: Tue, 5 Dec 2023 01:56:07 +0800 Subject: [PATCH 11/30] Update aiohttp/cookiejar.py Co-authored-by: Sam Bull --- aiohttp/cookiejar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 3dbaf87d875..41767d615be 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -283,7 +283,7 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": pairs.append(("", "/")) cookies = itertools.chain.from_iterable( - self._cookies[p].values() for p in pairs + self._cookies[p].values() for p in reversed(pairs) ) # try push From c7a0d3a8e2d0d66d525e8600da2213e253887f1a Mon Sep 17 00:00:00 2001 From: xiang_xli Date: Tue, 5 Dec 2023 02:03:42 +0800 Subject: [PATCH 12/30] modify tests to suggestions --- aiohttp/cookiejar.py | 2 -- tests/test_cookiejar.py | 46 ++++++++--------------------------------- 2 files changed, 9 insertions(+), 39 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 41767d615be..4f787dfaa48 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -286,8 +286,6 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": self._cookies[p].values() for p in reversed(pairs) ) - # try push - # Point 2: https://www.rfc-editor.org/rfc/rfc6265.html#section-5.4 for cookie in cookies: name = cookie.key diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index ddb4fc2b642..bda120f93ae 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -253,43 +253,6 @@ async def test_filter_cookies_str_deprecated(loop: Any) -> None: jar.filter_cookies("http://éé.com") -async def test_filter_cookies_with_domain_path_lookup(loop: Any) -> None: - jar = CookieJar() - cookies = SimpleCookie( - "shared-cookie=first; " - "domain-cookie=second; Domain=example.com; " - "subdomain1-cookie=third; Domain=test1.example.com; " - "subdomain2-cookie=fourth; Domain=test2.example.com; " - "dotted-domain-cookie=fifth; Domain=.example.com; " - "different-domain-cookie=sixth; Domain=different.org; " - "secure-cookie=seventh; Domain=secure.com; Secure; " - "no-path-cookie=eighth; Domain=pathtest.com; " - "path1-cookie=ninth; Domain=pathtest.com; Path=/; " - "path2-cookie=tenth; Domain=pathtest.com; Path=/one; " - "path3-cookie=eleventh; Domain=pathtest.com; Path=/one/two; " - "path4-cookie=twelfth; Domain=pathtest.com; Path=/one/two/; " - "expires-cookie=thirteenth; Domain=expirestest.com; Path=/;" - " Expires=Tue, 1 Jan 1980 12:00:00 GMT; " - "max-age-cookie=fourteenth; Domain=maxagetest.com; Path=/;" - " Max-Age=60; " - "invalid-max-age-cookie=fifteenth; Domain=invalid-values.com; " - " Max-Age=string; " - "invalid-expires-cookie=sixteenth; Domain=invalid-values.com; " - " Expires=string;" - ) - jar.update_cookies(cookies) - cookies = jar.filter_cookies(URL("http://pathtest.com/")) - - expected_cookies = [ - "shared-cookie", - "no-path-cookie", - "path1-cookie", - ] - assert len(cookies) == 3 - for c in cookies: - assert c in expected_cookies - - @pytest.mark.parametrize( ("url", "expected_cookies"), [ @@ -311,10 +274,19 @@ async def test_filter_cookies_with_domain_path_lookup(loop: Any) -> None: "dotted-domain-cookie", ], ), + ( + "http://pathtest.com/", + [ + "shared-cookie", + "no-path-cookie", + "path1-cookie", + ], + ) ], ids=( "/one/two/ path", "test1.example.com subdomain", + "pathtest.com", ), ) async def test_filter_cookies_with_domain_path_lookup_multilevelpath( From 72c88db06af4371402ee090e0ba392801371f241 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 18:05:10 +0000 Subject: [PATCH 13/30] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_cookiejar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index bda120f93ae..b238b2264c8 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -281,7 +281,7 @@ async def test_filter_cookies_str_deprecated(loop: Any) -> None: "no-path-cookie", "path1-cookie", ], - ) + ), ], ids=( "/one/two/ path", From 80c83e4aa8bd5fc657dca70110e18eb0503b3ad3 Mon Sep 17 00:00:00 2001 From: xiangxli Date: Tue, 5 Dec 2023 02:46:05 +0800 Subject: [PATCH 14/30] Update tests/test_cookiejar.py Co-authored-by: Sam Bull --- tests/test_cookiejar.py | 42 +++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index b238b2264c8..b33c202f407 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -255,38 +255,56 @@ async def test_filter_cookies_str_deprecated(loop: Any) -> None: @pytest.mark.parametrize( ("url", "expected_cookies"), - [ + ( ( "http://pathtest.com/one/two/", - [ + ( + "no-path-cookie", + "path1-cookie", + "path2-cookie", + "shared-cookie", + "path3-cookie", + "path4-cookie", + ), + ), + ( + "http://pathtest.com/one/two", + ( + "no-path-cookie", + "path1-cookie", + "path2-cookie", + "shared-cookie", + "path3-cookie", + ), + ), + ( + "http://pathtest.com/one/two/three/", + ( + "no-path-cookie", + "path1-cookie", "path2-cookie", "shared-cookie", "path3-cookie", "path4-cookie", - ], + ), ), ( "http://test1.example.com/", - [ + ( "shared-cookie", "domain-cookie", "subdomain1-cookie", "dotted-domain-cookie", - ], + ), ), ( "http://pathtest.com/", - [ + ( "shared-cookie", "no-path-cookie", "path1-cookie", - ], + ), ), - ], - ids=( - "/one/two/ path", - "test1.example.com subdomain", - "pathtest.com", ), ) async def test_filter_cookies_with_domain_path_lookup_multilevelpath( From f6b4d48046a35804c3404135b32553b335387ac6 Mon Sep 17 00:00:00 2001 From: xiang_xli Date: Tue, 5 Dec 2023 02:56:55 +0800 Subject: [PATCH 15/30] modify cookiejar to handle no-path-cookie --- aiohttp/cookiejar.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 4f787dfaa48..d165739ada8 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -210,6 +210,7 @@ def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> No # Cut everything from the last slash to the end path = "/" + path[1 : path.rfind("/")] cookie["path"] = path + path = path.rstrip("/") max_age = cookie["max-age"] if max_age: @@ -272,6 +273,7 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": while p: pairs.append((d, p)) p = p.rsplit("/", maxsplit=1)[0] + pairs.append((d, "")) try: d = d.split(".", maxsplit=1)[1] @@ -280,7 +282,7 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": d = "" # shared cookie, it should have max of 1 entry - pairs.append(("", "/")) + pairs.append(("", "")) cookies = itertools.chain.from_iterable( self._cookies[p].values() for p in reversed(pairs) From 9841d1955a15a9d12d10957cf067ae3618e32b13 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 4 Dec 2023 19:03:52 +0000 Subject: [PATCH 16/30] Nitpicks --- aiohttp/cookiejar.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index d165739ada8..ed341e2645a 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -79,7 +79,7 @@ def __init__( *, unsafe: bool = False, quote_cookie: bool = True, - treat_as_secure_origin: Union[StrOrURL, List[StrOrURL], None] = None, + treat_as_secure_origin: Union[StrOrURL, List[StrOrURL], None] = None ) -> None: self._cookies: DefaultDict[Tuple[str, str], SimpleCookie] = defaultdict( SimpleCookie @@ -263,11 +263,8 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": request_origin = request_url.origin() is_not_secure = request_origin not in self._treat_as_secure_origin - # d: domain, d could be subdomain - # p: path - d = hostname - pairs = [] + d = hostname while d: p = request_url.path while p: From 1d5c61515eb5e65db5ca2f97e849a0613c721fff Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 18 Dec 2023 19:21:59 -1000 Subject: [PATCH 17/30] Update aiohttp/cookiejar.py Co-authored-by: Sam Bull --- aiohttp/cookiejar.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index ed341e2645a..45b1fafd2fb 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -263,29 +263,19 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": request_origin = request_url.origin() is_not_secure = request_origin not in self._treat_as_secure_origin - pairs = [] - d = hostname - while d: - p = request_url.path - while p: - pairs.append((d, p)) - p = p.rsplit("/", maxsplit=1)[0] - pairs.append((d, "")) - - try: - d = d.split(".", maxsplit=1)[1] - except IndexError: - # handle last element for split - d = "" - - # shared cookie, it should have max of 1 entry - pairs.append(("", "")) - - cookies = itertools.chain.from_iterable( - self._cookies[p].values() for p in reversed(pairs) + # Get all the path prefixes that might match a cookie (e.g. "", "/foo", "/foo/bar") + paths = itertools.accumulate(request_url.path.split("/"), lambda x, y: f"{x}/{y}") + # Get all the subdomains that might match a cookie (e.g. "foo.bar.com", "bar.com", "com") + domains = itertools.accumulate( + reversed(hostname.split(".")), lambda x, y: f"{y}.{x}" ) + # Create every combination of (domain, path) pairs, plus the shared cookie. + pairs = itertools.chain((("", ""),), itertools.product(domains, paths)) # Point 2: https://www.rfc-editor.org/rfc/rfc6265.html#section-5.4 + cookies = itertools.chain.from_iterable( + self._cookies[p].values() for p in pairs + ) for cookie in cookies: name = cookie.key domain = cookie["domain"] From 736d768d8dbd621d9c2dbb8efa7924fe77c72ed6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 05:22:30 +0000 Subject: [PATCH 18/30] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/cookiejar.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 45b1fafd2fb..22525db1213 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -79,7 +79,7 @@ def __init__( *, unsafe: bool = False, quote_cookie: bool = True, - treat_as_secure_origin: Union[StrOrURL, List[StrOrURL], None] = None + treat_as_secure_origin: Union[StrOrURL, List[StrOrURL], None] = None, ) -> None: self._cookies: DefaultDict[Tuple[str, str], SimpleCookie] = defaultdict( SimpleCookie @@ -264,7 +264,9 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": is_not_secure = request_origin not in self._treat_as_secure_origin # Get all the path prefixes that might match a cookie (e.g. "", "/foo", "/foo/bar") - paths = itertools.accumulate(request_url.path.split("/"), lambda x, y: f"{x}/{y}") + paths = itertools.accumulate( + request_url.path.split("/"), lambda x, y: f"{x}/{y}" + ) # Get all the subdomains that might match a cookie (e.g. "foo.bar.com", "bar.com", "com") domains = itertools.accumulate( reversed(hostname.split(".")), lambda x, y: f"{y}.{x}" From c7ae0e16b423a1e39dca6b5c969a8ebdb2e8829a Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 15:31:03 +0000 Subject: [PATCH 19/30] Update test_cookiejar.py --- tests/test_cookiejar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index b33c202f407..7a0105c52a6 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -927,7 +927,7 @@ def test_pickle_format(cookies_to_send) -> None: with file_path.open("wb") as f: pickle.dump(cookies, f, pickle.HIGHEST_PROTOCOL) """ - pickled = b"\x80\x05\x95\xc5\x07\x00\x00\x00\x00\x00\x00\x8c\x0bcollections\x94\x8c\x0bdefaultdict\x94\x93\x94\x8c\x0chttp.cookies\x94\x8c\x0cSimpleCookie\x94\x93\x94\x85\x94R\x94(\x8c\x00\x94\x8c\x01/\x94\x86\x94h\x05)\x81\x94\x8c\rshared-cookie\x94h\x03\x8c\x06Morsel\x94\x93\x94)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\t\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\x08\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(\x8c\x03key\x94h\x0c\x8c\x05value\x94\x8c\x05first\x94\x8c\x0bcoded_value\x94h\x1cubs\x8c\x0bexample.com\x94h\t\x86\x94h\x05)\x81\x94(\x8c\rdomain-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13h\x1eh\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ah!h\x1b\x8c\x06second\x94h\x1dh$ub\x8c\x14dotted-domain-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13\x8c\x0bexample.com\x94h\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ah%h\x1b\x8c\x05fifth\x94h\x1dh)ubu\x8c\x11test1.example.com\x94h\t\x86\x94h\x05)\x81\x94\x8c\x11subdomain1-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13h*h\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ah-h\x1b\x8c\x05third\x94h\x1dh0ubs\x8c\x11test2.example.com\x94h\t\x86\x94h\x05)\x81\x94\x8c\x11subdomain2-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13h1h\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ah4h\x1b\x8c\x06fourth\x94h\x1dh7ubs\x8c\rdifferent.org\x94h\t\x86\x94h\x05)\x81\x94\x8c\x17different-domain-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13h8h\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ah;h\x1b\x8c\x05sixth\x94h\x1dh>ubs\x8c\nsecure.com\x94h\t\x86\x94h\x05)\x81\x94\x8c\rsecure-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13h?h\x14h\x08h\x15\x88h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ahBh\x1b\x8c\x07seventh\x94h\x1dhEubs\x8c\x0cpathtest.com\x94h\t\x86\x94h\x05)\x81\x94(\x8c\x0eno-path-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13hFh\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ahIh\x1b\x8c\x06eighth\x94h\x1dhLub\x8c\x0cpath1-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13\x8c\x0cpathtest.com\x94h\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ahMh\x1b\x8c\x05ninth\x94h\x1dhQubu\x8c\x0cpathtest.com\x94\x8c\x04/one\x94\x86\x94h\x05)\x81\x94\x8c\x0cpath2-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11hSh\x12h\x08h\x13hRh\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ahVh\x1b\x8c\x05tenth\x94h\x1dhYubs\x8c\x0cpathtest.com\x94\x8c\x08/one/two\x94\x86\x94h\x05)\x81\x94\x8c\x0cpath3-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h[h\x12h\x08h\x13hZh\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ah^h\x1b\x8c\x08eleventh\x94h\x1dhaubs\x8c\x0cpathtest.com\x94\x8c\t/one/two/\x94\x86\x94h\x05)\x81\x94\x8c\x0cpath4-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11hch\x12h\x08h\x13hbh\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ahfh\x1b\x8c\x07twelfth\x94h\x1dhiubs\x8c\x0fexpirestest.com\x94h\t\x86\x94h\x05)\x81\x94\x8c\x0eexpires-cookie\x94h\x0e)\x81\x94(h\x10\x8c\x1cTue, 1 Jan 2999 12:00:00 GMT\x94h\x11h\th\x12h\x08h\x13hjh\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ahmh\x1b\x8c\nthirteenth\x94h\x1dhqubs\x8c\x0emaxagetest.com\x94h\t\x86\x94h\x05)\x81\x94\x8c\x0emax-age-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13hrh\x14\x8c\x0260\x94h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ahuh\x1b\x8c\nfourteenth\x94h\x1dhyubs\x8c\x12invalid-values.com\x94h\t\x86\x94h\x05)\x81\x94(\x8c\x16invalid-max-age-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13hzh\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ah}h\x1b\x8c\tfifteenth\x94h\x1dh\x80ub\x8c\x16invalid-expires-cookie\x94h\x0e)\x81\x94(h\x10h\x08h\x11h\th\x12h\x08h\x13\x8c\x12invalid-values.com\x94h\x14h\x08h\x15h\x08h\x16h\x08h\x17h\x08h\x18h\x08u}\x94(h\x1ah\x81h\x1b\x8c\tsixteenth\x94h\x1dh\x85ubuu." + pickled = b"\x80\x04\x95\xd3\x0b\x00\x00\x00\x00\x00\x00\x8c\x0bcollections\x94\x8c\x0bdefaultdict\x94\x93\x94\x8c\x0chttp.cookies\x94\x8c\x0cSimpleCookie\x94\x93\x94\x85\x94R\x94(\x8c\x00\x94h\x08\x86\x94h\x05)\x81\x94\x8c\rshared-cookie\x94h\x03\x8c\x06Morsel\x94\x93\x94)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94\x8c\x01/\x94\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\x08\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(\x8c\x03key\x94h\x0b\x8c\x05value\x94\x8c\x05first\x94\x8c\x0bcoded_value\x94h\x1cubs\x8c\x0bexample.com\x94h\x08\x86\x94h\x05)\x81\x94(\x8c\rdomain-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\x1e\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah!h\x1b\x8c\x06second\x94h\x1dh-ub\x8c\x14dotted-domain-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94\x8c\x0bexample.com\x94\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah.h\x1b\x8c\x05fifth\x94h\x1dh;ubu\x8c\x11test1.example.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x11subdomain1-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h<\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah?h\x1b\x8c\x05third\x94h\x1dhKubs\x8c\x11test2.example.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x11subdomain2-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94hL\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ahOh\x1b\x8c\x06fourth\x94h\x1dh[ubs\x8c\rdifferent.org\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x17different-domain-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\\\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah_h\x1b\x8c\x05sixth\x94h\x1dhkubs\x8c\nsecure.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\rsecure-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94hl\x8c\x07max-age\x94h\x08\x8c\x06secure\x94\x88\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ahoh\x1b\x8c\x07seventh\x94h\x1dh{ubs\x8c\x0cpathtest.com\x94h\x08\x86\x94h\x05)\x81\x94(\x8c\x0eno-path-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h|\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\x7fh\x1b\x8c\x06eighth\x94h\x1dh\x8bub\x8c\x0cpath1-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94\x8c\x0cpathtest.com\x94\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\x8ch\x1b\x8c\x05ninth\x94h\x1dh\x99ubu\x8c\x0cpathtest.com\x94\x8c\x04/one\x94\x86\x94h\x05)\x81\x94\x8c\x0cpath2-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x9b\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\x9a\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\x9eh\x1b\x8c\x05tenth\x94h\x1dh\xaaubs\x8c\x0cpathtest.com\x94\x8c\x08/one/two\x94\x86\x94h\x05)\x81\x94\x8c\x0cpath3-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\xac\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xab\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xafh\x1b\x8c\x08eleventh\x94h\x1dh\xbbubs\x8c\x0cpathtest.com\x94\x8c\t/one/two/\x94\x86\x94h\x05)\x81\x94\x8c\x0cpath4-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\xbd\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xbc\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xc0h\x1b\x8c\x07twelfth\x94h\x1dh\xccubs\x8c\x0fexpirestest.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x0eexpires-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94\x8c\x1cTue, 1 Jan 2999 12:00:00 GMT\x94\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xcd\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xd0h\x1b\x8c\nthirteenth\x94h\x1dh\xddubs\x8c\x0emaxagetest.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x0emax-age-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xde\x8c\x07max-age\x94\x8c\x0260\x94\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xe1h\x1b\x8c\nfourteenth\x94h\x1dh\xeeubs\x8c\x12invalid-values.com\x94h\x08\x86\x94h\x05)\x81\x94(\x8c\x16invalid-max-age-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xef\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xf2h\x1b\x8c\tfifteenth\x94h\x1dh\xfeub\x8c\x16invalid-expires-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94\x8c\x12invalid-values.com\x94\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xffh\x1b\x8c\tsixteenth\x94h\x1dj\x0c\x01\x00\x00ubuu." cookies = pickle.loads(pickled) cj = CookieJar() From 95e02409abdbbbedbe1f2ad7aa89121f884b7643 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 15:35:48 +0000 Subject: [PATCH 20/30] Update 7583.feature --- CHANGES/7583.feature | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGES/7583.feature b/CHANGES/7583.feature index a700c5eb5fb..75de6b672dd 100644 --- a/CHANGES/7583.feature +++ b/CHANGES/7583.feature @@ -1 +1,14 @@ Implement filter_cookies() with domain-matching and path-matching on the keys, instead of testing every single cookie. +This may break existing cookies that have been saved with `CookieJar.save()`. Cookies can be migrated with this script:: + + import pickle + with file_path.open("rb") as f: + cookies = pickle.load(f) + + morsels = [(name, m) for c in cookies.values() for name, m in c.items()] + cookies.clear() + for name, m in morsels: + cookies[(m["domain"], "" if m["path"] == "/" else m["path"])][name] = m + + with file_path.open("wb") as f: + pickle.dump(cookies, f, pickle.HIGHEST_PROTOCOL) From 9ff4082d55fc511e00b24dcd8237ff98e718b134 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 15:40:48 +0000 Subject: [PATCH 21/30] Update tests/test_cookiejar.py --- tests/test_cookiejar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index 7a0105c52a6..7183309fa94 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -927,7 +927,7 @@ def test_pickle_format(cookies_to_send) -> None: with file_path.open("wb") as f: pickle.dump(cookies, f, pickle.HIGHEST_PROTOCOL) """ - pickled = b"\x80\x04\x95\xd3\x0b\x00\x00\x00\x00\x00\x00\x8c\x0bcollections\x94\x8c\x0bdefaultdict\x94\x93\x94\x8c\x0chttp.cookies\x94\x8c\x0cSimpleCookie\x94\x93\x94\x85\x94R\x94(\x8c\x00\x94h\x08\x86\x94h\x05)\x81\x94\x8c\rshared-cookie\x94h\x03\x8c\x06Morsel\x94\x93\x94)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94\x8c\x01/\x94\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\x08\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(\x8c\x03key\x94h\x0b\x8c\x05value\x94\x8c\x05first\x94\x8c\x0bcoded_value\x94h\x1cubs\x8c\x0bexample.com\x94h\x08\x86\x94h\x05)\x81\x94(\x8c\rdomain-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\x1e\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah!h\x1b\x8c\x06second\x94h\x1dh-ub\x8c\x14dotted-domain-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94\x8c\x0bexample.com\x94\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah.h\x1b\x8c\x05fifth\x94h\x1dh;ubu\x8c\x11test1.example.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x11subdomain1-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h<\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah?h\x1b\x8c\x05third\x94h\x1dhKubs\x8c\x11test2.example.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x11subdomain2-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94hL\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ahOh\x1b\x8c\x06fourth\x94h\x1dh[ubs\x8c\rdifferent.org\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x17different-domain-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\\\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah_h\x1b\x8c\x05sixth\x94h\x1dhkubs\x8c\nsecure.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\rsecure-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94hl\x8c\x07max-age\x94h\x08\x8c\x06secure\x94\x88\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ahoh\x1b\x8c\x07seventh\x94h\x1dh{ubs\x8c\x0cpathtest.com\x94h\x08\x86\x94h\x05)\x81\x94(\x8c\x0eno-path-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h|\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\x7fh\x1b\x8c\x06eighth\x94h\x1dh\x8bub\x8c\x0cpath1-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94\x8c\x0cpathtest.com\x94\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\x8ch\x1b\x8c\x05ninth\x94h\x1dh\x99ubu\x8c\x0cpathtest.com\x94\x8c\x04/one\x94\x86\x94h\x05)\x81\x94\x8c\x0cpath2-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x9b\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\x9a\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\x9eh\x1b\x8c\x05tenth\x94h\x1dh\xaaubs\x8c\x0cpathtest.com\x94\x8c\x08/one/two\x94\x86\x94h\x05)\x81\x94\x8c\x0cpath3-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\xac\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xab\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xafh\x1b\x8c\x08eleventh\x94h\x1dh\xbbubs\x8c\x0cpathtest.com\x94\x8c\t/one/two/\x94\x86\x94h\x05)\x81\x94\x8c\x0cpath4-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\xbd\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xbc\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xc0h\x1b\x8c\x07twelfth\x94h\x1dh\xccubs\x8c\x0fexpirestest.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x0eexpires-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94\x8c\x1cTue, 1 Jan 2999 12:00:00 GMT\x94\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xcd\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xd0h\x1b\x8c\nthirteenth\x94h\x1dh\xddubs\x8c\x0emaxagetest.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x0emax-age-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xde\x8c\x07max-age\x94\x8c\x0260\x94\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xe1h\x1b\x8c\nfourteenth\x94h\x1dh\xeeubs\x8c\x12invalid-values.com\x94h\x08\x86\x94h\x05)\x81\x94(\x8c\x16invalid-max-age-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xef\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xf2h\x1b\x8c\tfifteenth\x94h\x1dh\xfeub\x8c\x16invalid-expires-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94\x8c\x12invalid-values.com\x94\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xffh\x1b\x8c\tsixteenth\x94h\x1dj\x0c\x01\x00\x00ubuu." + pickled = b"\x80\x04\x95\xc8\x0b\x00\x00\x00\x00\x00\x00\x8c\x0bcollections\x94\x8c\x0bdefaultdict\x94\x93\x94\x8c\x0chttp.cookies\x94\x8c\x0cSimpleCookie\x94\x93\x94\x85\x94R\x94(\x8c\x00\x94h\x08\x86\x94h\x05)\x81\x94\x8c\rshared-cookie\x94h\x03\x8c\x06Morsel\x94\x93\x94)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94\x8c\x01/\x94\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\x08\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(\x8c\x03key\x94h\x0b\x8c\x05value\x94\x8c\x05first\x94\x8c\x0bcoded_value\x94h\x1cubs\x8c\x0bexample.com\x94h\x08\x86\x94h\x05)\x81\x94(\x8c\rdomain-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\x1e\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah!h\x1b\x8c\x06second\x94h\x1dh-ub\x8c\x14dotted-domain-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94\x8c\x0bexample.com\x94\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah.h\x1b\x8c\x05fifth\x94h\x1dh;ubu\x8c\x11test1.example.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x11subdomain1-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h<\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah?h\x1b\x8c\x05third\x94h\x1dhKubs\x8c\x11test2.example.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x11subdomain2-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94hL\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ahOh\x1b\x8c\x06fourth\x94h\x1dh[ubs\x8c\rdifferent.org\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x17different-domain-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\\\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah_h\x1b\x8c\x05sixth\x94h\x1dhkubs\x8c\nsecure.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\rsecure-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94hl\x8c\x07max-age\x94h\x08\x8c\x06secure\x94\x88\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ahoh\x1b\x8c\x07seventh\x94h\x1dh{ubs\x8c\x0cpathtest.com\x94h\x08\x86\x94h\x05)\x81\x94(\x8c\x0eno-path-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h|\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\x7fh\x1b\x8c\x06eighth\x94h\x1dh\x8bub\x8c\x0cpath1-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94\x8c\x0cpathtest.com\x94\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\x8ch\x1b\x8c\x05ninth\x94h\x1dh\x99ubu\x8c\x0cpathtest.com\x94\x8c\x04/one\x94\x86\x94h\x05)\x81\x94\x8c\x0cpath2-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x9b\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\x9a\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\x9eh\x1b\x8c\x05tenth\x94h\x1dh\xaaubs\x8c\x0cpathtest.com\x94\x8c\x08/one/two\x94\x86\x94h\x05)\x81\x94(\x8c\x0cpath3-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\xac\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xab\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xafh\x1b\x8c\x08eleventh\x94h\x1dh\xbbub\x8c\x0cpath4-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94\x8c\t/one/two/\x94\x8c\x07comment\x94h\x08\x8c\x06domain\x94\x8c\x0cpathtest.com\x94\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xbch\x1b\x8c\x07twelfth\x94h\x1dh\xcaubu\x8c\x0fexpirestest.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x0eexpires-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94\x8c\x1cTue, 1 Jan 2999 12:00:00 GMT\x94\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xcb\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xceh\x1b\x8c\nthirteenth\x94h\x1dh\xdbubs\x8c\x0emaxagetest.com\x94h\x08\x86\x94h\x05)\x81\x94\x8c\x0emax-age-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xdc\x8c\x07max-age\x94\x8c\x0260\x94\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xdfh\x1b\x8c\nfourteenth\x94h\x1dh\xecubs\x8c\x12invalid-values.com\x94h\x08\x86\x94h\x05)\x81\x94(\x8c\x16invalid-max-age-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94h\xed\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xf0h\x1b\x8c\tfifteenth\x94h\x1dh\xfcub\x8c\x16invalid-expires-cookie\x94h\r)\x81\x94(\x8c\x07expires\x94h\x08\x8c\x04path\x94h\x11\x8c\x07comment\x94h\x08\x8c\x06domain\x94\x8c\x12invalid-values.com\x94\x8c\x07max-age\x94h\x08\x8c\x06secure\x94h\x08\x8c\x08httponly\x94h\x08\x8c\x07version\x94h\x08\x8c\x08samesite\x94h\x08u}\x94(h\x1ah\xfdh\x1b\x8c\tsixteenth\x94h\x1dj\n\x01\x00\x00ubuu." cookies = pickle.loads(pickled) cj = CookieJar() From 6adae743e01417aa18689cd35618599669e1be9c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 15:41:29 +0000 Subject: [PATCH 22/30] Update CHANGES/7583.feature --- CHANGES/7583.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES/7583.feature b/CHANGES/7583.feature index 75de6b672dd..c05e64be780 100644 --- a/CHANGES/7583.feature +++ b/CHANGES/7583.feature @@ -8,7 +8,7 @@ This may break existing cookies that have been saved with `CookieJar.save()`. Co morsels = [(name, m) for c in cookies.values() for name, m in c.items()] cookies.clear() for name, m in morsels: - cookies[(m["domain"], "" if m["path"] == "/" else m["path"])][name] = m + cookies[(m["domain"], m["path"].rstrip("/"))][name] = m with file_path.open("wb") as f: pickle.dump(cookies, f, pickle.HIGHEST_PROTOCOL) From 84b772e9b7238f904e1cade2991a24b1b8b47fb9 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 16:24:43 +0000 Subject: [PATCH 23/30] Optimise by handling IP outside of loop --- aiohttp/cookiejar.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 22525db1213..20e0194136a 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -263,14 +263,19 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": request_origin = request_url.origin() is_not_secure = request_origin not in self._treat_as_secure_origin + if is_ip_address(hostname): + if not self._unsafe: + return filtered + domains = (hostname,) + else: + # Get all the subdomains that might match a cookie (e.g. "foo.bar.com", "bar.com", "com") + domains = itertools.accumulate( + reversed(hostname.split(".")), lambda x, y: f"{y}.{x}" + ) # Get all the path prefixes that might match a cookie (e.g. "", "/foo", "/foo/bar") paths = itertools.accumulate( request_url.path.split("/"), lambda x, y: f"{x}/{y}" ) - # Get all the subdomains that might match a cookie (e.g. "foo.bar.com", "bar.com", "com") - domains = itertools.accumulate( - reversed(hostname.split(".")), lambda x, y: f"{y}.{x}" - ) # Create every combination of (domain, path) pairs, plus the shared cookie. pairs = itertools.chain((("", ""),), itertools.product(domains, paths)) @@ -287,9 +292,6 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": filtered[name] = cookie.value continue - if not self._unsafe and is_ip_address(hostname): - continue - if (domain, name) in self._host_only_cookies: if domain != hostname: continue From 5b56087162fcbc534e9ad82dc7c7fa06251ea41c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 16:39:15 +0000 Subject: [PATCH 24/30] Also move shared cookie out of loop --- aiohttp/cookiejar.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 20e0194136a..dceba3b5b07 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -263,6 +263,10 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": request_origin = request_url.origin() is_not_secure = request_origin not in self._treat_as_secure_origin + # Send shared cookie + for c in self._cookies[("", "")].values(): + filtered[c.key] = c.value + if is_ip_address(hostname): if not self._unsafe: return filtered @@ -276,8 +280,8 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": paths = itertools.accumulate( request_url.path.split("/"), lambda x, y: f"{x}/{y}" ) - # Create every combination of (domain, path) pairs, plus the shared cookie. - pairs = itertools.chain((("", ""),), itertools.product(domains, paths)) + # Create every combination of (domain, path) pairs. + pairs = itertools.product(domains, paths) # Point 2: https://www.rfc-editor.org/rfc/rfc6265.html#section-5.4 cookies = itertools.chain.from_iterable( @@ -287,11 +291,6 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": name = cookie.key domain = cookie["domain"] - # Send shared cookies - if not domain: - filtered[name] = cookie.value - continue - if (domain, name) in self._host_only_cookies: if domain != hostname: continue From cd820e8ecec47e8aed73000eebacba90b624894d Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 17:02:10 +0000 Subject: [PATCH 25/30] Update test for coverage --- tests/test_cookiejar.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index 7183309fa94..9edfd3aa5cf 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -593,11 +593,11 @@ def test_domain_filter_diff_host(self) -> None: def test_domain_filter_host_only(self) -> None: self.jar.update_cookies(self.cookies_to_receive, URL("http://example.com/")) + sub_cookie = SimpleCookie("subdomain=spam; Path=/;") + self.jar.update_cookies(sub_cookie, URL("http://foo.example.com/")) - cookies_sent = self.jar.filter_cookies(URL("http://example.com/")) - self.assertIn("unconstrained-cookie", set(cookies_sent.keys())) - - cookies_sent = self.jar.filter_cookies(URL("http://different.org/")) + cookies_sent = self.jar.filter_cookies(URL("http://foo.example.com/")) + self.assertIn("subdomain", set(cookies_sent.keys())) self.assertNotIn("unconstrained-cookie", set(cookies_sent.keys())) def test_secure_filter(self) -> None: From 8ec5db32b8473ebaba5653b2af156fcf4ef3afb2 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 17:08:35 +0000 Subject: [PATCH 26/30] Remove now redundant check --- aiohttp/cookiejar.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index dceba3b5b07..b6f96637383 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -294,8 +294,6 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": if (domain, name) in self._host_only_cookies: if domain != hostname: continue - elif not self._is_domain_match(domain, hostname): - continue if not self._is_path_match(request_url.path, cookie["path"]): continue From 2f6f7304775427c4ca8955fc48e031fd4a6e5779 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 17:41:20 +0000 Subject: [PATCH 27/30] Replace redundant path match --- aiohttp/cookiejar.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index b6f96637383..2887c799b53 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -295,7 +295,8 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": if domain != hostname: continue - if not self._is_path_match(request_url.path, cookie["path"]): + # Skip edge case when the cookie has a trailing slash but request doesn't. + if len(cookie["path"]) > path_len: continue if is_not_secure and cookie["secure"]: @@ -325,25 +326,6 @@ def _is_domain_match(domain: str, hostname: str) -> bool: return not is_ip_address(hostname) - @staticmethod - def _is_path_match(req_path: str, cookie_path: str) -> bool: - """Implements path matching adhering to RFC 6265.""" - if not req_path.startswith("/"): - req_path = "/" - - if req_path == cookie_path: - return True - - if not req_path.startswith(cookie_path): - return False - - if cookie_path.endswith("/"): - return True - - non_matching = req_path[len(cookie_path) :] - - return non_matching.startswith("/") - @classmethod def _parse_date(cls, date_str: str) -> Optional[int]: """Implements date string parsing adhering to RFC 6265.""" From 1cfcd44f6b709f9d62f7329f7fdbc74c4f1e5676 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 17:43:30 +0000 Subject: [PATCH 28/30] Update cookiejar.py --- aiohttp/cookiejar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 2887c799b53..e5b3ec5ae75 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -270,7 +270,7 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": if is_ip_address(hostname): if not self._unsafe: return filtered - domains = (hostname,) + domains: Iterable[str] = (hostname,) else: # Get all the subdomains that might match a cookie (e.g. "foo.bar.com", "bar.com", "com") domains = itertools.accumulate( @@ -287,6 +287,7 @@ def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": cookies = itertools.chain.from_iterable( self._cookies[p].values() for p in pairs ) + path_len = len(request_url.path) for cookie in cookies: name = cookie.key domain = cookie["domain"] From 8557e39b7a35e08de11d4db5a686857f759b6698 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 17:43:59 +0000 Subject: [PATCH 29/30] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/cookiejar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index e5b3ec5ae75..aa63d280079 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -11,7 +11,7 @@ from collections import defaultdict from http.cookies import BaseCookie, Morsel, SimpleCookie from math import ceil -from typing import ( # noqa +from typing import ( DefaultDict, Dict, Iterable, From ac55d6466963b7bb2834fe3d19a4f3c2a67e82a3 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 20 Jan 2024 17:58:54 +0000 Subject: [PATCH 30/30] Update test_cookiejar.py --- tests/test_cookiejar.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index 9edfd3aa5cf..b2f8085ef21 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -155,28 +155,6 @@ def test_domain_matching() -> None: assert not test_func("test.com", "127.0.0.1") -def test_path_matching() -> None: - test_func = CookieJar._is_path_match - - assert test_func("/", "") - assert test_func("", "/") - assert test_func("/file", "") - assert test_func("/folder/file", "") - assert test_func("/", "/") - assert test_func("/file", "/") - assert test_func("/file", "/file") - assert test_func("/folder/", "/folder/") - assert test_func("/folder/", "/") - assert test_func("/folder/file", "/") - - assert not test_func("/", "/file") - assert not test_func("/", "/folder/") - assert not test_func("/file", "/folder/file") - assert not test_func("/folder/", "/folder/file") - assert not test_func("/different-file", "/file") - assert not test_func("/different-folder/", "/folder/") - - async def test_constructor(cookies_to_send: Any, cookies_to_receive: Any) -> None: jar = CookieJar() jar.update_cookies(cookies_to_send)