From 7cb8c8fe5b1cb6e8983123a15e0491c3dc2dff4c Mon Sep 17 00:00:00 2001 From: Behnam Esfahbod Date: Fri, 13 Jul 2018 17:39:09 -0700 Subject: [PATCH 1/8] test_url_query: Expand tests Expanding URL-query tests to cover more behaviors around special chars when parsing, setting, re-setting and getting values. Bug revealed are marked with `FIXME` to address separately. --- tests/test_url_query.py | 193 ++++++++++++++++++++++++++++++---------- 1 file changed, 148 insertions(+), 45 deletions(-) diff --git a/tests/test_url_query.py b/tests/test_url_query.py index 776fea1a8..633f12f46 100644 --- a/tests/test_url_query.py +++ b/tests/test_url_query.py @@ -1,36 +1,74 @@ -from urllib.parse import urlencode - +import pytest from multidict import MultiDict, MultiDictProxy +from urllib.parse import urlencode from yarl import URL -# query - - -def test_query_spaces(): - url = URL("http://example.com?a+b=c+d") - assert url.query == MultiDict({"a b": "c d"}) - - -def test_query_empty(): - url = URL("http://example.com") - assert isinstance(url.query, MultiDictProxy) - assert url.query == MultiDict() - - -def test_query(): - url = URL("http://example.com?a=1&b=2") - assert url.query == MultiDict([("a", "1"), ("b", "2")]) - -def test_query_repeated_args(): - url = URL("http://example.com?a=1&b=2&a=3") - assert url.query == MultiDict([("a", "1"), ("b", "2"), ("a", "3")]) - - -def test_query_empty_arg(): - url = URL("http://example.com?a") - assert url.query == MultiDict([("a", "")]) +# ======================================== +# Basic chars in query values +# ======================================== + +URLS_WITH_BASIC_QUERY_VALUES = [ + # Empty strings, keys and values + ( + URL("http://example.com"), + MultiDict(), + ), + ( + URL("http://example.com?a"), + MultiDict([("a", "")]), + ), + ( + URL("http://example.com?a="), + MultiDict([("a", "")]), + ), + # ASCII chars + ( + URL("http://example.com?a+b=c+d"), + MultiDict({"a b": "c d"}), + ), + ( + URL("http://example.com?a=1&b=2"), + MultiDict([("a", "1"), ("b", "2")]), + ), + ( + URL("http://example.com?a=1&b=2&a=3"), + MultiDict([("a", "1"), ("b", "2"), ("a", "3")]), + ), + # Non-ASCI BMP chars + ( + URL("http://example.com?ключ=знач"), + MultiDict({"ключ": "знач"}), + ), + ( + URL("http://example.com?foo=ᴜɴɪᴄᴏᴅᴇ"), + MultiDict({"foo": "ᴜɴɪᴄᴏᴅᴇ"}), + ), + # Non-BMP chars + ( + URL("http://example.com?bar=𝕦𝕟𝕚𝕔𝕠𝕕𝕖"), + MultiDict({"bar": "𝕦𝕟𝕚𝕔𝕠𝕕𝕖"}), + ), +] + + +@pytest.mark.parametrize( + "original_url, expected_query", + URLS_WITH_BASIC_QUERY_VALUES, +) +def test_query_basic_parsing(original_url, expected_query): + assert original_url.query == expected_query + + +@pytest.mark.parametrize( + "original_url, expected_query", + URLS_WITH_BASIC_QUERY_VALUES, +) +def test_query_basic_update_query(original_url, expected_query): + new_url = original_url.update_query({}) + # FIXME: `?a` becomes `?a=` right now. Maybe support `None` values? + # assert new_url == original_url def test_query_dont_unqoute_twice(): @@ -42,20 +80,85 @@ def test_query_dont_unqoute_twice(): assert url.query["url"] == sample_url -def test_query_nonascii(): - url = URL("http://example.com?ключ=знач") - assert url.query == MultiDict({"ключ": "знач"}) - - -# query separators - - -def test_ampersand_as_separator(): - u = URL("http://127.0.0.1/?a=1&b=2") - assert len(u.query) == 2 - - -def test_ampersand_as_value(): - u = URL("http://127.0.0.1/?a=1%26b=2") - assert len(u.query) == 1 - assert u.query["a"] == "1&b=2" +# ======================================== +# Reserved chars in query values +# ======================================== + +URLS_WITH_RESERVED_CHARS_IN_QUERY_VALUES = [ + # Ampersand + (URL("http://127.0.0.1/?a=10&b=20"), 2, "10"), + (URL("http://127.0.0.1/?a=10%26b=20"), 1, "10&b=20"), + (URL("http://127.0.0.1/?a=10%3Bb=20"), 1, "10;b=20"), + # Semicolon + (URL("http://127.0.0.1/?a=10;b=20"), 2, "10"), + (URL("http://127.0.0.1/?a=10%26b=20"), 1, "10&b=20"), + (URL("http://127.0.0.1/?a=10%3Bb=20"), 1, "10;b=20"), +] + + +@pytest.mark.parametrize( + "original_url, expected_query_len, expected_value_a", + URLS_WITH_RESERVED_CHARS_IN_QUERY_VALUES, +) +def test_query_separators_from_parsing( + original_url, + expected_query_len, + expected_value_a, +): + assert len(original_url.query) == expected_query_len + assert original_url.query["a"] == expected_value_a + + +@pytest.mark.parametrize( + "original_url, expected_query_len, expected_value_a", + URLS_WITH_RESERVED_CHARS_IN_QUERY_VALUES, +) +def test_query_separators_from_update_query( + original_url, + expected_query_len, + expected_value_a, +): + new_url = original_url.update_query( + { + "c": expected_value_a, + } + ) + assert new_url.query["a"] == expected_value_a + assert new_url.query["c"] == expected_value_a + + +@pytest.mark.parametrize( + "original_url, expected_query_len, expected_value_a", + URLS_WITH_RESERVED_CHARS_IN_QUERY_VALUES, +) +def test_query_separators_from_with_query( + original_url, + expected_query_len, + expected_value_a, +): + new_url = original_url.with_query( + { + "c": expected_value_a, + } + ) + assert new_url.query["c"] == expected_value_a + + +@pytest.mark.parametrize( + "original_url, expected_query_len, expected_value_a", + URLS_WITH_RESERVED_CHARS_IN_QUERY_VALUES, +) +def test_query_separators_from_with_query( + original_url, + expected_query_len, + expected_value_a, +): + new_url = original_url.update_query({}) + + assert new_url.query["a"] == original_url.query["a"] + + if "b" in original_url.query: + assert new_url.query["b"] == original_url.query["b"] + + # FIXME: Broken because of asymmetric query encoding + # assert new_url == original_url From e1245cb65d898f1459e83313045e37a6e7b12def Mon Sep 17 00:00:00 2001 From: Behnam Esfahbod Date: Sun, 12 Aug 2018 19:45:33 -0700 Subject: [PATCH 2/8] Update test_url_query.py --- tests/test_url_query.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_url_query.py b/tests/test_url_query.py index 633f12f46..e154e1631 100644 --- a/tests/test_url_query.py +++ b/tests/test_url_query.py @@ -1,5 +1,5 @@ import pytest -from multidict import MultiDict, MultiDictProxy +from multidict import MultiDict from urllib.parse import urlencode from yarl import URL @@ -69,6 +69,7 @@ def test_query_basic_update_query(original_url, expected_query): new_url = original_url.update_query({}) # FIXME: `?a` becomes `?a=` right now. Maybe support `None` values? # assert new_url == original_url + assert new_url is not None def test_query_dont_unqoute_twice(): @@ -148,7 +149,7 @@ def test_query_separators_from_with_query( "original_url, expected_query_len, expected_value_a", URLS_WITH_RESERVED_CHARS_IN_QUERY_VALUES, ) -def test_query_separators_from_with_query( +def test_query_from_empty_update_query( original_url, expected_query_len, expected_value_a, From b44ebb84b474c368f40a58a3d14e978abb07c0f9 Mon Sep 17 00:00:00 2001 From: Behnam Esfahbod Date: Mon, 13 Aug 2018 10:41:53 -0700 Subject: [PATCH 3/8] tests: Add type hints to make mypy happy --- tests/test_url_query.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_url_query.py b/tests/test_url_query.py index e154e1631..8108b0a10 100644 --- a/tests/test_url_query.py +++ b/tests/test_url_query.py @@ -1,6 +1,8 @@ +from typing import List, Tuple # noqa: F401 +from urllib.parse import urlencode + import pytest from multidict import MultiDict -from urllib.parse import urlencode from yarl import URL @@ -50,7 +52,7 @@ URL("http://example.com?bar=𝕦𝕟𝕚𝕔𝕠𝕕𝕖"), MultiDict({"bar": "𝕦𝕟𝕚𝕔𝕠𝕕𝕖"}), ), -] +] # type: List[Tuple[URL, MultiDict]] @pytest.mark.parametrize( From 842002f750822f6ffbbee91a000cc0457c8a821d Mon Sep 17 00:00:00 2001 From: Behnam Esfahbod Date: Mon, 13 Aug 2018 15:12:03 -0700 Subject: [PATCH 4/8] test_url_query: Add back assertion for url.query type --- tests/test_url_query.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_url_query.py b/tests/test_url_query.py index 8108b0a10..a543bd22f 100644 --- a/tests/test_url_query.py +++ b/tests/test_url_query.py @@ -2,7 +2,7 @@ from urllib.parse import urlencode import pytest -from multidict import MultiDict +from multidict import MultiDict, MultiDictProxy from yarl import URL @@ -60,6 +60,7 @@ URLS_WITH_BASIC_QUERY_VALUES, ) def test_query_basic_parsing(original_url, expected_query): + assert isinstance(original_url.query, MultiDictProxy) assert original_url.query == expected_query From e6b532543583a63e0ae794231e4200f8a7636163 Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Wed, 19 Apr 2023 15:44:17 +0100 Subject: [PATCH 5/8] Use an annotation for the typehint --- tests/test_url_query.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_url_query.py b/tests/test_url_query.py index a543bd22f..088cd8c6e 100644 --- a/tests/test_url_query.py +++ b/tests/test_url_query.py @@ -1,4 +1,4 @@ -from typing import List, Tuple # noqa: F401 +from typing import List, Tuple from urllib.parse import urlencode import pytest @@ -6,12 +6,11 @@ from yarl import URL - # ======================================== # Basic chars in query values # ======================================== -URLS_WITH_BASIC_QUERY_VALUES = [ +URLS_WITH_BASIC_QUERY_VALUES: List[Tuple[URL, MultiDict]] = [ # Empty strings, keys and values ( URL("http://example.com"), @@ -52,7 +51,7 @@ URL("http://example.com?bar=𝕦𝕟𝕚𝕔𝕠𝕕𝕖"), MultiDict({"bar": "𝕦𝕟𝕚𝕔𝕠𝕕𝕖"}), ), -] # type: List[Tuple[URL, MultiDict]] +] @pytest.mark.parametrize( From 152bfe1b2c9d7ad407da5e81b924011b2ff16cc6 Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Wed, 19 Apr 2023 16:11:14 +0100 Subject: [PATCH 6/8] Correct semi-colon test Semi-colons are _not_ reserved characters in the query string of a URL; the query string portion consists of `pchar` characters, and `pchar` includes `sub-delim`, which is the class that contains `;`. --- tests/test_url_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_url_query.py b/tests/test_url_query.py index 088cd8c6e..b6dcc3b24 100644 --- a/tests/test_url_query.py +++ b/tests/test_url_query.py @@ -93,7 +93,7 @@ def test_query_dont_unqoute_twice(): (URL("http://127.0.0.1/?a=10%26b=20"), 1, "10&b=20"), (URL("http://127.0.0.1/?a=10%3Bb=20"), 1, "10;b=20"), # Semicolon - (URL("http://127.0.0.1/?a=10;b=20"), 2, "10"), + (URL("http://127.0.0.1/?a=10;b=20"), 1, "10;b=20"), (URL("http://127.0.0.1/?a=10%26b=20"), 1, "10&b=20"), (URL("http://127.0.0.1/?a=10%3Bb=20"), 1, "10;b=20"), ] From f268a30c584cafca8d9f0ba417515938d304a676 Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Wed, 19 Apr 2023 17:03:41 +0100 Subject: [PATCH 7/8] Remove non-applicable FIXMEs - There is no way to distinguish between `?a=` and `?a` when rebuilding a query string. - The further encoding of query delimiters (`=` and `&` in query _values_ is entirely normal and expected. --- tests/test_url_query.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tests/test_url_query.py b/tests/test_url_query.py index b6dcc3b24..e858564f2 100644 --- a/tests/test_url_query.py +++ b/tests/test_url_query.py @@ -16,10 +16,6 @@ URL("http://example.com"), MultiDict(), ), - ( - URL("http://example.com?a"), - MultiDict([("a", "")]), - ), ( URL("http://example.com?a="), MultiDict([("a", "")]), @@ -69,9 +65,7 @@ def test_query_basic_parsing(original_url, expected_query): ) def test_query_basic_update_query(original_url, expected_query): new_url = original_url.update_query({}) - # FIXME: `?a` becomes `?a=` right now. Maybe support `None` values? - # assert new_url == original_url - assert new_url is not None + assert new_url == original_url def test_query_dont_unqoute_twice(): @@ -162,6 +156,3 @@ def test_query_from_empty_update_query( if "b" in original_url.query: assert new_url.query["b"] == original_url.query["b"] - - # FIXME: Broken because of asymmetric query encoding - # assert new_url == original_url From 7f12f4312d3c245dcecc07418f2b2719d5496afc Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Wed, 19 Apr 2023 17:07:04 +0100 Subject: [PATCH 8/8] Include a changelog entry --- CHANGES/220.misc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 CHANGES/220.misc.rst diff --git a/CHANGES/220.misc.rst b/CHANGES/220.misc.rst new file mode 100644 index 000000000..553b94925 --- /dev/null +++ b/CHANGES/220.misc.rst @@ -0,0 +1 @@ +Expanded the test suite to cover more query string behaviours.