From e445011aebc670d78163eeeb1e5da58fa851221f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 30 Sep 2021 21:48:23 +0100 Subject: [PATCH 01/66] [pre-commit.ci] pre-commit autoupdate (#5985) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/pre-commit/pre-commit-hooks: v3.3.0 → v4.0.1](https://github.com/pre-commit/pre-commit-hooks/compare/v3.3.0...v4.0.1) - [github.com/asottile/yesqa: v1.2.2 → v1.2.3](https://github.com/asottile/yesqa/compare/v1.2.2...v1.2.3) - https://github.com/pre-commit/mirrors-isort → https://github.com/PyCQA/isort - [github.com/PyCQA/isort: v5.6.4 → 5.9.3](https://github.com/PyCQA/isort/compare/v5.6.4...5.9.3) - [github.com/psf/black: 20.8b1 → 21.9b0](https://github.com/psf/black/compare/20.8b1...21.9b0) - [github.com/pre-commit/pre-commit-hooks: v3.3.0 → v4.0.1](https://github.com/pre-commit/pre-commit-hooks/compare/v3.3.0...v4.0.1) - [github.com/asottile/pyupgrade: v2.7.4 → v2.28.0](https://github.com/asottile/pyupgrade/compare/v2.7.4...v2.28.0) - https://gitlab.com/pycqa/flake8 → https://github.com/PyCQA/flake8 - [github.com/PyCQA/flake8: 3.8.4 → 3.9.2](https://github.com/PyCQA/flake8/compare/3.8.4...3.9.2) - git://github.com/Lucas-C/pre-commit-hooks-markup: v1.0.0 → v1.0.1 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update test_resolver.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update bench-asyncio-write.py * Update bench-asyncio-write.py * Update payload.py Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Sam Bull --- .pre-commit-config.yaml | 20 +++++++-------- aiohttp/client.py | 2 +- aiohttp/client_exceptions.py | 2 +- aiohttp/connector.py | 2 +- aiohttp/cookiejar.py | 2 +- aiohttp/helpers.py | 6 ++--- aiohttp/locks.py | 2 +- aiohttp/multipart.py | 2 +- aiohttp/payload.py | 7 ++---- aiohttp/streams.py | 2 +- aiohttp/tracing.py | 34 +++++++++++++------------- aiohttp/web_app.py | 2 +- aiohttp/web_exceptions.py | 2 +- aiohttp/web_routedef.py | 2 +- examples/legacy/tcp_protocol_parser.py | 2 +- tests/test_client_functional.py | 2 +- tests/test_helpers.py | 2 +- tests/test_http_parser.py | 2 +- tests/test_resolver.py | 2 +- tests/test_web_functional.py | 2 +- tools/bench-asyncio-write.py | 24 +++++++++--------- 21 files changed, 61 insertions(+), 62 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ba83762c34f..ef2d2d86b96 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,24 +7,24 @@ repos: entry: ./tools/check_changes.py pass_filenames: false - repo: https://github.com/pre-commit/pre-commit-hooks - rev: 'v3.3.0' + rev: 'v4.0.1' hooks: - id: check-merge-conflict - repo: https://github.com/asottile/yesqa - rev: v1.2.2 + rev: v1.2.3 hooks: - id: yesqa -- repo: https://github.com/pre-commit/mirrors-isort - rev: 'v5.6.4' +- repo: https://github.com/PyCQA/isort + rev: '5.9.3' hooks: - id: isort - repo: https://github.com/psf/black - rev: '20.8b1' + rev: '21.9b0' hooks: - id: black language_version: python3 # Should be a command that runs python3.6+ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: 'v3.3.0' + rev: 'v4.0.1' hooks: - id: end-of-file-fixer exclude: >- @@ -60,18 +60,18 @@ repos: - id: detect-private-key exclude: ^examples/ - repo: https://github.com/asottile/pyupgrade - rev: 'v2.7.4' + rev: 'v2.28.0' hooks: - id: pyupgrade args: ['--py36-plus'] -- repo: https://gitlab.com/pycqa/flake8 - rev: '3.8.4' +- repo: https://github.com/PyCQA/flake8 + rev: '3.9.2' hooks: - id: flake8 exclude: "^docs/" - repo: git://github.com/Lucas-C/pre-commit-hooks-markup - rev: v1.0.0 + rev: v1.0.1 hooks: - id: rst-linter files: >- diff --git a/aiohttp/client.py b/aiohttp/client.py index 1aab4829ffe..c14a682a404 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -263,7 +263,7 @@ def __init__( real_headers = CIMultiDict() self._default_headers = real_headers # type: CIMultiDict[str] if skip_auto_headers is not None: - self._skip_auto_headers = frozenset([istr(i) for i in skip_auto_headers]) + self._skip_auto_headers = frozenset(istr(i) for i in skip_auto_headers) else: self._skip_auto_headers = frozenset() diff --git a/aiohttp/client_exceptions.py b/aiohttp/client_exceptions.py index 808c1cc614e..5f2f8958e84 100644 --- a/aiohttp/client_exceptions.py +++ b/aiohttp/client_exceptions.py @@ -85,7 +85,7 @@ def __repr__(self) -> str: args += f", message={self.message!r}" if self.headers is not None: args += f", headers={self.headers!r}" - return "{}({})".format(type(self).__name__, args) + return f"{type(self).__name__}({args})" class ContentTypeError(ClientResponseError): diff --git a/aiohttp/connector.py b/aiohttp/connector.py index 58c528de6fb..ab5124966b5 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -150,7 +150,7 @@ def closed(self) -> bool: class _TransportPlaceholder: - """ placeholder for BaseConnector.connect function """ + """placeholder for BaseConnector.connect function""" def __init__(self, loop: asyncio.AbstractEventLoop) -> None: fut = loop.create_future() diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index 8d0a1edf49b..ef32f5faa53 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -320,7 +320,7 @@ def _parse_date(cls, date_str: str) -> Optional[datetime.datetime]: time_match = cls.DATE_HMS_TIME_RE.match(token) if time_match: found_time = True - hour, minute, second = [int(s) for s in time_match.groups()] + hour, minute, second = (int(s) for s in time_match.groups()) continue if not found_day: diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 418de0f6f9a..be76e0cec62 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -499,7 +499,7 @@ def _is_ip_address( elif isinstance(host, (bytes, bytearray, memoryview)): return bool(regexb.match(host)) else: - raise TypeError("{} [{}] is not a str or bytes".format(host, type(host))) + raise TypeError(f"{host} [{type(host)}] is not a str or bytes") is_ipv4_address = functools.partial(_is_ip_address, _ipv4_regex, _ipv4_regexb) @@ -593,7 +593,7 @@ def call_later( class TimeoutHandle: - """ Timeout handle """ + """Timeout handle""" def __init__( self, loop: asyncio.AbstractEventLoop, timeout: Optional[float] @@ -656,7 +656,7 @@ def __exit__( class TimerContext(BaseTimerContext): - """ Low resolution timeout context manager """ + """Low resolution timeout context manager""" def __init__(self, loop: asyncio.AbstractEventLoop) -> None: self._loop = loop diff --git a/aiohttp/locks.py b/aiohttp/locks.py index ce5b9c6f731..8ea99d70ce8 100644 --- a/aiohttp/locks.py +++ b/aiohttp/locks.py @@ -40,6 +40,6 @@ async def wait(self) -> Any: return val def cancel(self) -> None: - """ Cancel all waiters """ + """Cancel all waiters""" for waiter in self._waiters: waiter.cancel() diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index b5e78c835e6..b2c67baa45d 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -154,7 +154,7 @@ def unescape(text: str, *, chars: str = "".join(map(re.escape, CHAR))) -> str: elif parts: # maybe just ; in filename, in any case this is just # one case fix, for proper fix we need to redesign parser - _value = "{};{}".format(value, parts[0]) + _value = f"{value};{parts[0]}" if is_quoted(_value): parts.pop(0) value = unescape(_value[1:-1].lstrip("\\/")) diff --git a/aiohttp/payload.py b/aiohttp/payload.py index b88da2cd8ed..ace3dc2b995 100644 --- a/aiohttp/payload.py +++ b/aiohttp/payload.py @@ -15,7 +15,6 @@ Dict, Iterable, Optional, - Text, TextIO, Tuple, Type, @@ -221,9 +220,7 @@ async def write(self, writer: AbstractStreamWriter) -> None: class BytesPayload(Payload): def __init__(self, value: ByteString, *args: Any, **kwargs: Any) -> None: if not isinstance(value, (bytes, bytearray, memoryview)): - raise TypeError( - "value argument must be byte-ish, not {!r}".format(type(value)) - ) + raise TypeError(f"value argument must be byte-ish, not {type(value)!r}") if "content_type" not in kwargs: kwargs["content_type"] = "application/octet-stream" @@ -251,7 +248,7 @@ async def write(self, writer: AbstractStreamWriter) -> None: class StringPayload(BytesPayload): def __init__( self, - value: Text, + value: str, *args: Any, encoding: Optional[str] = None, content_type: Optional[str] = None, diff --git a/aiohttp/streams.py b/aiohttp/streams.py index a077b81b82d..185f46ecdab 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -480,7 +480,7 @@ def _read_nowait_chunk(self, n: int) -> bytes: return data def _read_nowait(self, n: int) -> bytes: - """ Read not more than n bytes, or whole buffer if n == -1 """ + """Read not more than n bytes, or whole buffer if n == -1""" chunks = [] while self._buffer: diff --git a/aiohttp/tracing.py b/aiohttp/tracing.py index 435bf3ddf73..7ffe93f8507 100644 --- a/aiohttp/tracing.py +++ b/aiohttp/tracing.py @@ -107,7 +107,7 @@ def __init__( def trace_config_ctx( self, trace_request_ctx: Optional[SimpleNamespace] = None ) -> SimpleNamespace: - """ Return a new trace_config_ctx instance """ + """Return a new trace_config_ctx instance""" return self._trace_config_ctx_factory(trace_request_ctx=trace_request_ctx) def freeze(self) -> None: @@ -219,7 +219,7 @@ def on_request_headers_sent( @dataclasses.dataclass(frozen=True) class TraceRequestStartParams: - """ Parameters sent by the `on_request_start` signal""" + """Parameters sent by the `on_request_start` signal""" method: str url: URL @@ -228,7 +228,7 @@ class TraceRequestStartParams: @dataclasses.dataclass(frozen=True) class TraceRequestChunkSentParams: - """ Parameters sent by the `on_request_chunk_sent` signal""" + """Parameters sent by the `on_request_chunk_sent` signal""" method: str url: URL @@ -237,7 +237,7 @@ class TraceRequestChunkSentParams: @dataclasses.dataclass(frozen=True) class TraceResponseChunkReceivedParams: - """ Parameters sent by the `on_response_chunk_received` signal""" + """Parameters sent by the `on_response_chunk_received` signal""" method: str url: URL @@ -246,7 +246,7 @@ class TraceResponseChunkReceivedParams: @dataclasses.dataclass(frozen=True) class TraceRequestEndParams: - """ Parameters sent by the `on_request_end` signal""" + """Parameters sent by the `on_request_end` signal""" method: str url: URL @@ -256,7 +256,7 @@ class TraceRequestEndParams: @dataclasses.dataclass(frozen=True) class TraceRequestExceptionParams: - """ Parameters sent by the `on_request_exception` signal""" + """Parameters sent by the `on_request_exception` signal""" method: str url: URL @@ -266,7 +266,7 @@ class TraceRequestExceptionParams: @dataclasses.dataclass(frozen=True) class TraceRequestRedirectParams: - """ Parameters sent by the `on_request_redirect` signal""" + """Parameters sent by the `on_request_redirect` signal""" method: str url: URL @@ -276,60 +276,60 @@ class TraceRequestRedirectParams: @dataclasses.dataclass(frozen=True) class TraceConnectionQueuedStartParams: - """ Parameters sent by the `on_connection_queued_start` signal""" + """Parameters sent by the `on_connection_queued_start` signal""" @dataclasses.dataclass(frozen=True) class TraceConnectionQueuedEndParams: - """ Parameters sent by the `on_connection_queued_end` signal""" + """Parameters sent by the `on_connection_queued_end` signal""" @dataclasses.dataclass(frozen=True) class TraceConnectionCreateStartParams: - """ Parameters sent by the `on_connection_create_start` signal""" + """Parameters sent by the `on_connection_create_start` signal""" @dataclasses.dataclass(frozen=True) class TraceConnectionCreateEndParams: - """ Parameters sent by the `on_connection_create_end` signal""" + """Parameters sent by the `on_connection_create_end` signal""" @dataclasses.dataclass(frozen=True) class TraceConnectionReuseconnParams: - """ Parameters sent by the `on_connection_reuseconn` signal""" + """Parameters sent by the `on_connection_reuseconn` signal""" @dataclasses.dataclass(frozen=True) class TraceDnsResolveHostStartParams: - """ Parameters sent by the `on_dns_resolvehost_start` signal""" + """Parameters sent by the `on_dns_resolvehost_start` signal""" host: str @dataclasses.dataclass(frozen=True) class TraceDnsResolveHostEndParams: - """ Parameters sent by the `on_dns_resolvehost_end` signal""" + """Parameters sent by the `on_dns_resolvehost_end` signal""" host: str @dataclasses.dataclass(frozen=True) class TraceDnsCacheHitParams: - """ Parameters sent by the `on_dns_cache_hit` signal""" + """Parameters sent by the `on_dns_cache_hit` signal""" host: str @dataclasses.dataclass(frozen=True) class TraceDnsCacheMissParams: - """ Parameters sent by the `on_dns_cache_miss` signal""" + """Parameters sent by the `on_dns_cache_miss` signal""" host: str @dataclasses.dataclass(frozen=True) class TraceRequestHeadersSentParams: - """ Parameters sent by the `on_request_headers_sent` signal""" + """Parameters sent by the `on_request_headers_sent` signal""" method: str url: URL diff --git a/aiohttp/web_app.py b/aiohttp/web_app.py index d78d603d38b..999ea9ceb95 100644 --- a/aiohttp/web_app.py +++ b/aiohttp/web_app.py @@ -359,7 +359,7 @@ def __call__(self) -> "Application": return self def __repr__(self) -> str: - return "".format(id(self)) + return f"" def __bool__(self) -> bool: return True diff --git a/aiohttp/web_exceptions.py b/aiohttp/web_exceptions.py index 28fe0aafcc5..b22995f39ac 100644 --- a/aiohttp/web_exceptions.py +++ b/aiohttp/web_exceptions.py @@ -431,7 +431,7 @@ def __init__( super().__init__( headers=headers, reason=reason, text=text, content_type=content_type ) - self.headers["Link"] = '<{}>; rel="blocked-by"'.format(str(link)) + self.headers["Link"] = f'<{str(link)}>; rel="blocked-by"' self._link = URL(link) @property diff --git a/aiohttp/web_routedef.py b/aiohttp/web_routedef.py index 787d9cbdeca..a22c3df4d0e 100644 --- a/aiohttp/web_routedef.py +++ b/aiohttp/web_routedef.py @@ -158,7 +158,7 @@ def __init__(self) -> None: self._items = [] # type: List[AbstractRouteDef] def __repr__(self) -> str: - return "".format(len(self._items)) + return f"" @overload def __getitem__(self, index: int) -> AbstractRouteDef: diff --git a/examples/legacy/tcp_protocol_parser.py b/examples/legacy/tcp_protocol_parser.py index ca49db7d8f9..1ef972758e5 100755 --- a/examples/legacy/tcp_protocol_parser.py +++ b/examples/legacy/tcp_protocol_parser.py @@ -60,7 +60,7 @@ def stop(self): self.transport.write(b"stop:\r\n") def send_text(self, text): - self.transport.write(f"text:{text.strip()}\r\n".encode("utf-8")) + self.transport.write(f"text:{text.strip()}\r\n".encode()) class EchoServer(asyncio.Protocol): diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 79e007537cd..b3a01957bf7 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -996,7 +996,7 @@ async def handler(request): async def redirect(request): count = int(request.match_info["count"]) if count: - raise web.HTTPFound(location="/redirect/{}".format(count - 1)) + raise web.HTTPFound(location=f"/redirect/{count - 1}") else: raise web.HTTPFound(location="/") diff --git a/tests/test_helpers.py b/tests/test_helpers.py index e9b99e12170..8f029f15666 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -172,7 +172,7 @@ def test_basic_auth_decode_invalid_credentials() -> None: ), ) def test_basic_auth_decode_blank_username(credentials, expected_auth) -> None: - header = "Basic {}".format(base64.b64encode(credentials.encode()).decode()) + header = f"Basic {base64.b64encode(credentials.encode()).decode()}" assert helpers.BasicAuth.decode(header) == expected_auth diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 172d7bc30cf..80913ae4360 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -978,7 +978,7 @@ async def test_http_payload_parser_deflate(self, stream: Any) -> None: assert out.is_eof() async def test_http_payload_parser_deflate_no_hdrs(self, stream: Any) -> None: - """Tests incorrectly formed data (no zlib headers) """ + """Tests incorrectly formed data (no zlib headers)""" # c=compressobj(wbits=-15); b''.join([c.compress(b'data'), c.flush()]) COMPRESSED = b"KI,I\x04\x00" diff --git a/tests/test_resolver.py b/tests/test_resolver.py index b74764525c2..3b984cb46c4 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -44,7 +44,7 @@ async def fake(*args: Any, **kwargs: Any) -> List[Any]: if not hosts: raise socket.gaierror - return list([(None, None, None, None, [h, 0]) for h in hosts]) + return [(None, None, None, None, [h, 0]) for h in hosts] return fake diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 6a3b7a3fee5..c54db16bdc2 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -445,7 +445,7 @@ async def handler(request): def test_repr_for_application() -> None: app = web.Application() - assert "".format(id(app)) == repr(app) + assert f"" == repr(app) async def test_expect_default_handler_unknown(aiohttp_client: Any) -> None: diff --git a/tools/bench-asyncio-write.py b/tools/bench-asyncio-write.py index 6b34782d909..8535219fe55 100644 --- a/tools/bench-asyncio-write.py +++ b/tools/bench-asyncio-write.py @@ -3,6 +3,7 @@ import math import os import signal +from typing import List, Tuple PORT = 8888 @@ -38,7 +39,7 @@ def fm_size(s, _fms=("", "K", "M", "G")): while s >= 1024: s /= 1024 i += 1 - return "{:.0f}{}B".format(s, _fms[i]) + return f"{s:.0f}{_fms[i]}B" def fm_time(s, _fms=("", "m", "µ", "n")): @@ -48,7 +49,14 @@ def fm_time(s, _fms=("", "m", "µ", "n")): while s < 1: s *= 1000 i += 1 - return "{:.2f}{}s".format(s, _fms[i]) + return f"{s:.2f}{_fms[i]}s" + + +def _job(j: List[int]) -> Tuple[str, List[bytes]]: + # Always start with a 256B headers chunk + body = [b"0" * s for s in [256] + list(j)] + job_title = f"{fm_size(sum(j))} / {len(j)}" + return (job_title, body) writes = [ @@ -71,14 +79,8 @@ def fm_time(s, _fms=("", "m", "µ", "n")): [10 * 2 ** 27 for _ in range(5)], ) -jobs = [ - ( - # always start with a 256B headers chunk - "{} / {}".format(fm_size(sum(j) if j else 0), len(j)), - [b"0" * s for s in [256] + list(j)], - ) - for j in bodies -] + +jobs = [_job(j) for j in bodies] async def time(loop, fn, *args): @@ -111,7 +113,7 @@ async def bench(job_title, w, body, base=None): fm_time(mean), fm_time(sd), str(it), - "{:.2%}".format(mean / base - 1) if base is not None else "", + f"{mean / base - 1:.2%}" if base is not None else "", ) ) return mean From d66e07c652322d280740106ebb9946a3dd7daf5b Mon Sep 17 00:00:00 2001 From: bmbouter Date: Sun, 3 Oct 2021 11:30:12 -0400 Subject: [PATCH 02/66] Add xfailing integration tests against ``proxy.py`` This patch adds full end-to-end tests for sending requests to HTTP and HTTPS endpoints through an HTTPS proxy. The first case is currently supported and the second one is not. This is why the latter test is marked as expected to fail. The support for TLS-in-TLS in the upstream stdlib asyncio is currently disabled but is available in Python 3.9 via monkey-patching which is demonstrated in the added tests. Refs: * https://bugs.python.org/issue37179 * https://github.com/python/cpython/pull/28073 * https://github.com/aio-libs/aiohttp/pull/5992 Co-authored-by: Sviatoslav Sydorenko PR #6002 --- .github/workflows/ci.yml | 2 + CHANGES/6002.misc | 2 + requirements/dev.txt | 3 + requirements/test.txt | 1 + tests/test_proxy_functional.py | 128 +++++++++++++++++++++++++++++++++ 5 files changed, 136 insertions(+) create mode 100644 CHANGES/6002.misc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 427950bcb6d..037bcbf0898 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,8 @@ jobs: path: ${{ steps.pip-cache.outputs.dir }} restore-keys: | pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}- + - name: Upgrade wheel # Needed for proxy.py install not to explode + run: pip install -U wheel - name: Cythonize if: ${{ matrix.no-extensions == '' }} run: | diff --git a/CHANGES/6002.misc b/CHANGES/6002.misc new file mode 100644 index 00000000000..5df927cf65d --- /dev/null +++ b/CHANGES/6002.misc @@ -0,0 +1,2 @@ +Implemented end-to-end testing of sending HTTP and HTTPS requests +via ``proxy.py``. diff --git a/requirements/dev.txt b/requirements/dev.txt index 59ca566d7fe..9364ad351e6 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -163,6 +163,8 @@ pluggy==0.13.1 # pytest pre-commit==2.15.0 # via -r requirements/lint.txt +proxy.py==2.3.1 + # via -r requirements/test.txt py==1.10.0 # via # -r requirements/lint.txt @@ -277,6 +279,7 @@ typing-extensions==3.7.4.3 # -r requirements/lint.txt # async-timeout # mypy + # proxy.py uritemplate==3.0.1 # via gidgethub urllib3==1.26.5 diff --git a/requirements/test.txt b/requirements/test.txt index 8ba2b11d792..d5c4f2d271c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -6,6 +6,7 @@ cryptography==3.3.1; platform_machine!="i686" and python_version<"3.9" # no 32-b freezegun==1.1.0 mypy==0.910; implementation_name=="cpython" mypy-extensions==0.4.3; implementation_name=="cpython" +proxy.py==2.3.1 pytest==6.2.2 pytest-cov==2.12.1 pytest-mock==3.6.1 diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py index e1c3c0095e7..a5091b0a827 100644 --- a/tests/test_proxy_functional.py +++ b/tests/test_proxy_functional.py @@ -5,12 +5,140 @@ from typing import Any from unittest import mock +import proxy import pytest from yarl import URL import aiohttp from aiohttp import web +ASYNCIO_SUPPORTS_TLS_IN_TLS = hasattr( + asyncio.sslproto._SSLProtocolTransport, + "_start_tls_compatible", +) + + +@pytest.fixture +def secure_proxy_url(monkeypatch, tls_certificate_pem_path): + """Return the URL of an instance of a running secure proxy. + + This fixture also spawns that instance and tears it down after the test. + """ + proxypy_args = [ + "--threadless", # use asyncio + "--num-workers", + "1", # the tests only send one query anyway + "--hostname", + "127.0.0.1", # network interface to listen to + "--port", + 0, # ephemeral port, so that kernel allocates a free one + "--cert-file", + tls_certificate_pem_path, # contains both key and cert + "--key-file", + tls_certificate_pem_path, # contains both key and cert + ] + + class PatchedAccetorPool(proxy.core.acceptor.AcceptorPool): + def listen(self): + super().listen() + self.socket_host, self.socket_port = self.socket.getsockname()[:2] + + monkeypatch.setattr(proxy.proxy, "AcceptorPool", PatchedAccetorPool) + + with proxy.Proxy(input_args=proxypy_args) as proxy_instance: + yield URL.build( + scheme="https", + host=proxy_instance.acceptors.socket_host, + port=proxy_instance.acceptors.socket_port, + ) + + +@pytest.fixture +def web_server_endpoint_payload(): + return "Test message" + + +@pytest.fixture(params=("http", "https")) +def web_server_endpoint_type(request): + return request.param + + +@pytest.fixture +async def web_server_endpoint_url( + aiohttp_server, + ssl_ctx, + web_server_endpoint_payload, + web_server_endpoint_type, +): + server_kwargs = ( + { + "ssl": ssl_ctx, + } + if web_server_endpoint_type == "https" + else {} + ) + + async def handler(*args, **kwargs): + return web.Response(text=web_server_endpoint_payload) + + app = web.Application() + app.router.add_route("GET", "/", handler) + server = await aiohttp_server(app, **server_kwargs) + + return URL.build( + scheme=web_server_endpoint_type, + host=server.host, + port=server.port, + ) + + +@pytest.fixture +def _pretend_asyncio_supports_tls_in_tls( + monkeypatch, + web_server_endpoint_type, +): + if web_server_endpoint_type != "https" or ASYNCIO_SUPPORTS_TLS_IN_TLS: + return + + # for https://github.com/python/cpython/pull/28073 + # and https://bugs.python.org/issue37179 + monkeypatch.setattr( + asyncio.sslproto._SSLProtocolTransport, + "_start_tls_compatible", + True, + raising=False, + ) + + +@pytest.mark.xfail( + reason="https://github.com/aio-libs/aiohttp/pull/5992", + raises=ValueError, +) +@pytest.mark.parametrize("web_server_endpoint_type", ("http", "https")) +@pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop") +async def test_secure_https_proxy_absolute_path( + client_ssl_ctx, + secure_proxy_url, + web_server_endpoint_url, + web_server_endpoint_payload, +) -> None: + """Test urls can be requested through a secure proxy.""" + conn = aiohttp.TCPConnector() + sess = aiohttp.ClientSession(connector=conn) + + response = await sess.get( + web_server_endpoint_url, + proxy=secure_proxy_url, + ssl=client_ssl_ctx, # used for both proxy and endpoint connections + ) + + assert response.status == 200 + assert await response.text() == web_server_endpoint_payload + + response.close() + await sess.close() + await conn.close() + @pytest.fixture def proxy_test_server(aiohttp_raw_server: Any, loop: Any, monkeypatch: Any): From f3eb699dc1b32cb0dd5428f3b9bf4e392643659e Mon Sep 17 00:00:00 2001 From: Slava Date: Mon, 4 Oct 2021 05:41:28 +0300 Subject: [PATCH 03/66] Remove duplicate entries in dependabot config (#6029) --- .github/dependabot.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 908f93d629f..cd8b2782b43 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -37,23 +37,3 @@ updates: schedule: interval: "daily" open-pull-requests-limit: 10 - - # Maintain dependencies for GitHub Actions aiohttp 3.8 - - package-ecosystem: "github-actions" - directory: "/" - labels: - - dependencies - target-branch: "3.8" - schedule: - interval: "daily" - open-pull-requests-limit: 10 - - # Maintain dependencies for Python aiohttp 3.8 - - package-ecosystem: "pip" - directory: "/" - labels: - - dependencies - target-branch: "3.8" - schedule: - interval: "daily" - open-pull-requests-limit: 10 From d4dafec753538615f982bbc891f0c463ecb32df9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 02:52:27 +0000 Subject: [PATCH 04/66] Bump pytest-cov from 2.12.1 to 3.0.0 (#6037) Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.12.1 to 3.0.0. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.12.1...v3.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index d5c4f2d271c..e9f3a9b4551 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,7 +8,7 @@ mypy==0.910; implementation_name=="cpython" mypy-extensions==0.4.3; implementation_name=="cpython" proxy.py==2.3.1 pytest==6.2.2 -pytest-cov==2.12.1 +pytest-cov==3.0.0 pytest-mock==3.6.1 re-assert==1.1.0 setuptools-git==1.2 From 6ed46f432c88c7718661fba9d48c5c0009da03d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 02:59:11 +0000 Subject: [PATCH 05/66] Bump multidict from 5.1.0 to 5.2.0 (#6036) Bumps [multidict](https://github.com/aio-libs/multidict) from 5.1.0 to 5.2.0. - [Release notes](https://github.com/aio-libs/multidict/releases) - [Changelog](https://github.com/aio-libs/multidict/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/multidict/compare/v5.1.0...v5.2.0) --- updated-dependencies: - dependency-name: multidict dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/multidict.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/multidict.txt b/requirements/multidict.txt index 7357d4643f0..d3e1b42f470 100644 --- a/requirements/multidict.txt +++ b/requirements/multidict.txt @@ -1 +1 @@ -multidict==5.1.0 +multidict==5.2.0 From 3ae013e40c7998e7771932ff7de25f33aaa36d36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:03 +0000 Subject: [PATCH 06/66] Bump coverage from 5.5 to 6.0 (#6038) Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.5 to 6.0. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.5...6.0) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index e9f3a9b4551..a32135ef9b4 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,7 +1,7 @@ -r base.txt Brotli==1.0.9 -coverage==5.5 +coverage==6.0 cryptography==3.3.1; platform_machine!="i686" and python_version<"3.9" # no 32-bit wheels; no python 3.9 wheels yet freezegun==1.1.0 mypy==0.910; implementation_name=="cpython" From ed49e882117150eb29650addfa232fd696633058 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 17:32:47 +0100 Subject: [PATCH 07/66] [pre-commit.ci] pre-commit autoupdate (#6039) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.28.0 → v2.29.0](https://github.com/asottile/pyupgrade/compare/v2.28.0...v2.29.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ef2d2d86b96..f29d6f1c58c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,7 +60,7 @@ repos: - id: detect-private-key exclude: ^examples/ - repo: https://github.com/asottile/pyupgrade - rev: 'v2.28.0' + rev: 'v2.29.0' hooks: - id: pyupgrade args: ['--py36-plus'] From 1b4b89cccbdd2d9418b6ab4dcf7a2d7472b60227 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Oct 2021 10:11:46 +0000 Subject: [PATCH 08/66] Bump sqren/backport-github-action from 1.0.40 to 1.0.41 (#6042) Bumps [sqren/backport-github-action](https://github.com/sqren/backport-github-action) from 1.0.40 to 1.0.41. - [Release notes](https://github.com/sqren/backport-github-action/releases) - [Commits](https://github.com/sqren/backport-github-action/compare/v1.0.40...v1.0.41) --- updated-dependencies: - dependency-name: sqren/backport-github-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index e5bea15f7c9..f6fc9608971 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -26,7 +26,7 @@ jobs: app_id: ${{ secrets.BOT_APP_ID }} private_key: ${{ secrets.BOT_PRIVATE_KEY }} - name: Backport - uses: sqren/backport-github-action@v1.0.40 + uses: sqren/backport-github-action@v1.0.41 with: # Required # Token to authenticate requests From 13c26be82aa884994b52add6ec7a476b199fe910 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 6 Oct 2021 00:29:55 +0200 Subject: [PATCH 09/66] Add custom RST roles extlinks to the docs setup (#6045) This patch declares the following roles within Sphinx: * issue * pr * commit * gh * user They all correspond to respective GitHub URLs. For example, the following will link a GitHub user page: :user:`webknjaz` --- CHANGES/6045.misc | 2 ++ docs/conf.py | 70 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 21 deletions(-) create mode 100644 CHANGES/6045.misc diff --git a/CHANGES/6045.misc b/CHANGES/6045.misc new file mode 100644 index 00000000000..020b8ef28a2 --- /dev/null +++ b/CHANGES/6045.misc @@ -0,0 +1,2 @@ +Added `commit`, `gh`, `issue`, `pr` and `user` +RST roles in Sphinx — :user:`webknjaz`. diff --git a/docs/conf.py b/docs/conf.py index b7096306d42..d56150ceeb3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -43,8 +43,11 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "sphinx.ext.viewcode", + # stdlib-party extensions: + "sphinx.ext.extlinks", "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + # Third-party extensions: "sphinxcontrib.asyncio", "sphinxcontrib.blockdiag", ] @@ -81,9 +84,17 @@ # The master toctree document. master_doc = "index" -# General information about the project. -project = "aiohttp" -copyright = "2013-2020, aiohttp maintainers" +# -- Project information ----------------------------------------------------- + +github_url = "https://github.com" +github_repo_org = "aio-libs" +github_repo_name = "aiohttp" +github_repo_slug = f"{github_repo_org}/{github_repo_name}" +github_repo_url = f"{github_url}/{github_repo_slug}" +github_sponsors_url = f"{github_url}/sponsors" + +project = github_repo_name +copyright = f"2013-2020, {project} maintainers" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -136,6 +147,17 @@ # keep_warnings = False +# -- Extension configuration ------------------------------------------------- + +# -- Options for extlinks extension --------------------------------------- +extlinks = { + "issue": (f"{github_repo_url}/issues/%s", "#"), + "pr": (f"{github_repo_url}/pull/%s", "PR #"), + "commit": (f"{github_repo_url}/commit/%s", ""), + "gh": (f"{github_url}/%s", "GitHub: "), + "user": (f"{github_sponsors_url}/%s", "@"), +} + # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -148,39 +170,39 @@ html_theme_options = { "description": "Async HTTP client/server for asyncio and Python", "canonical_url": "http://docs.aiohttp.org/en/stable/", - "github_user": "aio-libs", - "github_repo": "aiohttp", + "github_user": github_repo_org, + "github_repo": github_repo_name, "github_button": True, "github_type": "star", "github_banner": True, "badges": [ { - "image": "https://github.com/aio-libs/aiohttp/workflows/CI/badge.svg", - "target": "https://github.com/aio-libs/aiohttp/actions?query=workflow%3ACI", + "image": f"{github_repo_url}/workflows/CI/badge.svg", + "target": f"{github_repo_url}/actions?query=workflow%3ACI", "height": "20", "alt": "Azure Pipelines CI status", }, { - "image": "https://codecov.io/github/aio-libs/aiohttp/coverage.svg?branch=master", - "target": "https://codecov.io/github/aio-libs/aiohttp", + "image": f"https://codecov.io/github/{github_repo_slug}/coverage.svg?branch=master", + "target": f"https://codecov.io/github/{github_repo_slug}", "height": "20", "alt": "Code coverage status", }, { - "image": "https://badge.fury.io/py/aiohttp.svg", - "target": "https://badge.fury.io/py/aiohttp", + "image": f"https://badge.fury.io/py/{project}.svg", + "target": f"https://badge.fury.io/py/{project}", "height": "20", "alt": "Latest PyPI package version", }, { - "image": "https://img.shields.io/discourse/status?server=https%3A%2F%2Faio-libs.discourse.group", - "target": "https://aio-libs.discourse.group", + "image": f"https://img.shields.io/discourse/status?server=https%3A%2F%2F{github_repo_org}.discourse.group", + "target": f"https://{github_repo_org}.discourse.group", "height": "20", "alt": "Discourse status", }, { "image": "https://badges.gitter.im/Join%20Chat.svg", - "target": "https://gitter.im/aio-libs/Lobby", + "target": f"https://gitter.im/{github_repo_org}/Lobby", "height": "20", "alt": "Chat on Gitter", }, @@ -268,7 +290,7 @@ # html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = "aiohttpdoc" +htmlhelp_basename = f"{project}doc" # -- Options for LaTeX output --------------------------------------------- @@ -286,7 +308,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ("index", "aiohttp.tex", "aiohttp Documentation", "aiohttp contributors", "manual"), + ( + "index", + f"{project}.tex", + f"{project} Documentation", + f"{project} contributors", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of @@ -314,7 +342,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [("index", "aiohttp", "aiohttp Documentation", ["aiohttp"], 1)] +man_pages = [("index", project, f"{project} Documentation", [project], 1)] # If true, show URL addresses after external links. # man_show_urls = False @@ -328,10 +356,10 @@ texinfo_documents = [ ( "index", - "aiohttp", - "aiohttp Documentation", + project, + f"{project} Documentation", "Aiohttp contributors", - "aiohttp", + project, "One line description of project.", "Miscellaneous", ), From c29e5fb58efb65c6198ea75b95805ab972e98adc Mon Sep 17 00:00:00 2001 From: bmbouter Date: Tue, 5 Oct 2021 19:04:56 -0400 Subject: [PATCH 10/66] Add secure proxy support in the client This patch opens up the code path and adds the implementation that allows end-users to start sending HTTPS requests through HTTPS proxies. The support for TLS-in-TLS (needed for this to work) in the stdlib is kinda available since Python 3.7 but is disabled for `asyncio` with an attribute/flag/toggle. When the upstream CPython enables it finally, aiohttp v3.8+ will be able to work with it out of the box. Currently the tests monkey-patch `asyncio` in order to verify that this works. The users who are willing to do the same, will be able to take advantage of it right now. Eventually (hopefully starting Python 3.11), the need for monkey-patching should be eliminated. Refs: * https://bugs.python.org/issue37179 * https://github.com/python/cpython/pull/28073 * https://docs.aiohttp.org/en/stable/client_advanced.html#proxy-support * https://github.com/aio-libs/aiohttp/discussions/6044 PR #5992 Resolves #3816 Resolves #4268 Co-authored-by: Brian Bouterse Co-authored-by: Jordan Borean Co-authored-by: Sviatoslav Sydorenko --- CHANGES/5992.feature | 3 + CONTRIBUTORS.txt | 2 + aiohttp/client_reqrep.py | 2 - aiohttp/connector.py | 127 +++++++++++++++++++++++++++++---- docs/client_advanced.rst | 35 ++++++++- tests/test_client_request.py | 5 -- tests/test_proxy.py | 115 +++++------------------------ tests/test_proxy_functional.py | 74 +++++++++++++++++-- 8 files changed, 234 insertions(+), 129 deletions(-) create mode 100644 CHANGES/5992.feature diff --git a/CHANGES/5992.feature b/CHANGES/5992.feature new file mode 100644 index 00000000000..5667d2de24b --- /dev/null +++ b/CHANGES/5992.feature @@ -0,0 +1,3 @@ +Added support for HTTPS proxies to the extent CPython's +:py:mod:`asyncio` supports it -- by :user:`bmbouter`, +:user:`jborean93` and :user:`webknjaz`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 8669ccd3b2c..45d9a1a5fb5 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -57,6 +57,7 @@ Boris Feld Borys Vorona Boyi Chen Brett Cannon +Brian Bouterse Brian C. Lane Brian Muller Bruce Merry @@ -165,6 +166,7 @@ Jonas Obrist Jonathan Wright Jonny Tan Joongi Kim +Jordan Borean Josep Cugat Josh Junon Joshu Coats diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index c63b73bcdf8..8c64db600e6 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -481,8 +481,6 @@ def update_proxy( proxy_auth: Optional[BasicAuth], proxy_headers: Optional[LooseHeaders], ) -> None: - if proxy and not proxy.scheme == "http": - raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy diff --git a/aiohttp/connector.py b/aiohttp/connector.py index ab5124966b5..c1b0f89ce16 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -951,6 +951,100 @@ async def _wrap_create_connection( except OSError as exc: raise client_error(req.connection_key, exc) from exc + def _warn_about_tls_in_tls( + self, + underlying_transport: asyncio.Transport, + req: "ClientRequest", + ) -> None: + """Issue a warning if the requested URL has HTTPS scheme.""" + if req.request_info.url.scheme != "https": + return + + asyncio_supports_tls_in_tls = getattr( + underlying_transport, + "_start_tls_compatible", + False, + ) + + if asyncio_supports_tls_in_tls: + return + + warnings.warn( + "An HTTPS request is being sent through an HTTPS proxy. " + "This support for TLS in TLS is known to be disabled " + "in the stdlib asyncio. This is why you'll probably see " + "an error in the log below.\n\n" + "It is possible to enable it via monkeypatching under " + "Python 3.7 or higher. For more details, see:\n" + "* https://bugs.python.org/issue37179\n" + "* https://github.com/python/cpython/pull/28073\n\n" + "You can temporarily patch this as follows:\n" + "* https://docs.aiohttp.org/en/stable/client_advanced.html#proxy-support\n" + "* https://github.com/aio-libs/aiohttp/discussions/6044\n", + RuntimeWarning, + source=self, + # Why `4`? At least 3 of the calls in the stack originate + # from the methods in this class. + stacklevel=3, + ) + + async def _start_tls_connection( + self, + underlying_transport: asyncio.Transport, + req: "ClientRequest", + timeout: "ClientTimeout", + client_error: Type[Exception] = ClientConnectorError, + ) -> Tuple[asyncio.BaseTransport, ResponseHandler]: + """Wrap the raw TCP transport with TLS.""" + tls_proto = self._factory() # Create a brand new proto for TLS + + # Safety of the `cast()` call here is based on the fact that + # internally `_get_ssl_context()` only returns `None` when + # `req.is_ssl()` evaluates to `False` which is never gonna happen + # in this code path. Of course, it's rather fragile + # maintainability-wise but this is to be solved separately. + sslcontext = cast(ssl.SSLContext, self._get_ssl_context(req)) + + try: + async with ceil_timeout(timeout.sock_connect): + try: + tls_transport = await self._loop.start_tls( + underlying_transport, + tls_proto, + sslcontext, + server_hostname=req.host, + ssl_handshake_timeout=timeout.total, + ) + except BaseException: + # We need to close the underlying transport since + # `start_tls()` probably failed before it had a + # chance to do this: + underlying_transport.close() + raise + except cert_errors as exc: + raise ClientConnectorCertificateError(req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + raise client_error(req.connection_key, exc) from exc + except TypeError as type_err: + # Example cause looks like this: + # TypeError: transport is not supported by start_tls() + + raise ClientConnectionError( + "Cannot initialize a TLS-in-TLS connection to host " + f"{req.host!s}:{req.port:d} through an underlying connection " + f"to an HTTPS proxy {req.proxy!s} ssl:{req.ssl or 'default'} " + f"[{type_err!s}]" + ) from type_err + else: + tls_proto.connection_made( + tls_transport + ) # Kick the state machine of the new TLS protocol + + return tls_transport, tls_proto + async def _create_direct_connection( self, req: "ClientRequest", @@ -1028,7 +1122,7 @@ def drop_exception(fut: "asyncio.Future[List[Dict[str, Any]]]") -> None: async def _create_proxy_connection( self, req: "ClientRequest", traces: List["Trace"], timeout: "ClientTimeout" - ) -> Tuple[asyncio.Transport, ResponseHandler]: + ) -> Tuple[asyncio.BaseTransport, ResponseHandler]: headers = {} # type: Dict[str, str] if req.proxy_headers is not None: headers = req.proxy_headers # type: ignore[assignment] @@ -1063,7 +1157,8 @@ async def _create_proxy_connection( proxy_req.headers[hdrs.PROXY_AUTHORIZATION] = auth if req.is_ssl(): - sslcontext = self._get_ssl_context(req) + self._warn_about_tls_in_tls(transport, req) + # For HTTPS requests over HTTP proxy # we must notify proxy to tunnel connection # so we send CONNECT command: @@ -1083,7 +1178,11 @@ async def _create_proxy_connection( try: protocol = conn._protocol assert protocol is not None - protocol.set_response_params() + + # read_until_eof=True will ensure the connection isn't closed + # once the response is received and processed allowing + # START_TLS to work on the connection below. + protocol.set_response_params(read_until_eof=True) resp = await proxy_resp.start(conn) except BaseException: proxy_resp.close() @@ -1104,21 +1203,19 @@ async def _create_proxy_connection( message=message, headers=resp.headers, ) - rawsock = transport.get_extra_info("socket", default=None) - if rawsock is None: - raise RuntimeError("Transport does not expose socket instance") - # Duplicate the socket, so now we can close proxy transport - rawsock = rawsock.dup() - finally: + except BaseException: + # It shouldn't be closed in `finally` because it's fed to + # `loop.start_tls()` and the docs say not to touch it after + # passing there. transport.close() + raise - transport, proto = await self._wrap_create_connection( - self._factory, - timeout=timeout, - ssl=sslcontext, - sock=rawsock, - server_hostname=req.host, + return await self._start_tls_connection( + # Access the old transport for the last time before it's + # closed and forgotten forever: + transport, req=req, + timeout=timeout, ) finally: proxy_resp.close() diff --git a/docs/client_advanced.rst b/docs/client_advanced.rst index 7a2f4bef217..316f41cd1f3 100644 --- a/docs/client_advanced.rst +++ b/docs/client_advanced.rst @@ -533,9 +533,11 @@ DER with e.g:: Proxy support ------------- -aiohttp supports plain HTTP proxies and HTTP proxies that can be upgraded to HTTPS -via the HTTP CONNECT method. aiohttp does not support proxies that must be -connected to via ``https://``. To connect, use the *proxy* parameter:: +aiohttp supports plain HTTP proxies and HTTP proxies that can be +upgraded to HTTPS via the HTTP CONNECT method. aiohttp has a limited +support for proxies that must be connected to via ``https://`` — see +the info box below for more details. +To connect, use the *proxy* parameter:: async with aiohttp.ClientSession() as session: async with session.get("http://python.org", @@ -570,6 +572,33 @@ variables* (all are case insensitive):: Proxy credentials are given from ``~/.netrc`` file if present (see :class:`aiohttp.ClientSession` for more details). +.. attention:: + + CPython has introduced the support for TLS in TLS around Python 3.7. + But, as of now (Python 3.10), it's disabled for the transports that + :py:mod:`asyncio` uses. If the further release of Python (say v3.11) + toggles one attribute, it'll *just work™*. + + aiohttp v3.8 and higher is ready for this to happen and has code in + place supports TLS-in-TLS, hence sending HTTPS requests over HTTPS + proxy tunnels. + + ⚠️ For as long as your Python runtime doesn't declare the support for + TLS-in-TLS, please don't file bugs with aiohttp but rather try to + help the CPython upstream enable this feature. Meanwhile, if you + *really* need this to work, there's a patch that may help you make + it happen, include it into your app's code base: + https://github.com/aio-libs/aiohttp/discussions/6044#discussioncomment-1432443. + +.. important:: + + When supplying a custom :py:class:`ssl.SSLContext` instance, bear in + mind that it will be used not only to establish a TLS session with + the HTTPS endpoint you're hitting but also to establish a TLS tunnel + to the HTTPS proxy. To avoid surprises, make sure to set up the trust + chain that would recognize TLS certificates used by both the endpoint + and the proxy. + .. _aiohttp-persistent-session: Persistent session diff --git a/tests/test_client_request.py b/tests/test_client_request.py index cfe2a45edc7..6ab8761778a 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -119,11 +119,6 @@ def test_version_err(make_request: Any) -> None: make_request("get", "http://python.org/", version="1.c") -def test_https_proxy(make_request: Any) -> None: - with pytest.raises(ValueError): - make_request("get", "http://python.org/", proxy=URL("https://proxy.org")) - - def test_keep_alive(make_request: Any) -> None: req = make_request("get", "http://python.org/", version=(0, 9)) assert not req.keep_alive() diff --git a/tests/test_proxy.py b/tests/test_proxy.py index c778f85f531..af869ee88f7 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -228,6 +228,7 @@ async def make_conn(): tr, proto = mock.Mock(), mock.Mock() self.loop.create_connection = make_mocked_coro((tr, proto)) + self.loop.start_tls = make_mocked_coro(mock.Mock()) req = ClientRequest( "GET", @@ -242,8 +243,6 @@ async def make_conn(): self.assertEqual(req.url.path, "/") self.assertEqual(proxy_req.method, "CONNECT") self.assertEqual(proxy_req.url, URL("https://www.python.org")) - tr.close.assert_called_once_with() - tr.get_extra_info.assert_called_with("socket", default=None) self.loop.run_until_complete(proxy_req.close()) proxy_resp.close() @@ -287,22 +286,10 @@ async def make_conn(): ] ) - seq = 0 - - async def create_connection(*args, **kwargs): - nonlocal seq - seq += 1 - - # connection to http://proxy.example.com - if seq == 1: - return mock.Mock(), mock.Mock() - # connection to https://www.python.org - elif seq == 2: - raise ssl.CertificateError - else: - assert False - - self.loop.create_connection = create_connection + # Called on connection to http://proxy.example.com + self.loop.create_connection = make_mocked_coro((mock.Mock(), mock.Mock())) + # Called on connection to https://www.python.org + self.loop.start_tls = make_mocked_coro(raise_exception=ssl.CertificateError) req = ClientRequest( "GET", @@ -353,75 +340,12 @@ async def make_conn(): ] ) - seq = 0 - - async def create_connection(*args, **kwargs): - nonlocal seq - seq += 1 - - # connection to http://proxy.example.com - if seq == 1: - return mock.Mock(), mock.Mock() - # connection to https://www.python.org - elif seq == 2: - raise ssl.SSLError - else: - assert False - - self.loop.create_connection = create_connection - - req = ClientRequest( - "GET", - URL("https://www.python.org"), - proxy=URL("http://proxy.example.com"), - loop=self.loop, + # Called on connection to http://proxy.example.com + self.loop.create_connection = make_mocked_coro( + (mock.Mock(), mock.Mock()), ) - with self.assertRaises(aiohttp.ClientConnectorSSLError): - self.loop.run_until_complete( - connector._create_connection(req, None, aiohttp.ClientTimeout()) - ) - - @mock.patch("aiohttp.connector.ClientRequest") - def test_https_connect_runtime_error(self, ClientRequestMock: Any) -> None: - proxy_req = ClientRequest( - "GET", URL("http://proxy.example.com"), loop=self.loop - ) - ClientRequestMock.return_value = proxy_req - - proxy_resp = ClientResponse( - "get", - URL("http://proxy.example.com"), - request_info=mock.Mock(), - writer=mock.Mock(), - continue100=None, - timer=TimerNoop(), - traces=[], - loop=self.loop, - session=mock.Mock(), - ) - proxy_req.send = make_mocked_coro(proxy_resp) - proxy_resp.start = make_mocked_coro(mock.Mock(status=200)) - - async def make_conn(): - return aiohttp.TCPConnector() - - connector = self.loop.run_until_complete(make_conn()) - connector._resolve_host = make_mocked_coro( - [ - { - "hostname": "hostname", - "host": "127.0.0.1", - "port": 80, - "family": socket.AF_INET, - "proto": 0, - "flags": 0, - } - ] - ) - - tr, proto = mock.Mock(), mock.Mock() - tr.get_extra_info.return_value = None - self.loop.create_connection = make_mocked_coro((tr, proto)) + # Called on connection to https://www.python.org + self.loop.start_tls = make_mocked_coro(raise_exception=ssl.SSLError) req = ClientRequest( "GET", @@ -429,17 +353,11 @@ async def make_conn(): proxy=URL("http://proxy.example.com"), loop=self.loop, ) - with self.assertRaisesRegex( - RuntimeError, "Transport does not expose socket instance" - ): + with self.assertRaises(aiohttp.ClientConnectorSSLError): self.loop.run_until_complete( connector._create_connection(req, None, aiohttp.ClientTimeout()) ) - self.loop.run_until_complete(proxy_req.close()) - proxy_resp.close() - self.loop.run_until_complete(req.close()) - @mock.patch("aiohttp.connector.ClientRequest") def test_https_connect_http_proxy_error(self, ClientRequestMock: Any) -> None: proxy_req = ClientRequest( @@ -650,6 +568,7 @@ async def make_conn(): tr, proto = mock.Mock(), mock.Mock() self.loop.create_connection = make_mocked_coro((tr, proto)) + self.loop.start_tls = make_mocked_coro(mock.Mock()) req = ClientRequest( "GET", @@ -661,18 +580,17 @@ async def make_conn(): connector._create_connection(req, None, aiohttp.ClientTimeout()) ) - self.loop.create_connection.assert_called_with( + self.loop.start_tls.assert_called_with( + mock.ANY, mock.ANY, - ssl=connector._make_ssl_context(True), - sock=mock.ANY, + connector._make_ssl_context(True), server_hostname="www.python.org", + ssl_handshake_timeout=mock.ANY, ) self.assertEqual(req.url.path, "/") self.assertEqual(proxy_req.method, "CONNECT") self.assertEqual(proxy_req.url, URL("https://www.python.org")) - tr.close.assert_called_once_with() - tr.get_extra_info.assert_called_with("socket", default=None) self.loop.run_until_complete(proxy_req.close()) proxy_resp.close() @@ -721,6 +639,7 @@ async def make_conn(): tr, proto = mock.Mock(), mock.Mock() self.loop.create_connection = make_mocked_coro((tr, proto)) + self.loop.start_tls = make_mocked_coro(mock.Mock()) self.assertIn("AUTHORIZATION", proxy_req.headers) self.assertNotIn("PROXY-AUTHORIZATION", proxy_req.headers) diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py index a5091b0a827..eb44fd04630 100644 --- a/tests/test_proxy_functional.py +++ b/tests/test_proxy_functional.py @@ -2,8 +2,10 @@ import asyncio import os import pathlib +from re import match as match_regex from typing import Any from unittest import mock +from uuid import uuid4 import proxy import pytest @@ -11,6 +13,7 @@ import aiohttp from aiohttp import web +from aiohttp.client_exceptions import ClientConnectionError ASYNCIO_SUPPORTS_TLS_IN_TLS = hasattr( asyncio.sslproto._SSLProtocolTransport, @@ -55,7 +58,7 @@ def listen(self): @pytest.fixture def web_server_endpoint_payload(): - return "Test message" + return str(uuid4()) @pytest.fixture(params=("http", "https")) @@ -110,10 +113,6 @@ def _pretend_asyncio_supports_tls_in_tls( ) -@pytest.mark.xfail( - reason="https://github.com/aio-libs/aiohttp/pull/5992", - raises=ValueError, -) @pytest.mark.parametrize("web_server_endpoint_type", ("http", "https")) @pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop") async def test_secure_https_proxy_absolute_path( @@ -122,7 +121,7 @@ async def test_secure_https_proxy_absolute_path( web_server_endpoint_url, web_server_endpoint_payload, ) -> None: - """Test urls can be requested through a secure proxy.""" + """Ensure HTTP(S) sites are accessible through a secure proxy.""" conn = aiohttp.TCPConnector() sess = aiohttp.ClientSession(connector=conn) @@ -140,6 +139,69 @@ async def test_secure_https_proxy_absolute_path( await conn.close() +@pytest.mark.parametrize("web_server_endpoint_type", ("https",)) +@pytest.mark.usefixtures("loop") +async def test_https_proxy_unsupported_tls_in_tls( + client_ssl_ctx, + secure_proxy_url, + web_server_endpoint_type, +) -> None: + """Ensure connecting to TLS endpoints w/ HTTPS proxy needs patching. + + This also checks that a helpful warning on how to patch the env + is displayed. + """ + url = URL.build(scheme=web_server_endpoint_type, host="python.org") + + escaped_host_port = ":".join((url.host.replace(".", r"\."), str(url.port))) + escaped_proxy_url = str(secure_proxy_url).replace(".", r"\.") + + conn = aiohttp.TCPConnector() + sess = aiohttp.ClientSession(connector=conn) + + expected_warning_text = ( + r"^" + r"An HTTPS request is being sent through an HTTPS proxy\. " + "This support for TLS in TLS is known to be disabled " + r"in the stdlib asyncio\. This is why you'll probably see " + r"an error in the log below.\n\n" + "It is possible to enable it via monkeypatching under " + r"Python 3\.7 or higher\. For more details, see:\n" + r"* https://bugs\.python\.org/issue37179\n" + r"* https://github\.com/python/cpython/pull/28073\n\n" + r"You can temporarily patch this as follows:\n" + r"* https://docs\.aiohttp\.org/en/stable/client_advanced\.html#proxy-support\n", + r"* https://github\.com/aio-libs/aiohttp/discussions/6044\n$", + ) + type_err = ( + r"transport is not supported by start_tls\(\)" + ) + expected_exception_reason = ( + r"^" + "Cannot initialize a TLS-in-TLS connection to host " + f"{escaped_host_port!s} through an underlying connection " + f"to an HTTPS proxy {escaped_proxy_url!s} ssl:{client_ssl_ctx!s} " + f"[{type_err!s}]" + r"$" + ) + + with pytest.raises( + ClientConnectionError, + match=expected_exception_reason, + ) as conn_err, pytest.warns( + RuntimeWarning, + match=expected_warning_text, + ): + await sess.get(url, proxy=secure_proxy_url, ssl=client_ssl_ctx) + + assert type(conn_err.value.__cause__) == TypeError + assert match_regex(f"^{type_err!s}$", str(conn_err.value.__cause__)) + + await sess.close() + await conn.close() + + @pytest.fixture def proxy_test_server(aiohttp_raw_server: Any, loop: Any, monkeypatch: Any): # Handle all proxy requests and imitate remote server response. From 9518bba7fdd53a526dd419eb6cf5e426420f0781 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 6 Oct 2021 01:27:03 +0200 Subject: [PATCH 11/66] Add a README to the change notes dir (#6047) --- CHANGES/README.rst | 95 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 CHANGES/README.rst diff --git a/CHANGES/README.rst b/CHANGES/README.rst new file mode 100644 index 00000000000..c6b5153913a --- /dev/null +++ b/CHANGES/README.rst @@ -0,0 +1,95 @@ +.. _Adding change notes with your PRs: + +Adding change notes with your PRs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is very important to maintain a log for news of how +updating to the new version of the software will affect +end-users. This is why we enforce collection of the change +fragment files in pull requests as per `Towncrier philosophy`_. + +The idea is that when somebody makes a change, they must record +the bits that would affect end-users only including information +that would be useful to them. Then, when the maintainers publish +a new release, they'll automatically use these records to compose +a change log for the respective version. It is important to +understand that including unnecessary low-level implementation +related details generates noise that is not particularly useful +to the end-users most of the time. And so such details should be +recorded in the Git history rather than a changelog. + +Alright! So how to add a news fragment? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``aiohttp`` uses `towncrier `_ +for changelog management. +To submit a change note about your PR, add a text file into the +``CHANGES/`` folder. It should contain an +explanation of what applying this PR will change in the way +end-users interact with the project. One sentence is usually +enough but feel free to add as many details as you feel necessary +for the users to understand what it means. + +**Use the past tense** for the text in your fragment because, +combined with others, it will be a part of the "news digest" +telling the readers **what changed** in a specific version of +the library *since the previous version*. You should also use +reStructuredText syntax for highlighting code (inline or block), +linking parts of the docs or external sites. +If you wish to sign your change, feel free to add ``-- by +:user:`github-username``` at the end (replace ``github-username`` +with your own!). + +Finally, name your file following the convention that Towncrier +understands: it should start with the number of an issue or a +PR followed by a dot, then add a patch type, like ``feature``, +``doc``, ``misc`` etc., and add ``.rst`` as a suffix. If you +need to add more than one fragment, you may add an optional +sequence number (delimited with another period) between the type +and the suffix. + +In general the name will follow ``..rst`` pattern, +where the categories are: + +- ``feature``: Any new feature +- ``bugfix``: A bug fix +- ``doc``: A change to the documentation +- ``misc``: Changes internal to the repo like CI, test and build changes +- ``removal``: For deprecations and removals of an existing feature or behavior + +A pull request may have more than one of these components, for example +a code change may introduce a new feature that deprecates an old +feature, in which case two fragments should be added. It is not +necessary to make a separate documentation fragment for documentation +changes accompanying the relevant code changes. + +Examples for adding changelog entries to your Pull Requests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +File :file:`CHANGES/6045.doc.1.rst`: + +.. code-block:: rst + + Added a ``:user:`` role to Sphinx config -- by :user:`webknjaz` + +File :file:`CHANGES/4431.bugfix.rst`: + +.. code-block:: rst + + Fixed HTTP client requests to honor ``no_proxy`` environment + variables -- by :user:`scirelli` + +File :file:`CHANGES/4594.feature.rst`: + +.. code-block:: rst + + Added support for ``ETag`` to :py:class:`~aiohttp.web.FileResponse` + -- by :user:`greshilov`, :user:`serhiy-storchaka` and :user:`asvetlov` + +.. tip:: + + See :file:`pyproject.toml` for all available categories + (``tool.towncrier.type``). + +.. _Towncrier philosophy: + https://towncrier.readthedocs.io/en/actual-freaking-docs/#philosophy From 926292248151fda5bf107530bad31b6d104ba4c5 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 6 Oct 2021 01:53:36 +0200 Subject: [PATCH 12/66] Fix the `check_changes` script to allow README&rst (#6050) --- tools/check_changes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/check_changes.py b/tools/check_changes.py index 77ae2431051..cd488bebc9a 100755 --- a/tools/check_changes.py +++ b/tools/check_changes.py @@ -4,6 +4,7 @@ from pathlib import Path ALLOWED_SUFFIXES = [".feature", ".bugfix", ".doc", ".removal", ".misc"] +ALLOWED_SUFFIXES += [f"{suffix}.rst" for suffix in ALLOWED_SUFFIXES] def get_root(script_path): @@ -22,7 +23,7 @@ def main(argv): changes = root / "CHANGES" failed = False for fname in changes.iterdir(): - if fname.name in (".gitignore", ".TEMPLATE.rst"): + if fname.name in (".gitignore", ".TEMPLATE.rst", "README.rst"): continue if fname.suffix not in ALLOWED_SUFFIXES: if not failed: From 1aaac4e9a139ee7338af8bb4c6efedb3b56543c6 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 6 Oct 2021 01:58:15 +0200 Subject: [PATCH 13/66] Fix the change note RST syntax for PR #6045 (#6052) --- CHANGES/6045.misc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES/6045.misc b/CHANGES/6045.misc index 020b8ef28a2..a27f2d17d4b 100644 --- a/CHANGES/6045.misc +++ b/CHANGES/6045.misc @@ -1,2 +1,3 @@ -Added `commit`, `gh`, `issue`, `pr` and `user` -RST roles in Sphinx — :user:`webknjaz`. +Added ``commit``, ``gh``, ``issue``, ``pr`` +and ``user`` RST roles in Sphinx +-- :user:`webknjaz`. From 95039800d8fabd1e6bdd0363dd386ed3770bab61 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 6 Oct 2021 03:11:56 +0200 Subject: [PATCH 14/66] Enable test coverage collection in pytest (#6054) --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index a3a33ad2ba0..e6ada3b0f26 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,7 @@ addopts = # `pytest-cov`: --cov=aiohttp + --cov=tests/ filterwarnings = error ignore:module 'ssl' has no attribute 'OP_NO_COMPRESSION'. The Python interpreter is compiled against OpenSSL < 1.0.0. Ref. https.//docs.python.org/3/library/ssl.html#ssl.OP_NO_COMPRESSION:UserWarning From 079e9c5a7e6fede74be6fabd689b875f2065e084 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 6 Oct 2021 10:00:35 +0200 Subject: [PATCH 15/66] Implement future changelog previews in the docs (#6055) * Implement future changelog previews in the docs * Self-install on RTD * Fix spelling mistakes in the check notes --- .readthedocs.yml | 20 ++++++++++++++++++-- CHANGES.rst | 4 ---- CHANGES/3559.doc | 2 +- CHANGES/3828.feature | 4 ++-- CHANGES/4054.feature | 2 +- CHANGES/4277.feature | 2 +- CHANGES/4299.bugfix | 2 +- CHANGES/4302.bugfix | 2 +- CHANGES/4452.doc | 2 +- CHANGES/4700.feature | 8 ++++---- CHANGES/5326.doc | 2 +- CHANGES/5783.feature | 2 +- CHANGES/5905.bugfix | 2 +- docs/changes.rst | 12 ++++++++++++ docs/conf.py | 11 +++++++++++ docs/spelling_wordlist.txt | 5 +++++ requirements/dev.txt | 15 ++++++++++----- requirements/doc-spelling.txt | 7 ++++++- requirements/doc.txt | 1 + 19 files changed, 78 insertions(+), 27 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index e2e8d918392..90fe80896bc 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,5 +1,21 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html +# for details + +--- +version: 2 + +submodules: + include: all # [] + exclude: [] + recursive: true + build: image: latest python: - version: 3.6 - pip_install: false + version: 3.8 + install: + - method: pip + path: . + - requirements: requirements/doc.txt +... diff --git a/CHANGES.rst b/CHANGES.rst index f064f4895ce..6301a2a17a5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,3 @@ -========= -Changelog -========= - .. You should *NOT* be adding new change log entries to this file, this file is managed by towncrier. You *may* edit previous change logs to diff --git a/CHANGES/3559.doc b/CHANGES/3559.doc index f15261dd5e0..aa49970bc18 100644 --- a/CHANGES/3559.doc +++ b/CHANGES/3559.doc @@ -1 +1 @@ -Clarified WebSocketResponse closure in quickstart example. +Clarified ``WebSocketResponse`` closure in the quick start example. diff --git a/CHANGES/3828.feature b/CHANGES/3828.feature index 7d31a2bb243..9d78d813e95 100644 --- a/CHANGES/3828.feature +++ b/CHANGES/3828.feature @@ -1,4 +1,4 @@ -Disable implicit switch-back to pure python mode. The build fails loudly if aiohttp -cannot be compiled with C Accellerators. Use AIOHTTP_NO_EXTENSIONS=1 to explicitly +Disabled implicit switch-back to pure python mode. The build fails loudly if aiohttp +cannot be compiled with C Accelerators. Use `AIOHTTP_NO_EXTENSIONS=1` to explicitly disable C Extensions complication and switch to Pure-Python mode. Note that Pure-Python mode is significantly slower than compiled one. diff --git a/CHANGES/4054.feature b/CHANGES/4054.feature index 436bf352f6d..e34d741cba1 100644 --- a/CHANGES/4054.feature +++ b/CHANGES/4054.feature @@ -1 +1 @@ -Implemented readuntil in StreamResponse +Implemented ``readuntil`` in ``StreamResponse`` diff --git a/CHANGES/4277.feature b/CHANGES/4277.feature index bc5b360bf73..94677b1e855 100644 --- a/CHANGES/4277.feature +++ b/CHANGES/4277.feature @@ -1 +1 @@ -Add set_cookie and del_cookie methods to HTTPException +Added ``set_cookie`` and ``del_cookie`` methods to ``HTTPException`` diff --git a/CHANGES/4299.bugfix b/CHANGES/4299.bugfix index 8b82acbdacc..78ae9ec042a 100644 --- a/CHANGES/4299.bugfix +++ b/CHANGES/4299.bugfix @@ -1 +1 @@ -Delete older code in example (examples/web_classview.py) +Delete older code in example (:file:`examples/web_classview.py`) diff --git a/CHANGES/4302.bugfix b/CHANGES/4302.bugfix index af14834f2fc..eb72de901dd 100644 --- a/CHANGES/4302.bugfix +++ b/CHANGES/4302.bugfix @@ -1 +1 @@ -Fixed the support of route handlers wrapped by functools.partial() +Fixed the support of route handlers wrapped by :py:func:`functools.partial` diff --git a/CHANGES/4452.doc b/CHANGES/4452.doc index 306f751afc5..17681b2ee11 100644 --- a/CHANGES/4452.doc +++ b/CHANGES/4452.doc @@ -1 +1 @@ -Fix typo in client_quickstart docs. +Fixed a typo in the ``client_quickstart`` doc. diff --git a/CHANGES/4700.feature b/CHANGES/4700.feature index dfcd88ff960..da691aa7c0e 100644 --- a/CHANGES/4700.feature +++ b/CHANGES/4700.feature @@ -1,6 +1,6 @@ -AioHTTPTestCase is more async friendly now. +``AioHTTPTestCase`` is more async friendly now. -For people who use unittest and are used to use unittest.TestCase -it will be easier to write new test cases like the sync version of the TestCase class, +For people who use unittest and are used to use :py:exc:`~unittest.TestCase` +it will be easier to write new test cases like the sync version of the :py:exc:`~unittest.TestCase` class, without using the decorator `@unittest_run_loop`, just `async def test_*`. -The only difference is that for the people using python3.7 and below a new dependency is needed, it is `asynctestcase`. +The only difference is that for the people using python3.7 and below a new dependency is needed, it is ``asynctestcase``. diff --git a/CHANGES/5326.doc b/CHANGES/5326.doc index 74aff4c4225..5564425aff4 100644 --- a/CHANGES/5326.doc +++ b/CHANGES/5326.doc @@ -1 +1 @@ -Refactor OpenAPI/Swagger aiohttp addons, added aio-openapi +Refactored OpenAPI/Swagger aiohttp addons, added ``aio-openapi`` diff --git a/CHANGES/5783.feature b/CHANGES/5783.feature index 4be16c23343..6b5c534f66f 100644 --- a/CHANGES/5783.feature +++ b/CHANGES/5783.feature @@ -1 +1 @@ -Started keeping the ``Authorization`` header during http->https redirects when the host remains the same. +Started keeping the ``Authorization`` header during HTTP -> HTTPS redirects when the host remains the same. diff --git a/CHANGES/5905.bugfix b/CHANGES/5905.bugfix index b667968fe19..0e581b5cbf3 100644 --- a/CHANGES/5905.bugfix +++ b/CHANGES/5905.bugfix @@ -1 +1 @@ -remove deprecated loop argument for asnycio.sleep/gather calls +Removed the deprecated ``loop`` argument from the ``asyncio.sleep``/``gather`` calls diff --git a/docs/changes.rst b/docs/changes.rst index 0ecf1d76af8..6a61dfbcc1e 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,5 +1,17 @@ .. _aiohttp_changes: +========= +Changelog +========= + +To be included in v\ |release| (if present) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. towncrier-draft-entries:: |release| [UNRELEASED DRAFT] + +Released versions +^^^^^^^^^^^^^^^^^ + .. include:: ../CHANGES.rst .. include:: ../HISTORY.rst diff --git a/docs/conf.py b/docs/conf.py index d56150ceeb3..b7d80de93dc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,6 +14,9 @@ import os import re +from pathlib import Path + +PROJECT_ROOT_DIR = Path(__file__).parents[1].resolve() _docs_path = os.path.dirname(__file__) _version_path = os.path.abspath( @@ -50,6 +53,7 @@ # Third-party extensions: "sphinxcontrib.asyncio", "sphinxcontrib.blockdiag", + "sphinxcontrib.towncrier", # provides `towncrier-draft-entries` directive ] @@ -420,3 +424,10 @@ ("py:meth", "aiohttp.web.UrlDispatcher.register_resource"), # undocumented ("py:func", "aiohttp_debugtoolbar.setup"), # undocumented ] + +# -- Options for towncrier_draft extension ----------------------------------- + +towncrier_draft_autoversion_mode = "draft" # or: 'sphinx-version', 'sphinx-release' +towncrier_draft_include_empty = True +towncrier_draft_working_directory = PROJECT_ROOT_DIR +# Not yet supported: towncrier_draft_config_path = 'pyproject.toml' # relative to cwd diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 4a18a93b0b5..e7a608fd658 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -82,6 +82,7 @@ WSMsgType Websockets Workflow abc +addons aiodns aioes aiohttp @@ -117,6 +118,7 @@ brotli bugfix builtin cChardet +callables cancelled canonicalization canonicalize @@ -204,6 +206,7 @@ login lookup lookups lossless +lowercased manylinux metadata microservice @@ -261,6 +264,7 @@ redirections refactor refactored refactoring +referenceable regex regexps regexs @@ -319,6 +323,7 @@ unittest unix unsets unstripped +uppercased upstr url urldispatcher diff --git a/requirements/dev.txt b/requirements/dev.txt index 9364ad351e6..a2e54a04764 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -61,7 +61,7 @@ click==7.1.2 # towncrier click-default-group==1.2.2 # via towncrier -coverage==5.5 +coverage[toml]==6.0 # via # -r requirements/test.txt # pytest-cov @@ -128,7 +128,7 @@ mccabe==0.6.1 # via # -r requirements/lint.txt # flake8 -multidict==5.1.0 +multidict==5.2.0 # via # -r requirements/multidict.txt # yarl @@ -198,7 +198,7 @@ pytest==6.2.2 # -r requirements/test.txt # pytest-cov # pytest-mock -pytest-cov==2.12.1 +pytest-cov==3.0.0 # via -r requirements/test.txt pytest-mock==3.6.1 # via -r requirements/test.txt @@ -236,6 +236,7 @@ sphinx==4.2.0 # -r requirements/doc.txt # sphinxcontrib-asyncio # sphinxcontrib-blockdiag + # sphinxcontrib-towncrier sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-asyncio==0.3.0 @@ -252,6 +253,8 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx +sphinxcontrib-towncrier==0.2.0a0 + # via -r requirements/doc.txt toml==0.10.2 # via # -r requirements/lint.txt @@ -259,14 +262,16 @@ toml==0.10.2 # mypy # pre-commit # pytest - # pytest-cov # towncrier tomli==1.2.1 # via # -r requirements/lint.txt # black + # coverage towncrier==21.3.0 - # via -r requirements/doc.txt + # via + # -r requirements/doc.txt + # sphinxcontrib-towncrier trustme==0.9.0 ; platform_machine != "i686" # via -r requirements/test.txt types-chardet==0.1.3 diff --git a/requirements/doc-spelling.txt b/requirements/doc-spelling.txt index e371d76e30a..a9800eef0e6 100644 --- a/requirements/doc-spelling.txt +++ b/requirements/doc-spelling.txt @@ -64,6 +64,7 @@ sphinx==4.2.0 # sphinxcontrib-asyncio # sphinxcontrib-blockdiag # sphinxcontrib-spelling + # sphinxcontrib-towncrier sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-asyncio==0.3.0 @@ -82,10 +83,14 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-spelling==7.2.1 ; platform_system != "Windows" # via -r requirements/doc-spelling.in +sphinxcontrib-towncrier==0.2.0a0 + # via -r requirements/doc.txt toml==0.10.2 # via towncrier towncrier==21.3.0 - # via -r requirements/doc.txt + # via + # -r requirements/doc.txt + # sphinxcontrib-towncrier urllib3==1.26.5 # via requests webcolors==1.11.1 diff --git a/requirements/doc.txt b/requirements/doc.txt index 8d1cd5453d4..29b493c8b37 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -5,4 +5,5 @@ pygments==2.10.0 sphinx==4.2.0 sphinxcontrib-asyncio==0.3.0 sphinxcontrib-blockdiag==2.0.0 +sphinxcontrib-towncrier==0.2.0a0 towncrier==21.3.0 From e5af76acf9c229c0a45ebf499ba3ee6b5e2490be Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Wed, 6 Oct 2021 12:10:47 +0100 Subject: [PATCH 16/66] Add a warning when a cookie's length exceeds 4096 bytes (#5959) --- CHANGES/5634.feature | 1 + Makefile | 2 ++ aiohttp/helpers.py | 11 +++++++++++ setup.cfg | 5 +++++ tests/test_web_response.py | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+) create mode 100644 CHANGES/5634.feature diff --git a/CHANGES/5634.feature b/CHANGES/5634.feature new file mode 100644 index 00000000000..1240147019e --- /dev/null +++ b/CHANGES/5634.feature @@ -0,0 +1 @@ +A warning was added, when a cookie's length exceeds the :rfc:`6265` minimum client support -- :user:`anesabml`. diff --git a/Makefile b/Makefile index 0e7189760bb..81b8a0325c3 100644 --- a/Makefile +++ b/Makefile @@ -92,10 +92,12 @@ test: .develop .PHONY: vtest vtest: .develop @pytest -s -v + @python -X dev -m pytest -s -v -m dev_mode .PHONY: vvtest vvtest: .develop @pytest -vv + @python -X dev -m pytest -s -vv -m dev_mode .PHONY: cov-dev cov-dev: .develop diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index be76e0cec62..ad80ffe47bb 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -13,6 +13,7 @@ import re import sys import time +import warnings import weakref from collections import namedtuple from contextlib import suppress @@ -55,6 +56,7 @@ PY_38 = sys.version_info >= (3, 8) +COOKIE_MAX_LENGTH = 4096 try: from typing import ContextManager @@ -876,6 +878,15 @@ def set_cookie( if samesite is not None: c["samesite"] = samesite + if DEBUG: + cookie_length = len(c.output(header="")[1:]) + if cookie_length > COOKIE_MAX_LENGTH: + warnings.warn( + "The size of is too large, it might get ignored by the client.", + UserWarning, + stacklevel=2, + ) + def del_cookie( self, name: str, *, domain: Optional[str] = None, path: str = "/" ) -> None: diff --git a/setup.cfg b/setup.cfg index e6ada3b0f26..5f5dc0bd3bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,6 +51,9 @@ addopts = # `pytest-cov`: --cov=aiohttp --cov=tests/ + + # run tests that are not marked with dev_mode + -m "not dev_mode" filterwarnings = error ignore:module 'ssl' has no attribute 'OP_NO_COMPRESSION'. The Python interpreter is compiled against OpenSSL < 1.0.0. Ref. https.//docs.python.org/3/library/ssl.html#ssl.OP_NO_COMPRESSION:UserWarning @@ -66,3 +69,5 @@ minversion = 3.8.2 testpaths = tests/ junit_family=xunit2 xfail_strict = true +markers = + dev_mode: mark test to run in dev mode. diff --git a/tests/test_web_response.py b/tests/test_web_response.py index 5ff0aabbe6e..ddda01aae7d 100644 --- a/tests/test_web_response.py +++ b/tests/test_web_response.py @@ -3,6 +3,7 @@ import datetime import gzip import json +import re import weakref from concurrent.futures import ThreadPoolExecutor from typing import Any, Optional @@ -1167,3 +1168,34 @@ def test_text_is_json_encoded(self) -> None: def test_content_type_is_overrideable(self) -> None: resp = json_response({"foo": 42}, content_type="application/vnd.json+api") assert "application/vnd.json+api" == resp.content_type + + +@pytest.mark.dev_mode +async def test_no_warn_small_cookie(buf: Any, writer: Any) -> None: + resp = Response() + resp.set_cookie("foo", "ÿ" + "8" * 4064, max_age=2600) # No warning + req = make_request("GET", "/", writer=writer) + + await resp.prepare(req) + await resp.write_eof() + + cookie = re.search(b"Set-Cookie: (.*?)\r\n", buf).group(1) + assert len(cookie) == 4096 + + +@pytest.mark.dev_mode +async def test_warn_large_cookie(buf: Any, writer: Any) -> None: + resp = Response() + + with pytest.warns( + UserWarning, + match="The size of is too large, it might get ignored by the client.", + ): + resp.set_cookie("foo", "ÿ" + "8" * 4065, max_age=2600) + req = make_request("GET", "/", writer=writer) + + await resp.prepare(req) + await resp.write_eof() + + cookie = re.search(b"Set-Cookie: (.*?)\r\n", buf).group(1) + assert len(cookie) == 4097 From a3c85e9d68f5555391a03824a1ab8c4b6e5aace2 Mon Sep 17 00:00:00 2001 From: rigens <68144278+rigens@users.noreply.github.com> Date: Wed, 6 Oct 2021 19:52:55 +0300 Subject: [PATCH 17/66] Docs: Add aiohttp-socks to third party libraries (#5977) --- docs/third_party.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/third_party.rst b/docs/third_party.rst index 48dd0c6d354..212543cca65 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -290,3 +290,6 @@ ask to raise the status. - `aiohttp-retry `_ Wrapper for aiohttp client for retrying requests. Python 3.6+ required. + +- `aiohttp-socks `_ + SOCKS proxy connector for aiohttp. From 5fab03a523010c0fb531c064f300e6cfe0d5b67b Mon Sep 17 00:00:00 2001 From: Shady Goat <48590492+ShadiestGoat@users.noreply.github.com> Date: Wed, 6 Oct 2021 23:45:33 +0100 Subject: [PATCH 18/66] Further optimize the logo/icon (#5914) * Further optimized, now using viewbox of 0 0 24 24 It's not exactly required, but projects such as simple-icons use a 24 24 viewbox, and dont move the viewbox into the middle for no reason (instead starting at 0 0) * Added color to the icon * Update aiohttp-plain.svg * Added same height width as original * Update aiohttp-plain.svg * Create 5914.misc * Make the change note clearer Co-authored-by: Sviatoslav Sydorenko --- CHANGES/5914.misc | 1 + docs/aiohttp-icon.svg | 2 +- docs/aiohttp-plain.svg | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 CHANGES/5914.misc diff --git a/CHANGES/5914.misc b/CHANGES/5914.misc new file mode 100644 index 00000000000..f876dc3ca92 --- /dev/null +++ b/CHANGES/5914.misc @@ -0,0 +1 @@ +Changed the SVG logos to be more optimized and the viewbox to 0 0 24 24, while keeping the same height and width -- :user:`ShadiestGoat`. diff --git a/docs/aiohttp-icon.svg b/docs/aiohttp-icon.svg index 9e2009f2994..0b3ebacb0bf 100644 --- a/docs/aiohttp-icon.svg +++ b/docs/aiohttp-icon.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/docs/aiohttp-plain.svg b/docs/aiohttp-plain.svg index 520727a369c..aec1b00c1e5 100644 --- a/docs/aiohttp-plain.svg +++ b/docs/aiohttp-plain.svg @@ -1 +1 @@ - \ No newline at end of file + From a45c7c53e20ccf77f4f6b721157d4a449c0f1208 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Wed, 6 Oct 2021 23:57:16 +0100 Subject: [PATCH 19/66] Integrate autobahn tests with pytest (#5809) * Integrate autobahn tests with pytest * Fix docker compose file paths * Fix typo in CHANGES file * Fix add python-on-whales dependency to .in file instead of .txt * Use pathlib instead of os * Use buildx instead of compose build * Regenerate dev requirements * Rename changes file * Use request fspath instead of hard coded path * Create a sepearte builder when building aiohttp * Use subprocess instead of python-on-whales * Extract failed tests and make assertions on them * Fix lint issues * Fix fixture scope * Add ports to docker-compose files * Add wait-for-it package * Use xfail instead of fail * Use wstest cmd tool instead of the docker image * Fix lint issues * Use assert statement with custom output Co-authored-by: Sviatoslav Sydorenko * Code cleanup * Use docker instead of docker-compose * Add xfail decorator * Add tmp_path * Remove gitignore * Skip tests only on macOS * Check if docker is available * Regenerate dev.txt Co-authored-by: Sviatoslav Sydorenko --- .mypy.ini | 3 + CHANGES/4247.1.misc | 1 + requirements/dev.in | 2 + requirements/dev.txt | 14 +++ tests/autobahn/.gitignore | 1 - tests/autobahn/Dockerfile.autobahn | 6 ++ tests/autobahn/client/client.py | 2 +- tests/autobahn/client/docker-compose.yml | 17 --- tests/autobahn/docker-compose.yml | 6 -- tests/autobahn/run-tests.sh | 12 --- tests/autobahn/server/docker-compose.yml | 18 ---- tests/autobahn/server/fuzzingclient.json | 2 +- tests/autobahn/server/server.py | 3 + tests/autobahn/test_autobahn.py | 132 +++++++++++++++++++++++ 14 files changed, 163 insertions(+), 56 deletions(-) create mode 100644 CHANGES/4247.1.misc delete mode 100644 tests/autobahn/.gitignore create mode 100644 tests/autobahn/Dockerfile.autobahn delete mode 100644 tests/autobahn/client/docker-compose.yml delete mode 100644 tests/autobahn/docker-compose.yml delete mode 100755 tests/autobahn/run-tests.sh delete mode 100644 tests/autobahn/server/docker-compose.yml create mode 100644 tests/autobahn/test_autobahn.py diff --git a/.mypy.ini b/.mypy.ini index 5b5796e4a51..ebcd461441c 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -41,3 +41,6 @@ ignore_missing_imports = True [mypy-uvloop] ignore_missing_imports = True + +[mypy-python_on_whales] +ignore_missing_imports = True diff --git a/CHANGES/4247.1.misc b/CHANGES/4247.1.misc new file mode 100644 index 00000000000..86463d0577d --- /dev/null +++ b/CHANGES/4247.1.misc @@ -0,0 +1 @@ +Automated running autobahn test suite by integrating with pytest. diff --git a/requirements/dev.in b/requirements/dev.in index 31b14be9997..d827d37d65f 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -2,3 +2,5 @@ -r test.txt -r doc.txt cherry_picker==2.0.0; python_version>="3.6" +python-on-whales==0.19.0 +wait-for-it==2.2.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index a2e54a04764..57f7f360845 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -59,6 +59,8 @@ click==7.1.2 # cherry-picker # click-default-group # towncrier + # typer + # wait-for-it click-default-group==1.2.2 # via towncrier coverage[toml]==6.0 @@ -177,6 +179,8 @@ pycodestyle==2.7.0 # flake8 pycparser==2.20 # via cffi +pydantic==1.8.2 + # via python-on-whales pyflakes==2.3.0 # via # -r requirements/lint.txt @@ -204,6 +208,8 @@ pytest-mock==3.6.1 # via -r requirements/test.txt python-dateutil==2.8.1 # via freezegun +python-on-whales==0.19.0 + # via -r requirements/dev.in pytz==2020.5 # via babel pyyaml==5.4.1 @@ -220,6 +226,7 @@ regex==2020.11.13 requests==2.25.1 # via # cherry-picker + # python-on-whales # sphinx setuptools-git==1.2 # via -r requirements/test.txt @@ -272,8 +279,12 @@ towncrier==21.3.0 # via # -r requirements/doc.txt # sphinxcontrib-towncrier +tqdm==4.62.2 + # via python-on-whales trustme==0.9.0 ; platform_machine != "i686" # via -r requirements/test.txt +typer==0.4.0 + # via python-on-whales types-chardet==0.1.3 # via # -r requirements/lint.txt @@ -285,6 +296,7 @@ typing-extensions==3.7.4.3 # async-timeout # mypy # proxy.py + # pydantic uritemplate==3.0.1 # via gidgethub urllib3==1.26.5 @@ -293,6 +305,8 @@ virtualenv==20.4.2 # via # -r requirements/lint.txt # pre-commit +wait-for-it==2.2.0 + # via -r requirements/dev.in webcolors==1.11.1 # via blockdiag yarl==1.6.3 diff --git a/tests/autobahn/.gitignore b/tests/autobahn/.gitignore deleted file mode 100644 index 08ab34c5253..00000000000 --- a/tests/autobahn/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/reports diff --git a/tests/autobahn/Dockerfile.autobahn b/tests/autobahn/Dockerfile.autobahn new file mode 100644 index 00000000000..45f18182804 --- /dev/null +++ b/tests/autobahn/Dockerfile.autobahn @@ -0,0 +1,6 @@ +FROM crossbario/autobahn-testsuite:0.8.2 + +RUN apt-get update && apt-get install python3 python3-pip -y +RUN pip3 install wait-for-it + +CMD ["wstest", "--mode", "fuzzingserver", "--spec", "/config/fuzzingserver.json"] diff --git a/tests/autobahn/client/client.py b/tests/autobahn/client/client.py index 107c183070e..dfca77d12b2 100644 --- a/tests/autobahn/client/client.py +++ b/tests/autobahn/client/client.py @@ -38,4 +38,4 @@ async def run(url: str, name: str) -> None: if __name__ == "__main__": - asyncio.run(run("http://autobahn:9001", "aiohttp")) + asyncio.run(run("http://localhost:9001", "aiohttp")) diff --git a/tests/autobahn/client/docker-compose.yml b/tests/autobahn/client/docker-compose.yml deleted file mode 100644 index ac6a8bf3ab7..00000000000 --- a/tests/autobahn/client/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.9" -services: - autobahn: - image: crossbario/autobahn-testsuite:0.8.2 - volumes: - - type: bind - source: ./fuzzingserver.json - target: /config/fuzzingserver.json - - type: bind - source: ../reports - target: /reports - - aiohttp: - image: aiohttp-autobahn_aiohttp - depends_on: - - autobahn - command: ["python", "tests/autobahn/client/client.py"] diff --git a/tests/autobahn/docker-compose.yml b/tests/autobahn/docker-compose.yml deleted file mode 100644 index ea6b640810d..00000000000 --- a/tests/autobahn/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3.9" -services: - aiohttp: - build: - context: ../.. - dockerfile: tests/autobahn/Dockerfile.aiohttp diff --git a/tests/autobahn/run-tests.sh b/tests/autobahn/run-tests.sh deleted file mode 100755 index d48894d8cb8..00000000000 --- a/tests/autobahn/run-tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -rm -rf $PWD/reports -mkdir $PWD/reports - -docker-compose -p aiohttp-autobahn build - -docker-compose -f $PWD/client/docker-compose.yml up --abort-on-container-exit -docker-compose -f $PWD/client/docker-compose.yml down - -docker-compose -f $PWD/server/docker-compose.yml up --abort-on-container-exit -docker-compose -f $PWD/server/docker-compose.yml down diff --git a/tests/autobahn/server/docker-compose.yml b/tests/autobahn/server/docker-compose.yml deleted file mode 100644 index 8f12f2d19cc..00000000000 --- a/tests/autobahn/server/docker-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3.9" -services: - autobahn: - image: crossbario/autobahn-testsuite:0.8.2 - depends_on: - - aiohttp - volumes: - - type: bind - source: ./fuzzingclient.json - target: /config/fuzzingclient.json - - type: bind - source: ../reports - target: /reports - command: ["wstest", "--mode", "fuzzingclient", "--spec", "/config/fuzzingclient.json"] - - aiohttp: - image: aiohttp-autobahn_aiohttp - command: ["python", "tests/autobahn/server/server.py"] diff --git a/tests/autobahn/server/fuzzingclient.json b/tests/autobahn/server/fuzzingclient.json index e9bef9591dc..0ed2f84acf8 100644 --- a/tests/autobahn/server/fuzzingclient.json +++ b/tests/autobahn/server/fuzzingclient.json @@ -5,7 +5,7 @@ "servers": [ { "agent": "AutobahnServer", - "url": "ws://aiohttp:9001", + "url": "ws://localhost:9001", "options": { "version": 18 } } ], diff --git a/tests/autobahn/server/server.py b/tests/autobahn/server/server.py index d4ca04b1d5f..c0e50259b47 100644 --- a/tests/autobahn/server/server.py +++ b/tests/autobahn/server/server.py @@ -13,6 +13,8 @@ async def wshandler(request: web.Request) -> web.WebSocketResponse: await ws.prepare(request) + request.app["websockets"].append(ws) + while True: msg = await ws.receive() @@ -40,6 +42,7 @@ async def on_shutdown(app: web.Application) -> None: ) app = web.Application() + app["websockets"] = [] app.router.add_route("GET", "/", wshandler) app.on_shutdown.append(on_shutdown) try: diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py new file mode 100644 index 00000000000..5d72e37a17a --- /dev/null +++ b/tests/autobahn/test_autobahn.py @@ -0,0 +1,132 @@ +import json +import subprocess +import sys +from pathlib import Path +from typing import Any, Dict, Generator, List + +import pytest +from pytest import TempPathFactory +from python_on_whales import DockerException, docker + + +@pytest.fixture(scope="session") +def report_dir(tmp_path_factory: TempPathFactory) -> Path: + return tmp_path_factory.mktemp("reports") + + +@pytest.fixture(scope="session", autouse=True) +def build_autobahn_testsuite() -> Generator[None, None, None]: + + try: + docker.build( + file="tests/autobahn/Dockerfile.autobahn", + tags=["autobahn-testsuite"], + context_path=".", + ) + except DockerException: + pytest.skip(msg="The docker daemon is not running.") + + try: + yield + finally: + docker.image.remove(x="autobahn-testsuite") + + +def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]: + path = Path(report_path) + result_summary = json.loads((path / "index.json").read_text())[name] + failed_messages = [] + PASS = {"OK", "INFORMATIONAL"} + entry_fields = {"case", "description", "expectation", "expected", "received"} + for results in result_summary.values(): + if results["behavior"] in PASS and results["behaviorClose"] in PASS: + continue + report = json.loads((path / results["reportfile"]).read_text()) + failed_messages.append({field: report[field] for field in entry_fields}) + return failed_messages + + +@pytest.mark.skipif(sys.platform == "darwin", reason="Don't run on macOS") +@pytest.mark.xfail +def test_client(report_dir: Path, request: Any) -> None: + try: + print("Starting autobahn-testsuite server") + autobahn_container = docker.run( + detach=True, + image="autobahn-testsuite", + name="autobahn", + publish=[(9001, 9001)], + remove=True, + volumes=[ + (f"{request.fspath.dirname}/client", "/config"), + (f"{report_dir}", "/reports"), + ], + ) + print("Running aiohttp test client") + client = subprocess.Popen( + ["wait-for-it", "-s", "localhost:9001", "--"] + + [sys.executable] + + ["tests/autobahn/client/client.py"] + ) + client.wait() + finally: + print("Stopping client and server") + client.terminate() + client.wait() + autobahn_container.stop() + + failed_messages = get_failed_tests(f"{report_dir}/clients", "aiohttp") + + assert not failed_messages, "\n".join( + "\n\t".join( + f"{field}: {msg[field]}" + for field in ("case", "description", "expectation", "expected", "received") + ) + for msg in failed_messages + ) + + +@pytest.mark.skipif(sys.platform == "darwin", reason="Don't run on macOS") +@pytest.mark.xfail +def test_server(report_dir: Path, request: Any) -> None: + try: + print("Starting aiohttp test server") + server = subprocess.Popen( + [sys.executable] + ["tests/autobahn/server/server.py"] + ) + print("Starting autobahn-testsuite client") + docker.run( + image="autobahn-testsuite", + name="autobahn", + remove=True, + volumes=[ + (f"{request.fspath.dirname}/server", "/config"), + (f"{report_dir}", "/reports"), + ], + networks=["host"], + command=[ + "wait-for-it", + "-s", + "localhost:9001", + "--", + "wstest", + "--mode", + "fuzzingclient", + "--spec", + "/config/fuzzingclient.json", + ], + ) + finally: + print("Stopping client and server") + server.terminate() + server.wait() + + failed_messages = get_failed_tests(f"{report_dir}/servers", "AutobahnServer") + + assert not failed_messages, "\n".join( + "\n\t".join( + f"{field}: {msg[field]}" + for field in ("case", "description", "expectation", "expected", "received") + ) + for msg in failed_messages + ) From a75ad30cb750493786cd00f27c23a4c9905df6b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Oct 2021 10:17:07 +0000 Subject: [PATCH 20/66] Bump python-on-whales from 0.19.0 to 0.27.0 (#6061) * Bump python-on-whales from 0.19.0 to 0.27.0 Bumps [python-on-whales](https://github.com/gabrieldemarmiesse/python-on-whales) from 0.19.0 to 0.27.0. - [Release notes](https://github.com/gabrieldemarmiesse/python-on-whales/releases) - [Commits](https://github.com/gabrieldemarmiesse/python-on-whales/compare/v0.19.0...v0.27.0) --- updated-dependencies: - dependency-name: python-on-whales dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/third_party.rst | 2 +- requirements/dev.in | 2 +- requirements/dev.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/third_party.rst b/docs/third_party.rst index 212543cca65..c180322dc15 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -290,6 +290,6 @@ ask to raise the status. - `aiohttp-retry `_ Wrapper for aiohttp client for retrying requests. Python 3.6+ required. - + - `aiohttp-socks `_ SOCKS proxy connector for aiohttp. diff --git a/requirements/dev.in b/requirements/dev.in index d827d37d65f..62d8335373e 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -2,5 +2,5 @@ -r test.txt -r doc.txt cherry_picker==2.0.0; python_version>="3.6" -python-on-whales==0.19.0 +python-on-whales==0.27.0 wait-for-it==2.2.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index 57f7f360845..8d44d1feb73 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -208,7 +208,7 @@ pytest-mock==3.6.1 # via -r requirements/test.txt python-dateutil==2.8.1 # via freezegun -python-on-whales==0.19.0 +python-on-whales==0.27.0 # via -r requirements/dev.in pytz==2020.5 # via babel From c9c9bec65c03c9db602f99b16b3d3c20e3c53edb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Oct 2021 10:21:13 +0000 Subject: [PATCH 21/66] Bump coverage from 6.0 to 6.0.1 (#6063) * Bump coverage from 6.0 to 6.0.1 Bumps [coverage](https://github.com/nedbat/coveragepy) from 6.0 to 6.0.1. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/6.0...6.0.1) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index a32135ef9b4..6a476516ed2 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,7 +1,7 @@ -r base.txt Brotli==1.0.9 -coverage==6.0 +coverage==6.0.1 cryptography==3.3.1; platform_machine!="i686" and python_version<"3.9" # no 32-bit wheels; no python 3.9 wheels yet freezegun==1.1.0 mypy==0.910; implementation_name=="cpython" From f2597a478cc4bccdd5b9cdcb2af3d8e62ab014f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Oct 2021 14:59:28 +0000 Subject: [PATCH 22/66] Bump wait-for-it from 2.2.0 to 2.2.1 (#6062) Bumps [wait-for-it](https://github.com/clarketm/wait-for-it) from 2.2.0 to 2.2.1. - [Release notes](https://github.com/clarketm/wait-for-it/releases) - [Commits](https://github.com/clarketm/wait-for-it/compare/v2.2.0...v2.2.1) --- updated-dependencies: - dependency-name: wait-for-it dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev.in | 2 +- requirements/dev.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/dev.in b/requirements/dev.in index 62d8335373e..3da176a4bf5 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -3,4 +3,4 @@ -r doc.txt cherry_picker==2.0.0; python_version>="3.6" python-on-whales==0.27.0 -wait-for-it==2.2.0 +wait-for-it==2.2.1 diff --git a/requirements/dev.txt b/requirements/dev.txt index 8d44d1feb73..62d56075e52 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -63,7 +63,7 @@ click==7.1.2 # wait-for-it click-default-group==1.2.2 # via towncrier -coverage[toml]==6.0 +coverage[toml]==6.0.1 # via # -r requirements/test.txt # pytest-cov @@ -305,7 +305,7 @@ virtualenv==20.4.2 # via # -r requirements/lint.txt # pre-commit -wait-for-it==2.2.0 +wait-for-it==2.2.1 # via -r requirements/dev.in webcolors==1.11.1 # via blockdiag From f362679b0e8082a21fa5454cc1ca4ed246734ff9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Oct 2021 10:14:27 +0000 Subject: [PATCH 23/66] Bump yarl from 1.6.3 to 1.7.0 (#6067) Bumps [yarl](https://github.com/aio-libs/yarl) from 1.6.3 to 1.7.0. - [Release notes](https://github.com/aio-libs/yarl/releases) - [Changelog](https://github.com/aio-libs/yarl/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/yarl/compare/v1.6.3...v1.7.0) --- updated-dependencies: - dependency-name: yarl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 68e007c65b2..c4cdc49189e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -11,4 +11,4 @@ frozenlist==1.1.1 gunicorn==20.1.0 typing_extensions==3.7.4.3 uvloop==0.14.0; platform_system!="Windows" and implementation_name=="cpython" and python_version<"3.9" # MagicStack/uvloop#14 -yarl==1.6.3 +yarl==1.7.0 From 84babebc96e08f1ea2b7c31406eb5a1ed20cd5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A7=8B=E8=91=89?= Date: Sat, 9 Oct 2021 07:41:32 +0800 Subject: [PATCH 24/66] Add support for Python 3.10 (#5927) Co-authored-by: Sviatoslav Sydorenko --- .github/workflows/ci.yml | 2 +- CHANGES/5927.feature | 1 + aiohttp/connector.py | 4 +++- aiohttp/helpers.py | 1 + aiohttp/test_utils.py | 6 ++++-- requirements/dev.txt | 6 +++--- requirements/lint.in | 2 +- requirements/lint.txt | 4 ++-- requirements/test.txt | 2 +- tests/conftest.py | 2 +- tests/test_client_functional.py | 2 +- tests/test_connector.py | 14 +++++++++----- tests/test_loop.py | 6 +++--- tests/test_proxy_functional.py | 17 ++++++++++++++++- tests/test_worker.py | 6 +++--- 15 files changed, 50 insertions(+), 25 deletions(-) create mode 100644 CHANGES/5927.feature diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 037bcbf0898..cba1198ecb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: needs: lint strategy: matrix: - pyver: [3.7, 3.8, 3.9] + pyver: [3.7, 3.8, 3.9, '3.10'] no-extensions: ['', 'Y'] os: [ubuntu, macos, windows] exclude: diff --git a/CHANGES/5927.feature b/CHANGES/5927.feature new file mode 100644 index 00000000000..dac4f3e5eb9 --- /dev/null +++ b/CHANGES/5927.feature @@ -0,0 +1 @@ +Added support for Python 3.10 to Github Actions CI/CD workflows and fix the related deprecation warnings -- :user:`Hanaasagi`. diff --git a/aiohttp/connector.py b/aiohttp/connector.py index c1b0f89ce16..f1cae659f5e 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -876,9 +876,11 @@ def _make_ssl_context(verified: bool) -> SSLContext: if verified: return ssl.create_default_context() else: - sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext.options |= ssl.OP_NO_SSLv2 sslcontext.options |= ssl.OP_NO_SSLv3 + sslcontext.check_hostname = False + sslcontext.verify_mode = ssl.CERT_NONE try: sslcontext.options |= ssl.OP_NO_COMPRESSION except AttributeError as attr_err: diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index ad80ffe47bb..948b12f99d8 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -55,6 +55,7 @@ __all__ = ("BasicAuth", "ChainMapProxy", "ETag") PY_38 = sys.version_info >= (3, 8) +PY_310 = sys.version_info >= (3, 10) COOKIE_MAX_LENGTH = 4096 diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index 36a860b0171..48c276791f8 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -428,8 +428,10 @@ def get_app(self) -> Application: raise RuntimeError("Did you forget to define get_application()?") def setUp(self) -> None: - if PY_38: - self.loop = asyncio.get_event_loop() + try: + self.loop = asyncio.get_running_loop() + except RuntimeError: + self.loop = asyncio.get_event_loop_policy().get_event_loop() self.loop.run_until_complete(self.setUpAsync()) diff --git a/requirements/dev.txt b/requirements/dev.txt index 62d56075e52..48c73bed7a0 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -196,7 +196,7 @@ pyparsing==2.4.7 # via # -r requirements/lint.txt # packaging -pytest==6.2.2 +pytest==6.2.5 # via # -r requirements/lint.txt # -r requirements/test.txt @@ -230,7 +230,7 @@ requests==2.25.1 # sphinx setuptools-git==1.2 # via -r requirements/test.txt -six==1.15.0 +six==1.16.0 # via # -r requirements/lint.txt # cryptography @@ -313,7 +313,7 @@ yarl==1.6.3 # via -r requirements/base.txt # The following packages are considered to be unsafe in a requirements file: -setuptools==51.3.1 +setuptools==57.4.0 # via # blockdiag # gunicorn diff --git a/requirements/lint.in b/requirements/lint.in index e76aeff67f3..0405d406548 100644 --- a/requirements/lint.in +++ b/requirements/lint.in @@ -4,5 +4,5 @@ flake8-pyi==20.10.0 isort==5.9.3 mypy==0.910; implementation_name=="cpython" pre-commit==2.15.0 -pytest==6.2.2 +pytest==6.2.5 types-chardet==0.1.3 diff --git a/requirements/lint.txt b/requirements/lint.txt index 5a9e0d5e6ad..422328f395f 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -62,13 +62,13 @@ pyflakes==2.3.0 # flake8-pyi pyparsing==2.4.7 # via packaging -pytest==6.2.2 +pytest==6.2.5 # via -r requirements/lint.in pyyaml==5.4.1 # via pre-commit regex==2020.11.13 # via black -six==1.15.0 +six==1.16.0 # via virtualenv toml==0.10.2 # via diff --git a/requirements/test.txt b/requirements/test.txt index 6a476516ed2..a7710f2edca 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -7,7 +7,7 @@ freezegun==1.1.0 mypy==0.910; implementation_name=="cpython" mypy-extensions==0.4.3; implementation_name=="cpython" proxy.py==2.3.1 -pytest==6.2.2 +pytest==6.2.5 pytest-cov==3.0.0 pytest-mock==3.6.1 re-assert==1.1.0 diff --git a/tests/conftest.py b/tests/conftest.py index eda5c60b727..0922d3b21f3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,7 +55,7 @@ def tls_certificate(tls_certificate_authority: Any) -> Any: @pytest.fixture def ssl_ctx(tls_certificate: Any) -> ssl.SSLContext: - ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) tls_certificate.configure_cert(ssl_ctx) return ssl_ctx diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index b3a01957bf7..058d1594ec3 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -2348,7 +2348,7 @@ def create(url: URL, srv: Any): cert = tls_certificate_authority.issue_cert( url.host, "localhost", "127.0.0.1" ) - ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) cert.configure_cert(ssl_ctx) kwargs["ssl"] = ssl_ctx return aiohttp_server(app, **kwargs) diff --git a/tests/test_connector.py b/tests/test_connector.py index 12ab38cb5be..fa8d6f6ff88 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -81,7 +81,11 @@ async def go(app): def create_mocked_conn(conn_closing_result: Optional[Any] = None, **kwargs: Any): assert "loop" not in kwargs - loop = asyncio.get_event_loop() + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.get_event_loop_policy().get_event_loop() + proto = mock.Mock(**kwargs) proto.closed = loop.create_future() proto.closed.set_result(conn_closing_result) @@ -1267,7 +1271,7 @@ async def test___get_ssl_context1(loop: Any) -> None: async def test___get_ssl_context2(loop: Any) -> None: - ctx = ssl.SSLContext() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) conn = aiohttp.TCPConnector() req = mock.Mock() req.is_ssl.return_value = True @@ -1276,7 +1280,7 @@ async def test___get_ssl_context2(loop: Any) -> None: async def test___get_ssl_context3(loop: Any) -> None: - ctx = ssl.SSLContext() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) conn = aiohttp.TCPConnector(ssl=ctx) req = mock.Mock() req.is_ssl.return_value = True @@ -1285,7 +1289,7 @@ async def test___get_ssl_context3(loop: Any) -> None: async def test___get_ssl_context4(loop: Any) -> None: - ctx = ssl.SSLContext() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) conn = aiohttp.TCPConnector(ssl=ctx) req = mock.Mock() req.is_ssl.return_value = True @@ -1294,7 +1298,7 @@ async def test___get_ssl_context4(loop: Any) -> None: async def test___get_ssl_context5(loop: Any) -> None: - ctx = ssl.SSLContext() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) conn = aiohttp.TCPConnector(ssl=ctx) req = mock.Mock() req.is_ssl.return_value = True diff --git a/tests/test_loop.py b/tests/test_loop.py index b72771a175a..f5a4c7774e1 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -39,11 +39,11 @@ async def test_on_startup_hook(self) -> None: self.assertTrue(self.on_startup_called) def test_default_loop(self) -> None: - self.assertIs(self.loop, asyncio.get_event_loop()) + self.assertIs(self.loop, asyncio.get_event_loop_policy().get_event_loop()) def test_default_loop(loop: Any) -> None: - assert asyncio.get_event_loop() is loop + assert asyncio.get_event_loop_policy().get_event_loop() is loop @pytest.mark.xfail(not PY_38, reason="ThreadedChildWatcher is only available in 3.8+") @@ -53,7 +53,7 @@ def test_setup_loop_non_main_thread() -> None: def target() -> None: try: with loop_context() as loop: - assert asyncio.get_event_loop() is loop + assert asyncio.get_event_loop_policy().get_event_loop() is loop loop.run_until_complete(test_subprocess_co(loop)) except Exception as exc: nonlocal child_exc diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py index eb44fd04630..8d0d488929c 100644 --- a/tests/test_proxy_functional.py +++ b/tests/test_proxy_functional.py @@ -1,7 +1,9 @@ # type: ignore import asyncio +import functools import os import pathlib +import platform from re import match as match_regex from typing import Any from unittest import mock @@ -13,7 +15,18 @@ import aiohttp from aiohttp import web -from aiohttp.client_exceptions import ClientConnectionError +from aiohttp.client_exceptions import ClientConnectionError, ClientProxyConnectionError +from aiohttp.helpers import PY_310 + +secure_proxy_xfail_under_py310_except_macos = functools.partial( + pytest.mark.xfail, + PY_310 and platform.system() != "Darwin", + reason=( + "The secure proxy fixture does not seem to work " + "under Python 3.10 on Linux or Windows. " + "See https://github.com/abhinavsingh/proxy.py/issues/622." + ), +) ASYNCIO_SUPPORTS_TLS_IN_TLS = hasattr( asyncio.sslproto._SSLProtocolTransport, @@ -113,6 +126,7 @@ def _pretend_asyncio_supports_tls_in_tls( ) +@secure_proxy_xfail_under_py310_except_macos(raises=ClientProxyConnectionError) @pytest.mark.parametrize("web_server_endpoint_type", ("http", "https")) @pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop") async def test_secure_https_proxy_absolute_path( @@ -139,6 +153,7 @@ async def test_secure_https_proxy_absolute_path( await conn.close() +@secure_proxy_xfail_under_py310_except_macos(raises=AssertionError) @pytest.mark.parametrize("web_server_endpoint_type", ("https",)) @pytest.mark.usefixtures("loop") async def test_https_proxy_unsupported_tls_in_tls( diff --git a/tests/test_worker.py b/tests/test_worker.py index 317945f895a..5f973179228 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -250,7 +250,7 @@ def test__create_ssl_context_without_certs_and_ciphers( worker, tls_certificate_pem_path, ) -> None: - worker.cfg.ssl_version = ssl.PROTOCOL_SSLv23 + worker.cfg.ssl_version = ssl.PROTOCOL_TLS_CLIENT worker.cfg.cert_reqs = ssl.CERT_OPTIONAL worker.cfg.certfile = tls_certificate_pem_path worker.cfg.keyfile = tls_certificate_pem_path @@ -264,7 +264,7 @@ def test__create_ssl_context_with_ciphers( worker, tls_certificate_pem_path, ) -> None: - worker.cfg.ssl_version = ssl.PROTOCOL_SSLv23 + worker.cfg.ssl_version = ssl.PROTOCOL_TLS_CLIENT worker.cfg.cert_reqs = ssl.CERT_OPTIONAL worker.cfg.certfile = tls_certificate_pem_path worker.cfg.keyfile = tls_certificate_pem_path @@ -279,7 +279,7 @@ def test__create_ssl_context_with_ca_certs( tls_ca_certificate_pem_path, tls_certificate_pem_path, ) -> None: - worker.cfg.ssl_version = ssl.PROTOCOL_SSLv23 + worker.cfg.ssl_version = ssl.PROTOCOL_TLS_CLIENT worker.cfg.cert_reqs = ssl.CERT_OPTIONAL worker.cfg.certfile = tls_certificate_pem_path worker.cfg.keyfile = tls_certificate_pem_path From 4ebd52ed1c29c7db52039f213cd2ab268c85bc74 Mon Sep 17 00:00:00 2001 From: Jiyeon Seo Date: Mon, 11 Oct 2021 02:12:43 +0900 Subject: [PATCH 25/66] doc : remove a broken link (#6069) Co-authored-by: USER --- docs/third_party.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/third_party.rst b/docs/third_party.rst index c180322dc15..c6115130d79 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -237,9 +237,6 @@ ask to raise the status. - `aiogram `_ A fully asynchronous library for Telegram Bot API written with asyncio and aiohttp. -- `vk.py `_ - Extremely-fast Python 3.6+ toolkit for create applications work`s with VKAPI. - - `aiohttp-graphql `_ GraphQL and GraphIQL interface for aiohttp. From 9723abf600dc85f28c06422e3481d342742286ba Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 11 Oct 2021 03:01:35 +0200 Subject: [PATCH 26/66] Fix the order of checks in `test_proxy_functional` (#6072) --- tests/test_proxy_functional.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py index 8d0d488929c..ab3f5986ac9 100644 --- a/tests/test_proxy_functional.py +++ b/tests/test_proxy_functional.py @@ -179,14 +179,14 @@ async def test_https_proxy_unsupported_tls_in_tls( r"An HTTPS request is being sent through an HTTPS proxy\. " "This support for TLS in TLS is known to be disabled " r"in the stdlib asyncio\. This is why you'll probably see " - r"an error in the log below.\n\n" + r"an error in the log below\.\n\n" "It is possible to enable it via monkeypatching under " r"Python 3\.7 or higher\. For more details, see:\n" - r"* https://bugs\.python\.org/issue37179\n" - r"* https://github\.com/python/cpython/pull/28073\n\n" + r"\* https://bugs\.python\.org/issue37179\n" + r"\* https://github\.com/python/cpython/pull/28073\n\n" r"You can temporarily patch this as follows:\n" - r"* https://docs\.aiohttp\.org/en/stable/client_advanced\.html#proxy-support\n", - r"* https://github\.com/aio-libs/aiohttp/discussions/6044\n$", + r"\* https://docs\.aiohttp\.org/en/stable/client_advanced\.html#proxy-support\n" + r"\* https://github\.com/aio-libs/aiohttp/discussions/6044\n$" ) type_err = ( r"transport Date: Mon, 11 Oct 2021 10:15:10 +0000 Subject: [PATCH 27/66] Bump multidict from 5.1.0 to 5.2.0 (#6076) Bumps [multidict](https://github.com/aio-libs/multidict) from 5.1.0 to 5.2.0. - [Release notes](https://github.com/aio-libs/multidict/releases) - [Changelog](https://github.com/aio-libs/multidict/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/multidict/compare/v5.1.0...v5.2.0) --- updated-dependencies: - dependency-name: multidict dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/cython.txt | 2 +- requirements/dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/cython.txt b/requirements/cython.txt index c981d498131..8d63ed4b4f9 100644 --- a/requirements/cython.txt +++ b/requirements/cython.txt @@ -6,7 +6,7 @@ # cython==0.29.24 # via -r requirements/cython.in -multidict==5.1.0 +multidict==5.2.0 # via -r requirements/multidict.txt typing_extensions==3.7.4.3 # via -r requirements/cython.in diff --git a/requirements/dev.txt b/requirements/dev.txt index 48c73bed7a0..bcfc0925004 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -309,7 +309,7 @@ wait-for-it==2.2.1 # via -r requirements/dev.in webcolors==1.11.1 # via blockdiag -yarl==1.6.3 +yarl==1.7.0 # via -r requirements/base.txt # The following packages are considered to be unsafe in a requirements file: From 68bae50eb73b11c76e46602407e29fc0815ca673 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 10:18:46 +0000 Subject: [PATCH 28/66] Bump flake8 from 3.9.2 to 4.0.0 (#6075) Bumps [flake8](https://github.com/pycqa/flake8) from 3.9.2 to 4.0.0. - [Release notes](https://github.com/pycqa/flake8/releases) - [Commits](https://github.com/pycqa/flake8/compare/3.9.2...4.0.0) --- updated-dependencies: - dependency-name: flake8 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev.txt | 6 +++--- requirements/lint.in | 2 +- requirements/lint.txt | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index bcfc0925004..e803890af6e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -81,7 +81,7 @@ filelock==3.0.12 # via # -r requirements/lint.txt # virtualenv -flake8==3.9.2 +flake8==4.0.0 # via # -r requirements/lint.txt # flake8-pyi @@ -173,7 +173,7 @@ py==1.10.0 # pytest pycares==4.0.0 # via aiodns -pycodestyle==2.7.0 +pycodestyle==2.8.0 # via # -r requirements/lint.txt # flake8 @@ -181,7 +181,7 @@ pycparser==2.20 # via cffi pydantic==1.8.2 # via python-on-whales -pyflakes==2.3.0 +pyflakes==2.4.0 # via # -r requirements/lint.txt # flake8 diff --git a/requirements/lint.in b/requirements/lint.in index 0405d406548..1fa4c0784be 100644 --- a/requirements/lint.in +++ b/requirements/lint.in @@ -1,5 +1,5 @@ black==21.7b0; implementation_name=="cpython" -flake8==3.9.2 +flake8==4.0.0 flake8-pyi==20.10.0 isort==5.9.3 mypy==0.910; implementation_name=="cpython" diff --git a/requirements/lint.txt b/requirements/lint.txt index 422328f395f..79dab5f5b1f 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -22,7 +22,7 @@ distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -flake8==3.9.2 +flake8==4.0.0 # via # -r requirements/lint.in # flake8-pyi @@ -54,9 +54,9 @@ pre-commit==2.15.0 # via -r requirements/lint.in py==1.10.0 # via pytest -pycodestyle==2.7.0 +pycodestyle==2.8.0 # via flake8 -pyflakes==2.3.0 +pyflakes==2.4.0 # via # flake8 # flake8-pyi From 7b3384ecfdb1edb20dc8849307af7762c7bd0203 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 19:15:02 +0100 Subject: [PATCH 29/66] [pre-commit.ci] pre-commit autoupdate (#6077) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/flake8: 3.9.2 → 4.0.1](https://github.com/PyCQA/flake8/compare/3.9.2...4.0.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f29d6f1c58c..1cf75a7fb3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,7 +65,7 @@ repos: - id: pyupgrade args: ['--py36-plus'] - repo: https://github.com/PyCQA/flake8 - rev: '3.9.2' + rev: '4.0.1' hooks: - id: flake8 exclude: "^docs/" From 02bb2009cda3814c5cbaddc5b4a466bd0067dbdc Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 12 Oct 2021 03:40:28 +0300 Subject: [PATCH 30/66] Add support for Python 3.10 (#6079) --- .github/workflows/ci.yml | 4 ++-- CHANGES/6079.feature | 1 + CONTRIBUTORS.txt | 1 + setup.py | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 CHANGES/6079.feature diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cba1198ecb9..6ce6a67adfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,7 +176,7 @@ jobs: name: Linux strategy: matrix: - pyver: [cp37-cp37m, cp38-cp38, cp39-cp39] + pyver: [cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] arch: [x86_64, aarch64, i686, ppc64le, s390x] fail-fast: false runs-on: ubuntu-latest @@ -224,7 +224,7 @@ jobs: name: Binary wheels strategy: matrix: - pyver: [3.7, 3.8, 3.9] + pyver: [3.7, 3.8, 3.9, '3.10'] os: [macos, windows] arch: [x86, x64] exclude: diff --git a/CHANGES/6079.feature b/CHANGES/6079.feature new file mode 100644 index 00000000000..25dc6039b44 --- /dev/null +++ b/CHANGES/6079.feature @@ -0,0 +1 @@ +Add Trove classifier and create binary wheels for 3.10. -- :user:`hugovk`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 45d9a1a5fb5..f94ab0f7438 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -135,6 +135,7 @@ Hrishikesh Paranjape Hu Bo Hugh Young Hugo Herter +Hugo van Kemenade Hynek Schlawack Igor Alexandrov Igor Davydenko diff --git a/setup.py b/setup.py index 54b548c7b44..d9c7ef68a04 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,7 @@ def read(f): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Development Status :: 5 - Production/Stable", "Operating System :: POSIX", "Operating System :: MacOS :: MacOS X", From e46b40e5c41a0759539dab5187cc344b8b0ee658 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Oct 2021 10:12:28 +0000 Subject: [PATCH 31/66] Bump coverage from 6.0.1 to 6.0.2 (#6082) Bumps [coverage](https://github.com/nedbat/coveragepy) from 6.0.1 to 6.0.2. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/6.0.1...6.0.2) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index a7710f2edca..ccdb1b8ff68 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,7 +1,7 @@ -r base.txt Brotli==1.0.9 -coverage==6.0.1 +coverage==6.0.2 cryptography==3.3.1; platform_machine!="i686" and python_version<"3.9" # no 32-bit wheels; no python 3.9 wheels yet freezegun==1.1.0 mypy==0.910; implementation_name=="cpython" From 36c76011510bb926333796b5969f24332a215899 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Oct 2021 10:20:18 +0000 Subject: [PATCH 32/66] Bump python-on-whales from 0.27.0 to 0.28.0 (#6085) Bumps [python-on-whales](https://github.com/gabrieldemarmiesse/python-on-whales) from 0.27.0 to 0.28.0. - [Release notes](https://github.com/gabrieldemarmiesse/python-on-whales/releases) - [Commits](https://github.com/gabrieldemarmiesse/python-on-whales/compare/v0.27.0...v0.28.0) --- updated-dependencies: - dependency-name: python-on-whales dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev.in | 2 +- requirements/dev.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/dev.in b/requirements/dev.in index 3da176a4bf5..1da2b9d5bc0 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -2,5 +2,5 @@ -r test.txt -r doc.txt cherry_picker==2.0.0; python_version>="3.6" -python-on-whales==0.27.0 +python-on-whales==0.28.0 wait-for-it==2.2.1 diff --git a/requirements/dev.txt b/requirements/dev.txt index e803890af6e..7cbd5a8876b 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -63,7 +63,7 @@ click==7.1.2 # wait-for-it click-default-group==1.2.2 # via towncrier -coverage[toml]==6.0.1 +coverage[toml]==6.0.2 # via # -r requirements/test.txt # pytest-cov @@ -208,7 +208,7 @@ pytest-mock==3.6.1 # via -r requirements/test.txt python-dateutil==2.8.1 # via freezegun -python-on-whales==0.27.0 +python-on-whales==0.28.0 # via -r requirements/dev.in pytz==2020.5 # via babel From 42254889dead2fc504dbcd7a05ca6c463e3d6005 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Oct 2021 15:09:51 +0300 Subject: [PATCH 33/66] Bump flake8 from 4.0.0 to 4.0.1 (#6083) Bumps [flake8](https://github.com/pycqa/flake8) from 4.0.0 to 4.0.1. - [Release notes](https://github.com/pycqa/flake8/releases) - [Commits](https://github.com/pycqa/flake8/compare/4.0.0...4.0.1) --- updated-dependencies: - dependency-name: flake8 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev.txt | 2 +- requirements/lint.in | 2 +- requirements/lint.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 7cbd5a8876b..52612089bb9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -81,7 +81,7 @@ filelock==3.0.12 # via # -r requirements/lint.txt # virtualenv -flake8==4.0.0 +flake8==4.0.1 # via # -r requirements/lint.txt # flake8-pyi diff --git a/requirements/lint.in b/requirements/lint.in index 1fa4c0784be..daea0fba678 100644 --- a/requirements/lint.in +++ b/requirements/lint.in @@ -1,5 +1,5 @@ black==21.7b0; implementation_name=="cpython" -flake8==4.0.0 +flake8==4.0.1 flake8-pyi==20.10.0 isort==5.9.3 mypy==0.910; implementation_name=="cpython" diff --git a/requirements/lint.txt b/requirements/lint.txt index 79dab5f5b1f..38c1c2c3492 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -22,7 +22,7 @@ distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -flake8==4.0.0 +flake8==4.0.1 # via # -r requirements/lint.in # flake8-pyi From cc872be89fef230eb6b8daad86d8d6ea20b519ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Oct 2021 10:14:14 +0000 Subject: [PATCH 34/66] Bump actions/checkout from 2.3.4 to 2.3.5 (#6094) Bumps [actions/checkout](https://github.com/actions/checkout) from 2.3.4 to 2.3.5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2.3.4...v2.3.5) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 10 +++++----- .github/workflows/update-pre-commit.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ce6a67adfa..0b92a258785 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 with: submodules: true - name: Setup Python 3.8 @@ -98,7 +98,7 @@ jobs: timeout-minutes: 15 steps: - name: Checkout - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 with: submodules: true - name: Setup Python ${{ matrix.pyver }} @@ -153,7 +153,7 @@ jobs: needs: pre-deploy steps: - name: Checkout - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 with: submodules: true - name: Setup Python 3.8 @@ -186,7 +186,7 @@ jobs: needs: pre-deploy steps: - name: Checkout - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 with: submodules: true - name: Set up QEMU @@ -235,7 +235,7 @@ jobs: needs: pre-deploy steps: - name: Checkout - uses: actions/checkout@v2.3.4 + uses: actions/checkout@v2.3.5 with: submodules: true - name: Setup Python 3.8 diff --git a/.github/workflows/update-pre-commit.yml b/.github/workflows/update-pre-commit.yml index 56a964401d8..44c784e6aa5 100644 --- a/.github/workflows/update-pre-commit.yml +++ b/.github/workflows/update-pre-commit.yml @@ -7,7 +7,7 @@ jobs: if: github.repository_owner == 'aiohttp' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.3.5 - name: Set up Python uses: actions/setup-python@v2 with: From 86e6e35e36fea068c2fe02b615411614df0d305a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Oct 2021 10:20:25 +0000 Subject: [PATCH 35/66] Bump aiosignal from 1.1.2 to 1.2.0 (#6096) Bumps [aiosignal](https://github.com/aio-libs/aiosignal) from 1.1.2 to 1.2.0. - [Release notes](https://github.com/aio-libs/aiosignal/releases) - [Changelog](https://github.com/aio-libs/aiosignal/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiosignal/compare/v1.1.2...v1.2.0) --- updated-dependencies: - dependency-name: aiosignal dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index c4cdc49189e..f486f27580f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,7 +1,7 @@ -r multidict.txt # required c-ares will not build on windows and has build problems on Macos Python<3.7 aiodns==3.0.0; sys_platform=="linux" or sys_platform=="darwin" and python_version>="3.7" -aiosignal==1.1.2 +aiosignal==1.2.0 async-timeout==4.0.0a3 asynctest==0.13.0; python_version<"3.8" Brotli==1.0.9 From 6542ac82689bab211578c04ecf54216470829d5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Oct 2021 10:26:39 +0000 Subject: [PATCH 36/66] Bump frozenlist from 1.1.1 to 1.2.0 (#6097) Bumps [frozenlist](https://github.com/aio-libs/frozenlist) from 1.1.1 to 1.2.0. - [Release notes](https://github.com/aio-libs/frozenlist/releases) - [Changelog](https://github.com/aio-libs/frozenlist/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/frozenlist/compare/v1.1.1...v1.2.0) --- updated-dependencies: - dependency-name: frozenlist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index f486f27580f..add03f1d06a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,7 +7,7 @@ asynctest==0.13.0; python_version<"3.8" Brotli==1.0.9 cchardet==2.1.7 chardet==4.0.0 -frozenlist==1.1.1 +frozenlist==1.2.0 gunicorn==20.1.0 typing_extensions==3.7.4.3 uvloop==0.14.0; platform_system!="Windows" and implementation_name=="cpython" and python_version<"3.9" # MagicStack/uvloop#14 From 2904573dc4d40883812fce2778352f1b088ac9f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Oct 2021 10:26:09 +0000 Subject: [PATCH 37/66] Bump aiosignal from 1.1.2 to 1.2.0 (#6102) Bumps [aiosignal](https://github.com/aio-libs/aiosignal) from 1.1.2 to 1.2.0. - [Release notes](https://github.com/aio-libs/aiosignal/releases) - [Changelog](https://github.com/aio-libs/aiosignal/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiosignal/compare/v1.1.2...v1.2.0) --- updated-dependencies: - dependency-name: aiosignal dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 52612089bb9..bd79518db7d 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,7 +8,7 @@ aiodns==3.0.0 ; sys_platform == "linux" or sys_platform == "darwin" and python_v # via -r requirements/base.txt aiohttp-theme==0.1.6 # via -r requirements/doc.txt -aiosignal==1.1.2 +aiosignal==1.2.0 # via -r requirements/base.txt alabaster==0.7.12 # via sphinx @@ -89,7 +89,7 @@ flake8-pyi==20.10.0 # via -r requirements/lint.txt freezegun==1.1.0 # via -r requirements/test.txt -frozenlist==1.1.1 +frozenlist==1.2.0 # via # -r requirements/base.txt # aiosignal From 2d5597e6743bb4c579adb6f9b67482d5d35978c7 Mon Sep 17 00:00:00 2001 From: TAHRI Ahmed R Date: Wed, 20 Oct 2021 13:29:26 +0200 Subject: [PATCH 38/66] Switch default fallback encoding detection lib to `charset-normalizer` This change improves the performance of the encoding detection by substituting the backend lib with the new `Charset-Normalizer` (used to be `Chardet`). The patch is backward-compatible API wise, except that the dependency is different. PR #5930 Co-authored-by: Sviatoslav Sydorenko --- CHANGES/5930.feature | 1 + CONTRIBUTORS.txt | 1 + README.rst | 4 ++-- aiohttp/client_reqrep.py | 2 +- docs/client_reference.rst | 16 ++++++++-------- docs/glossary.rst | 7 ++++--- docs/index.rst | 8 ++++---- docs/spelling_wordlist.txt | 2 ++ requirements/base.txt | 2 +- requirements/dev.txt | 2 +- setup.py | 2 +- 11 files changed, 26 insertions(+), 21 deletions(-) create mode 100644 CHANGES/5930.feature diff --git a/CHANGES/5930.feature b/CHANGES/5930.feature new file mode 100644 index 00000000000..17cecee40d9 --- /dev/null +++ b/CHANGES/5930.feature @@ -0,0 +1 @@ +Switched ``chardet`` to ``charset-normalizer`` for guessing the HTTP payload body encoding -- :user:`Ousret`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index f94ab0f7438..f42331eeaef 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -7,6 +7,7 @@ Adam Horacek Adam Mills Adrian Krupa Adrián Chaves +Ahmed Tahri Alan Tse Alec Hanefeld Alejandro Gómez diff --git a/README.rst b/README.rst index 6abb34bef56..143c3a59baf 100644 --- a/README.rst +++ b/README.rst @@ -164,14 +164,14 @@ Requirements - Python >= 3.7 - async-timeout_ -- chardet_ +- charset-normalizer_ - multidict_ - yarl_ Optionally you may install the cChardet_ and aiodns_ libraries (highly recommended for sake of speed). -.. _chardet: https://pypi.python.org/pypi/chardet +.. _charset-normalizer: https://pypi.org/project/charset-normalizer .. _aiodns: https://pypi.python.org/pypi/aiodns .. _multidict: https://pypi.python.org/pypi/multidict .. _yarl: https://pypi.python.org/pypi/yarl diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 8c64db600e6..41602afe703 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -70,7 +70,7 @@ try: import cchardet as chardet except ImportError: # pragma: no cover - import chardet # type: ignore[no-redef] + import charset_normalizer as chardet # type: ignore[no-redef] __all__ = ("ClientRequest", "ClientResponse", "RequestInfo", "Fingerprint") diff --git a/docs/client_reference.rst b/docs/client_reference.rst index ed935a2da1a..86bad7f0c95 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1374,10 +1374,10 @@ Response object specified *encoding* parameter. If *encoding* is ``None`` content encoding is autocalculated - using ``Content-Type`` HTTP header and *chardet* tool if the + using ``Content-Type`` HTTP header and *charset-normalizer* tool if the header is not provided by server. - :term:`cchardet` is used with fallback to :term:`chardet` if + :term:`cchardet` is used with fallback to :term:`charset-normalizer` if *cchardet* is not available. Close underlying connection if data reading gets an error, @@ -1389,14 +1389,14 @@ Response object :return str: decoded *BODY* - :raise LookupError: if the encoding detected by chardet or cchardet is + :raise LookupError: if the encoding detected by cchardet is unknown by Python (e.g. VISCII). .. note:: If response has no ``charset`` info in ``Content-Type`` HTTP - header :term:`cchardet` / :term:`chardet` is used for content - encoding autodetection. + header :term:`cchardet` / :term:`charset-normalizer` is used for + content encoding autodetection. It may hurt performance. If page encoding is known passing explicit *encoding* parameter might help:: @@ -1411,7 +1411,7 @@ Response object a ``read`` call will be done, If *encoding* is ``None`` content encoding is autocalculated - using :term:`cchardet` or :term:`chardet` as fallback if + using :term:`cchardet` or :term:`charset-normalizer` as fallback if *cchardet* is not available. if response's `content-type` does not match `content_type` parameter @@ -1449,11 +1449,11 @@ Response object Automatically detect content encoding using ``charset`` info in ``Content-Type`` HTTP header. If this info is not exists or there are no appropriate codecs for encoding then :term:`cchardet` / - :term:`chardet` is used. + :term:`charset-normalizer` is used. Beware that it is not always safe to use the result of this function to decode a response. Some encodings detected by cchardet are not known by - Python (e.g. VISCII). + Python (e.g. VISCII). *charset-normalizer* is not concerned by that issue. :raise RuntimeError: if called before the body has been read, for :term:`cchardet` usage diff --git a/docs/glossary.rst b/docs/glossary.rst index c2da11817af..1de13dc7d04 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -45,11 +45,12 @@ Any object that can be called. Use :func:`callable` to check that. - chardet + charset-normalizer - The Universal Character Encoding Detector + The Real First Universal Charset Detector. + Open, modern and actively maintained alternative to Chardet. - https://pypi.python.org/pypi/chardet/ + https://pypi.org/project/charset-normalizer/ cchardet diff --git a/docs/index.rst b/docs/index.rst index 0f627bd170f..6be4898e029 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,7 +34,7 @@ Library Installation $ pip install aiohttp You may want to install *optional* :term:`cchardet` library as faster -replacement for :term:`chardet`: +replacement for :term:`charset-normalizer`: .. code-block:: bash @@ -51,7 +51,7 @@ This option is highly recommended: Installing speedups altogether ------------------------------ -The following will get you ``aiohttp`` along with :term:`chardet`, +The following will get you ``aiohttp`` along with :term:`charset-normalizer`, :term:`aiodns` and ``Brotli`` in one bundle. No need to type separate commands anymore! @@ -148,11 +148,11 @@ Dependencies - Python 3.7+ - *async_timeout* -- *chardet* +- *charset-normalizer* - *multidict* - *yarl* - *Optional* :term:`cchardet` as faster replacement for - :term:`chardet`. + :term:`charset-normalizer`. Install it explicitly via: diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index e7a608fd658..c4182c2f06d 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -124,6 +124,7 @@ canonicalization canonicalize cchardet ceil +Chardet charset charsetdetect chunked @@ -228,6 +229,7 @@ namespace netrc nginx noop +normalizer nowait optimizations os diff --git a/requirements/base.txt b/requirements/base.txt index add03f1d06a..4c995d352d6 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,7 +6,7 @@ async-timeout==4.0.0a3 asynctest==0.13.0; python_version<"3.8" Brotli==1.0.9 cchardet==2.1.7 -chardet==4.0.0 +charset-normalizer==2.0.4 frozenlist==1.2.0 gunicorn==20.1.0 typing_extensions==3.7.4.3 diff --git a/requirements/dev.txt b/requirements/dev.txt index bd79518db7d..df6a12a9ad4 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -46,7 +46,7 @@ cfgv==3.2.0 # via # -r requirements/lint.txt # pre-commit -chardet==4.0.0 +charset-normalizer==2.0.4 # via # -r requirements/base.txt # requests diff --git a/setup.py b/setup.py index d9c7ef68a04..a73d331ea07 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ raise RuntimeError("Unable to determine version.") install_requires = [ - "chardet>=2.0,<5.0", + "charset-normalizer>=2.0,<3.0", "multidict>=4.5,<7.0", "async_timeout>=4.0a2,<5.0", 'asynctest==0.13.0; python_version<"3.8"', From cd4c700d3fc19f15403b5c0de674f39768b61000 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 20 Oct 2021 13:43:49 +0200 Subject: [PATCH 39/66] Free resp conn @ `test_mark_formdata_as_processed` (#6107) --- tests/test_formdata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_formdata.py b/tests/test_formdata.py index 4b1e4cc855f..4655f484464 100644 --- a/tests/test_formdata.py +++ b/tests/test_formdata.py @@ -94,8 +94,10 @@ async def test_mark_formdata_as_processed() -> None: data = FormData() data.add_field("test", "test_value", content_type="application/json") - await session.post(url, data=data) + resp = await session.post(url, data=data) assert len(data._writer._parts) == 1 with pytest.raises(RuntimeError): await session.post(url, data=data) + + resp.release() From 816e0bc81bb809f54fd7a88cd699a93a75b82885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 20 Oct 2021 15:12:09 +0300 Subject: [PATCH 40/66] Spelling fixes (#5897) --- HISTORY.rst | 2 +- aiohttp/http_parser.py | 4 ++-- tests/test_cookiejar.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 44c1484f917..b3c3b97f886 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -950,7 +950,7 @@ Misc - Added session's `raise_for_status` parameter, automatically calls raise_for_status() on any request. (`#1724 `_) -- `response.json()` raises `ClientReponseError` exception if response's +- `response.json()` raises `ClientResponseError` exception if response's content type does not match (`#1723 `_) - Cleanup timer and loop handle on any client exception. diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 1045b6c0926..3feac7ac1bb 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -316,7 +316,7 @@ def feed_data( finally: self._lines.clear() - def get_content_lenght() -> Optional[int]: + def get_content_length() -> Optional[int]: # payload length length_hdr = msg.headers.get(CONTENT_LENGTH) if length_hdr is None: @@ -332,7 +332,7 @@ def get_content_lenght() -> Optional[int]: return length - length = get_content_lenght() + length = get_content_length() # do not support old websocket spec if SEC_WEBSOCKET_KEY1 in msg.headers: raise InvalidHeader(SEC_WEBSOCKET_KEY1) diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index 5a740e0dfb8..a0212dcd049 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -26,7 +26,7 @@ def cookies_to_send(): "different-domain-cookie=sixth; Domain=different.org; " "secure-cookie=seventh; Domain=secure.com; Secure; " "no-path-cookie=eighth; Domain=pathtest.com; " - "path1-cookie=nineth; Domain=pathtest.com; Path=/; " + "path1-cookie=ninth; Domain=pathtest.com; Path=/; " "path2-cookie=tenth; Domain=pathtest.com; Path=/one; " "path3-cookie=eleventh; Domain=pathtest.com; Path=/one/two; " "path4-cookie=twelfth; Domain=pathtest.com; Path=/one/two/; " @@ -52,7 +52,7 @@ def cookies_to_send_with_expired(): "different-domain-cookie=sixth; Domain=different.org; " "secure-cookie=seventh; Domain=secure.com; Secure; " "no-path-cookie=eighth; Domain=pathtest.com; " - "path1-cookie=nineth; Domain=pathtest.com; Path=/; " + "path1-cookie=ninth; Domain=pathtest.com; Path=/; " "path2-cookie=tenth; Domain=pathtest.com; Path=/one; " "path3-cookie=eleventh; Domain=pathtest.com; Path=/one/two; " "path4-cookie=twelfth; Domain=pathtest.com; Path=/one/two/; " @@ -78,7 +78,7 @@ def cookies_to_receive(): "different-domain-cookie=sixth; Domain=different.org; Path=/; " "no-path-cookie=seventh; Domain=pathtest.com; " "path-cookie=eighth; Domain=pathtest.com; Path=/somepath; " - "wrong-path-cookie=nineth; Domain=pathtest.com; Path=somepath;" + "wrong-path-cookie=ninth; Domain=pathtest.com; Path=somepath;" ) @@ -254,7 +254,7 @@ async def test_domain_filter_ip_cookie_send(loop: Any) -> None: "different-domain-cookie=sixth; Domain=different.org; " "secure-cookie=seventh; Domain=secure.com; Secure; " "no-path-cookie=eighth; Domain=pathtest.com; " - "path1-cookie=nineth; Domain=pathtest.com; Path=/; " + "path1-cookie=ninth; Domain=pathtest.com; Path=/; " "path2-cookie=tenth; Domain=pathtest.com; Path=/one; " "path3-cookie=eleventh; Domain=pathtest.com; Path=/one/two; " "path4-cookie=twelfth; Domain=pathtest.com; Path=/one/two/; " @@ -371,7 +371,7 @@ def setUp(self): "different-domain-cookie=sixth; Domain=different.org; " "secure-cookie=seventh; Domain=secure.com; Secure; " "no-path-cookie=eighth; Domain=pathtest.com; " - "path1-cookie=nineth; Domain=pathtest.com; Path=/; " + "path1-cookie=ninth; Domain=pathtest.com; Path=/; " "path2-cookie=tenth; Domain=pathtest.com; Path=/one; " "path3-cookie=eleventh; Domain=pathtest.com; Path=/one/two; " "path4-cookie=twelfth; Domain=pathtest.com; Path=/one/two/; " @@ -394,7 +394,7 @@ def setUp(self): "different-domain-cookie=sixth; Domain=different.org; Path=/; " "no-path-cookie=seventh; Domain=pathtest.com; " "path-cookie=eighth; Domain=pathtest.com; Path=/somepath; " - "wrong-path-cookie=nineth; Domain=pathtest.com; Path=somepath;" + "wrong-path-cookie=ninth; Domain=pathtest.com; Path=somepath;" ) async def make_jar(): From 728505c119e6abe96968f13567aaa7e4cbcdbe8f Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Wed, 20 Oct 2021 14:35:25 +0200 Subject: [PATCH 41/66] Remove external test dependency to http://httpbin.org (#5843) --- CHANGES/5840.bugfix | 1 + tests/test_formdata.py | 28 +++++++++++++++++----------- 2 files changed, 18 insertions(+), 11 deletions(-) create mode 100644 CHANGES/5840.bugfix diff --git a/CHANGES/5840.bugfix b/CHANGES/5840.bugfix new file mode 100644 index 00000000000..7380aa246f2 --- /dev/null +++ b/CHANGES/5840.bugfix @@ -0,0 +1 @@ +Remove external test dependency to http://httpbin.org diff --git a/tests/test_formdata.py b/tests/test_formdata.py index 4655f484464..e1c642c65c3 100644 --- a/tests/test_formdata.py +++ b/tests/test_formdata.py @@ -4,7 +4,7 @@ import pytest -from aiohttp import ClientSession, FormData +from aiohttp import FormData, web @pytest.fixture @@ -88,16 +88,22 @@ async def test_formdata_field_name_is_not_quoted(buf: Any, writer: Any) -> None: assert b'name="email 1"' in buf -async def test_mark_formdata_as_processed() -> None: - async with ClientSession() as session: - url = "http://httpbin.org/anything" - data = FormData() - data.add_field("test", "test_value", content_type="application/json") +async def test_mark_formdata_as_processed(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response() - resp = await session.post(url, data=data) - assert len(data._writer._parts) == 1 + app = web.Application() + app.add_routes([web.post("/", handler)]) - with pytest.raises(RuntimeError): - await session.post(url, data=data) + client = await aiohttp_client(app) - resp.release() + data = FormData() + data.add_field("test", "test_value", content_type="application/json") + + resp = await client.post("/", data=data) + assert len(data._writer._parts) == 1 + + with pytest.raises(RuntimeError): + await client.post("/", data=data) + + resp.release() From bdcbc2d06382805be6c8e0b13c9ca2ebdc62a848 Mon Sep 17 00:00:00 2001 From: Nick Gregory Date: Wed, 20 Oct 2021 09:28:24 -0400 Subject: [PATCH 42/66] fix typo in ServerDisconnectedError docs (#5410) * fix typo in ServerDisconnectedError docs * Unignore `ServerDisconnectionError` in Sphinx Co-authored-by: Sviatoslav Sydorenko --- docs/client_reference.rst | 2 +- docs/conf.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 86bad7f0c95..accd4f627de 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -2163,7 +2163,7 @@ Connection errors Server disconnected. - Derived from :exc:`~aiohttp.ServerDisconnectionError` + Derived from :exc:`~aiohttp.ServerConnectionError` .. attribute:: message diff --git a/docs/conf.py b/docs/conf.py index b7d80de93dc..4a0130a0172 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -396,7 +396,6 @@ ("py:class", "aiohttp.abc.AbstractResolver"), # undocumented ("py:func", "aiohttp.ws_connect"), # undocumented ("py:meth", "start"), # undocumented - ("py:exc", "aiohttp.ServerDisconnectionError"), # undocumented ("py:exc", "aiohttp.ClientHttpProxyError"), # undocumented ("py:class", "asyncio.AbstractServer"), # undocumented ("py:mod", "aiohttp.test_tools"), # undocumented From 3eb01213d38d51a1a631786e03def631563140fc Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Thu, 21 Oct 2021 00:41:40 +1100 Subject: [PATCH 43/66] docs: fix simple typo, pipelinig -> pipelining (#5363) --- aiohttp/web_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index aab4f31f297..e3f15329841 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -393,7 +393,7 @@ def keep_alive(self, val: bool) -> None: self._keepalive_handle = None def close(self) -> None: - """Stop accepting new pipelinig messages and close + """Stop accepting new pipelining messages and close connection when handlers done processing messages""" self._close = True if self._waiter: From abdb14277854663f7aac3fad6dc99fd14deaf536 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Oct 2021 10:22:35 +0000 Subject: [PATCH 44/66] Bump charset-normalizer from 2.0.4 to 2.0.7 (#6115) Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.4 to 2.0.7. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.4...2.0.7) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 4c995d352d6..20021bd8f54 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,7 +6,7 @@ async-timeout==4.0.0a3 asynctest==0.13.0; python_version<"3.8" Brotli==1.0.9 cchardet==2.1.7 -charset-normalizer==2.0.4 +charset-normalizer==2.0.7 frozenlist==1.2.0 gunicorn==20.1.0 typing_extensions==3.7.4.3 From 7b5611c642e16afdfcaceebfa5bbe6d2c3330f3f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 23 Oct 2021 16:11:05 +0300 Subject: [PATCH 45/66] Fix MacOS tests (#6119) --- tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 0922d3b21f3..b8ba9ccf8b3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,9 @@ try: import trustme + # Check if the CA is available in runtime, MacOS on Py3.10 fails somehow + trustme.CA() + TRUSTME: bool = True except ImportError: TRUSTME = False From f5bc200938ebe83551fa8dff2a528f5e2a92e4ab Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 23 Oct 2021 16:30:21 +0300 Subject: [PATCH 46/66] Fix skip reason when trustme is not available --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index b8ba9ccf8b3..ecb26c3f161 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,7 +43,7 @@ @pytest.fixture def tls_certificate_authority() -> Any: if not TRUSTME: - pytest.xfail("trustme fails on 32bit Linux") + pytest.xfail("trustme is not supported") return trustme.CA() From b8da14cb3bcad6d3a2990a518102e1175df824f8 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 23 Oct 2021 14:34:49 +0100 Subject: [PATCH 47/66] Remove legacy examples (#5625) * Remove legacy examples. * Add changes file. * Revert exclude. Co-authored-by: Andrew Svetlov --- .mypy.ini | 1 - CHANGES/5625.misc | 1 + examples/legacy/crawl.py | 106 --------------- examples/legacy/srv.py | 178 ------------------------- examples/legacy/tcp_protocol_parser.py | 172 ------------------------ 5 files changed, 1 insertion(+), 457 deletions(-) create mode 100644 CHANGES/5625.misc delete mode 100755 examples/legacy/crawl.py delete mode 100755 examples/legacy/srv.py delete mode 100755 examples/legacy/tcp_protocol_parser.py diff --git a/.mypy.ini b/.mypy.ini index ebcd461441c..9a888ebffff 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,7 +1,6 @@ [mypy] files = aiohttp, examples, tests check_untyped_defs = True -exclude = examples/legacy/ follow_imports_for_stubs = True #disallow_any_decorated = True disallow_any_generics = True diff --git a/CHANGES/5625.misc b/CHANGES/5625.misc new file mode 100644 index 00000000000..69e2723e6d6 --- /dev/null +++ b/CHANGES/5625.misc @@ -0,0 +1 @@ +Remove (broken) legacy examples. diff --git a/examples/legacy/crawl.py b/examples/legacy/crawl.py deleted file mode 100755 index 026de536d9a..00000000000 --- a/examples/legacy/crawl.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python3 - -import asyncio -import logging -import re -import signal -import sys -import urllib.parse - -import aiohttp - - -class Crawler: - def __init__(self, rooturl, maxtasks=100): - self.rooturl = rooturl - self.todo = set() - self.busy = set() - self.done = {} - self.tasks = set() - self.sem = asyncio.Semaphore(maxtasks) - - # connector stores cookies between requests and uses connection pool - self.session = aiohttp.ClientSession() - - async def run(self): - t = asyncio.ensure_future(self.addurls([(self.rooturl, "")])) - await asyncio.sleep(1) - while self.busy: - await asyncio.sleep(1) - - await t - await self.session.close() - - async def addurls(self, urls): - for url, parenturl in urls: - url = urllib.parse.urljoin(parenturl, url) - url, frag = urllib.parse.urldefrag(url) - if ( - url.startswith(self.rooturl) - and url not in self.busy - and url not in self.done - and url not in self.todo - ): - self.todo.add(url) - await self.sem.acquire() - task = asyncio.ensure_future(self.process(url)) - task.add_done_callback(lambda t: self.sem.release()) - task.add_done_callback(self.tasks.remove) - self.tasks.add(task) - - async def process(self, url): - print("processing:", url) - - self.todo.remove(url) - self.busy.add(url) - try: - resp = await self.session.get(url) - except Exception as exc: - print("...", url, "has error", repr(str(exc))) - self.done[url] = False - else: - if resp.status == 200 and ("text/html" in resp.headers.get("content-type")): - data = (await resp.read()).decode("utf-8", "replace") - urls = re.findall(r'(?i)href=["\']?([^\s"\'<>]+)', data) - asyncio.Task(self.addurls([(u, url) for u in urls])) - - resp.close() - self.done[url] = True - - self.busy.remove(url) - print( - len(self.done), - "completed tasks,", - len(self.tasks), - "still pending, todo", - len(self.todo), - ) - - -def main(): - loop = asyncio.get_event_loop() - - c = Crawler(sys.argv[1]) - asyncio.ensure_future(c.run()) - - try: - loop.add_signal_handler(signal.SIGINT, loop.stop) - except RuntimeError: - pass - loop.run_forever() - print("todo:", len(c.todo)) - print("busy:", len(c.busy)) - print("done:", len(c.done), "; ok:", sum(c.done.values())) - print("tasks:", len(c.tasks)) - - -if __name__ == "__main__": - if "--iocp" in sys.argv: - from asyncio import events, windows_events - - sys.argv.remove("--iocp") - logging.info("using iocp") - el = windows_events.ProactorEventLoop() - events.set_event_loop(el) - - main() diff --git a/examples/legacy/srv.py b/examples/legacy/srv.py deleted file mode 100755 index 628b6f332f1..00000000000 --- a/examples/legacy/srv.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python3 -"""Simple server written using an event loop.""" - -import argparse -import asyncio -import logging -import os -import sys - -import aiohttp -import aiohttp.server - -try: - import ssl -except ImportError: # pragma: no cover - ssl = None - - -class HttpRequestHandler(aiohttp.server.ServerHttpProtocol): - async def handle_request(self, message, payload): - print( - "method = {!r}; path = {!r}; version = {!r}".format( - message.method, message.path, message.version - ) - ) - - path = message.path - - if not (path.isprintable() and path.startswith("/")) or "/." in path: - print("bad path", repr(path)) - path = None - else: - path = "." + path - if not os.path.exists(path): - print("no file", repr(path)) - path = None - else: - isdir = os.path.isdir(path) - - if not path: - raise aiohttp.HttpProcessingError(code=404) - - for hdr, val in message.headers.items(): - print(hdr, val) - - if isdir and not path.endswith("/"): - path = path + "/" - raise aiohttp.HttpProcessingError( - code=302, headers=(("URI", path), ("Location", path)) - ) - - response = aiohttp.Response(self.writer, 200, http_version=message.version) - response.add_header("Transfer-Encoding", "chunked") - - # content encoding - accept_encoding = message.headers.get("accept-encoding", "").lower() - if "deflate" in accept_encoding: - response.add_header("Content-Encoding", "deflate") - response.add_compression_filter("deflate") - elif "gzip" in accept_encoding: - response.add_header("Content-Encoding", "gzip") - response.add_compression_filter("gzip") - - response.add_chunking_filter(1025) - - if isdir: - response.add_header("Content-type", "text/html") - response.send_headers() - - response.write(b"
    \r\n") - for name in sorted(os.listdir(path)): - if name.isprintable() and not name.startswith("."): - try: - bname = name.encode("ascii") - except UnicodeError: - pass - else: - if os.path.isdir(os.path.join(path, name)): - response.write( - b'
  • ' - + bname - + b"/
    • \r\n" - ) - else: - response.write( - b'
  • ' - + bname - + b"
    • \r\n" - ) - response.write(b"
") - else: - response.add_header("Content-type", "text/plain") - response.send_headers() - - try: - with open(path, "rb") as fp: - chunk = fp.read(8192) - while chunk: - response.write(chunk) - chunk = fp.read(8192) - except OSError: - response.write(b"Cannot open") - - await response.write_eof() - if response.keep_alive(): - self.keep_alive(True) - - -ARGS = argparse.ArgumentParser(description="Run simple HTTP server.") -ARGS.add_argument( - "--host", action="store", dest="host", default="127.0.0.1", help="Host name" -) -ARGS.add_argument( - "--port", action="store", dest="port", default=8080, type=int, help="Port number" -) -# make iocp and ssl mutually exclusive because ProactorEventLoop is -# incompatible with SSL -group = ARGS.add_mutually_exclusive_group() -group.add_argument( - "--iocp", action="store_true", dest="iocp", help="Windows IOCP event loop" -) -group.add_argument("--ssl", action="store_true", dest="ssl", help="Run ssl mode.") -ARGS.add_argument("--sslcert", action="store", dest="certfile", help="SSL cert file.") -ARGS.add_argument("--sslkey", action="store", dest="keyfile", help="SSL key file.") - - -def main(): - args = ARGS.parse_args() - - if ":" in args.host: - args.host, port = args.host.split(":", 1) - args.port = int(port) - - if args.iocp: - from asyncio import windows_events - - sys.argv.remove("--iocp") - logging.info("using iocp") - el = windows_events.ProactorEventLoop() - asyncio.set_event_loop(el) - - if args.ssl: - here = os.path.join(os.path.dirname(__file__), "tests") - - if args.certfile: - certfile = args.certfile or os.path.join(here, "sample.crt") - keyfile = args.keyfile or os.path.join(here, "sample.key") - else: - certfile = os.path.join(here, "sample.crt") - keyfile = os.path.join(here, "sample.key") - - sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - sslcontext.load_cert_chain(certfile, keyfile) - else: - sslcontext = None - - loop = asyncio.get_event_loop() - f = loop.create_server( - lambda: HttpRequestHandler(debug=True, keep_alive=75), - args.host, - args.port, - ssl=sslcontext, - ) - svr = loop.run_until_complete(f) - socks = svr.sockets - print("serving on", socks[0].getsockname()) - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - -if __name__ == "__main__": - main() diff --git a/examples/legacy/tcp_protocol_parser.py b/examples/legacy/tcp_protocol_parser.py deleted file mode 100755 index 1ef972758e5..00000000000 --- a/examples/legacy/tcp_protocol_parser.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python3 -"""Protocol parser example.""" -import argparse -import asyncio -import collections - -import aiohttp - -try: - import signal -except ImportError: - signal = None - - -MSG_TEXT = b"text:" -MSG_PING = b"ping:" -MSG_PONG = b"pong:" -MSG_STOP = b"stop:" - -Message = collections.namedtuple("Message", ("tp", "data")) - - -def my_protocol_parser(out, buf): - """Parser is used with StreamParser for incremental protocol parsing. - Parser is a generator function, but it is not a coroutine. Usually - parsers are implemented as a state machine. - - more details in asyncio/parsers.py - existing parsers: - * HTTP protocol parsers asyncio/http/protocol.py - * websocket parser asyncio/http/websocket.py - """ - while True: - tp = yield from buf.read(5) - if tp in (MSG_PING, MSG_PONG): - # skip line - yield from buf.skipuntil(b"\r\n") - out.feed_data(Message(tp, None)) - elif tp == MSG_STOP: - out.feed_data(Message(tp, None)) - elif tp == MSG_TEXT: - # read text - text = yield from buf.readuntil(b"\r\n") - out.feed_data(Message(tp, text.strip().decode("utf-8"))) - else: - raise ValueError("Unknown protocol prefix.") - - -class MyProtocolWriter: - def __init__(self, transport): - self.transport = transport - - def ping(self): - self.transport.write(b"ping:\r\n") - - def pong(self): - self.transport.write(b"pong:\r\n") - - def stop(self): - self.transport.write(b"stop:\r\n") - - def send_text(self, text): - self.transport.write(f"text:{text.strip()}\r\n".encode()) - - -class EchoServer(asyncio.Protocol): - def connection_made(self, transport): - print("Connection made") - self.transport = transport - self.stream = aiohttp.StreamParser() - asyncio.Task(self.dispatch()) - - def data_received(self, data): - self.stream.feed_data(data) - - def eof_received(self): - self.stream.feed_eof() - - def connection_lost(self, exc): - print("Connection lost") - - async def dispatch(self): - reader = self.stream.set_parser(my_protocol_parser) - writer = MyProtocolWriter(self.transport) - - while True: - try: - msg = await reader.read() - except aiohttp.ConnectionError: - # client has been disconnected - break - - print(f"Message received: {msg}") - - if msg.type == MSG_PING: - writer.pong() - elif msg.type == MSG_TEXT: - writer.send_text("Re: " + msg.data) - elif msg.type == MSG_STOP: - self.transport.close() - break - - -async def start_client(loop, host, port): - transport, stream = await loop.create_connection(aiohttp.StreamProtocol, host, port) - reader = stream.reader.set_parser(my_protocol_parser) - writer = MyProtocolWriter(transport) - writer.ping() - - message = "This is the message. It will be echoed." - - while True: - try: - msg = await reader.read() - except aiohttp.ConnectionError: - print("Server has been disconnected.") - break - - print(f"Message received: {msg}") - if msg.type == MSG_PONG: - writer.send_text(message) - print("data sent:", message) - elif msg.type == MSG_TEXT: - writer.stop() - print("stop sent") - break - - transport.close() - - -def start_server(loop, host, port): - f = loop.create_server(EchoServer, host, port) - srv = loop.run_until_complete(f) - x = srv.sockets[0] - print("serving on", x.getsockname()) - loop.run_forever() - - -ARGS = argparse.ArgumentParser(description="Protocol parser example.") -ARGS.add_argument( - "--server", action="store_true", dest="server", default=False, help="Run tcp server" -) -ARGS.add_argument( - "--client", action="store_true", dest="client", default=False, help="Run tcp client" -) -ARGS.add_argument( - "--host", action="store", dest="host", default="127.0.0.1", help="Host name" -) -ARGS.add_argument( - "--port", action="store", dest="port", default=9999, type=int, help="Port number" -) - - -if __name__ == "__main__": - args = ARGS.parse_args() - - if ":" in args.host: - args.host, port = args.host.split(":", 1) - args.port = int(port) - - if (not (args.server or args.client)) or (args.server and args.client): - print("Please specify --server or --client\n") - ARGS.print_help() - else: - loop = asyncio.get_event_loop() - if signal is not None: - loop.add_signal_handler(signal.SIGINT, loop.stop) - - if args.server: - start_server(loop, args.host, args.port) - else: - loop.run_until_complete(start_client(loop, args.host, args.port)) From fa2b50b8211d4700f81d519dd7f751a228761098 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Oct 2021 13:39:47 +0000 Subject: [PATCH 48/66] Bump charset-normalizer from 2.0.4 to 2.0.7 (#6118) Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.4 to 2.0.7. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.4...2.0.7) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index df6a12a9ad4..8e1e1c5cf5c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -46,10 +46,10 @@ cfgv==3.2.0 # via # -r requirements/lint.txt # pre-commit -charset-normalizer==2.0.4 - # via - # -r requirements/base.txt - # requests +chardet==4.0.0 + # via requests +charset-normalizer==2.0.7 + # via -r requirements/base.txt cherry_picker==2.0.0 ; python_version >= "3.6" # via -r requirements/dev.in click==7.1.2 From 09c96e73289f45d204c61676c5bbcc14e9031b77 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Oct 2021 14:19:53 +0000 Subject: [PATCH 49/66] Bump babel from 2.9.0 to 2.9.1 in /requirements (#6116) Bumps [babel](https://github.com/python-babel/babel) from 2.9.0 to 2.9.1. - [Release notes](https://github.com/python-babel/babel/releases) - [Changelog](https://github.com/python-babel/babel/blob/master/CHANGES) - [Commits](https://github.com/python-babel/babel/compare/v2.9.0...v2.9.1) --- updated-dependencies: - dependency-name: babel dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev.txt | 161 +++++++++++++++++++--------------- requirements/doc-spelling.txt | 26 +++--- 2 files changed, 107 insertions(+), 80 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 8e1e1c5cf5c..b2e7dd10f8e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,37 +5,39 @@ # pip-compile --allow-unsafe requirements/dev.in # aiodns==3.0.0 ; sys_platform == "linux" or sys_platform == "darwin" and python_version >= "3.7" - # via -r requirements/base.txt + # via -r base.txt aiohttp-theme==0.1.6 - # via -r requirements/doc.txt + # via -r doc.txt aiosignal==1.2.0 - # via -r requirements/base.txt + # via -r base.txt alabaster==0.7.12 # via sphinx appdirs==1.4.4 # via - # -r requirements/lint.txt + # -r lint.txt # black # virtualenv async-timeout==4.0.0a3 - # via -r requirements/base.txt + # via -r base.txt +asynctest==0.13.0 ; python_version < "3.8" + # via -r base.txt attrs==20.3.0 # via - # -r requirements/lint.txt + # -r lint.txt # flake8-pyi # pytest -babel==2.9.0 +babel==2.9.1 # via sphinx black==21.7b0 ; implementation_name == "cpython" - # via -r requirements/lint.txt + # via -r lint.txt blockdiag==2.0.1 # via sphinxcontrib-blockdiag brotli==1.0.9 # via - # -r requirements/base.txt - # -r requirements/test.txt + # -r base.txt + # -r test.txt cchardet==2.1.7 - # via -r requirements/base.txt + # via -r base.txt certifi==2020.12.5 # via requests cffi==1.14.4 @@ -44,17 +46,17 @@ cffi==1.14.4 # pycares cfgv==3.2.0 # via - # -r requirements/lint.txt + # -r lint.txt # pre-commit chardet==4.0.0 # via requests charset-normalizer==2.0.7 - # via -r requirements/base.txt + # via -r base.txt cherry_picker==2.0.0 ; python_version >= "3.6" - # via -r requirements/dev.in + # via -r dev.in click==7.1.2 # via - # -r requirements/lint.txt + # -r lint.txt # black # cherry-picker # click-default-group @@ -65,45 +67,46 @@ click-default-group==1.2.2 # via towncrier coverage[toml]==6.0.2 # via - # -r requirements/test.txt + # -r test.txt # pytest-cov -cryptography==3.3.1 +cryptography==3.3.1 ; platform_machine != "i686" and python_version < "3.9" # via + # -r test.txt # pyjwt # trustme distlib==0.3.1 # via - # -r requirements/lint.txt + # -r lint.txt # virtualenv docutils==0.16 # via sphinx filelock==3.0.12 # via - # -r requirements/lint.txt + # -r lint.txt # virtualenv flake8==4.0.1 # via - # -r requirements/lint.txt + # -r lint.txt # flake8-pyi flake8-pyi==20.10.0 - # via -r requirements/lint.txt + # via -r lint.txt freezegun==1.1.0 - # via -r requirements/test.txt + # via -r test.txt frozenlist==1.2.0 # via - # -r requirements/base.txt + # -r base.txt # aiosignal funcparserlib==1.0.0a0 # via - # -r requirements/doc.txt + # -r doc.txt # blockdiag gidgethub==5.0.0 # via cherry-picker gunicorn==20.1.0 - # via -r requirements/base.txt + # via -r base.txt identify==1.5.14 # via - # -r requirements/lint.txt + # -r lint.txt # pre-commit idna==2.10 # via @@ -112,14 +115,21 @@ idna==2.10 # yarl imagesize==1.2.0 # via sphinx +importlib-metadata==4.2.0 + # via + # flake8 + # pluggy + # pre-commit + # pytest + # virtualenv incremental==17.5.0 # via towncrier iniconfig==1.1.1 # via - # -r requirements/lint.txt + # -r lint.txt # pytest isort==5.9.3 - # via -r requirements/lint.txt + # via -r lint.txt jinja2==2.11.3 # via # sphinx @@ -128,54 +138,54 @@ markupsafe==1.1.1 # via jinja2 mccabe==0.6.1 # via - # -r requirements/lint.txt + # -r lint.txt # flake8 multidict==5.2.0 # via - # -r requirements/multidict.txt + # -r multidict.txt # yarl mypy==0.910 ; implementation_name == "cpython" # via - # -r requirements/lint.txt - # -r requirements/test.txt + # -r lint.txt + # -r test.txt mypy-extensions==0.4.3 ; implementation_name == "cpython" # via - # -r requirements/lint.txt - # -r requirements/test.txt + # -r lint.txt + # -r test.txt # black # mypy nodeenv==1.5.0 # via - # -r requirements/lint.txt + # -r lint.txt # pre-commit packaging==20.9 # via - # -r requirements/lint.txt + # -r lint.txt # pytest # sphinx pathspec==0.8.1 # via - # -r requirements/lint.txt + # -r lint.txt # black pillow==8.3.2 # via blockdiag pluggy==0.13.1 # via - # -r requirements/lint.txt + # -r lint.txt # pytest pre-commit==2.15.0 - # via -r requirements/lint.txt + # via -r lint.txt proxy.py==2.3.1 - # via -r requirements/test.txt + # via -r test.txt py==1.10.0 # via - # -r requirements/lint.txt + # -r lint.txt # pytest pycares==4.0.0 # via aiodns pycodestyle==2.8.0 # via - # -r requirements/lint.txt + # -r lint.txt # flake8 pycparser==2.20 # via cffi @@ -183,44 +193,44 @@ pydantic==1.8.2 # via python-on-whales pyflakes==2.4.0 # via - # -r requirements/lint.txt + # -r lint.txt # flake8 # flake8-pyi pygments==2.10.0 # via - # -r requirements/doc.txt + # -r doc.txt # sphinx pyjwt[crypto]==2.0.0 # via gidgethub pyparsing==2.4.7 # via - # -r requirements/lint.txt + # -r lint.txt # packaging pytest==6.2.5 # via - # -r requirements/lint.txt - # -r requirements/test.txt + # -r lint.txt + # -r test.txt # pytest-cov # pytest-mock pytest-cov==3.0.0 - # via -r requirements/test.txt + # via -r test.txt pytest-mock==3.6.1 - # via -r requirements/test.txt + # via -r test.txt python-dateutil==2.8.1 # via freezegun python-on-whales==0.28.0 - # via -r requirements/dev.in + # via -r dev.in pytz==2020.5 # via babel pyyaml==5.4.1 # via - # -r requirements/lint.txt + # -r lint.txt # pre-commit re-assert==1.1.0 - # via -r requirements/test.txt + # via -r test.txt regex==2020.11.13 # via - # -r requirements/lint.txt + # -r lint.txt # black # re-assert requests==2.25.1 @@ -229,10 +239,10 @@ requests==2.25.1 # python-on-whales # sphinx setuptools-git==1.2 - # via -r requirements/test.txt + # via -r test.txt six==1.16.0 # via - # -r requirements/lint.txt + # -r lint.txt # cryptography # python-dateutil # virtualenv @@ -240,16 +250,16 @@ snowballstemmer==2.0.0 # via sphinx sphinx==4.2.0 # via - # -r requirements/doc.txt + # -r doc.txt # sphinxcontrib-asyncio # sphinxcontrib-blockdiag # sphinxcontrib-towncrier sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-asyncio==0.3.0 - # via -r requirements/doc.txt + # via -r doc.txt sphinxcontrib-blockdiag==2.0.0 - # via -r requirements/doc.txt + # via -r doc.txt sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==2.0.0 @@ -261,10 +271,10 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-towncrier==0.2.0a0 - # via -r requirements/doc.txt + # via -r doc.txt toml==0.10.2 # via - # -r requirements/lint.txt + # -r lint.txt # cherry-picker # mypy # pre-commit @@ -272,45 +282,56 @@ toml==0.10.2 # towncrier tomli==1.2.1 # via - # -r requirements/lint.txt + # -r lint.txt # black # coverage towncrier==21.3.0 # via - # -r requirements/doc.txt + # -r doc.txt # sphinxcontrib-towncrier tqdm==4.62.2 # via python-on-whales trustme==0.9.0 ; platform_machine != "i686" - # via -r requirements/test.txt + # via -r test.txt +typed-ast==1.4.3 + # via + # black + # mypy typer==0.4.0 # via python-on-whales types-chardet==0.1.3 # via - # -r requirements/lint.txt - # -r requirements/test.txt + # -r lint.txt + # -r test.txt typing-extensions==3.7.4.3 # via - # -r requirements/base.txt - # -r requirements/lint.txt + # -r base.txt + # -r lint.txt # async-timeout + # black + # importlib-metadata # mypy # proxy.py # pydantic + # yarl uritemplate==3.0.1 # via gidgethub urllib3==1.26.5 # via requests +uvloop==0.14.0 ; platform_system != "Windows" and implementation_name == "cpython" and python_version < "3.9" + # via -r base.txt virtualenv==20.4.2 # via - # -r requirements/lint.txt + # -r lint.txt # pre-commit wait-for-it==2.2.1 - # via -r requirements/dev.in + # via -r dev.in webcolors==1.11.1 # via blockdiag yarl==1.7.0 - # via -r requirements/base.txt + # via -r base.txt +zipp==3.6.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: setuptools==57.4.0 diff --git a/requirements/doc-spelling.txt b/requirements/doc-spelling.txt index a9800eef0e6..494335f9644 100644 --- a/requirements/doc-spelling.txt +++ b/requirements/doc-spelling.txt @@ -5,10 +5,10 @@ # pip-compile --allow-unsafe requirements/doc-spelling.in # aiohttp-theme==0.1.6 - # via -r requirements/doc.txt + # via -r doc.txt alabaster==0.7.12 # via sphinx -babel==2.9.0 +babel==2.9.1 # via sphinx blockdiag==2.0.1 # via sphinxcontrib-blockdiag @@ -26,12 +26,14 @@ docutils==0.16 # via sphinx funcparserlib==1.0.0a0 # via - # -r requirements/doc.txt + # -r doc.txt # blockdiag idna==2.10 # via requests imagesize==1.2.0 # via sphinx +importlib-metadata==4.8.1 + # via sphinxcontrib-spelling incremental==17.5.0 # via towncrier jinja2==2.11.3 @@ -48,7 +50,7 @@ pyenchant==3.2.0 # via sphinxcontrib-spelling pygments==2.10.0 # via - # -r requirements/doc.txt + # -r doc.txt # sphinx pyparsing==2.4.7 # via packaging @@ -60,7 +62,7 @@ snowballstemmer==2.1.0 # via sphinx sphinx==4.2.0 # via - # -r requirements/doc.txt + # -r doc.txt # sphinxcontrib-asyncio # sphinxcontrib-blockdiag # sphinxcontrib-spelling @@ -68,9 +70,9 @@ sphinx==4.2.0 sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-asyncio==0.3.0 - # via -r requirements/doc.txt + # via -r doc.txt sphinxcontrib-blockdiag==2.0.0 - # via -r requirements/doc.txt + # via -r doc.txt sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==2.0.0 @@ -82,19 +84,23 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-spelling==7.2.1 ; platform_system != "Windows" - # via -r requirements/doc-spelling.in + # via -r doc-spelling.in sphinxcontrib-towncrier==0.2.0a0 - # via -r requirements/doc.txt + # via -r doc.txt toml==0.10.2 # via towncrier towncrier==21.3.0 # via - # -r requirements/doc.txt + # -r doc.txt # sphinxcontrib-towncrier +typing-extensions==3.10.0.2 + # via importlib-metadata urllib3==1.26.5 # via requests webcolors==1.11.1 # via blockdiag +zipp==3.6.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: setuptools==53.0.0 From c091db813c01e48140bc8146d3ca1d80bb1e9578 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 23 Oct 2021 17:33:50 +0100 Subject: [PATCH 50/66] Don't cancel tasks when entering a timer context (#5853) --- CHANGES/5853.bugfix | 1 + aiohttp/helpers.py | 1 - tests/test_helpers.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 CHANGES/5853.bugfix diff --git a/CHANGES/5853.bugfix b/CHANGES/5853.bugfix new file mode 100644 index 00000000000..3b3b03a374f --- /dev/null +++ b/CHANGES/5853.bugfix @@ -0,0 +1 @@ +Don't cancel current task when entering a cancelled timer. diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 948b12f99d8..c60397931fb 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -675,7 +675,6 @@ def __enter__(self) -> BaseTimerContext: ) if self._cancelled: - task.cancel() raise asyncio.TimeoutError from None self._tasks.append(task) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 8f029f15666..564ed783848 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -362,7 +362,7 @@ def test_timeout_handle_cb_exc(loop) -> None: assert not handle._callbacks -def test_timer_context_cancelled() -> None: +def test_timer_context_not_cancelled() -> None: with mock.patch("aiohttp.helpers.asyncio") as m_asyncio: m_asyncio.TimeoutError = asyncio.TimeoutError loop = mock.Mock() @@ -373,7 +373,7 @@ def test_timer_context_cancelled() -> None: with ctx: pass - assert m_asyncio.current_task.return_value.cancel.called + assert not m_asyncio.current_task.return_value.cancel.called def test_timer_context_no_task(loop) -> None: From c6eb5307aef3df6e4b4c8fa4292eaafeea265ed2 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 24 Oct 2021 08:58:27 +0100 Subject: [PATCH 51/66] Fix content-type on empty string. (#5393) Co-authored-by: Andrew Svetlov --- CHANGES/5392.bugfix | 1 + aiohttp/client_reqrep.py | 6 +++--- tests/test_client_functional.py | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 CHANGES/5392.bugfix diff --git a/CHANGES/5392.bugfix b/CHANGES/5392.bugfix new file mode 100644 index 00000000000..99a0fa38938 --- /dev/null +++ b/CHANGES/5392.bugfix @@ -0,0 +1 @@ +Set "text/plain" when data is an empty string in client requests. diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 41602afe703..3a924769ae0 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -245,7 +245,7 @@ def __init__( self.update_proxy(proxy, proxy_auth, proxy_headers) self.update_body_from_data(data) - if data or self.method not in self.GET_METHODS: + if data is not None or self.method not in self.GET_METHODS: self.update_transfer_encoding() self.update_expect_continue(expect100) if traces is None: @@ -383,7 +383,7 @@ def update_cookies(self, cookies: Optional[LooseCookies]) -> None: def update_content_encoding(self, data: Any) -> None: """Set request content encoding.""" - if not data: + if data is None: return enc = self.headers.get(hdrs.CONTENT_ENCODING, "").lower() @@ -433,7 +433,7 @@ def update_auth(self, auth: Optional[BasicAuth]) -> None: self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, body: Any) -> None: - if not body: + if body is None: return # FormData diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 058d1594ec3..95067ecc9e4 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -3022,3 +3022,22 @@ async def handler(request): async with await client.get("/", read_bufsize=4) as resp: assert resp.content.get_read_buffer_limits() == (4, 8) + + +async def test_http_empty_data_text(aiohttp_client: Any) -> None: + async def handler(request): + data = await request.read() + ret = "ok" if data == b"" else "fail" + resp = web.Response(text=ret) + resp.headers["Content-Type"] = request.headers["Content-Type"] + return resp + + app = web.Application() + app.add_routes([web.post("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.post("/", data="") as resp: + assert resp.status == 200 + assert await resp.text() == "ok" + assert resp.headers["Content-Type"] == "text/plain; charset=utf-8" From 4744923f8070baa364d5bfd3edd8510a4394303c Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Sun, 24 Oct 2021 17:33:57 +0900 Subject: [PATCH 52/66] fix(5925): suppress ValueError due to invalid datetime header (#6012) Co-authored-by: Andrew Svetlov --- CHANGES/5925.bugfix | 1 + CONTRIBUTORS.txt | 1 + aiohttp/helpers.py | 11 +++++++++++ aiohttp/web_request.py | 17 ++++------------- aiohttp/web_response.py | 9 ++------- tests/test_helpers.py | 28 +++++++++++++++++++++++++++- tests/test_web_request.py | 26 ++++++++++++++++++++++++++ tests/test_web_response.py | 13 +++++++++++++ 8 files changed, 85 insertions(+), 21 deletions(-) create mode 100644 CHANGES/5925.bugfix diff --git a/CHANGES/5925.bugfix b/CHANGES/5925.bugfix new file mode 100644 index 00000000000..297033b0f44 --- /dev/null +++ b/CHANGES/5925.bugfix @@ -0,0 +1 @@ +Return ``None`` from ``request.if_modified_since``, ``request.if_unmodified_since``, ``request.if_range`` and ``response.last_modified`` when corresponding http date headers are invalid. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index f42331eeaef..f18943eb5bc 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -132,6 +132,7 @@ Gustavo Carneiro Günther Jena Hans Adema Harmon Y. +Hiroshi Ogawa Hrishikesh Paranjape Hu Bo Hugh Young diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index c60397931fb..627bd98ff99 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -17,6 +17,7 @@ import weakref from collections import namedtuple from contextlib import suppress +from email.utils import parsedate from http.cookies import SimpleCookie from math import ceil from pathlib import Path @@ -935,3 +936,13 @@ def validate_etag_value(value: str) -> None: raise ValueError( f"Value {value!r} is not a valid etag. Maybe it contains '\"'?" ) + + +def parse_http_date(date_str: Optional[str]) -> Optional[datetime.datetime]: + """Process a date string, return a datetime object""" + if date_str is not None: + timetuple = parsedate(date_str) + if timetuple is not None: + with suppress(ValueError): + return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) + return None diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index d4d941d26e7..0424d7757c8 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -7,7 +7,6 @@ import string import tempfile import types -from email.utils import parsedate from http.cookies import SimpleCookie from types import MappingProxyType from typing import ( @@ -40,6 +39,7 @@ ETag, HeadersMixin, is_expected_content_type, + parse_http_date, reify, sentinel, set_result, @@ -478,22 +478,13 @@ def raw_headers(self) -> RawHeaders: """A sequence of pairs for all headers.""" return self._message.raw_headers - @staticmethod - def _http_date(_date_str: Optional[str]) -> Optional[datetime.datetime]: - """Process a date string, return a datetime object""" - if _date_str is not None: - timetuple = parsedate(_date_str) - if timetuple is not None: - return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) - return None - @reify def if_modified_since(self) -> Optional[datetime.datetime]: """The value of If-Modified-Since HTTP header, or None. This header is represented as a `datetime` object. """ - return self._http_date(self.headers.get(hdrs.IF_MODIFIED_SINCE)) + return parse_http_date(self.headers.get(hdrs.IF_MODIFIED_SINCE)) @reify def if_unmodified_since(self) -> Optional[datetime.datetime]: @@ -501,7 +492,7 @@ def if_unmodified_since(self) -> Optional[datetime.datetime]: This header is represented as a `datetime` object. """ - return self._http_date(self.headers.get(hdrs.IF_UNMODIFIED_SINCE)) + return parse_http_date(self.headers.get(hdrs.IF_UNMODIFIED_SINCE)) @staticmethod def _etag_values(etag_header: str) -> Iterator[ETag]: @@ -555,7 +546,7 @@ def if_range(self) -> Optional[datetime.datetime]: This header is represented as a `datetime` object. """ - return self._http_date(self.headers.get(hdrs.IF_RANGE)) + return parse_http_date(self.headers.get(hdrs.IF_RANGE)) @reify def keep_alive(self) -> bool: diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index 634e38b0725..e7118a631a4 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -8,7 +8,6 @@ import warnings import zlib from concurrent.futures import Executor -from email.utils import parsedate from http.cookies import Morsel from typing import ( TYPE_CHECKING, @@ -34,6 +33,7 @@ CookieMixin, ETag, HeadersMixin, + parse_http_date, populate_with_cookies, rfc822_formatted_time, sentinel, @@ -251,12 +251,7 @@ def last_modified(self) -> Optional[datetime.datetime]: This header is represented as a `datetime` object. """ - httpdate = self._headers.get(hdrs.LAST_MODIFIED) - if httpdate is not None: - timetuple = parsedate(httpdate) - if timetuple is not None: - return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) - return None + return parse_http_date(self._headers.get(hdrs.LAST_MODIFIED)) @last_modified.setter def last_modified( diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 564ed783848..4582cfe6e3a 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,6 +1,7 @@ # type: ignore import asyncio import base64 +import datetime import gc import os import platform @@ -14,7 +15,7 @@ from yarl import URL from aiohttp import helpers -from aiohttp.helpers import is_expected_content_type +from aiohttp.helpers import is_expected_content_type, parse_http_date IS_PYPY = platform.python_implementation() == "PyPy" @@ -840,3 +841,28 @@ def test_populate_with_cookies(): helpers.populate_with_cookies(headers, cookies_mixin.cookies) assert headers == CIMultiDict({"Set-Cookie": "name=value; Path=/"}) + + +@pytest.mark.parametrize( + ["value", "expected"], + [ + # email.utils.parsedate returns None + pytest.param("xxyyzz", None), + # datetime.datetime fails with ValueError("year 4446413 is out of range") + pytest.param("Tue, 08 Oct 4446413 00:56:40 GMT", None), + # datetime.datetime fails with ValueError("second must be in 0..59") + pytest.param("Tue, 08 Oct 2000 00:56:80 GMT", None), + # OK + pytest.param( + "Tue, 08 Oct 2000 00:56:40 GMT", + datetime.datetime(2000, 10, 8, 0, 56, 40, tzinfo=datetime.timezone.utc), + ), + # OK (ignore timezone and overwrite to UTC) + pytest.param( + "Tue, 08 Oct 2000 00:56:40 +0900", + datetime.datetime(2000, 10, 8, 0, 56, 40, tzinfo=datetime.timezone.utc), + ), + ], +) +def test_parse_http_date(value, expected): + assert parse_http_date(value) == expected diff --git a/tests/test_web_request.py b/tests/test_web_request.py index eeeba2b44db..b4d6514d16a 100644 --- a/tests/test_web_request.py +++ b/tests/test_web_request.py @@ -1,5 +1,6 @@ # type: ignore import asyncio +import datetime import socket import weakref from collections.abc import MutableMapping @@ -867,3 +868,28 @@ async def invalid_handler_1(request): def test_etag_headers(header, header_attr, header_val, expected) -> None: req = make_mocked_request("GET", "/", headers={header: header_val}) assert getattr(req, header_attr) == expected + + +@pytest.mark.parametrize( + ["header", "header_attr"], + [ + pytest.param("If-Modified-Since", "if_modified_since"), + pytest.param("If-Unmodified-Since", "if_unmodified_since"), + pytest.param("If-Range", "if_range"), + ], +) +@pytest.mark.parametrize( + ["header_val", "expected"], + [ + pytest.param("xxyyzz", None), + pytest.param("Tue, 08 Oct 4446413 00:56:40 GMT", None), + pytest.param("Tue, 08 Oct 2000 00:56:80 GMT", None), + pytest.param( + "Tue, 08 Oct 2000 00:56:40 GMT", + datetime.datetime(2000, 10, 8, 0, 56, 40, tzinfo=datetime.timezone.utc), + ), + ], +) +def test_datetime_headers(header, header_attr, header_val, expected) -> None: + req = make_mocked_request("GET", "/", headers={header: header_val}) + assert getattr(req, header_attr) == expected diff --git a/tests/test_web_response.py b/tests/test_web_response.py index ddda01aae7d..00721ed0716 100644 --- a/tests/test_web_response.py +++ b/tests/test_web_response.py @@ -255,6 +255,19 @@ def test_last_modified_reset() -> None: assert resp.last_modified is None +@pytest.mark.parametrize( + ["header_val", "expected"], + [ + pytest.param("xxyyzz", None), + pytest.param("Tue, 08 Oct 4446413 00:56:40 GMT", None), + pytest.param("Tue, 08 Oct 2000 00:56:80 GMT", None), + ], +) +def test_last_modified_string_invalid(header_val, expected) -> None: + resp = StreamResponse(headers={"Last-Modified": header_val}) + assert resp.last_modified == expected + + def test_etag_initial() -> None: resp = StreamResponse() assert resp.etag is None From ccb9c0f2dca226945ca064434dc134868c0b8077 Mon Sep 17 00:00:00 2001 From: PluieElectrique <41453973+PluieElectrique@users.noreply.github.com> Date: Sun, 24 Oct 2021 01:51:24 -0700 Subject: [PATCH 53/66] Use iter_chunked to stream response content (#5880) It takes fewer lines than the while loop, which makes it a bit cleaner. Co-authored-by: Andrew Svetlov --- docs/client_quickstart.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/client_quickstart.rst b/docs/client_quickstart.rst index 7f7d44caad0..7ba0bb9e4f1 100644 --- a/docs/client_quickstart.rst +++ b/docs/client_quickstart.rst @@ -245,10 +245,7 @@ In general, however, you should use a pattern like this to save what is being streamed to a file:: with open(filename, 'wb') as fd: - while True: - chunk = await resp.content.read(chunk_size) - if not chunk: - break + async for chunk in resp.content.iter_chunked(chunk_size): fd.write(chunk) It is not possible to use :meth:`~ClientResponse.read`, From 3dea782aa8c407c89bd2e065c502974e49f6b972 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 24 Oct 2021 12:15:26 +0300 Subject: [PATCH 54/66] Cleanup pre-commit hooks (#6123) --- .pre-commit-config.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1cf75a7fb3c..351e064cf0c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,17 +44,13 @@ repos: - id: check-xml - id: check-executables-have-shebangs - id: check-toml - - id: check-xml - id: check-yaml - id: debug-statements - id: check-added-large-files - id: check-symlinks - - id: debug-statements - id: fix-byte-order-marker - id: fix-encoding-pragma args: ['--remove'] - - id: check-executables-have-shebangs - - id: check-case-conflict - id: detect-aws-credentials args: ['--allow-missing-credentials'] - id: detect-private-key @@ -69,7 +65,6 @@ repos: hooks: - id: flake8 exclude: "^docs/" - - repo: git://github.com/Lucas-C/pre-commit-hooks-markup rev: v1.0.1 hooks: From 28a806d8a972fded464e3dc78b2b033a619ee059 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Sun, 24 Oct 2021 11:47:32 +0200 Subject: [PATCH 55/66] Don't send secure cookies by insecure transports (#5839) By default, the transport is secure if https or wss scheme is used. Use `CookieJar(treat_as_secure_origin="http://127.0.0.1")` to override the default security checker. Co-authored-by: Andrew Svetlov --- CHANGES/5571.feature | 4 ++++ aiohttp/cookiejar.py | 33 ++++++++++++++++++++++++++++++--- docs/client_reference.rst | 14 ++++++++++---- docs/spelling_wordlist.txt | 1 + docs/testing.rst | 3 ++- tests/test_cookiejar.py | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 CHANGES/5571.feature diff --git a/CHANGES/5571.feature b/CHANGES/5571.feature new file mode 100644 index 00000000000..625651f5ffc --- /dev/null +++ b/CHANGES/5571.feature @@ -0,0 +1,4 @@ +Don't send secure cookies by insecure transports. + +By default, the transport is secure if https or wss scheme is used. +Use `CookieJar(treat_as_secure_origin="http://127.0.0.1")` to override the default security checker. diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index ef32f5faa53..f519fff8dd7 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -1,4 +1,5 @@ import asyncio +import contextlib import datetime import os # noqa import pathlib @@ -12,6 +13,7 @@ Dict, Iterable, Iterator, + List, Mapping, Optional, Set, @@ -24,7 +26,7 @@ from .abc import AbstractCookieJar, ClearCookiePredicate from .helpers import is_ip_address, next_whole_second -from .typedefs import LooseCookies, PathLike +from .typedefs import LooseCookies, PathLike, StrOrURL __all__ = ("CookieJar", "DummyCookieJar") @@ -55,7 +57,13 @@ class CookieJar(AbstractCookieJar): MAX_32BIT_TIME = datetime.datetime.utcfromtimestamp(2 ** 31 - 1) - def __init__(self, *, unsafe: bool = False, quote_cookie: bool = True) -> None: + def __init__( + self, + *, + unsafe: bool = False, + quote_cookie: bool = True, + treat_as_secure_origin: Union[StrOrURL, List[StrOrURL], None] = None + ) -> None: self._loop = asyncio.get_running_loop() self._cookies = defaultdict( SimpleCookie @@ -63,6 +71,18 @@ def __init__(self, *, unsafe: bool = False, quote_cookie: bool = True) -> None: self._host_only_cookies = set() # type: Set[Tuple[str, str]] self._unsafe = unsafe self._quote_cookie = quote_cookie + if treat_as_secure_origin is None: + treat_as_secure_origin = [] + elif isinstance(treat_as_secure_origin, URL): + treat_as_secure_origin = [treat_as_secure_origin.origin()] + elif isinstance(treat_as_secure_origin, str): + treat_as_secure_origin = [URL(treat_as_secure_origin).origin()] + else: + treat_as_secure_origin = [ + URL(url).origin() if isinstance(url, str) else url.origin() + for url in treat_as_secure_origin + ] + self._treat_as_secure_origin = treat_as_secure_origin self._next_expiration = next_whole_second() self._expirations = {} # type: Dict[Tuple[str, str], datetime.datetime] # #4515: datetime.max may not be representable on 32-bit platforms @@ -227,7 +247,14 @@ def filter_cookies( SimpleCookie() if self._quote_cookie else BaseCookie() ) hostname = request_url.raw_host or "" - is_not_secure = request_url.scheme not in ("https", "wss") + request_origin = URL() + with contextlib.suppress(ValueError): + request_origin = request_url.origin() + + is_not_secure = ( + request_url.scheme not in ("https", "wss") + and request_origin not in self._treat_as_secure_origin + ) for cookie in self: name = cookie.key diff --git a/docs/client_reference.rst b/docs/client_reference.rst index accd4f627de..cb5bbecae39 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1778,7 +1778,7 @@ BasicAuth CookieJar ^^^^^^^^^ -.. class:: CookieJar(*, unsafe=False, quote_cookie=True, loop=None) +.. class:: CookieJar(*, unsafe=False, quote_cookie=True, treat_as_secure_origin = []) The cookie jar instance is available as :attr:`ClientSession.cookie_jar`. @@ -1810,11 +1810,17 @@ CookieJar .. versionadded:: 3.7 - :param bool loop: an :ref:`event loop` instance. - See :class:`aiohttp.abc.AbstractCookieJar` + :param treat_as_secure_origin: (optional) Mark origins as secure + for cookies marked as Secured. Possible types are - .. deprecated:: 2.0 + Possible types are: + + - :class:`tuple` or :class:`list` of + :class:`str` or :class:`yarl.URL` + - :class:`str` + - :class:`yarl.URL` + .. versionadded:: 3.8 .. method:: update_cookies(cookies, response_url=None) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index c4182c2f06d..da65cda5bb5 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -165,6 +165,7 @@ fallback fallbacks filename finalizers +formatter formatters frontend getall diff --git a/docs/testing.rst b/docs/testing.rst index 59c1cbe439d..ff6026f7bcb 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -744,7 +744,8 @@ Test Client first with ``TestServer(app)``. :param cookie_jar: an optional :class:`aiohttp.CookieJar` instance, - may be useful with ``CookieJar(unsafe=True)`` + may be useful with + ``CookieJar(unsafe=True, treat_as_secure_origin="http://127.0.0.1")`` option. :param str scheme: HTTP scheme, non-protected ``"http"`` by default. diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index a0212dcd049..a82b0d471b2 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -756,3 +756,35 @@ async def test_cookie_jar_clear_domain() -> None: assert morsel.value == "bar" with pytest.raises(StopIteration): next(iterator) + + +@pytest.mark.parametrize( + "url", + [ + "http://127.0.0.1/index.html", + URL("http://127.0.0.1/index.html"), + ["http://127.0.0.1/index.html"], + [URL("http://127.0.0.1/index.html")], + ], +) +async def test_treat_as_secure_origin_init(url) -> None: + jar = CookieJar(unsafe=True, treat_as_secure_origin=url) + assert jar._treat_as_secure_origin == [URL("http://127.0.0.1")] + + +async def test_treat_as_secure_origin() -> None: + endpoint = URL("http://127.0.0.1/") + + jar = CookieJar(unsafe=True, treat_as_secure_origin=[endpoint]) + secure_cookie = SimpleCookie( + "cookie-key=cookie-value; HttpOnly; Path=/; Secure", + ) + + jar.update_cookies( + secure_cookie, + endpoint, + ) + + assert len(jar) == 1 + filtered_cookies = jar.filter_cookies(request_url=endpoint) + assert len(filtered_cookies) == 1 From 55f02560724dd56a2c3e0f05bb116e4ca5369de0 Mon Sep 17 00:00:00 2001 From: quthla Date: Sun, 24 Oct 2021 11:56:35 +0200 Subject: [PATCH 56/66] Pass timeout to initial websocket request (#5793) --- CHANGES/5793.bugfix | 1 + aiohttp/client.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 CHANGES/5793.bugfix diff --git a/CHANGES/5793.bugfix b/CHANGES/5793.bugfix new file mode 100644 index 00000000000..5f7f7076cb7 --- /dev/null +++ b/CHANGES/5793.bugfix @@ -0,0 +1 @@ +Properly adhere to timeout passed to ws_connect diff --git a/aiohttp/client.py b/aiohttp/client.py index c14a682a404..cf853789a0c 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -752,6 +752,7 @@ async def _ws_connect( proxy_auth=proxy_auth, ssl=ssl, proxy_headers=proxy_headers, + timeout=timeout, ) try: From 4cc6922b0af83019eae8a7e9dcf35f40cfb6297f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 24 Oct 2021 13:12:50 +0300 Subject: [PATCH 57/66] Autoupdate pre-commit hooks (#6125) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 351e064cf0c..9fa49e9406c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: check-merge-conflict - repo: https://github.com/asottile/yesqa - rev: v1.2.3 + rev: v1.3.0 hooks: - id: yesqa - repo: https://github.com/PyCQA/isort From bd008ca646df25e043738aa85acca0d30ddcf128 Mon Sep 17 00:00:00 2001 From: Austin Scola Date: Sun, 24 Oct 2021 06:16:31 -0400 Subject: [PATCH 58/66] Add middleware type alias (#5898) --- CHANGES/5898.feature | 1 + aiohttp/typedefs.py | 1 + aiohttp/web_app.py | 13 +++++-------- aiohttp/web_middlewares.py | 11 ++++------- 4 files changed, 11 insertions(+), 15 deletions(-) create mode 100644 CHANGES/5898.feature diff --git a/CHANGES/5898.feature b/CHANGES/5898.feature new file mode 100644 index 00000000000..9d1d72bf23c --- /dev/null +++ b/CHANGES/5898.feature @@ -0,0 +1 @@ +Add a middleware type alias ``aiohttp.typedefs.Middleware``. diff --git a/aiohttp/typedefs.py b/aiohttp/typedefs.py index 1b13a4dbd0f..57d95b384f2 100644 --- a/aiohttp/typedefs.py +++ b/aiohttp/typedefs.py @@ -49,5 +49,6 @@ ] Handler = Callable[["Request"], Awaitable["StreamResponse"]] +Middleware = Callable[["Request", Handler], Awaitable["StreamResponse"]] PathLike = Union[str, "os.PathLike[str]"] diff --git a/aiohttp/web_app.py b/aiohttp/web_app.py index 999ea9ceb95..d5e5b88365e 100644 --- a/aiohttp/web_app.py +++ b/aiohttp/web_app.py @@ -28,6 +28,7 @@ from . import hdrs from .log import web_logger +from .typedefs import Middleware from .web_middlewares import _fix_request_current_app from .web_request import Request from .web_response import StreamResponse @@ -46,20 +47,16 @@ if TYPE_CHECKING: # pragma: no cover - from .typedefs import Handler - _AppSignal = Signal[Callable[["Application"], Awaitable[None]]] _RespPrepareSignal = Signal[Callable[[Request, StreamResponse], Awaitable[None]]] - _Middleware = Callable[[Request, Handler], Awaitable[StreamResponse]] - _Middlewares = FrozenList[_Middleware] - _MiddlewaresHandlers = Sequence[_Middleware] + _Middlewares = FrozenList[Middleware] + _MiddlewaresHandlers = Sequence[Middleware] _Subapps = List["Application"] else: # No type checker mode, skip types _AppSignal = Signal _RespPrepareSignal = Signal _Handler = Callable - _Middleware = Callable _Middlewares = FrozenList _MiddlewaresHandlers = Sequence _Subapps = List @@ -92,7 +89,7 @@ def __init__( self, *, logger: logging.Logger = web_logger, - middlewares: Iterable[_Middleware] = (), + middlewares: Iterable[Middleware] = (), handler_args: Optional[Mapping[str, Any]] = None, client_max_size: int = 1024 ** 2, debug: Any = ..., # mypy doesn't support ellipsis @@ -325,7 +322,7 @@ async def cleanup(self) -> None: # If an exception occurs in startup, ensure cleanup contexts are completed. await self._cleanup_ctx._on_cleanup(self) - def _prepare_middleware(self) -> Iterator[_Middleware]: + def _prepare_middleware(self) -> Iterator[Middleware]: yield from reversed(self._middlewares) yield _fix_request_current_app(self) diff --git a/aiohttp/web_middlewares.py b/aiohttp/web_middlewares.py index 4d28ff76307..f9801879fab 100644 --- a/aiohttp/web_middlewares.py +++ b/aiohttp/web_middlewares.py @@ -1,8 +1,8 @@ import re import warnings -from typing import TYPE_CHECKING, Awaitable, Callable, Tuple, Type, TypeVar +from typing import TYPE_CHECKING, Tuple, Type, TypeVar -from .typedefs import Handler +from .typedefs import Handler, Middleware from .web_exceptions import HTTPMove, HTTPPermanentRedirect from .web_request import Request from .web_response import StreamResponse @@ -42,16 +42,13 @@ def middleware(f: _Func) -> _Func: return f -_Middleware = Callable[[Request, Handler], Awaitable[StreamResponse]] - - def normalize_path_middleware( *, append_slash: bool = True, remove_slash: bool = False, merge_slashes: bool = True, redirect_class: Type[HTTPMove] = HTTPPermanentRedirect, -) -> _Middleware: +) -> Middleware: """ Middleware factory which produces a middleware that normalizes the path of a request. By normalizing it means: @@ -118,7 +115,7 @@ async def impl(request: Request, handler: Handler) -> StreamResponse: return impl -def _fix_request_current_app(app: "Application") -> _Middleware: +def _fix_request_current_app(app: "Application") -> Middleware: async def impl(request: Request, handler: Handler) -> StreamResponse: with request.match_info.set_current_app(app): return await handler(request) From bcb74cb00e1ba454dbcf25b27833deff23c2cbd9 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Sun, 24 Oct 2021 11:20:07 +0100 Subject: [PATCH 59/66] Feat: Add compression strategy (#5948) Co-authored-by: Anes Abismail --- CHANGES/5909.feature | 1 + aiohttp/http_writer.py | 6 ++++-- aiohttp/multipart.py | 6 ++++-- 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 CHANGES/5909.feature diff --git a/CHANGES/5909.feature b/CHANGES/5909.feature new file mode 100644 index 00000000000..d820979dac9 --- /dev/null +++ b/CHANGES/5909.feature @@ -0,0 +1 @@ +Add compression strategy parameter to enable_compression method. diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index 428a7929b1a..e09144736c5 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -61,9 +61,11 @@ def protocol(self) -> BaseProtocol: def enable_chunking(self) -> None: self.chunked = True - def enable_compression(self, encoding: str = "deflate") -> None: + def enable_compression( + self, encoding: str = "deflate", strategy: int = zlib.Z_DEFAULT_STRATEGY + ) -> None: zlib_mode = 16 + zlib.MAX_WBITS if encoding == "gzip" else zlib.MAX_WBITS - self._compress = zlib.compressobj(wbits=zlib_mode) + self._compress = zlib.compressobj(wbits=zlib_mode, strategy=strategy) def _write(self, chunk: bytes) -> None: size = len(chunk) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index b2c67baa45d..f6695fe9b69 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -978,9 +978,11 @@ def enable_encoding(self, encoding: str) -> None: elif encoding == "quoted-printable": self._encoding = "quoted-printable" - def enable_compression(self, encoding: str = "deflate") -> None: + def enable_compression( + self, encoding: str = "deflate", strategy: int = zlib.Z_DEFAULT_STRATEGY + ) -> None: zlib_mode = 16 + zlib.MAX_WBITS if encoding == "gzip" else -zlib.MAX_WBITS - self._compress = zlib.compressobj(wbits=zlib_mode) + self._compress = zlib.compressobj(wbits=zlib_mode, strategy=strategy) async def write_eof(self) -> None: if self._compress is not None: From 47ab6dfd9453a858d2a849834c1a025d3e3d4bf4 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 24 Oct 2021 13:29:51 +0300 Subject: [PATCH 60/66] Revert "Pass timeout to initial websocket request (#5793)" This reverts commit 55f02560724dd56a2c3e0f05bb116e4ca5369de0. --- CHANGES/5793.bugfix | 1 - aiohttp/client.py | 1 - 2 files changed, 2 deletions(-) delete mode 100644 CHANGES/5793.bugfix diff --git a/CHANGES/5793.bugfix b/CHANGES/5793.bugfix deleted file mode 100644 index 5f7f7076cb7..00000000000 --- a/CHANGES/5793.bugfix +++ /dev/null @@ -1 +0,0 @@ -Properly adhere to timeout passed to ws_connect diff --git a/aiohttp/client.py b/aiohttp/client.py index cf853789a0c..c14a682a404 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -752,7 +752,6 @@ async def _ws_connect( proxy_auth=proxy_auth, ssl=ssl, proxy_headers=proxy_headers, - timeout=timeout, ) try: From 50b8830d6cf395fc1fcca903d3ad19987e18be8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavol=20Vargov=C4=8D=C3=ADk?= Date: Sun, 24 Oct 2021 14:02:14 +0200 Subject: [PATCH 61/66] auto_decompress is optional in server (#5958) Co-authored-by: Pavol Vargovcik Co-authored-by: Andrew Svetlov Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES/5957.feature | 1 + CONTRIBUTORS.txt | 1 + aiohttp/_http_parser.pyx | 4 +++- aiohttp/web_protocol.py | 2 ++ docs/web_reference.rst | 4 ++++ tests/test_web_functional.py | 26 ++++++++++++++++++++++++++ 6 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 CHANGES/5957.feature diff --git a/CHANGES/5957.feature b/CHANGES/5957.feature new file mode 100644 index 00000000000..80a1adb1dc4 --- /dev/null +++ b/CHANGES/5957.feature @@ -0,0 +1 @@ +Added optional auto_decompress argument for HttpRequestParser diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index f18943eb5bc..1e8b3ec3614 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -243,6 +243,7 @@ Paulus Schoutsen Pavel Kamaev Pavel Polyakov Pavel Sapezhko +Pavol Vargovčík Pawel Kowalski Pawel Miech Pepe Osca diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index c24e31057a8..2a6d1ffa6f5 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -566,10 +566,12 @@ cdef class HttpRequestParser(HttpParser): size_t max_line_size=8190, size_t max_headers=32768, size_t max_field_size=8190, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, + bint auto_decompress=True, ): self._init(cparser.HTTP_REQUEST, protocol, loop, limit, timer, max_line_size, max_headers, max_field_size, - payload_exception, response_with_body, read_until_eof) + payload_exception, response_with_body, read_until_eof, + auto_decompress) cdef object _on_status_complete(self): cdef Py_buffer py_buf diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index e3f15329841..fb9f363e3b1 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -194,6 +194,7 @@ def __init__( max_field_size: int = 8190, lingering_time: float = 10.0, read_bufsize: int = 2 ** 16, + auto_decompress: bool = True, ): super().__init__(loop) @@ -227,6 +228,7 @@ def __init__( max_field_size=max_field_size, max_headers=max_headers, payload_exception=RequestPayloadError, + auto_decompress=auto_decompress, ) # type: Optional[HttpRequestParser] self.logger = logger diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 9b50a1c1412..2d1c3af1a7d 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -2583,6 +2583,10 @@ application on specific TCP or Unix socket, e.g.:: it means that the session global value is used. .. versionadded:: 3.7 + :param bool auto_decompress: Automatically decompress request body, + ``True`` by default. + + .. versionadded:: 3.8 diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index c54db16bdc2..802f1940e0c 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -1896,6 +1896,32 @@ async def handler(request): assert await resp.text() == "data (2, 4)" +@pytest.mark.parametrize( + "auto_decompress,len_of", [(True, "uncompressed"), (False, "compressed")] +) +async def test_auto_decompress( + aiohttp_client: Any, + auto_decompress: bool, + len_of: str, +) -> None: + async def handler(request): + data = await request.read() + return web.Response(text=str(len(data))) + + app = web.Application(handler_args={"auto_decompress": auto_decompress}) + app.router.add_post("/", handler) + + client = await aiohttp_client(app) + uncompressed = b"dataaaaaaaaaaaaaaaaaaaaaaaaa" + compressor = zlib.compressobj(wbits=16 + zlib.MAX_WBITS) + compressed = compressor.compress(uncompressed) + compressor.flush() + assert len(compressed) != len(uncompressed) + headers = {"content-encoding": "gzip"} + resp = await client.post("/", data=compressed, headers=headers) + assert resp.status == 200 + assert await resp.text() == str(len(locals()[len_of])) + + @pytest.mark.parametrize( "status", [101, 204], From 5ec6aec1baef1ff68d0bf3066b4f81f64c367376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A7=8B=E8=91=89?= Date: Sun, 24 Oct 2021 20:05:03 +0800 Subject: [PATCH 62/66] Fix: fix the code about handling `getaddrinfo` when disable ipv6 in (#5906) CPython and enable ipv6 in system. Related to #5901; `getaddrinfo` will return an `(int, bytes)` tuple, if CPython could not handle the address family. It will cause a index out of range error in aiohttp. --- CHANGES/5901.bugfix | 4 ++++ aiohttp/resolver.py | 4 +++- tests/test_resolver.py | 24 ++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 CHANGES/5901.bugfix diff --git a/CHANGES/5901.bugfix b/CHANGES/5901.bugfix new file mode 100644 index 00000000000..5c5940fde1b --- /dev/null +++ b/CHANGES/5901.bugfix @@ -0,0 +1,4 @@ +Fix the error in handling the return value of `getaddrinfo`. +`getaddrinfo` will return an `(int, bytes)` tuple, if CPython could not handle the address family. +It will cause a index out of range error in aiohttp. For example, if user compile CPython with +`--disable-ipv6` option but his system enable the ipv6. diff --git a/aiohttp/resolver.py b/aiohttp/resolver.py index e62e7f377b6..e2220010826 100644 --- a/aiohttp/resolver.py +++ b/aiohttp/resolver.py @@ -37,7 +37,9 @@ async def resolve( hosts = [] for family, _, proto, _, address in infos: - if family == socket.AF_INET6 and address[3]: # type: ignore[misc] + if family == socket.AF_INET6: + if not (socket.has_ipv6 and address[3]): # type: ignore[misc] + continue # This is essential for link-local IPv6 addresses. # LL IPv6 is a VERY rare case. Strictly speaking, we should use # getnameinfo() unconditionally, but performance makes sense. diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 3b984cb46c4..e84733ff4fe 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -119,6 +119,30 @@ async def test_threaded_negative_lookup() -> None: await resolver.resolve("doesnotexist.bla") +async def test_threaded_negative_lookup_with_unknown_result() -> None: + loop = Mock() + + # If compile CPython with `--disable-ipv6` option, + # we will get an (int, bytes) tuple, instead of a Exception. + async def unknown_addrinfo(*args: Any, **kwargs: Any) -> List[Any]: + return [ + ( + socket.AF_INET6, + socket.SOCK_STREAM, + 6, + "", + (10, b"\x01\xbb\x00\x00\x00\x00*\x04NB\x00\x1a\x00\x00"), + ) + ] + + loop.getaddrinfo = unknown_addrinfo + resolver = ThreadedResolver() + resolver._loop = loop + with patch("socket.has_ipv6", False): + res = await resolver.resolve("www.python.org") + assert len(res) == 0 + + async def test_close_for_threaded_resolver(loop: Any) -> None: resolver = ThreadedResolver() await resolver.close() From 7a3a367396fe16a367a13a04f70ad188883330bf Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 24 Oct 2021 16:10:04 +0300 Subject: [PATCH 63/66] Fix linter --- tests/autobahn/test_autobahn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 5d72e37a17a..ee97c246dbb 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -6,7 +6,7 @@ import pytest from pytest import TempPathFactory -from python_on_whales import DockerException, docker +from python_on_whales import DockerException, docker # type: ignore[attr-defined] @pytest.fixture(scope="session") @@ -73,7 +73,7 @@ def test_client(report_dir: Path, request: Any) -> None: print("Stopping client and server") client.terminate() client.wait() - autobahn_container.stop() + autobahn_container.stop() # type: ignore[union-attr] failed_messages = get_failed_tests(f"{report_dir}/clients", "aiohttp") From 179868300773b113697004fd1dc810fda8cb0664 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 24 Oct 2021 16:16:15 +0300 Subject: [PATCH 64/66] Remove redundant type ignores --- tests/autobahn/test_autobahn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index ee97c246dbb..5d72e37a17a 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -6,7 +6,7 @@ import pytest from pytest import TempPathFactory -from python_on_whales import DockerException, docker # type: ignore[attr-defined] +from python_on_whales import DockerException, docker @pytest.fixture(scope="session") @@ -73,7 +73,7 @@ def test_client(report_dir: Path, request: Any) -> None: print("Stopping client and server") client.terminate() client.wait() - autobahn_container.stop() # type: ignore[union-attr] + autobahn_container.stop() failed_messages = get_failed_tests(f"{report_dir}/clients", "aiohttp") From 545c9759d51be43700ef0f3664642faacd1b5dbf Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 24 Oct 2021 16:47:55 +0300 Subject: [PATCH 65/66] Fix spelling --- docs/spelling_wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index da65cda5bb5..8a39c37e4da 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -189,6 +189,7 @@ intaking io ip ipdb +ipv ish iterable iterables From 58f86f52176e3504a1a88723bd77bc91fa8eae74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 10:20:37 +0000 Subject: [PATCH 66/66] Bump python-on-whales from 0.28.0 to 0.29.0 (#6133) Bumps [python-on-whales](https://github.com/gabrieldemarmiesse/python-on-whales) from 0.28.0 to 0.29.0. - [Release notes](https://github.com/gabrieldemarmiesse/python-on-whales/releases) - [Commits](https://github.com/gabrieldemarmiesse/python-on-whales/compare/v0.28.0...v0.29.0) --- updated-dependencies: - dependency-name: python-on-whales dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev.in | 2 +- requirements/dev.txt | 161 +++++++++++++++++++------------------------ 2 files changed, 71 insertions(+), 92 deletions(-) diff --git a/requirements/dev.in b/requirements/dev.in index 1da2b9d5bc0..5e2b1470e48 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -2,5 +2,5 @@ -r test.txt -r doc.txt cherry_picker==2.0.0; python_version>="3.6" -python-on-whales==0.28.0 +python-on-whales==0.29.0 wait-for-it==2.2.1 diff --git a/requirements/dev.txt b/requirements/dev.txt index b2e7dd10f8e..7eb5f73664c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,39 +5,37 @@ # pip-compile --allow-unsafe requirements/dev.in # aiodns==3.0.0 ; sys_platform == "linux" or sys_platform == "darwin" and python_version >= "3.7" - # via -r base.txt + # via -r requirements/base.txt aiohttp-theme==0.1.6 - # via -r doc.txt + # via -r requirements/doc.txt aiosignal==1.2.0 - # via -r base.txt + # via -r requirements/base.txt alabaster==0.7.12 # via sphinx appdirs==1.4.4 # via - # -r lint.txt + # -r requirements/lint.txt # black # virtualenv async-timeout==4.0.0a3 - # via -r base.txt -asynctest==0.13.0 ; python_version < "3.8" - # via -r base.txt + # via -r requirements/base.txt attrs==20.3.0 # via - # -r lint.txt + # -r requirements/lint.txt # flake8-pyi # pytest babel==2.9.1 # via sphinx black==21.7b0 ; implementation_name == "cpython" - # via -r lint.txt + # via -r requirements/lint.txt blockdiag==2.0.1 # via sphinxcontrib-blockdiag brotli==1.0.9 # via - # -r base.txt - # -r test.txt + # -r requirements/base.txt + # -r requirements/test.txt cchardet==2.1.7 - # via -r base.txt + # via -r requirements/base.txt certifi==2020.12.5 # via requests cffi==1.14.4 @@ -46,17 +44,17 @@ cffi==1.14.4 # pycares cfgv==3.2.0 # via - # -r lint.txt + # -r requirements/lint.txt # pre-commit chardet==4.0.0 # via requests charset-normalizer==2.0.7 - # via -r base.txt + # via -r requirements/base.txt cherry_picker==2.0.0 ; python_version >= "3.6" - # via -r dev.in + # via -r requirements/dev.in click==7.1.2 # via - # -r lint.txt + # -r requirements/lint.txt # black # cherry-picker # click-default-group @@ -67,46 +65,45 @@ click-default-group==1.2.2 # via towncrier coverage[toml]==6.0.2 # via - # -r test.txt + # -r requirements/test.txt # pytest-cov -cryptography==3.3.1 ; platform_machine != "i686" and python_version < "3.9" +cryptography==3.3.1 # via - # -r test.txt # pyjwt # trustme distlib==0.3.1 # via - # -r lint.txt + # -r requirements/lint.txt # virtualenv docutils==0.16 # via sphinx filelock==3.0.12 # via - # -r lint.txt + # -r requirements/lint.txt # virtualenv flake8==4.0.1 # via - # -r lint.txt + # -r requirements/lint.txt # flake8-pyi flake8-pyi==20.10.0 - # via -r lint.txt + # via -r requirements/lint.txt freezegun==1.1.0 - # via -r test.txt + # via -r requirements/test.txt frozenlist==1.2.0 # via - # -r base.txt + # -r requirements/base.txt # aiosignal funcparserlib==1.0.0a0 # via - # -r doc.txt + # -r requirements/doc.txt # blockdiag gidgethub==5.0.0 # via cherry-picker gunicorn==20.1.0 - # via -r base.txt + # via -r requirements/base.txt identify==1.5.14 # via - # -r lint.txt + # -r requirements/lint.txt # pre-commit idna==2.10 # via @@ -115,21 +112,14 @@ idna==2.10 # yarl imagesize==1.2.0 # via sphinx -importlib-metadata==4.2.0 - # via - # flake8 - # pluggy - # pre-commit - # pytest - # virtualenv incremental==17.5.0 # via towncrier iniconfig==1.1.1 # via - # -r lint.txt + # -r requirements/lint.txt # pytest isort==5.9.3 - # via -r lint.txt + # via -r requirements/lint.txt jinja2==2.11.3 # via # sphinx @@ -138,54 +128,54 @@ markupsafe==1.1.1 # via jinja2 mccabe==0.6.1 # via - # -r lint.txt + # -r requirements/lint.txt # flake8 multidict==5.2.0 # via - # -r multidict.txt + # -r requirements/multidict.txt # yarl mypy==0.910 ; implementation_name == "cpython" # via - # -r lint.txt - # -r test.txt + # -r requirements/lint.txt + # -r requirements/test.txt mypy-extensions==0.4.3 ; implementation_name == "cpython" # via - # -r lint.txt - # -r test.txt + # -r requirements/lint.txt + # -r requirements/test.txt # black # mypy nodeenv==1.5.0 # via - # -r lint.txt + # -r requirements/lint.txt # pre-commit packaging==20.9 # via - # -r lint.txt + # -r requirements/lint.txt # pytest # sphinx pathspec==0.8.1 # via - # -r lint.txt + # -r requirements/lint.txt # black pillow==8.3.2 # via blockdiag pluggy==0.13.1 # via - # -r lint.txt + # -r requirements/lint.txt # pytest pre-commit==2.15.0 - # via -r lint.txt + # via -r requirements/lint.txt proxy.py==2.3.1 - # via -r test.txt + # via -r requirements/test.txt py==1.10.0 # via - # -r lint.txt + # -r requirements/lint.txt # pytest pycares==4.0.0 # via aiodns pycodestyle==2.8.0 # via - # -r lint.txt + # -r requirements/lint.txt # flake8 pycparser==2.20 # via cffi @@ -193,44 +183,44 @@ pydantic==1.8.2 # via python-on-whales pyflakes==2.4.0 # via - # -r lint.txt + # -r requirements/lint.txt # flake8 # flake8-pyi pygments==2.10.0 # via - # -r doc.txt + # -r requirements/doc.txt # sphinx pyjwt[crypto]==2.0.0 # via gidgethub pyparsing==2.4.7 # via - # -r lint.txt + # -r requirements/lint.txt # packaging pytest==6.2.5 # via - # -r lint.txt - # -r test.txt + # -r requirements/lint.txt + # -r requirements/test.txt # pytest-cov # pytest-mock pytest-cov==3.0.0 - # via -r test.txt + # via -r requirements/test.txt pytest-mock==3.6.1 - # via -r test.txt + # via -r requirements/test.txt python-dateutil==2.8.1 # via freezegun -python-on-whales==0.28.0 - # via -r dev.in +python-on-whales==0.29.0 + # via -r requirements/dev.in pytz==2020.5 # via babel pyyaml==5.4.1 # via - # -r lint.txt + # -r requirements/lint.txt # pre-commit re-assert==1.1.0 - # via -r test.txt + # via -r requirements/test.txt regex==2020.11.13 # via - # -r lint.txt + # -r requirements/lint.txt # black # re-assert requests==2.25.1 @@ -239,10 +229,10 @@ requests==2.25.1 # python-on-whales # sphinx setuptools-git==1.2 - # via -r test.txt + # via -r requirements/test.txt six==1.16.0 # via - # -r lint.txt + # -r requirements/lint.txt # cryptography # python-dateutil # virtualenv @@ -250,16 +240,16 @@ snowballstemmer==2.0.0 # via sphinx sphinx==4.2.0 # via - # -r doc.txt + # -r requirements/doc.txt # sphinxcontrib-asyncio # sphinxcontrib-blockdiag # sphinxcontrib-towncrier sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-asyncio==0.3.0 - # via -r doc.txt + # via -r requirements/doc.txt sphinxcontrib-blockdiag==2.0.0 - # via -r doc.txt + # via -r requirements/doc.txt sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==2.0.0 @@ -271,10 +261,10 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-towncrier==0.2.0a0 - # via -r doc.txt + # via -r requirements/doc.txt toml==0.10.2 # via - # -r lint.txt + # -r requirements/lint.txt # cherry-picker # mypy # pre-commit @@ -282,56 +272,45 @@ toml==0.10.2 # towncrier tomli==1.2.1 # via - # -r lint.txt + # -r requirements/lint.txt # black # coverage towncrier==21.3.0 # via - # -r doc.txt + # -r requirements/doc.txt # sphinxcontrib-towncrier tqdm==4.62.2 # via python-on-whales trustme==0.9.0 ; platform_machine != "i686" - # via -r test.txt -typed-ast==1.4.3 - # via - # black - # mypy + # via -r requirements/test.txt typer==0.4.0 # via python-on-whales types-chardet==0.1.3 # via - # -r lint.txt - # -r test.txt + # -r requirements/lint.txt + # -r requirements/test.txt typing-extensions==3.7.4.3 # via - # -r base.txt - # -r lint.txt + # -r requirements/base.txt + # -r requirements/lint.txt # async-timeout - # black - # importlib-metadata # mypy # proxy.py # pydantic - # yarl uritemplate==3.0.1 # via gidgethub urllib3==1.26.5 # via requests -uvloop==0.14.0 ; platform_system != "Windows" and implementation_name == "cpython" and python_version < "3.9" - # via -r base.txt virtualenv==20.4.2 # via - # -r lint.txt + # -r requirements/lint.txt # pre-commit wait-for-it==2.2.1 - # via -r dev.in + # via -r requirements/dev.in webcolors==1.11.1 # via blockdiag yarl==1.7.0 - # via -r base.txt -zipp==3.6.0 - # via importlib-metadata + # via -r requirements/base.txt # The following packages are considered to be unsafe in a requirements file: setuptools==57.4.0