From d66e07c652322d280740106ebb9946a3dd7daf5b Mon Sep 17 00:00:00 2001 From: bmbouter Date: Sun, 3 Oct 2021 11:30:12 -0400 Subject: [PATCH] Add xfailing integration tests against ``proxy.py`` This patch adds full end-to-end tests for sending requests to HTTP and HTTPS endpoints through an HTTPS proxy. The first case is currently supported and the second one is not. This is why the latter test is marked as expected to fail. The support for TLS-in-TLS in the upstream stdlib asyncio is currently disabled but is available in Python 3.9 via monkey-patching which is demonstrated in the added tests. Refs: * https://bugs.python.org/issue37179 * https://github.com/python/cpython/pull/28073 * https://github.com/aio-libs/aiohttp/pull/5992 Co-authored-by: Sviatoslav Sydorenko PR #6002 --- .github/workflows/ci.yml | 2 + CHANGES/6002.misc | 2 + requirements/dev.txt | 3 + requirements/test.txt | 1 + tests/test_proxy_functional.py | 128 +++++++++++++++++++++++++++++++++ 5 files changed, 136 insertions(+) create mode 100644 CHANGES/6002.misc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 427950bcb6d..037bcbf0898 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,8 @@ jobs: path: ${{ steps.pip-cache.outputs.dir }} restore-keys: | pip-ci-${{ runner.os }}-${{ matrix.pyver }}-${{ matrix.no-extensions }}- + - name: Upgrade wheel # Needed for proxy.py install not to explode + run: pip install -U wheel - name: Cythonize if: ${{ matrix.no-extensions == '' }} run: | diff --git a/CHANGES/6002.misc b/CHANGES/6002.misc new file mode 100644 index 00000000000..5df927cf65d --- /dev/null +++ b/CHANGES/6002.misc @@ -0,0 +1,2 @@ +Implemented end-to-end testing of sending HTTP and HTTPS requests +via ``proxy.py``. diff --git a/requirements/dev.txt b/requirements/dev.txt index 59ca566d7fe..9364ad351e6 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -163,6 +163,8 @@ pluggy==0.13.1 # pytest pre-commit==2.15.0 # via -r requirements/lint.txt +proxy.py==2.3.1 + # via -r requirements/test.txt py==1.10.0 # via # -r requirements/lint.txt @@ -277,6 +279,7 @@ typing-extensions==3.7.4.3 # -r requirements/lint.txt # async-timeout # mypy + # proxy.py uritemplate==3.0.1 # via gidgethub urllib3==1.26.5 diff --git a/requirements/test.txt b/requirements/test.txt index 8ba2b11d792..d5c4f2d271c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -6,6 +6,7 @@ cryptography==3.3.1; platform_machine!="i686" and python_version<"3.9" # no 32-b freezegun==1.1.0 mypy==0.910; implementation_name=="cpython" mypy-extensions==0.4.3; implementation_name=="cpython" +proxy.py==2.3.1 pytest==6.2.2 pytest-cov==2.12.1 pytest-mock==3.6.1 diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py index e1c3c0095e7..a5091b0a827 100644 --- a/tests/test_proxy_functional.py +++ b/tests/test_proxy_functional.py @@ -5,12 +5,140 @@ from typing import Any from unittest import mock +import proxy import pytest from yarl import URL import aiohttp from aiohttp import web +ASYNCIO_SUPPORTS_TLS_IN_TLS = hasattr( + asyncio.sslproto._SSLProtocolTransport, + "_start_tls_compatible", +) + + +@pytest.fixture +def secure_proxy_url(monkeypatch, tls_certificate_pem_path): + """Return the URL of an instance of a running secure proxy. + + This fixture also spawns that instance and tears it down after the test. + """ + proxypy_args = [ + "--threadless", # use asyncio + "--num-workers", + "1", # the tests only send one query anyway + "--hostname", + "127.0.0.1", # network interface to listen to + "--port", + 0, # ephemeral port, so that kernel allocates a free one + "--cert-file", + tls_certificate_pem_path, # contains both key and cert + "--key-file", + tls_certificate_pem_path, # contains both key and cert + ] + + class PatchedAccetorPool(proxy.core.acceptor.AcceptorPool): + def listen(self): + super().listen() + self.socket_host, self.socket_port = self.socket.getsockname()[:2] + + monkeypatch.setattr(proxy.proxy, "AcceptorPool", PatchedAccetorPool) + + with proxy.Proxy(input_args=proxypy_args) as proxy_instance: + yield URL.build( + scheme="https", + host=proxy_instance.acceptors.socket_host, + port=proxy_instance.acceptors.socket_port, + ) + + +@pytest.fixture +def web_server_endpoint_payload(): + return "Test message" + + +@pytest.fixture(params=("http", "https")) +def web_server_endpoint_type(request): + return request.param + + +@pytest.fixture +async def web_server_endpoint_url( + aiohttp_server, + ssl_ctx, + web_server_endpoint_payload, + web_server_endpoint_type, +): + server_kwargs = ( + { + "ssl": ssl_ctx, + } + if web_server_endpoint_type == "https" + else {} + ) + + async def handler(*args, **kwargs): + return web.Response(text=web_server_endpoint_payload) + + app = web.Application() + app.router.add_route("GET", "/", handler) + server = await aiohttp_server(app, **server_kwargs) + + return URL.build( + scheme=web_server_endpoint_type, + host=server.host, + port=server.port, + ) + + +@pytest.fixture +def _pretend_asyncio_supports_tls_in_tls( + monkeypatch, + web_server_endpoint_type, +): + if web_server_endpoint_type != "https" or ASYNCIO_SUPPORTS_TLS_IN_TLS: + return + + # for https://github.com/python/cpython/pull/28073 + # and https://bugs.python.org/issue37179 + monkeypatch.setattr( + asyncio.sslproto._SSLProtocolTransport, + "_start_tls_compatible", + True, + raising=False, + ) + + +@pytest.mark.xfail( + reason="https://github.com/aio-libs/aiohttp/pull/5992", + raises=ValueError, +) +@pytest.mark.parametrize("web_server_endpoint_type", ("http", "https")) +@pytest.mark.usefixtures("_pretend_asyncio_supports_tls_in_tls", "loop") +async def test_secure_https_proxy_absolute_path( + client_ssl_ctx, + secure_proxy_url, + web_server_endpoint_url, + web_server_endpoint_payload, +) -> None: + """Test urls can be requested through a secure proxy.""" + conn = aiohttp.TCPConnector() + sess = aiohttp.ClientSession(connector=conn) + + response = await sess.get( + web_server_endpoint_url, + proxy=secure_proxy_url, + ssl=client_ssl_ctx, # used for both proxy and endpoint connections + ) + + assert response.status == 200 + assert await response.text() == web_server_endpoint_payload + + response.close() + await sess.close() + await conn.close() + @pytest.fixture def proxy_test_server(aiohttp_raw_server: Any, loop: Any, monkeypatch: Any):