[3.7] Fix python 3.8 tests under Windows (#4513)

* Add Python 3.8 to build matrix

* fix tests

* flake fix

* fixes

* fix mypy

* one more assert

* Move selector_loop fixture to conftest.py

* Add CHANGELOG

* Fix tests on Python 3.8.1

Co-authored-by: hh-h <[email protected]>.
(cherry picked from commit ec493d6)

Co-authored-by: Andrew Svetlov <[email protected]>

.azure-pipelines/deploy.yml

@@ -60,6 +60,9 @@ stages:
         py37 x64:
           python.code: 'cp37-cp37m'
           manylinux: 'manylinux_64'
+        py38 x64:
+          python.code: 'cp38-cp38'
+          manylinux: 'manylinux_64'
     pool:
       vmImage: 'ubuntu-latest'
     container: manylinux
@@ -106,9 +109,15 @@ stages:
           python.version: '3.7'
           python.architecture: 'x64'
           image: 'windows-latest'
+<<<<<<< HEAD
         Win py35 x86:
           python.version: '3.5'
           python.architecture: 'x86'
+=======
+        Win py38 x64:
+          python.version: '3.8'
+          python.architecture: 'x64'
+>>>>>>> ec493d6c... Fix python 3.8 tests under Windows (#4513)
           image: 'windows-latest'
         Win py36 x86:
           python.version: '3.6'
@@ -118,10 +127,17 @@ stages:
           python.version: '3.7'
           python.architecture: 'x86'
           image: 'windows-latest'
+<<<<<<< HEAD
         Mac py35:
           python.version: '3.5'
           image: 'macos-latest'
           python.architecture: 'x64'
+=======
+        Win py38 x86:
+          python.version: '3.8'
+          python.architecture: 'x86'
+          image: 'windows-latest'
+>>>>>>> ec493d6c... Fix python 3.8 tests under Windows (#4513)
         Mac py36:
           python.version: '3.6'
           image: 'macos-latest'
@@ -130,6 +146,10 @@ stages:
           python.version: '3.7'
           image: 'macos-latest'
           python.architecture: 'x64'
+        Mac py38:
+          python.version: '3.8'
+          image: 'macos-latest'
+          python.architecture: 'x64'
     pool:
       vmImage: '$(image)'
     steps:

.azure-pipelines/stage-test.yml

@@ -21,9 +21,15 @@ stages:
           python.version: '3.7'
           no_extensions: ''
           image: 'ubuntu-latest'
+<<<<<<< HEAD
         Py35-Pure-Linux:
           python.version: '3.5'
           no_extensions: 'Y'
+=======
+        Py38-Cython-Linux:
+          python.version: '3.8'
+          no_extensions: ''
+>>>>>>> ec493d6c... Fix python 3.8 tests under Windows (#4513)
           image: 'ubuntu-latest'
         Py36-Pure-Linux:
           python.version: '3.6'
@@ -33,6 +39,10 @@ stages:
           python.version: '3.7'
           no_extensions: 'Y'
           image: 'ubuntu-latest'
+        Py38-Pure-Linux:
+          python.version: '3.8'
+          no_extensions: 'Y'
+          image: 'ubuntu-latest'
 #        PyPy3-Linux:
 #          python.version: 'pypy3'
 #          no_extensions: 'Y'
@@ -49,10 +59,17 @@ stages:
           python.version: '3.7'
           no_extensions: ''
           image: 'windows-latest'
+<<<<<<< HEAD
         Py35-Cython-Mac:
           python.version: '3.5'
           no_extensions: ''
           image: 'macos-latest'
+=======
+        Py38-Cython-Win:
+          python.version: '3.8'
+          no_extensions: ''
+          image: 'windows-latest'
+>>>>>>> ec493d6c... Fix python 3.8 tests under Windows (#4513)
         Py36-Cython-Mac:
           python.version: '3.6'
           no_extensions: ''
@@ -61,6 +78,10 @@ stages:
           python.version: '3.7'
           no_extensions: ''
           image: 'macos-latest'
+        Py38-Cython-Mac:
+          python.version: '3.8'
+          no_extensions: ''
+          image: 'macos-latest'
     pool:
       vmImage: '$(image)'
 

CHANGES/4513.feature

@@ -0,0 +1 @@
+Pass tests on Python 3.8 for Windows.

aiohttp/payload_streamer.py

@@ -21,7 +21,7 @@ async def file_sender(writer, file_name=None):
 
 """
 
-import asyncio
+import types
 import warnings
 from typing import Any, Awaitable, Callable, Dict, Tuple
 
@@ -37,12 +37,12 @@ def __init__(self,
                  coro: Callable[..., Awaitable[None]],
                  args: Tuple[Any, ...],
                  kwargs: Dict[str, Any]) -> None:
-        self.coro = asyncio.coroutine(coro)
+        self.coro = types.coroutine(coro)
         self.args = args
         self.kwargs = kwargs
 
     async def __call__(self, writer: AbstractStreamWriter) -> None:
-        await self.coro(writer, *self.args, **self.kwargs)
+        await self.coro(writer, *self.args, **self.kwargs)  # type: ignore
 
 
 class streamer:

tests/conftest.py

@@ -1,12 +1,16 @@
+import asyncio
 import hashlib
 import pathlib
 import shutil
 import ssl
+import sys
 import tempfile
 import uuid
 
 import pytest
 
+from aiohttp.test_utils import loop_context
+
 try:
     import trustme
     TRUSTME = True
@@ -85,3 +89,18 @@ def tls_certificate_fingerprint_sha256(tls_certificate_pem_bytes):
 def pipe_name():
     name = r'\\.\pipe\{}'.format(uuid.uuid4().hex)
     return name
+@pytest.fixture
+def selector_loop():
+    if sys.version_info < (3, 7):
+        policy = asyncio.get_event_loop_policy()
+        policy._loop_factory = asyncio.SelectorEventLoop  # type: ignore
+    else:
+        if sys.version_info >= (3, 8):
+            policy = asyncio.WindowsSelectorEventLoopPolicy()  # type: ignore
+        else:
+            policy = asyncio.DefaultEventLoopPolicy()
+        asyncio.set_event_loop_policy(policy)
+
+    with loop_context(policy.new_event_loop) as _loop:
+        asyncio.set_event_loop(_loop)
+        yield _loop

tests/test_client_request.py

@@ -964,33 +964,6 @@ async def throw_exc():
     await req.close()
 
 
-async def test_data_stream_exc_deprecated(loop, conn) -> None:
-    fut = loop.create_future()
-
-    with pytest.warns(DeprecationWarning):
-        @aiohttp.streamer
-        async def gen(writer):
-            await writer.write(b'binary data')
-            await fut
-
-    req = ClientRequest(
-        'POST', URL('http://python.org/'), data=gen(), loop=loop)
-    assert req.chunked
-    assert req.headers['TRANSFER-ENCODING'] == 'chunked'
-
-    async def throw_exc():
-        await asyncio.sleep(0.01, loop=loop)
-        fut.set_exception(ValueError)
-
-    loop.create_task(throw_exc())
-
-    await req.send(conn)
-    await req._writer
-    # assert conn.close.called
-    assert conn.protocol.set_exception.called
-    await req.close()
-
-
 async def test_data_stream_exc_chain(loop, conn) -> None:
     fut = loop.create_future()
 
@@ -1020,36 +993,6 @@ async def throw_exc():
     await req.close()
 
 
-async def test_data_stream_exc_chain_deprecated(loop, conn) -> None:
-    fut = loop.create_future()
-
-    with pytest.warns(DeprecationWarning):
-        @aiohttp.streamer
-        async def gen(writer):
-            await fut
-
-    req = ClientRequest('POST', URL('http://python.org/'),
-                        data=gen(), loop=loop)
-
-    inner_exc = ValueError()
-
-    async def throw_exc():
-        await asyncio.sleep(0.01, loop=loop)
-        fut.set_exception(inner_exc)
-
-    loop.create_task(throw_exc())
-
-    await req.send(conn)
-    await req._writer
-    # assert connection.close.called
-    assert conn.protocol.set_exception.called
-    outer_exc = conn.protocol.set_exception.call_args[0][0]
-    assert isinstance(outer_exc, ValueError)
-    assert inner_exc is outer_exc
-    assert inner_exc is outer_exc
-    await req.close()
-
-
 async def test_data_stream_continue(loop, buf, conn) -> None:
     @async_generator
     async def gen():
@@ -1075,33 +1018,6 @@ async def coro():
     resp.close()
 
 
-async def test_data_stream_continue_deprecated(loop, buf, conn) -> None:
-    with pytest.warns(DeprecationWarning):
-        @aiohttp.streamer
-        async def gen(writer):
-            await writer.write(b'binary data')
-            await writer.write(b' result')
-            await writer.write_eof()
-
-    req = ClientRequest(
-        'POST', URL('http://python.org/'), data=gen(),
-        expect100=True, loop=loop)
-    assert req.chunked
-
-    async def coro():
-        await asyncio.sleep(0.0001, loop=loop)
-        req._continue.set_result(1)
-
-    loop.create_task(coro())
-
-    resp = await req.send(conn)
-    await req._writer
-    assert buf.split(b'\r\n\r\n', 1)[1] == \
-        b'b\r\nbinary data\r\n7\r\n result\r\n0\r\n\r\n'
-    await req.close()
-    resp.close()
-
-
 async def test_data_continue(loop, buf, conn) -> None:
     req = ClientRequest(
         'POST', URL('http://python.org/'), data=b'data',
@@ -1136,22 +1052,6 @@ async def gen():
     resp.close()
 
 
-async def test_close_deprecated(loop, buf, conn) -> None:
-    with pytest.warns(DeprecationWarning):
-        @aiohttp.streamer
-        async def gen(writer):
-            await asyncio.sleep(0.00001, loop=loop)
-            await writer.write(b'result')
-
-    req = ClientRequest(
-        'POST', URL('http://python.org/'), data=gen(), loop=loop)
-    resp = await req.send(conn)
-    await req.close()
-    assert buf.split(b'\r\n\r\n', 1)[1] == b'6\r\nresult\r\n0\r\n\r\n'
-    await req.close()
-    resp.close()
-
-
 async def test_custom_response_class(loop, conn) -> None:
     class CustomResponse(ClientResponse):
         def read(self, decode=False):

tests/test_client_session.py

@@ -3,6 +3,7 @@
 import gc
 import json
 import re
+import sys
 from http.cookies import SimpleCookie
 from io import BytesIO
 from unittest import mock
@@ -608,7 +609,8 @@ async def on_response_chunk_received(session, context, params):
             {'ok': True}).encode('utf8')
 
 
-async def test_request_tracing_exception(loop) -> None:
+async def test_request_tracing_exception() -> None:
+    loop = asyncio.get_event_loop()
     on_request_end = mock.Mock(side_effect=make_mocked_coro(mock.Mock()))
     on_request_exception = mock.Mock(
         side_effect=make_mocked_coro(mock.Mock())
@@ -620,9 +622,13 @@ async def test_request_tracing_exception(loop) -> None:
 
     with mock.patch("aiohttp.client.TCPConnector.connect") as connect_patched:
         error = Exception()
-        f = loop.create_future()
-        f.set_exception(error)
-        connect_patched.return_value = f
+        if sys.version_info >= (3, 8, 1):
+            connect_patched.side_effect = error
+        else:
+            loop = asyncio.get_event_loop()
+            f = loop.create_future()
+            f.set_exception(error)
+            connect_patched.return_value = f
 
         session = aiohttp.ClientSession(
             loop=loop,