From 72dcbf97a27a44e87123630fe78ff828eafa4d4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20K=C3=A4ufl?= Date: Fri, 4 Nov 2022 23:50:12 +0100 Subject: [PATCH 1/4] Use fixture `event_loop` instead of the deprecated fixture `loop` Fixes DeprecationWarning: 'loop' fixture is deprecated and scheduled for removal, please use 'event_loop' instead Except for `ext-aiohttp` because `pytest-aiohttp < 1.0.0` is too old for that. --- tests/ext/aiobotocore/test_aiobotocore.py | 24 +++++++++++++---------- tests/test_async_local_storage.py | 10 +++++----- tests/test_async_recorder.py | 18 +++++++++++------ tox.ini | 6 ++---- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/tests/ext/aiobotocore/test_aiobotocore.py b/tests/ext/aiobotocore/test_aiobotocore.py index ba43c5d2..7c4f3cb7 100644 --- a/tests/ext/aiobotocore/test_aiobotocore.py +++ b/tests/ext/aiobotocore/test_aiobotocore.py @@ -12,17 +12,19 @@ @pytest.fixture(scope='function') -def recorder(loop): +def recorder(event_loop): """ Clean up before and after each test run """ - xray_recorder.configure(service='test', sampling=False, context=AsyncContext(loop=loop)) + xray_recorder.configure( + service='test', sampling=False, context=AsyncContext(loop=event_loop) + ) xray_recorder.clear_trace_entities() yield xray_recorder xray_recorder.clear_trace_entities() -async def test_describe_table(loop, recorder): +async def test_describe_table(event_loop, recorder): segment = recorder.begin_segment('name') req_id = '1234' @@ -45,7 +47,7 @@ async def test_describe_table(loop, recorder): assert aws_meta['operation'] == 'DescribeTable' -async def test_s3_parameter_capture(loop, recorder): +async def test_s3_parameter_capture(event_loop, recorder): segment = recorder.begin_segment('name') bucket_name = 'mybucket' @@ -70,7 +72,7 @@ async def test_s3_parameter_capture(loop, recorder): assert aws_meta['operation'] == 'GetObject' -async def test_list_parameter_counting(loop, recorder): +async def test_list_parameter_counting(event_loop, recorder): """ Test special parameters that have shape of list are recorded as count based on `para_whitelist.json` @@ -103,7 +105,7 @@ async def test_list_parameter_counting(loop, recorder): assert aws_meta['queue_name_prefix'] == queue_name_prefix -async def test_map_parameter_grouping(loop, recorder): +async def test_map_parameter_grouping(event_loop, recorder): """ Test special parameters that have shape of map are recorded as a list of keys based on `para_whitelist.json` @@ -131,9 +133,10 @@ async def test_map_parameter_grouping(loop, recorder): assert sorted(aws_meta['table_names']) == ['table1', 'table2'] -async def test_context_missing_not_swallow_return(loop, recorder): +async def test_context_missing_not_swallow_return(event_loop, recorder): xray_recorder.configure(service='test', sampling=False, - context=AsyncContext(loop=loop), context_missing='LOG_ERROR') + context=AsyncContext(loop=event_loop), + context_missing='LOG_ERROR') response = {'ResponseMetadata': {'RequestId': '1234', 'HTTPStatusCode': 403}} @@ -146,9 +149,10 @@ async def test_context_missing_not_swallow_return(loop, recorder): assert actual_resp == response -async def test_context_missing_not_suppress_exception(loop, recorder): +async def test_context_missing_not_suppress_exception(event_loop, recorder): xray_recorder.configure(service='test', sampling=False, - context=AsyncContext(loop=loop), context_missing='LOG_ERROR') + context=AsyncContext(loop=event_loop), + context_missing='LOG_ERROR') session = get_session() async with session.create_client('dynamodb', region_name='eu-west-2') as client: diff --git a/tests/test_async_local_storage.py b/tests/test_async_local_storage.py index b43cc0ec..27a4519e 100644 --- a/tests/test_async_local_storage.py +++ b/tests/test_async_local_storage.py @@ -4,8 +4,8 @@ from aws_xray_sdk.core.async_context import TaskLocalStorage -def test_localstorage_isolation(loop): - local_storage = TaskLocalStorage(loop=loop) +def test_localstorage_isolation(event_loop): + local_storage = TaskLocalStorage(loop=event_loop) async def _test(): """ @@ -19,7 +19,7 @@ async def _test(): random_int = random.random() local_storage.randint = random_int - await asyncio.sleep(0.0, loop=loop) + await asyncio.sleep(0.0, loop=event_loop) current_random_int = local_storage.randint assert random_int == current_random_int @@ -29,8 +29,8 @@ async def _test(): return False # Run loads of concurrent tasks - results = loop.run_until_complete( - asyncio.wait([_test() for _ in range(0, 100)], loop=loop) + results = event_loop.run_until_complete( + asyncio.wait([_test() for _ in range(0, 100)], loop=event_loop) ) results = [item.result() for item in results[0]] diff --git a/tests/test_async_recorder.py b/tests/test_async_recorder.py index 0367fb3c..96a2fb90 100644 --- a/tests/test_async_recorder.py +++ b/tests/test_async_recorder.py @@ -19,8 +19,10 @@ async def async_method(): await async_method2() -async def test_capture(loop): - xray_recorder.configure(service='test', sampling=False, context=AsyncContext(loop=loop)) +async def test_capture(event_loop): + xray_recorder.configure( + service='test', sampling=False, context=AsyncContext(loop=event_loop) + ) segment = xray_recorder.begin_segment('name') @@ -44,8 +46,10 @@ async def test_capture(loop): assert platform.python_implementation() == service.get('runtime') assert platform.python_version() == service.get('runtime_version') -async def test_concurrent_calls(loop): - xray_recorder.configure(service='test', sampling=False, context=AsyncContext(loop=loop)) +async def test_concurrent_calls(event_loop): + xray_recorder.configure( + service='test', sampling=False, context=AsyncContext(loop=event_loop) + ) async with xray_recorder.in_segment_async('segment') as segment: global counter counter = 0 @@ -67,8 +71,10 @@ async def assert_task(): assert subseg_parent_id == segment.id -async def test_async_context_managers(loop): - xray_recorder.configure(service='test', sampling=False, context=AsyncContext(loop=loop)) +async def test_async_context_managers(event_loop): + xray_recorder.configure( + service='test', sampling=False, context=AsyncContext(loop=event_loop) + ) async with xray_recorder.in_segment_async('segment') as segment: async with xray_recorder.capture_async('aio_capture') as subsegment: diff --git a/tox.ini b/tox.ini index f973c697..6c06e23f 100644 --- a/tox.ini +++ b/tox.ini @@ -68,12 +68,10 @@ deps = py34: typing >= 3.7.4.3 ; Python 3.5+ only deps - ; for some reason pytest-aiohttp is required for "core" tests - ; TODO: find and replace by more direct dependency - py{35,36,37,38,39}: pytest-aiohttp + py{35,36,37,38,39}: pytest-asyncio ext-aiobotocore: aiobotocore >= 0.10.0 - ext-aiobotocore: pytest-aiohttp + ext-aiobotocore: pytest-asyncio ext-aiohttp: aiohttp >= 3.0.0 ; Breaking change where the `test_client` fixture was renamed. From dde50155e7a912aef80199a1d0c2e9900002af5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20K=C3=A4ufl?= Date: Sat, 5 Nov 2022 00:01:52 +0100 Subject: [PATCH 2/4] Do not pass event loop to `asyncio.sleep()` and `wait()` on Python 3.8+ Fixes The loop argument is deprecated since Python 3.8, and scheduled for removal in Python 3.10. See https://docs.python.org/3.8/library/asyncio-task.html#asyncio.sleep and https://docs.python.org/3.8/library/asyncio-task.html#asyncio.wait --- tests/ext/aiohttp/test_middleware.py | 19 ++++++++++++++----- tests/test_async_local_storage.py | 17 +++++++++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/ext/aiohttp/test_middleware.py b/tests/ext/aiohttp/test_middleware.py index 4c875de9..4e527d45 100644 --- a/tests/ext/aiohttp/test_middleware.py +++ b/tests/ext/aiohttp/test_middleware.py @@ -4,6 +4,7 @@ Expects pytest-aiohttp """ import asyncio +import sys from aws_xray_sdk import global_sdk_config try: from unittest.mock import patch @@ -84,7 +85,10 @@ async def handle_delay(self, request: web.Request) -> web.Response: """ Handle /delay request """ - await asyncio.sleep(0.3, loop=self._loop) + if sys.version_info >= (3, 8): + await asyncio.sleep(0.3) + else: + await asyncio.sleep(0.3, loop=self._loop) return web.Response(text="ok") def get_app(self) -> web.Application: @@ -282,10 +286,15 @@ async def get_delay(): resp = await client.get('/delay') assert resp.status == 200 - await asyncio.wait([get_delay(), get_delay(), get_delay(), - get_delay(), get_delay(), get_delay(), - get_delay(), get_delay(), get_delay()], - loop=loop) + if sys.version_info >= (3, 8): + await asyncio.wait([get_delay(), get_delay(), get_delay(), + get_delay(), get_delay(), get_delay(), + get_delay(), get_delay(), get_delay()]) + else: + await asyncio.wait([get_delay(), get_delay(), get_delay(), + get_delay(), get_delay(), get_delay(), + get_delay(), get_delay(), get_delay()], + loop=loop) # Ensure all ID's are different ids = [item.id for item in recorder.emitter.local] diff --git a/tests/test_async_local_storage.py b/tests/test_async_local_storage.py index 27a4519e..423b9026 100644 --- a/tests/test_async_local_storage.py +++ b/tests/test_async_local_storage.py @@ -1,5 +1,6 @@ import asyncio import random +import sys from aws_xray_sdk.core.async_context import TaskLocalStorage @@ -19,7 +20,10 @@ async def _test(): random_int = random.random() local_storage.randint = random_int - await asyncio.sleep(0.0, loop=event_loop) + if sys.version_info >= (3, 8): + await asyncio.sleep(0.0) + else: + await asyncio.sleep(0.0, loop=event_loop) current_random_int = local_storage.randint assert random_int == current_random_int @@ -29,9 +33,14 @@ async def _test(): return False # Run loads of concurrent tasks - results = event_loop.run_until_complete( - asyncio.wait([_test() for _ in range(0, 100)], loop=event_loop) - ) + if sys.version_info >= (3, 8): + results = event_loop.run_until_complete( + asyncio.wait([_test() for _ in range(0, 100)]) + ) + else: + results = event_loop.run_until_complete( + asyncio.wait([_test() for _ in range(0, 100)], loop=event_loop) + ) results = [item.result() for item in results[0]] # Double check all is good From b3d605d145dda82b9bcaa2fdac7eb593770886b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20K=C3=A4ufl?= Date: Sat, 5 Nov 2022 00:08:37 +0100 Subject: [PATCH 3/4] Do not pass coroutine objects directly to `asyncio.wait()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes The explicit passing of coroutine objects to asyncio.wait() is deprecated since Python 3.8, and scheduled for removal in Python 3.11. This has been “deprecated as it leads to confusing behavior”. See https://docs.python.org/3.8/library/asyncio-task.html#asyncio-example-wait-coroutine and https://stackoverflow.com/questions/69889075/asyncio-wait-confusion-when-passed-a-coroutine --- tests/ext/aiohttp/test_middleware.py | 9 ++------- tests/test_async_local_storage.py | 7 +++++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/ext/aiohttp/test_middleware.py b/tests/ext/aiohttp/test_middleware.py index 4e527d45..910e6639 100644 --- a/tests/ext/aiohttp/test_middleware.py +++ b/tests/ext/aiohttp/test_middleware.py @@ -287,14 +287,9 @@ async def get_delay(): assert resp.status == 200 if sys.version_info >= (3, 8): - await asyncio.wait([get_delay(), get_delay(), get_delay(), - get_delay(), get_delay(), get_delay(), - get_delay(), get_delay(), get_delay()]) + await asyncio.wait([loop.create_task(get_delay()) for i in range(9)]) else: - await asyncio.wait([get_delay(), get_delay(), get_delay(), - get_delay(), get_delay(), get_delay(), - get_delay(), get_delay(), get_delay()], - loop=loop) + await asyncio.wait([loop.create_task(get_delay()) for i in range(9)], loop=loop) # Ensure all ID's are different ids = [item.id for item in recorder.emitter.local] diff --git a/tests/test_async_local_storage.py b/tests/test_async_local_storage.py index 423b9026..d54e6cc6 100644 --- a/tests/test_async_local_storage.py +++ b/tests/test_async_local_storage.py @@ -35,11 +35,14 @@ async def _test(): # Run loads of concurrent tasks if sys.version_info >= (3, 8): results = event_loop.run_until_complete( - asyncio.wait([_test() for _ in range(0, 100)]) + asyncio.wait([event_loop.create_task(_test()) for _ in range(0, 100)]) ) else: results = event_loop.run_until_complete( - asyncio.wait([_test() for _ in range(0, 100)], loop=event_loop) + asyncio.wait( + [event_loop.create_task(_test()) for _ in range(0, 100)], + loop=event_loop, + ) ) results = [item.result() for item in results[0]] From b7f48eb7030f862ea3bf2d0efaee38d44d4ef22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20K=C3=A4ufl?= Date: Sat, 5 Nov 2022 00:25:52 +0100 Subject: [PATCH 4/4] Drop upper version bound of `pytest-aiohttp` Use fixture `aiohttp_client` instead of `test_client`. All the tests back to Python 3.5 complained that `test_client` was deprecated. --- tests/ext/aiohttp/test_middleware.py | 52 ++++++++++++++-------------- tox.ini | 6 ++-- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/tests/ext/aiohttp/test_middleware.py b/tests/ext/aiohttp/test_middleware.py index 910e6639..025af6a1 100644 --- a/tests/ext/aiohttp/test_middleware.py +++ b/tests/ext/aiohttp/test_middleware.py @@ -124,15 +124,15 @@ def recorder(loop): patcher.stop() -async def test_ok(test_client, loop, recorder): +async def test_ok(aiohttp_client, loop, recorder): """ Test a normal response - :param test_client: AioHttp test client fixture + :param aiohttp_client: AioHttp test client fixture :param loop: Eventloop fixture :param recorder: X-Ray recorder fixture """ - client = await test_client(ServerTest.app(loop=loop)) + client = await aiohttp_client(ServerTest.app(loop=loop)) resp = await client.get('/') assert resp.status == 200 @@ -148,15 +148,15 @@ async def test_ok(test_client, loop, recorder): assert response['status'] == 200 -async def test_ok_x_forwarded_for(test_client, loop, recorder): +async def test_ok_x_forwarded_for(aiohttp_client, loop, recorder): """ Test a normal response with x_forwarded_for headers - :param test_client: AioHttp test client fixture + :param aiohttp_client: AioHttp test client fixture :param loop: Eventloop fixture :param recorder: X-Ray recorder fixture """ - client = await test_client(ServerTest.app(loop=loop)) + client = await aiohttp_client(ServerTest.app(loop=loop)) resp = await client.get('/', headers={'X-Forwarded-For': 'foo'}) assert resp.status == 200 @@ -166,15 +166,15 @@ async def test_ok_x_forwarded_for(test_client, loop, recorder): assert segment.http['request']['x_forwarded_for'] -async def test_ok_content_length(test_client, loop, recorder): +async def test_ok_content_length(aiohttp_client, loop, recorder): """ Test a normal response with content length as response header - :param test_client: AioHttp test client fixture + :param aiohttp_client: AioHttp test client fixture :param loop: Eventloop fixture :param recorder: X-Ray recorder fixture """ - client = await test_client(ServerTest.app(loop=loop)) + client = await aiohttp_client(ServerTest.app(loop=loop)) resp = await client.get('/?content_length=100') assert resp.status == 200 @@ -183,15 +183,15 @@ async def test_ok_content_length(test_client, loop, recorder): assert segment.http['response']['content_length'] == 100 -async def test_error(test_client, loop, recorder): +async def test_error(aiohttp_client, loop, recorder): """ Test a 4XX response - :param test_client: AioHttp test client fixture + :param aiohttp_client: AioHttp test client fixture :param loop: Eventloop fixture :param recorder: X-Ray recorder fixture """ - client = await test_client(ServerTest.app(loop=loop)) + client = await aiohttp_client(ServerTest.app(loop=loop)) resp = await client.get('/error') assert resp.status == 404 @@ -208,15 +208,15 @@ async def test_error(test_client, loop, recorder): assert response['status'] == 404 -async def test_exception(test_client, loop, recorder): +async def test_exception(aiohttp_client, loop, recorder): """ Test handling an exception - :param test_client: AioHttp test client fixture + :param aiohttp_client: AioHttp test client fixture :param loop: Eventloop fixture :param recorder: X-Ray recorder fixture """ - client = await test_client(ServerTest.app(loop=loop)) + client = await aiohttp_client(ServerTest.app(loop=loop)) with pytest.raises(Exception): await client.get('/exception') @@ -235,15 +235,15 @@ async def test_exception(test_client, loop, recorder): assert exception.type == 'CancelledError' -async def test_unhauthorized(test_client, loop, recorder): +async def test_unhauthorized(aiohttp_client, loop, recorder): """ Test a 401 response - :param test_client: AioHttp test client fixture + :param aiohttp_client: AioHttp test client fixture :param loop: Eventloop fixture :param recorder: X-Ray recorder fixture """ - client = await test_client(ServerTest.app(loop=loop)) + client = await aiohttp_client(ServerTest.app(loop=loop)) resp = await client.get('/unauthorized') assert resp.status == 401 @@ -260,8 +260,8 @@ async def test_unhauthorized(test_client, loop, recorder): assert response['status'] == 401 -async def test_response_trace_header(test_client, loop, recorder): - client = await test_client(ServerTest.app(loop=loop)) +async def test_response_trace_header(aiohttp_client, loop, recorder): + client = await aiohttp_client(ServerTest.app(loop=loop)) resp = await client.get('/') xray_header = resp.headers[http.XRAY_HEADER] segment = recorder.emitter.pop() @@ -270,15 +270,15 @@ async def test_response_trace_header(test_client, loop, recorder): assert expected in xray_header -async def test_concurrent(test_client, loop, recorder): +async def test_concurrent(aiohttp_client, loop, recorder): """ Test multiple concurrent requests - :param test_client: AioHttp test client fixture + :param aiohttp_client: AioHttp test client fixture :param loop: Eventloop fixture :param recorder: X-Ray recorder fixture """ - client = await test_client(ServerTest.app(loop=loop)) + client = await aiohttp_client(ServerTest.app(loop=loop)) recorder.emitter = CustomStubbedEmitter() @@ -296,16 +296,16 @@ async def get_delay(): assert len(ids) == len(set(ids)) -async def test_disabled_sdk(test_client, loop, recorder): +async def test_disabled_sdk(aiohttp_client, loop, recorder): """ Test a normal response when the SDK is disabled. - :param test_client: AioHttp test client fixture + :param aiohttp_client: AioHttp test client fixture :param loop: Eventloop fixture :param recorder: X-Ray recorder fixture """ global_sdk_config.set_sdk_enabled(False) - client = await test_client(ServerTest.app(loop=loop)) + client = await aiohttp_client(ServerTest.app(loop=loop)) resp = await client.get('/') assert resp.status == 200 diff --git a/tox.ini b/tox.ini index 6c06e23f..678e8737 100644 --- a/tox.ini +++ b/tox.ini @@ -73,10 +73,8 @@ deps = ext-aiobotocore: aiobotocore >= 0.10.0 ext-aiobotocore: pytest-asyncio - ext-aiohttp: aiohttp >= 3.0.0 - ; Breaking change where the `test_client` fixture was renamed. - ; Also, the stable version is only supported for Python 3.7+ - ext-aiohttp: pytest-aiohttp < 1.0.0 + ext-aiohttp: aiohttp >= 3.3.0 + ext-aiohttp: pytest-aiohttp ext-httpx: httpx >= 0.20 ext-httpx: pytest-asyncio >= 0.19