From aa8353c2976deb2a7b8678f5748b54b4be90eb9f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 6 Dec 2024 20:32:08 +0100 Subject: [PATCH] Add benchmarks (#706) Use codspeed.io as the infrastructure --- .github/workflows/ci.yml | 36 +++++++++- requirements-dev.txt | 2 + setup.cfg | 4 ++ tests/test_benchmarks.py | 152 +++++++++++++++++++++++++++++++++++++++ tests/test_mixed.py | 6 +- 5 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 tests/test_benchmarks.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34415ca..b68d171 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,9 +79,43 @@ jobs: with: key: unit-${{ matrix.python-version }} + benchmark: + name: Benchmark + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: 3.13 + - name: Get pip cache dir + id: pip-cache + run: | + echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT # - name: Cache + shell: bash + - name: Cache PyPI + uses: actions/cache@v4 + with: + key: pip-ci-ubuntu-3.13-${{ hashFiles('requirements-dev.txt') }} + path: ${{ steps.pip-cache.outputs.dir }} + restore-keys: | + pip-ci-ubuntu-3.13- + - name: Install dependencies + uses: py-actions/py-dependency-install@v4.1.0 + with: + path: requirements-dev.txt + - name: Run benchmarks + uses: CodSpeedHQ/action@v3 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: python -Im pytest --no-cov -vvvvv --codspeed + + check: # The branch protection check if: always() - needs: [lint, unit] + needs: [lint, unit, benchmark] runs-on: ubuntu-latest steps: - name: Decide whether the needed jobs succeeded or failed diff --git a/requirements-dev.txt b/requirements-dev.txt index 2a418d7..41fdec6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ -e . +backports.asyncio.runner==1.1.0; python_version < "3.11" black==24.10.0 bandit==1.8.0 coverage==7.6.8 @@ -9,6 +10,7 @@ pyroma==4.2 pytest-cov==6.0.0 pytest==8.3.4 pytest-asyncio==0.24.0 +pytest-codspeed==3.0.0 isort==5.13.2 tox==4.23.2 wheel==0.45.1 diff --git a/setup.cfg b/setup.cfg index 7fa4606..ceb42b6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,3 +59,7 @@ junit_family=xunit2 asyncio_mode=strict asyncio_default_fixture_loop_scope=function filterwarnings=error + # pytest-asyncio 0.24.0 does't close the event loop for some reason + # it is the source of unclosed loop and socket warnings + ignore:unclosed event loop:ResourceWarning + ignore:unclosed :ResourceWarning diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py new file mode 100644 index 0000000..41a3b94 --- /dev/null +++ b/tests/test_benchmarks.py @@ -0,0 +1,152 @@ +import asyncio +import sys +import janus + +if sys.version_info >= (3, 11): + from asyncio import Runner +else: + from backports.asyncio.runner import Runner + + +def test_bench_sync_put_async_get(benchmark): + q: janus.Queue + + async def init(): + nonlocal q + q = janus.Queue() + + def threaded(): + for i in range(5): + q.sync_q.put(i) + + async def go(): + for i in range(100): + f = asyncio.get_running_loop().run_in_executor(None, threaded) + for i in range(5): + val = await q.async_q.get() + assert val == i + + await f + assert q.async_q.empty() + + async def finish(): + q.close() + await q.wait_closed() + + with Runner(debug=True) as runner: + runner.run(init()) + + @benchmark + def _run(): + runner.run(go()) + + runner.run(finish()) + + +def test_bench_sync_put_async_join(benchmark): + q: janus.Queue + + async def init(): + nonlocal q + q = janus.Queue() + + async def go(): + for i in range(100): + for i in range(5): + q.sync_q.put(i) + + async def do_work(): + await asyncio.sleep(0.01) + while not q.async_q.empty(): + await q.async_q.get() + q.async_q.task_done() + + task = asyncio.create_task(do_work()) + + await q.async_q.join() + await task + + async def finish(): + q.close() + await q.wait_closed() + + with Runner(debug=True) as runner: + runner.run(init()) + + @benchmark + def _run(): + runner.run(go()) + + runner.run(finish()) + + +def test_bench_async_put_sync_get(benchmark): + q: janus.Queue + + async def init(): + nonlocal q + q = janus.Queue() + + def threaded(): + for i in range(5): + val = q.sync_q.get() + assert val == i + + async def go(): + for i in range(100): + f = asyncio.get_running_loop().run_in_executor(None, threaded) + for i in range(5): + await q.async_q.put(i) + + await f + assert q.async_q.empty() + + async def finish(): + q.close() + await q.wait_closed() + + with Runner(debug=True) as runner: + runner.run(init()) + + @benchmark + def _run(): + runner.run(go()) + + runner.run(finish()) + + +def test_sync_join_async_done(benchmark): + q: janus.Queue + + async def init(): + nonlocal q + q = janus.Queue() + + def threaded(): + for i in range(5): + q.sync_q.put(i) + q.sync_q.join() + + async def go(): + for i in range(100): + f = asyncio.get_running_loop().run_in_executor(None, threaded) + for i in range(5): + val = await q.async_q.get() + assert val == i + q.async_q.task_done() + + await f + assert q.async_q.empty() + + async def finish(): + q.close() + await q.wait_closed() + + with Runner(debug=True) as runner: + runner.run(init()) + + @benchmark + def _run(): + runner.run(go()) + + runner.run(finish()) diff --git a/tests/test_mixed.py b/tests/test_mixed.py index 48686ba..ba1e359 100644 --- a/tests/test_mixed.py +++ b/tests/test_mixed.py @@ -77,8 +77,8 @@ async def test_sync_put_async_join(self): q.sync_q.put(i) async def do_work(): - await asyncio.sleep(1) - while True: + await asyncio.sleep(0.1) + while not q.async_q.empty(): await q.async_q.get() q.async_q.task_done() @@ -86,7 +86,7 @@ async def do_work(): async def wait_for_empty_queue(): await q.async_q.join() - task.cancel() + await task await wait_for_empty_queue()