From 3fa34adad8314b301a982c509e88e88491706312 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Fri, 13 Oct 2023 12:00:05 +0530 Subject: [PATCH] feat: utility to reset application state before processing event chore: bump version and update changelog chore: remove utility function from coverage docs: move how to add new implementation doc from atlassian docs: simplify consuming docs in how-to --- CHANGELOG.rst | 6 ++ ...-new-event-bus-concrete-implementation.rst | 35 +++++++++++ docs/how-tos/index.rst | 1 + openedx_events/__init__.py | 2 +- openedx_events/tooling.py | 44 +++++++++++++ requirements/base.in | 1 + requirements/base.txt | 29 ++++++++- requirements/ci.txt | 2 +- requirements/dev.txt | 54 ++++++++++++---- requirements/doc.txt | 58 +++++++++++++---- requirements/pip.txt | 2 +- requirements/quality.txt | 62 ++++++++++++++----- requirements/test.txt | 55 +++++++++++++--- 13 files changed, 295 insertions(+), 56 deletions(-) create mode 100644 docs/how-tos/add-new-event-bus-concrete-implementation.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 639217b9..615dc20a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,12 @@ Change Log Unreleased ---------- +[9.5.0] - 2024-02-07 +-------------------- +Added +~~~~~ +* Adds utility function to reset application state similar to setup/teardown in Django request/response cycle. + [9.4.0] - 2024-01-29 -------------------- Added diff --git a/docs/how-tos/add-new-event-bus-concrete-implementation.rst b/docs/how-tos/add-new-event-bus-concrete-implementation.rst new file mode 100644 index 00000000..a4d21eac --- /dev/null +++ b/docs/how-tos/add-new-event-bus-concrete-implementation.rst @@ -0,0 +1,35 @@ +How to add a new concrete implementation of the event bus +========================================================= + +Context +------- + +Here is a list of the existing concrete implementations of the event bus: + +- `Kafka `_ +- `Redis Streams `_ + +This how-to is to help you add a new concrete implementation, for example using Pulsar or some other technology. + +Producing +--------- + +There should be a producer class that inherits from `EventBusProducer `_ in openedx-events. + +The defined ``send`` method is meant to be called from within a signal receiver in the producing service. + +Consuming +--------- + +At a high level, the consumer should be a process that takes the signals and events from the broker and emits the signal with the event. There should be a consumer class that inherits from `EventBusConsumer `_ in openedx-events. + +The consumer class then needs to implement ``consume_indefinitely`` loop, which will stay running and listen to events as they come in. + +We have included an utility function called `prepare_for_new_work_cycle <../../openedx_events/tooling.py#L323>`_ in openedx-events which needs to be called before processing any signal. Currently, it reconnects the db connection if required as well as clears RequestCache and there may be later, more comprehensive changes. These steps mimic some setup/teardown that is normally performed by Django in its request/response based architecture. + +Checkout `consumer.py `_ in event bus redis implementation. + +Abstraction tickets +------------------- + +The known remaining work for a fully abstracted event bus is captured in the `Abstraction tickets `_ diff --git a/docs/how-tos/index.rst b/docs/how-tos/index.rst index f8908b33..3c210b0c 100644 --- a/docs/how-tos/index.rst +++ b/docs/how-tos/index.rst @@ -9,3 +9,4 @@ How-tos adding-events-to-a-service adding-events-to-event-bus using-events + add-new-event-bus-concrete-implementation diff --git a/openedx_events/__init__.py b/openedx_events/__init__.py index 35cdf1da..b1f959b2 100644 --- a/openedx_events/__init__.py +++ b/openedx_events/__init__.py @@ -5,4 +5,4 @@ more information about the project. """ -__version__ = "9.4.0" +__version__ = "9.5.0" diff --git a/openedx_events/tooling.py b/openedx_events/tooling.py index bee23a9a..4994c833 100644 --- a/openedx_events/tooling.py +++ b/openedx_events/tooling.py @@ -7,7 +7,9 @@ from logging import getLogger from django.conf import settings +from django.db import connection from django.dispatch import Signal +from edx_django_utils.cache import RequestCache from openedx_events.data import EventsMetadata from openedx_events.exceptions import SenderValidationError @@ -292,3 +294,45 @@ def load_all_signals(): Loads all non-test signals.py modules. """ _process_all_signals_modules(import_module) + + +def _reconnect_to_db_if_needed(): # pragma: no cover + """ + Reconnects the db connection if needed. + + This is important because Django only does connection validity/age checks as part of + its request/response cycle, which isn't in effect for the consume-loop. If we don't + force these checks, a broken connection will remain broken indefinitely. For most + consumers, this will cause event processing to fail. + """ + has_connection = bool(connection.connection) + requires_reconnect = has_connection and not connection.is_usable() + if requires_reconnect: + connection.connect() + + +def _clear_request_cache(): # pragma: no cover + """ + Clear the RequestCache so that each event consumption starts fresh. + + Signal handlers may be written with the assumption that they are called in the context + of a web request, so we clear the request cache just in case. + """ + RequestCache.clear_all_namespaces() + + +def prepare_for_new_work_cycle(): # pragma: no cover + """ + Ensure that the application state is appropriate for performing a new unit of work. + + This mimics some setup/teardown that is normally performed by Django in its + request/response based architecture and that is needed for ensuring a clean and + usable state in this worker-based application. + + See https://github.com/openedx/openedx-events/issues/236 for details. + """ + # Ensure that the database connection is active and usable. + _reconnect_to_db_if_needed() + + # Clear the request cache, in case anything in the signal handlers rely on it. + _clear_request_cache() diff --git a/requirements/base.in b/requirements/base.in index 7c12f67c..5afd59f7 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,6 +1,7 @@ # Core requirements for using this application -c constraints.txt +edx_django_utils django attrs fastavro diff --git a/requirements/base.txt b/requirements/base.txt index fcf62dd3..ec1d36fa 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,24 +8,47 @@ asgiref==3.7.2 # via django attrs==23.2.0 # via -r requirements/base.in -django==3.2.23 +cffi==1.16.0 + # via pynacl +click==8.1.7 + # via edx-django-utils +django==3.2.24 # via # -c requirements/common_constraints.txt # -r requirements/base.in + # django-crum + # django-waffle + # edx-django-utils +django-crum==0.7.9 + # via edx-django-utils +django-waffle==4.1.0 + # via edx-django-utils +edx-django-utils==5.10.1 + # via -r requirements/base.in edx-opaque-keys[django]==2.5.1 # via -r requirements/base.in fastavro==1.9.3 # via -r requirements/base.in +newrelic==9.6.0 + # via edx-django-utils pbr==6.0.0 # via stevedore +psutil==5.9.8 + # via edx-django-utils +pycparser==2.21 + # via cffi pymongo==3.13.0 # via edx-opaque-keys -pytz==2023.3.post1 +pynacl==1.5.0 + # via edx-django-utils +pytz==2024.1 # via django sqlparse==0.4.4 # via django stevedore==5.1.0 - # via edx-opaque-keys + # via + # edx-django-utils + # edx-opaque-keys typing-extensions==4.9.0 # via # asgiref diff --git a/requirements/ci.txt b/requirements/ci.txt index b20990d5..742e39c1 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -20,7 +20,7 @@ packaging==23.2 # via # pyproject-api # tox -platformdirs==4.1.0 +platformdirs==4.2.0 # via # tox # virtualenv diff --git a/requirements/dev.txt b/requirements/dev.txt index cb4f7cb3..026272a9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -24,7 +24,7 @@ cachetools==5.3.2 # via # -r requirements/ci.txt # tox -certifi==2023.11.17 +certifi==2024.2.2 # via # -r requirements/quality.txt # requests @@ -32,6 +32,7 @@ cffi==1.16.0 # via # -r requirements/quality.txt # cryptography + # pynacl chardet==5.2.0 # via # -r requirements/ci.txt @@ -47,13 +48,14 @@ click==8.1.7 # -r requirements/quality.txt # click-log # code-annotations + # edx-django-utils # edx-lint # pip-tools click-log==0.4.0 # via # -r requirements/quality.txt # edx-lint -code-annotations==1.5.0 +code-annotations==1.6.0 # via # -r requirements/quality.txt # edx-lint @@ -61,12 +63,12 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.4.0 +coverage[toml]==7.4.1 # via # -r requirements/quality.txt # coverage # pytest-cov -cryptography==42.0.0 +cryptography==42.0.2 # via # -r requirements/quality.txt # secretstorage @@ -74,7 +76,7 @@ ddt==1.7.1 # via -r requirements/quality.txt diff-cover==8.0.3 # via -r requirements/dev.in -dill==0.3.7 +dill==0.3.8 # via # -r requirements/quality.txt # pylint @@ -82,14 +84,27 @@ distlib==0.3.8 # via # -r requirements/ci.txt # virtualenv -django==3.2.23 +django==3.2.24 # via # -c requirements/common_constraints.txt # -r requirements/quality.txt + # django-crum + # django-waffle + # edx-django-utils +django-crum==0.7.9 + # via + # -r requirements/quality.txt + # edx-django-utils +django-waffle==4.1.0 + # via + # -r requirements/quality.txt + # edx-django-utils docutils==0.20.1 # via # -r requirements/quality.txt # readme-renderer +edx-django-utils==5.10.1 + # via -r requirements/quality.txt edx-lint==5.3.6 # via -r requirements/quality.txt edx-opaque-keys[django]==2.5.1 @@ -156,7 +171,7 @@ markdown-it-py==3.0.0 # via # -r requirements/quality.txt # rich -markupsafe==2.1.4 +markupsafe==2.1.5 # via # -r requirements/quality.txt # jinja2 @@ -172,6 +187,10 @@ more-itertools==10.2.0 # via # -r requirements/quality.txt # jaraco-classes +newrelic==9.6.0 + # via + # -r requirements/quality.txt + # edx-django-utils nh3==0.2.15 # via # -r requirements/quality.txt @@ -195,7 +214,7 @@ pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==4.1.0 +platformdirs==4.2.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -209,6 +228,10 @@ pluggy==1.4.0 # diff-cover # pytest # tox +psutil==5.9.8 + # via + # -r requirements/quality.txt + # edx-django-utils pycodestyle==2.11.1 # via -r requirements/quality.txt pycparser==2.21 @@ -247,6 +270,10 @@ pymongo==3.13.0 # via # -r requirements/quality.txt # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/quality.txt + # edx-django-utils pyproject-api==1.6.1 # via # -r requirements/ci.txt @@ -255,20 +282,20 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.4.4 +pytest==8.0.0 # via # -r requirements/quality.txt # pytest-cov # pytest-django pytest-cov==4.1.0 # via -r requirements/quality.txt -pytest-django==4.7.0 +pytest-django==4.8.0 # via -r requirements/quality.txt -python-slugify==8.0.1 +python-slugify==8.0.3 # via # -r requirements/quality.txt # code-annotations -pytz==2023.3.post1 +pytz==2024.1 # via # -r requirements/quality.txt # django @@ -317,6 +344,7 @@ stevedore==5.1.0 # via # -r requirements/quality.txt # code-annotations + # edx-django-utils # edx-opaque-keys text-unidecode==1.3 # via @@ -351,7 +379,7 @@ typing-extensions==4.9.0 # edx-opaque-keys # pylint # rich -urllib3==2.1.0 +urllib3==2.2.0 # via # -r requirements/quality.txt # requests diff --git a/requirements/doc.txt b/requirements/doc.txt index d010d6b0..7aee4ef1 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -22,33 +22,48 @@ beautifulsoup4==4.12.3 # via pydata-sphinx-theme build==1.0.3 # via -r requirements/doc.in -certifi==2023.11.17 +certifi==2024.2.2 # via requests cffi==1.16.0 - # via cryptography + # via + # -r requirements/test.txt + # cryptography + # pynacl charset-normalizer==3.3.2 # via requests click==8.1.7 # via # -r requirements/test.txt # code-annotations -code-annotations==1.5.0 + # edx-django-utils +code-annotations==1.6.0 # via -r requirements/test.txt colorama==0.4.6 # via sphinx-autobuild -coverage[toml]==7.4.0 +coverage[toml]==7.4.1 # via # -r requirements/test.txt # coverage # pytest-cov -cryptography==42.0.0 +cryptography==42.0.2 # via secretstorage ddt==1.7.1 # via -r requirements/test.txt -django==3.2.23 +django==3.2.24 # via # -c requirements/common_constraints.txt # -r requirements/test.txt + # django-crum + # django-waffle + # edx-django-utils +django-crum==0.7.9 + # via + # -r requirements/test.txt + # edx-django-utils +django-waffle==4.1.0 + # via + # -r requirements/test.txt + # edx-django-utils doc8==1.1.1 # via -r requirements/doc.in docutils==0.19 @@ -58,6 +73,8 @@ docutils==0.19 # readme-renderer # restructuredtext-lint # sphinx +edx-django-utils==5.10.1 + # via -r requirements/test.txt edx-opaque-keys[django]==2.5.1 # via # -r requirements/test.txt @@ -101,7 +118,7 @@ livereload==2.6.3 # via sphinx-autobuild markdown-it-py==3.0.0 # via rich -markupsafe==2.1.4 +markupsafe==2.1.5 # via # -r requirements/test.txt # jinja2 @@ -109,6 +126,10 @@ mdurl==0.1.2 # via markdown-it-py more-itertools==10.2.0 # via jaraco-classes +newrelic==9.6.0 + # via + # -r requirements/test.txt + # edx-django-utils nh3==0.2.15 # via readme-renderer packaging==23.2 @@ -128,8 +149,14 @@ pluggy==1.4.0 # via # -r requirements/test.txt # pytest +psutil==5.9.8 + # via + # -r requirements/test.txt + # edx-django-utils pycparser==2.21 - # via cffi + # via + # -r requirements/test.txt + # cffi pydata-sphinx-theme==0.14.4 # via sphinx-book-theme pygments==2.17.2 @@ -144,22 +171,26 @@ pymongo==3.13.0 # via # -r requirements/test.txt # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/test.txt + # edx-django-utils pyproject-hooks==1.0.0 # via build -pytest==7.4.4 +pytest==8.0.0 # via # -r requirements/test.txt # pytest-cov # pytest-django pytest-cov==4.1.0 # via -r requirements/test.txt -pytest-django==4.7.0 +pytest-django==4.8.0 # via -r requirements/test.txt -python-slugify==8.0.1 +python-slugify==8.0.3 # via # -r requirements/test.txt # code-annotations -pytz==2023.3.post1 +pytz==2024.1 # via # -r requirements/test.txt # babel @@ -230,6 +261,7 @@ stevedore==5.1.0 # -r requirements/test.txt # code-annotations # doc8 + # edx-django-utils # edx-opaque-keys text-unidecode==1.3 # via @@ -254,7 +286,7 @@ typing-extensions==4.9.0 # edx-opaque-keys # pydata-sphinx-theme # rich -urllib3==2.1.0 +urllib3==2.2.0 # via # requests # twine diff --git a/requirements/pip.txt b/requirements/pip.txt index a4cf5307..dfa2b778 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -8,7 +8,7 @@ wheel==0.42.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==23.3.2 +pip==24.0 # via -r requirements/pip.in setuptools==69.0.3 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index cf075b15..b7b0d436 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -15,10 +15,13 @@ astroid==2.15.8 # pylint-celery attrs==23.2.0 # via -r requirements/test.txt -certifi==2023.11.17 +certifi==2024.2.2 # via requests cffi==1.16.0 - # via cryptography + # via + # -r requirements/test.txt + # cryptography + # pynacl charset-normalizer==3.3.2 # via requests click==8.1.7 @@ -26,30 +29,44 @@ click==8.1.7 # -r requirements/test.txt # click-log # code-annotations + # edx-django-utils # edx-lint click-log==0.4.0 # via edx-lint -code-annotations==1.5.0 +code-annotations==1.6.0 # via # -r requirements/test.txt # edx-lint -coverage[toml]==7.4.0 +coverage[toml]==7.4.1 # via # -r requirements/test.txt # coverage # pytest-cov -cryptography==42.0.0 +cryptography==42.0.2 # via secretstorage ddt==1.7.1 # via -r requirements/test.txt -dill==0.3.7 +dill==0.3.8 # via pylint -django==3.2.23 +django==3.2.24 # via # -c requirements/common_constraints.txt # -r requirements/test.txt + # django-crum + # django-waffle + # edx-django-utils +django-crum==0.7.9 + # via + # -r requirements/test.txt + # edx-django-utils +django-waffle==4.1.0 + # via + # -r requirements/test.txt + # edx-django-utils docutils==0.20.1 # via readme-renderer +edx-django-utils==5.10.1 + # via -r requirements/test.txt edx-lint==5.3.6 # via -r requirements/quality.in edx-opaque-keys[django]==2.5.1 @@ -94,7 +111,7 @@ lazy-object-proxy==1.10.0 # via astroid markdown-it-py==3.0.0 # via rich -markupsafe==2.1.4 +markupsafe==2.1.5 # via # -r requirements/test.txt # jinja2 @@ -104,6 +121,10 @@ mdurl==0.1.2 # via markdown-it-py more-itertools==10.2.0 # via jaraco-classes +newrelic==9.6.0 + # via + # -r requirements/test.txt + # edx-django-utils nh3==0.2.15 # via readme-renderer packaging==23.2 @@ -116,16 +137,22 @@ pbr==6.0.0 # stevedore pkginfo==1.9.6 # via twine -platformdirs==4.1.0 +platformdirs==4.2.0 # via pylint pluggy==1.4.0 # via # -r requirements/test.txt # pytest +psutil==5.9.8 + # via + # -r requirements/test.txt + # edx-django-utils pycodestyle==2.11.1 # via -r requirements/quality.in pycparser==2.21 - # via cffi + # via + # -r requirements/test.txt + # cffi pydocstyle==6.3.0 # via -r requirements/quality.in pygments==2.17.2 @@ -150,20 +177,24 @@ pymongo==3.13.0 # via # -r requirements/test.txt # edx-opaque-keys -pytest==7.4.4 +pynacl==1.5.0 + # via + # -r requirements/test.txt + # edx-django-utils +pytest==8.0.0 # via # -r requirements/test.txt # pytest-cov # pytest-django pytest-cov==4.1.0 # via -r requirements/test.txt -pytest-django==4.7.0 +pytest-django==4.8.0 # via -r requirements/test.txt -python-slugify==8.0.1 +python-slugify==8.0.3 # via # -r requirements/test.txt # code-annotations -pytz==2023.3.post1 +pytz==2024.1 # via # -r requirements/test.txt # django @@ -197,6 +228,7 @@ stevedore==5.1.0 # via # -r requirements/test.txt # code-annotations + # edx-django-utils # edx-opaque-keys text-unidecode==1.3 # via @@ -220,7 +252,7 @@ typing-extensions==4.9.0 # edx-opaque-keys # pylint # rich -urllib3==2.1.0 +urllib3==2.2.0 # via # requests # twine diff --git a/requirements/test.txt b/requirements/test.txt index b3ba2309..dfce36ff 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -10,20 +10,40 @@ asgiref==3.7.2 # django attrs==23.2.0 # via -r requirements/base.txt +cffi==1.16.0 + # via + # -r requirements/base.txt + # pynacl click==8.1.7 - # via code-annotations -code-annotations==1.5.0 + # via + # -r requirements/base.txt + # code-annotations + # edx-django-utils +code-annotations==1.6.0 # via -r requirements/test.in -coverage[toml]==7.4.0 +coverage[toml]==7.4.1 # via # coverage # pytest-cov ddt==1.7.1 # via -r requirements/test.in -django==3.2.23 +django==3.2.24 # via # -c requirements/common_constraints.txt # -r requirements/base.txt + # django-crum + # django-waffle + # edx-django-utils +django-crum==0.7.9 + # via + # -r requirements/base.txt + # edx-django-utils +django-waffle==4.1.0 + # via + # -r requirements/base.txt + # edx-django-utils +edx-django-utils==5.10.1 + # via -r requirements/base.txt edx-opaque-keys[django]==2.5.1 # via # -r requirements/base.txt @@ -36,8 +56,12 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.3 # via code-annotations -markupsafe==2.1.4 +markupsafe==2.1.5 # via jinja2 +newrelic==9.6.0 + # via + # -r requirements/base.txt + # edx-django-utils packaging==23.2 # via pytest pbr==6.0.0 @@ -46,21 +70,33 @@ pbr==6.0.0 # stevedore pluggy==1.4.0 # via pytest +psutil==5.9.8 + # via + # -r requirements/base.txt + # edx-django-utils +pycparser==2.21 + # via + # -r requirements/base.txt + # cffi pymongo==3.13.0 # via # -r requirements/base.txt # edx-opaque-keys -pytest==7.4.4 +pynacl==1.5.0 + # via + # -r requirements/base.txt + # edx-django-utils +pytest==8.0.0 # via # pytest-cov # pytest-django pytest-cov==4.1.0 # via -r requirements/test.in -pytest-django==4.7.0 +pytest-django==4.8.0 # via -r requirements/test.in -python-slugify==8.0.1 +python-slugify==8.0.3 # via code-annotations -pytz==2023.3.post1 +pytz==2024.1 # via # -r requirements/base.txt # django @@ -74,6 +110,7 @@ stevedore==5.1.0 # via # -r requirements/base.txt # code-annotations + # edx-django-utils # edx-opaque-keys text-unidecode==1.3 # via python-slugify