diff --git a/cms/__init__.py b/cms/__init__.py index fd763b62c236..e022c752acaf 100644 --- a/cms/__init__.py +++ b/cms/__init__.py @@ -18,6 +18,7 @@ import kombu.utils kombu.utils.entrypoints = lambda namespace: iter([]) -# This will make sure the app is always imported when -# Django starts so that shared_task will use this app. +# This will make sure the app is always imported when Django starts so +# that shared_task will use this app, and also ensures that the celery +# singleton is always configured for the CMS. from .celery import APP as CELERY_APP diff --git a/cms/celery.py b/cms/celery.py index 046760436bf8..7936d04fe3d6 100644 --- a/cms/celery.py +++ b/cms/celery.py @@ -8,20 +8,12 @@ import os -from celery import Celery - from openedx.core.lib.celery.routers import AlternateEnvironmentRouter -# set the default Django settings module for the 'celery' program. +# Set the default Django settings module for the 'celery' program +# and then instantiate the Celery singleton. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cms.envs.production') - -APP = Celery('proj') - -APP.conf.task_protocol = 1 -# Using a string here means the worker will not have to -# pickle the object when using Windows. -APP.config_from_object('django.conf:settings') -APP.autodiscover_tasks() +from openedx.core.lib.celery import APP # pylint: disable=wrong-import-position,unused-import # Import after autodiscovery has had a chance to connect to the import_module signal # so celery doesn't miss any apps getting installed. diff --git a/lms/__init__.py b/lms/__init__.py index 5508a6a75c32..2481eb2689b4 100644 --- a/lms/__init__.py +++ b/lms/__init__.py @@ -14,6 +14,7 @@ import kombu.utils kombu.utils.entrypoints = lambda namespace: iter([]) -# This will make sure the app is always imported when -# Django starts so that shared_task will use this app. +# This will make sure the app is always imported when Django starts so +# that shared_task will use this app, and also ensures that the celery +# singleton is always configured for the LMS. from .celery import APP as CELERY_APP diff --git a/lms/celery.py b/lms/celery.py index dd9a23a8cf8b..7f4c44c9b2eb 100644 --- a/lms/celery.py +++ b/lms/celery.py @@ -8,20 +8,12 @@ import os -from celery import Celery - from openedx.core.lib.celery.routers import AlternateEnvironmentRouter -# set the default Django settings module for the 'celery' program. +# Set the default Django settings module for the 'celery' program +# and then instantiate the Celery singleton. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lms.envs.production') - -APP = Celery('proj') - -APP.conf.task_protocol = 1 -# Using a string here means the worker will not have to -# pickle the object when using Windows. -APP.config_from_object('django.conf:settings') -APP.autodiscover_tasks() +from openedx.core.lib.celery import APP # pylint: disable=wrong-import-position,unused-import class Router(AlternateEnvironmentRouter): diff --git a/openedx/core/lib/celery/__init__.py b/openedx/core/lib/celery/__init__.py index e69de29bb2d1..855970c1df3e 100644 --- a/openedx/core/lib/celery/__init__.py +++ b/openedx/core/lib/celery/__init__.py @@ -0,0 +1,30 @@ +"""Instantiate the singleton Celery instance that is used by either lms or cms. + +WARNING: Do not import this module directly! + +This module should only be imported by cms.celery and lms.celery, which perform +setup in a particular order before and after Celery is instantiated. Otherwise, +it might be possible for the Celery singleton to be created without variant- +specific configuration. + +The module is intended as a way to have a Celery singleton shared between cms +and lms code. Not having a guaranteed singleton means that it is possible for +each of cms and lms to instantiate Celery, and tasks may be nondeterministically +registered to one or the other of the instances and therefore sometimes lost +to the running process. The root ``__init__.py``s of cms and lms both ensure that +this module is loaded when any cms or lms code runs, effectively using the +Python module system as a singleton loader. (This is an incremental improvement +over older code, and there is probably a better mechanism to be had.) +""" + +from celery import Celery + +# WARNING: Do not refer to this unless you are cms.celery or +# lms.celery. See module docstring! +APP = Celery('proj') + +APP.conf.task_protocol = 1 +# Using a string here means the worker will not have to +# pickle the object when using Windows. +APP.config_from_object('django.conf:settings') +APP.autodiscover_tasks()