From 3133364f16d686889c574947ec77e1a49fde173f Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Fri, 12 Jul 2019 10:08:16 -0600 Subject: [PATCH 01/28] Add support for new lower-case settings --- pyramid_celery/__init__.py | 47 +++++++++++---------------------- pyramid_celery/loaders.py | 54 +++++++++++++++++++++++++++++--------- 2 files changed, 57 insertions(+), 44 deletions(-) diff --git a/pyramid_celery/__init__.py b/pyramid_celery/__init__.py index 3fe281d..7320445 100644 --- a/pyramid_celery/__init__.py +++ b/pyramid_celery/__init__.py @@ -3,8 +3,7 @@ from celery import VERSION as celery_version from celery.bin import Option from pyramid.paster import bootstrap, setup_logging -from pyramid_celery.loaders import INILoader -from pyramid.settings import asbool +from pyramid_celery.loaders import INILoader, get_any, set_all def add_preload_arguments(parser): @@ -33,17 +32,6 @@ def add_preload_arguments(parser): ini_file = None -def boolify(config, *names): - """Make config variables boolean. - - Celery wants ``False`` instead of ``"false"`` for CELERY_ALWAYS_EAGER. - """ - - for n in names: - if n in config: - config[n] = asbool(config[n]) - - def configure_logging(*args, **kwargs): setup_logging(ini_file) @@ -52,22 +40,12 @@ def setup_app(app, root, request, registry, closer, ini_location): loader = INILoader(celery_app, ini_file=ini_location) celery_config = loader.read_configuration() - #: TODO: There might be other variables requiring special handling - boolify( - celery_config, 'CELERY_ALWAYS_EAGER', 'CELERY_ENABLE_UTC', - 'CELERY_RESULT_PERSISTENT' - ) - - if asbool(celery_config.get('USE_CELERYCONFIG', False)) is True: - config_path = 'celeryconfig' - celery_app.config_from_object(config_path) + if get_any(celery_config, ('use_celeryconfig', 'USE_CELERYCONFIG')): + celery_app.config_from_object('celeryconfig') else: # TODO: Couldn't find a way with celery to do this - hijack_logger = asbool( - celery_config.get('CELERYD_HIJACK_ROOT_LOGGER', False) - ) - - celery_config['CELERYD_HIJACK_ROOT_LOGGER'] = hijack_logger + hijack_logger = get_any(celery_config, ( + 'worker_hijack_root_logger', 'CELERYD_HIJACK_ROOT_LOGGER'), False) if hijack_logger is False: global ini_file @@ -76,11 +54,16 @@ def setup_app(app, root, request, registry, closer, ini_location): celery_app.config_from_object(celery_config) - celery_app.conf.update({'PYRAMID_APP': app}) - celery_app.conf.update({'PYRAMID_ROOT': root}) - celery_app.conf.update({'PYRAMID_REQUEST': request}) - celery_app.conf.update({'PYRAMID_REGISTRY': registry}) - celery_app.conf.update({'PYRAMID_CLOSER': closer}) + # include custom pyramid_* settings + pyramid_conf = ( + ('pyramid_app', app), + ('pyramid_root', root), + ('pyramid_request', request), + ('pyramid_registry', registry), + ('pyramid_closer', closer), + ) + for k, v in pyramid_conf: + set_all(celery_app.conf, (k, k.upper()), v) @signals.user_preload_options.connect diff --git a/pyramid_celery/loaders.py b/pyramid_celery/loaders.py index 71549a8..f8fc513 100644 --- a/pyramid_celery/loaders.py +++ b/pyramid_celery/loaders.py @@ -4,10 +4,25 @@ import celery.schedules from pyramid.compat import configparser from pyramid.exceptions import ConfigurationError +from pyramid.settings import asbool from functools import partial +def get_any(dict_, keys, default=None): + for key in keys: + try: + return dict_[key] + except KeyError: + pass + return default + + +def set_all(dict_, keys, value): + for key in keys: + dict_[key] = value + + def crontab(value): return celery.schedules.crontab(**value) @@ -84,30 +99,43 @@ def get_route_config(parser, section): class INILoader(celery.loaders.base.BaseLoader): - ConfigParser = configparser.SafeConfigParser - def __init__(self, app, **kwargs): self.celery_conf = kwargs.pop('ini_file') - self.parser = self.ConfigParser() + self.parser = configparser.SafeConfigParser() + self.parser.optionxform = str super(INILoader, self).__init__(app, **kwargs) def read_configuration(self, fail_silently=True): self.parser.read(self.celery_conf) + config_dict = dict(self.parser.items('celery')) + + #: TODO: There might be other variables requiring special handling + bool_settings = [ + 'always_eager', 'CELERY_ALWAYS_EAGER', + 'enable_utc', 'CELERY_ENABLE_UTC', + 'result_persistent', 'CELERY_RESULT_PERSISTENT', + 'worker_hijack_root_logger', 'CELERYD_HIJACK_ROOT_LOGGER', + 'use_celeryconfig', 'USE_CELERYCONFIG', + ] + + for setting in bool_settings: + if setting in config_dict: + config_dict[setting] = asbool(config_dict[setting]) - config_dict = {} - - for key, value in self.parser.items('celery'): - config_dict[key.upper()] = value - - list_settings = ['CELERY_IMPORTS', 'CELERY_ACCEPT_CONTENT'] + list_settings = [ + 'imports', 'CELERY_IMPORTS', + 'accept_content', 'CELERY_ACCEPT_CONTENT', + ] for setting in list_settings: if setting in config_dict: split_setting = config_dict[setting].split() config_dict[setting] = split_setting - tuple_list_settings = ['ADMINS'] + tuple_list_settings = [ + 'admins', 'ADMINS', + ] for setting in tuple_list_settings: if setting in config_dict: @@ -127,9 +155,11 @@ def read_configuration(self, fail_silently=True): route_config[name] = get_route_config(self.parser, section) if beat_config: - config_dict['CELERYBEAT_SCHEDULE'] = beat_config + set_all(config_dict, ( + 'beat_schedule', 'CELERYBEAT_SCHEDULE'), beat_config) if route_config: - config_dict['CELERY_ROUTES'] = route_config + set_all(config_dict, ( + 'task_routes', 'CELERY_ROUTES'), route_config) return config_dict From 33def1c44b775603868e961b975d5e95abe20ae8 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Fri, 12 Jul 2019 10:10:05 -0600 Subject: [PATCH 02/28] Test Celery version x Python version matrix with tox --- setup.cfg | 2 +- tox.ini | 36 ++++++++++++++++++------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/setup.cfg b/setup.cfg index f7aea29..1ddb900 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ -[pytest] +[tool:pytest] norecursedirs = .git .tox diff --git a/tox.ini b/tox.ini index b62ffdb..a12befa 100644 --- a/tox.ini +++ b/tox.ini @@ -1,31 +1,31 @@ [tox] -skipsdist = True -envlist = py27, py34, py35, pypy, flake8, celery3, celery4 - -[base] -commands = - pip install -e . -r test-requirements.txt +skipsdist = true +envlist = clean, py{27,34,35,37,py}-celery{3,4}, flake8 [testenv] -pip_pre=False +usedevelop = true +deps = + -r test-requirements.txt + celery3: celery<4 + celery4: celery>=4 commands = - {[base]commands} - py.test {posargs} py.test {posargs} \ --cov=pyramid_celery \ --cov-report=xml \ - --cov-report=term-missing + --cov-report=term-missing \ + --cov-append [testenv:flake8] +deps = + flake8 +skip_install = true basepython = python3 commands = - pip install flake8 flake8 pyramid_celery/ tests/ -[testenv:celery4] -commands = - pip install 'celery<4' - -[testenv:celery3] -commands = - pip install 'celery>=4' \ No newline at end of file +[testenv:clean] +deps = + coverage +skip_install = true +commands = + coverage erase From 3b64ef42f5c757ca2be472b5e720f00ba4fd82fd Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Fri, 12 Jul 2019 13:03:27 -0600 Subject: [PATCH 03/28] Fix bootstrap and preload options tests --- pyramid_celery/__init__.py | 29 +++++++++++++++++------------ tests/conftest.py | 5 ++--- tests/test_celery.py | 18 ++++++++++++++---- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/pyramid_celery/__init__.py b/pyramid_celery/__init__.py index 7320445..324716d 100644 --- a/pyramid_celery/__init__.py +++ b/pyramid_celery/__init__.py @@ -17,18 +17,23 @@ def add_preload_arguments(parser): ) -celery_app = Celery() -if celery_version.major > 3: - celery_app.user_options['preload'].add(add_preload_arguments) -else: - celery_app.user_options['preload'].add(Option( - '-i', '--ini', default=None, - help='Paste ini configuration file.' - )) - celery_app.user_options['preload'].add(Option( - '--ini-var', default=None, - help='Comma separated list of key=value to pass to ini.' - )) +def make_app(): + app = Celery() + if celery_version.major > 3: + app.user_options['preload'].add(add_preload_arguments) + else: + app.user_options['preload'].add(Option( + '-i', '--ini', default=None, + help='Paste ini configuration file.' + )) + app.user_options['preload'].add(Option( + '--ini-var', default=None, + help='Comma separated list of key=value to pass to ini.' + )) + return app + + +celery_app = make_app() ini_file = None diff --git a/tests/conftest.py b/tests/conftest.py index 21a19f0..af57a8c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,9 @@ import pyramid_celery import pytest -from celery import Celery - @pytest.fixture(autouse=True) def setup_celery_app(monkeypatch): - app = Celery() + # use a fresh app instance for each test + app = pyramid_celery.make_app() monkeypatch.setattr(pyramid_celery, 'celery_app', app) diff --git a/tests/test_celery.py b/tests/test_celery.py index deba4a0..9e3ea84 100644 --- a/tests/test_celery.py +++ b/tests/test_celery.py @@ -69,7 +69,18 @@ def test_preload_ini(): with mock.patch('pyramid_celery.bootstrap') as boot: on_preload_parsed(options) - assert boot.called_with('dev.ini') + boot.assert_called_with('tests/configs/dev.ini') + + +@pytest.mark.unit +def test_preload_options(): + from pyramid_celery import celery_app + from celery.bin.celery import Command + + with mock.patch('pyramid_celery.bootstrap') as boot: + cmd = Command(celery_app) + cmd.setup_app_from_commandline(['--ini', 'tests/configs/dev.ini']) + boot.assert_called_with('tests/configs/dev.ini') @pytest.mark.unit @@ -101,7 +112,7 @@ def test_preload_with_ini_vars(): with mock.patch('pyramid_celery.bootstrap') as boot: on_preload_parsed(options) expected_vars = {'database': 'foo', 'password': 'bar'} - assert boot.called_with('dev.ini', expected_vars) + boot.assert_called_with('tests/configs/dev.ini', options=expected_vars) @pytest.mark.unit @@ -121,8 +132,7 @@ def test_ini_logging(): sender=None, loglevel='INFO', logfile=None, format='', colorize=False, ) - - assert setup_logging.called_with('tests/configs/dev.ini') + setup_logging.assert_called_with('tests/configs/dev.ini') @pytest.mark.unit From c601004a6ae316fbea05a2a9f84619e9cd3b4c90 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Fri, 12 Jul 2019 14:27:08 -0600 Subject: [PATCH 04/28] Reduce production/dev dependencies to setup.py --- requirements.txt | 4 ---- setup.cfg | 20 -------------------- setup.py | 6 +++--- test-requirements.txt | 5 ----- tox.ini | 15 ++++++++++++--- 5 files changed, 15 insertions(+), 35 deletions(-) delete mode 100644 requirements.txt delete mode 100644 setup.cfg delete mode 100644 test-requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4f34d3e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -pyramid -celery -pytest -mock diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 1ddb900..0000000 --- a/setup.cfg +++ /dev/null @@ -1,20 +0,0 @@ -[tool:pytest] -norecursedirs = - .git - .tox - dist - build - *.egg - examples/ - -addopts = - -rxEfs - --strict - --doctest-modules - --doctest-glob=\*.rst - --tb=short - --ignore=setup.py - - tests/ -markers = - unit diff --git a/setup.py b/setup.py index ff8db75..8b30630 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ CHANGES = open(os.path.join(here, 'CHANGES.txt')).read() requires = ['pyramid', 'celery'] - if sys.version_info < (2, 7): requires.append('argparse') @@ -40,6 +39,7 @@ include_package_data=True, zip_safe=False, install_requires=requires, - tests_require=requires + ['pytest', 'mock'], - test_suite="pyramid_celery", + extras_require={ + 'dev': ['tox'] + } ) diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 9b090cd..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -pytest -pytest-cov -mock -tox -redis diff --git a/tox.ini b/tox.ini index a12befa..a6e8963 100644 --- a/tox.ini +++ b/tox.ini @@ -5,14 +5,14 @@ envlist = clean, py{27,34,35,37,py}-celery{3,4}, flake8 [testenv] usedevelop = true deps = - -r test-requirements.txt + mock + pytest + pytest-cov celery3: celery<4 celery4: celery>=4 commands = py.test {posargs} \ --cov=pyramid_celery \ - --cov-report=xml \ - --cov-report=term-missing \ --cov-append [testenv:flake8] @@ -29,3 +29,12 @@ deps = skip_install = true commands = coverage erase + +[tool:pytest] +addopts = + -rxEfs + --strict + --tb=short + --ignore=setup.py +markers = + unit From b79ab95de9256baecaa3977a522a0cdc068528a8 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Fri, 12 Jul 2019 15:14:24 -0600 Subject: [PATCH 05/28] Update CI configuration --- .travis.yml | 19 +++++++++++++------ setup.cfg | 8 ++++++++ tox.ini | 17 ++++++++--------- 3 files changed, 29 insertions(+), 15 deletions(-) create mode 100644 setup.cfg diff --git a/.travis.yml b/.travis.yml index c7fce74..6dc3668 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,24 @@ language: python sudo: false +python: + - '2.7' + - '3.4' + - '3.5' + - '3.6' + - '3.7' + - pypy + env: - - TOXENV=py27 - - TOXENV=py34 - - TOXENV=pypy - - TOXENV=flake8 + - CELERY='<4' + - CELERY='>=4' install: - - travis_retry pip install tox coveralls + - pip install . + - pip install celery$CELERY script: - - travis_retry tox + - pytest --cov=pyramid_celery --cov-report= --cov-append after_success: - coveralls diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..03f770c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,8 @@ +[tool:pytest] +addopts = + -rxEfs + --strict + --tb=short + --ignore=setup.py +markers = + unit diff --git a/tox.ini b/tox.ini index a6e8963..ad182b8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist = true -envlist = clean, py{27,34,35,37,py}-celery{3,4}, flake8 +envlist = clean, py{27,34,35,37,py}-celery{3,4}, flake8, report [testenv] usedevelop = true @@ -13,6 +13,7 @@ deps = commands = py.test {posargs} \ --cov=pyramid_celery \ + --cov-report= \ --cov-append [testenv:flake8] @@ -30,11 +31,9 @@ skip_install = true commands = coverage erase -[tool:pytest] -addopts = - -rxEfs - --strict - --tb=short - --ignore=setup.py -markers = - unit +[testenv:report] +deps = + coverage +skip_install = true +commands = + coverage report From 48c2f259915490e73a4fc58281175c0478763075 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Fri, 12 Jul 2019 15:18:34 -0600 Subject: [PATCH 06/28] Install test dependencies in travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6dc3668..2af145d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: - CELERY='>=4' install: + - pip install mock pytest pytest-cov - pip install . - pip install celery$CELERY From 5f616fa4403f17e9b4716bd2bd497051fd6a25ff Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Fri, 12 Jul 2019 15:25:36 -0600 Subject: [PATCH 07/28] Try tox-travis --- .travis.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2af145d..6291a03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,21 +5,11 @@ python: - '2.7' - '3.4' - '3.5' - - '3.6' - '3.7' - pypy -env: - - CELERY='<4' - - CELERY='>=4' - install: - - pip install mock pytest pytest-cov - - pip install . - - pip install celery$CELERY + - pip install tox-travis script: - - pytest --cov=pyramid_celery --cov-report= --cov-append - -after_success: - - coveralls + - tox From d8f0782c90447203011ee402dd86134e5c3f0edc Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Fri, 12 Jul 2019 15:28:33 -0600 Subject: [PATCH 08/28] Don't test 3.7 on Travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6291a03..2cb5868 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ python: - '2.7' - '3.4' - '3.5' - - '3.7' - pypy install: From afc66b6039515ea11d7e9e6e66c874beffb4c685 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Fri, 12 Jul 2019 15:57:58 -0600 Subject: [PATCH 09/28] Try codecov integration --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2cb5868..4a9b22d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,10 @@ python: - pypy install: - - pip install tox-travis + - pip install codecov tox-travis script: - tox + +after_success: + - codecov From ea2250b6fd788920e0d3fd23861c2423948117aa Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Fri, 12 Jul 2019 16:12:18 -0600 Subject: [PATCH 10/28] Add coveralls integration --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4a9b22d..bfbf49f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,11 @@ python: - pypy install: - - pip install codecov tox-travis + - pip install tox-travis codecov coveralls script: - tox after_success: - codecov + - coveralls From c396978fb4fa0c7c41f96211ac1d3af550896542 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Mon, 15 Jul 2019 12:11:27 -0600 Subject: [PATCH 11/28] Update README --- README.rst | 204 +++++++++++++++++++++++------------------------------ 1 file changed, 89 insertions(+), 115 deletions(-) diff --git a/README.rst b/README.rst index a7ccf46..3b58ca7 100644 --- a/README.rst +++ b/README.rst @@ -1,30 +1,30 @@ Getting Started -===================== -.. image:: https://travis-ci.org/sontek/pyramid_celery.png?branch=master - :target: https://travis-ci.org/sontek/pyramid_celery +=============== +.. image:: https://travis-ci.org/aarki/pyramid_celery.png?branch=master + :target: https://travis-ci.org/aarki/pyramid_celery -.. image:: https://coveralls.io/repos/sontek/pyramid_celery/badge.png?branch=master - :target: https://coveralls.io/r/sontek/pyramid_celery?branch=master +.. image:: https://codecov.io/gh/aarki/pyramid_celery/branch/master/graph/badge.svg + :target: https://codecov.io/gh/aarki/project-name) -.. image:: https://img.shields.io/pypi/v/pyramid_celery.svg - :target: https://pypi.python.org/pypi/pyramid_celery - -Include pyramid_celery either by setting your includes in your .ini, -or by calling ``config.include('pyramid_celery')``: +Include ``pyramid_celery``, either in your ``.ini``: .. code-block:: ini pyramid.includes = pyramid_celery +or, equivalently, using ``config.include``: + +.. code-block:: python + + config.include('pyramid_celery') -Then you just need to tell **pyramid_celery** what ini file your **[celery]** -section is in: +Then you just need to tell ``pyramid_celery`` where to find the ``[celery]`` section: .. code-block:: python config.configure_celery('development.ini') -Then you are free to use celery, for example class based: +Then you are free to use Celery, class-based: .. code-block:: python @@ -34,7 +34,7 @@ Then you are free to use celery, for example class based: def run(self, x, y): print x+y -or decorator based: +or decorator-based: .. code-block:: python @@ -44,39 +44,46 @@ or decorator based: def add(x, y): print x+y -To get pyramid settings you may access them in ``app.conf['PYRAMID_REGISTRY']``. +To get pyramid settings you may access them in ``app.conf.pyramid_registry``. Configuration -===================== -By default **pyramid_celery** assumes you want to configure celery via an ini -settings. You can do this by calling **config.configure_celery('development.ini')** -but if you are already in the **main** of your application and want to use the ini -used to configure the app you can do the following: +============= + +**Note on lower-case settings**: Celery version 4.0 introduced new lower-case settings and setting organization. +Examples in this documentation use the new lower case settings, but ``pyramid_celery`` continues to support old setting +names, as does Celery. + +By default, ``pyramid_celery`` assumes you want to configure celery via ``.ini`` file +settings. You can do this by calling + +.. code-block:: python + + config.configure_celery('development.ini') + +but if you are already in the ``main`` entry point of your application, and want to use the ``.ini`` +used to configure the app, you can do the following: .. code-block:: python config.configure_celery(global_config['__file__']) -If you want to use the standard **celeryconfig** python file you can set the -**USE_CELERYCONFIG = True** like this: +If you want to configure Celery from the standard ``celeryconfig`` Python file, you can specify .. code-block:: ini [celery] - USE_CELERYCONFIG = True + use_celeryconfig = True -You can get more information for celeryconfig.py here: +You can get more information on ``celeryconfig.py`` `here `_. -http://celery.readthedocs.io/en/latest/userguide/configuration.html - -An example ini configuration looks like this: +An example ``.ini`` configuration looks like this: .. code-block:: ini [celery] - BROKER_URL = redis://localhost:1337/0 - CELERY_IMPORTS = app1.tasks - app2.tasks + broker_url = redis://localhost:1337/0 + imports = app1.tasks + app2.tasks [celerybeat:task1] task = app1.tasks.Task1 @@ -84,18 +91,21 @@ An example ini configuration looks like this: schedule = {"minute": 0} Scheduled/Periodic Tasks ------------------------------ -To use celerybeat (periodic tasks) you need to declare 1 ``celerybeat`` config -section per task. The options are: - -- **task** - The python task you need executed. -- **type** - The type of scheduling your configuration uses, options are - ``crontab``, ``timedelta``, and ``integer``. -- **schedule** - The actual schedule for your ``type`` of configuration. -- **args** - Additional positional arguments. -- **kwargs** - Additional keyword arguments. - -Example configuration for this: +------------------------ +To use celery beat (periodic tasks), declare one ``[celerybeat:...]`` config section per task. The options are: + +:``task``: + The python task you need executed. +:``type``: + The type of scheduling your configuration uses, one of ``crontab``, ``timedelta``, or ``integer``. +:``schedule``: + The actual schedule for your ``type`` of configuration, parsed as JSON. +:``args``: + Additional positional arguments, parsed as JSON. +:``kwargs``: + Additional keyword arguments, parsed as JSON. + +Example configuration: .. code-block:: ini @@ -121,14 +131,16 @@ Example configuration for this: type = integer schedule = 30 -A gotcha you want to watchout for is that the date/time in scheduled tasks -is UTC by default. If you want to schedule for an exact date/time for your -local timezone you need to set ``CELERY_TIMEZONE``. Documentation for that -can be found here: +Tasks are scheduled in UTC by default. If you want to schedule at a specific date/time in a different time zone, +use the ``timezone`` +`setting `_: -http://celery.readthedocs.org/en/latest/userguide/periodic-tasks.html#time-zones +.. code-block:: ini + + [celery] + timezone = US/Pacific -If you need to find out what timezones are available you can do the following: +To get a list of available time zones, do .. code-block:: python @@ -136,24 +148,8 @@ If you need to find out what timezones are available you can do the following: from pytz import all_timezones pprint(all_timezones) -Worker Execution ----------------- -The celerybeat worker will read your configuration and schedule tasks in the -queue to be executed at the time defined. This means if you are using -celerybeat you will end up running *2* workers: - -.. code-block:: bash - - $ celery worker -A pyramid_celery.celery_app --ini development.ini - $ celery beat -A pyramid_celery.celery_app --ini development.ini - -The first command is the standard worker command that will read messages off -of the queue and run the task. The second command will read the celerybeat -configuration and periodically schedule tasks on the queue. - - Routing ------------------------------ +------- If you would like to route a task to a specific queue you can define a route per task by declaring their ``queue`` and/or ``routing_key`` in a ``celeryroute`` section. @@ -170,50 +166,41 @@ An example configuration for this: queue = fast_tasks Running the worker -============================= -To run the worker we just use the standard celery command with an additional -argument: +================== + +To run the worker, use the ``celery worker`` command, and pass an additional ``--ini`` argument. .. code-block:: bash celery worker -A pyramid_celery.celery_app --ini development.ini -If you've defined variables in your .ini like %(database_username)s you can use -the *--ini-var* argument, which is a comma separated list of key value pairs: +To run the celery beat task scheduler, use the ``--beat`` option (during development), or the ``celery beat`` command +(in production). .. code-block:: bash - celery worker -A pyramid_celery.celery_app --ini development.ini --ini-var=database_username=sontek,database_password=OhYeah! - -The values in *ini-var* cannot have spaces in them, this will break celery's -parser. - -The reason it is a csv instead of using *--ini-var* multiple times is because of -a bug in celery itself. When they fix the bug we will re-work the API. Ticket -is here: - -https://github.com/celery/celery/pull/2435 + celery beat -A pyramid_celery.celery_app --ini development.ini -If you use celerybeat scheduler you need to run with the *--beat* flag to run -beat and the worker at the same time. +To expand variables in your ``.ini`` (e.g. ``%(database_username)s``), use the ``--ini-var`` option, and pass a +comma-separated list of key-value pairs. .. code-block:: bash - celery worker --beat -A pyramid_celery.celery_app --ini development.ini + celery worker -A pyramid_celery.celery_app --ini development.ini --ini-var=database_username=sontek,database_password=OhYeah! -Or you can launch it separately like this: +The ``--ini-var`` values cannot contain spaces, as this will break command-line argument parsing. -.. code-block:: bash - - celery beat -A pyramid_celery.celery_app --ini development.ini +Using ``--ini-var`` multiple times is not supported, due to a bug in Celery. The issue can be tracked +`here `. Logging -===================== -If you use the **.ini** configuration (i.e don't use celeryconfig.py) then the -logging configuration will be loaded from the .ini and will not use the default -celery loggers. +======= -You most likely want to add a logging section to your ini for celery as well: +If you use ``.ini`` configuration (rather than ``celeryconfig.py``), then the +logging configuration will be loaded from the ``.ini``, and the default Celery +loggers will not be used. + +You most likely want to add a ``[logger_celery]`` section to your ``.ini``. .. code-block:: ini @@ -224,36 +211,23 @@ You most likely want to add a logging section to your ini for celery as well: and then update your ``[loggers]`` section to include it. -If you want use the default celery loggers then you can set -**CELERYD_HIJACK_ROOT_LOGGER=True** in the [celery] section of your .ini. +If you want to use the default Celery loggers, use the ``worker_hijack_root_logger`` setting. + +.. code-block:: ini + + [celery] + worker_hijack_root_logger = True -Celery worker processes do not propagade exceptions inside tasks, but swallow them -silently by default. This is related to the behavior of reading asynchronous -task results back. To see if your tasks fail you might need to configure -``celery.worker.job`` logger to propagate exceptions: +Celery worker processes do not propagade exceptions inside tasks, swallowing them silently by default. +To fix, this, configure the ``celery.worker.job`` logger to propagate exceptions: .. code-block:: ini # Make sure Celery worker doesn't silently swallow exceptions - # See http://stackoverflow.com/a/20719461/315168 + # See http://stackoverflow.com/a/20719461/315168 # https://github.com/celery/celery/issues/2437 [logger_celery_worker_job] level = ERROR - handlers = + handlers = qualname = celery.worker.job propagate = 1 - -If you want use the default celery loggers then you can set -**CELERYD_HIJACK_ROOT_LOGGER=True** in the [celery] section of your .ini - -Demo -===================== -To see it all in action check out examples/long_running_with_tm, run -redis-server and then do: - -.. code-block:: bash - - $ python setup.py develop - $ populate_long_running_with_tm development.ini - $ pserve ./development.ini - $ celery worker -A pyramid_celery.celery_app --ini development.ini From 1d9b46e23169a19a27ea42a6b5454804e3de9ec8 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Mon, 15 Jul 2019 14:34:02 -0600 Subject: [PATCH 12/28] Fix link in README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3b58ca7..0448cc9 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ Getting Started :target: https://travis-ci.org/aarki/pyramid_celery .. image:: https://codecov.io/gh/aarki/pyramid_celery/branch/master/graph/badge.svg - :target: https://codecov.io/gh/aarki/project-name) + :target: https://codecov.io/gh/aarki/pyramid_celery Include ``pyramid_celery``, either in your ``.ini``: From dd7d741675578140672b58abb138ef7709678ed9 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Mon, 15 Jul 2019 15:23:52 -0600 Subject: [PATCH 13/28] Convert README.rst to README.md --- README.md | 237 +++++++++++++++++++++++++++++++++++++++++++++++++++++ README.rst | 233 ---------------------------------------------------- setup.py | 2 +- 3 files changed, 238 insertions(+), 234 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/README.md b/README.md new file mode 100644 index 0000000..8486399 --- /dev/null +++ b/README.md @@ -0,0 +1,237 @@ +# Getting Started + +[![image](https://travis-ci.org/aarki/pyramid_celery.png?branch=master)](https://travis-ci.org/aarki/pyramid_celery) +[![image](https://codecov.io/gh/aarki/pyramid_celery/branch/master/graph/badge.svg)](https://codecov.io/gh/aarki/pyramid_celery) + +Include `pyramid_celery`, either in your `.ini`: + +``` ini +pyramid.includes = pyramid_celery +``` + +or, equivalently, using `config.include`: + +``` python +config.include('pyramid_celery') +``` + +Then you just need to tell `pyramid_celery` where to find the `[celery]` +section: + +``` python +config.configure_celery('development.ini') +``` + +Then you are free to use Celery, class-based: + +``` python +from pyramid_celery import celery_app as app + +class AddTask(app.Task): + def run(self, x, y): + print x+y +``` + +or decorator-based: + +``` python +from pyramid_celery import celery_app as app + +@app.task +def add(x, y): + print x+y +``` + +To get pyramid settings you may access them in +`app.conf.pyramid_registry`. + +# Configuration + +**Note on lower-case settings**: Celery version 4.0 introduced new +lower-case settings and setting organization. Examples in this +documentation use the new lower case settings, but `pyramid_celery` +continues to support old setting names, as does Celery. + +By default, `pyramid_celery` assumes you want to configure celery via +`.ini` file settings. You can do this by calling + +``` python +config.configure_celery('development.ini') +``` + +but if you are already in the `main` entry point of your application, +and want to use the `.ini` used to configure the app, you can do the +following: + +``` python +config.configure_celery(global_config['__file__']) +``` + +If you want to configure Celery from the standard `celeryconfig` Python +file, you can specify + +``` ini +[celery] +use_celeryconfig = True +``` + +You can get more information on `celeryconfig.py` +[here](http://celery.readthedocs.io/en/latest/userguide/configuration.html/). + +An example `.ini` configuration looks like this: + +``` ini +[celery] +broker_url = redis://localhost:1337/0 +imports = app1.tasks + app2.tasks + +[celerybeat:task1] +task = app1.tasks.Task1 +type = crontab +schedule = {"minute": 0} +``` + +## Scheduled/Periodic Tasks + +To use celery beat (periodic tasks), declare one `[celerybeat:...]` +config section per task. Within each section, the following settings are available: + + - `task`: the Python task you need executed. + - `type`: the type of scheduling your configuration uses, one of `crontab`, + `timedelta`, or `integer`. + - `schedule`: the actual schedule for your `type` of configuration, parsed as + JSON. + - `args`: additional positional arguments, parsed as JSON. + - `kwargs`: additional keyword arguments, parsed as JSON. + +Example configuration: + +``` ini +[celerybeat:task1] +task = app1.tasks.Task1 +type = crontab +schedule = {"minute": 0} + +[celerybeat:task2] +task = app1.tasks.Task2 +type = timedelta +schedule = {"seconds": 30} +args = [16, 16] + +[celerybeat:task3] +task = app2.tasks.Task1 +type = crontab +schedule = {"hour": 0, "minute": 0} +kwargs = {"boom": "shaka"} + +[celerybeat:task4] +task = myapp.tasks.Task4 +type = integer +schedule = 30 +``` + +Tasks are scheduled in UTC by default. If you want to schedule at a +specific date/time in a different time zone, use the +[`timezone` setting](https://celery.readthedocs.io/en/latest/userguide/configuration.html#std:setting-timezone/): + +``` ini +[celery] +timezone = US/Pacific +``` + +To get a list of available time zones, do + +``` python +from pprint import pprint +from pytz import all_timezones +pprint(all_timezones) +``` + +## Routing + +If you would like to route a task to a specific queue you can define a +route per task by declaring their `queue` and/or `routing_key` in a +`[celeryroute:...]` section. + +An example configuration for this: + +``` ini +[celeryroute:otherapp.tasks.Task3] +queue = slow_tasks +routing_key = turtle + +[celeryroute:myapp.tasks.Task1] +queue = fast_tasks +``` + +# Running the worker + +To run the worker, use the `celery worker` command, and pass an +additional `--ini` argument. + +``` bash +celery worker -A pyramid_celery.celery_app --ini development.ini +``` + +To run the celery beat task scheduler, use the `--beat` option (during +development), or the `celery beat` command (in production). + +``` bash +celery beat -A pyramid_celery.celery_app --ini development.ini +``` + +To expand variables in your `.ini` (e.g. `%(database_username)s`), use +the `--ini-var` option, and pass a comma-separated list of key-value +pairs. + +``` bash +celery worker -A pyramid_celery.celery_app --ini development.ini --ini-var=database_username=sontek,database_password=OhYeah! +``` + +The `--ini-var` values cannot contain spaces, as this will break +command-line argument parsing. + +Using `--ini-var` multiple times is not supported, due to a bug in +Celery. The issue can be tracked here +\. + +# Logging + +If you use `.ini` configuration (rather than `celeryconfig.py`), then +the logging configuration will be loaded from the `.ini`, and the +default Celery loggers will not be used. + +You most likely want to add a `[logger_celery]` section to your `.ini`. + +``` ini +[logger_celery] +level = INFO +handlers = +qualname = celery +``` + +and then update your `[loggers]` section to include it. + +If you want to use the default Celery loggers, use the +[`worker_hijack_root_logger` setting](https://celery.readthedocs.io/en/latest/userguide/configuration.html#std:setting-worker_hijack_root_logger). + +``` ini +[celery] +worker_hijack_root_logger = True +``` + +Celery worker processes do not propagade exceptions inside tasks, +swallowing them silently by default. To fix, this, configure the +`celery.worker.job` logger to propagate exceptions: + +``` ini +# Make sure Celery worker doesn't silently swallow exceptions +# See http://stackoverflow.com/a/20719461/315168 +# https://github.com/celery/celery/issues/2437 +[logger_celery_worker_job] +level = ERROR +handlers = +qualname = celery.worker.job +propagate = 1 +``` diff --git a/README.rst b/README.rst deleted file mode 100644 index 0448cc9..0000000 --- a/README.rst +++ /dev/null @@ -1,233 +0,0 @@ -Getting Started -=============== -.. image:: https://travis-ci.org/aarki/pyramid_celery.png?branch=master - :target: https://travis-ci.org/aarki/pyramid_celery - -.. image:: https://codecov.io/gh/aarki/pyramid_celery/branch/master/graph/badge.svg - :target: https://codecov.io/gh/aarki/pyramid_celery - -Include ``pyramid_celery``, either in your ``.ini``: - -.. code-block:: ini - - pyramid.includes = pyramid_celery - -or, equivalently, using ``config.include``: - -.. code-block:: python - - config.include('pyramid_celery') - -Then you just need to tell ``pyramid_celery`` where to find the ``[celery]`` section: - -.. code-block:: python - - config.configure_celery('development.ini') - -Then you are free to use Celery, class-based: - -.. code-block:: python - - from pyramid_celery import celery_app as app - - class AddTask(app.Task): - def run(self, x, y): - print x+y - -or decorator-based: - -.. code-block:: python - - from pyramid_celery import celery_app as app - - @app.task - def add(x, y): - print x+y - -To get pyramid settings you may access them in ``app.conf.pyramid_registry``. - -Configuration -============= - -**Note on lower-case settings**: Celery version 4.0 introduced new lower-case settings and setting organization. -Examples in this documentation use the new lower case settings, but ``pyramid_celery`` continues to support old setting -names, as does Celery. - -By default, ``pyramid_celery`` assumes you want to configure celery via ``.ini`` file -settings. You can do this by calling - -.. code-block:: python - - config.configure_celery('development.ini') - -but if you are already in the ``main`` entry point of your application, and want to use the ``.ini`` -used to configure the app, you can do the following: - -.. code-block:: python - - config.configure_celery(global_config['__file__']) - -If you want to configure Celery from the standard ``celeryconfig`` Python file, you can specify - -.. code-block:: ini - - [celery] - use_celeryconfig = True - -You can get more information on ``celeryconfig.py`` `here `_. - -An example ``.ini`` configuration looks like this: - -.. code-block:: ini - - [celery] - broker_url = redis://localhost:1337/0 - imports = app1.tasks - app2.tasks - - [celerybeat:task1] - task = app1.tasks.Task1 - type = crontab - schedule = {"minute": 0} - -Scheduled/Periodic Tasks ------------------------- -To use celery beat (periodic tasks), declare one ``[celerybeat:...]`` config section per task. The options are: - -:``task``: - The python task you need executed. -:``type``: - The type of scheduling your configuration uses, one of ``crontab``, ``timedelta``, or ``integer``. -:``schedule``: - The actual schedule for your ``type`` of configuration, parsed as JSON. -:``args``: - Additional positional arguments, parsed as JSON. -:``kwargs``: - Additional keyword arguments, parsed as JSON. - -Example configuration: - -.. code-block:: ini - - [celerybeat:task1] - task = app1.tasks.Task1 - type = crontab - schedule = {"minute": 0} - - [celerybeat:task2] - task = app1.tasks.Task2 - type = timedelta - schedule = {"seconds": 30} - args = [16, 16] - - [celerybeat:task3] - task = app2.tasks.Task1 - type = crontab - schedule = {"hour": 0, "minute": 0} - kwargs = {"boom": "shaka"} - - [celerybeat:task4] - task = myapp.tasks.Task4 - type = integer - schedule = 30 - -Tasks are scheduled in UTC by default. If you want to schedule at a specific date/time in a different time zone, -use the ``timezone`` -`setting `_: - -.. code-block:: ini - - [celery] - timezone = US/Pacific - -To get a list of available time zones, do - -.. code-block:: python - - from pprint import pprint - from pytz import all_timezones - pprint(all_timezones) - -Routing -------- -If you would like to route a task to a specific queue you can define a route -per task by declaring their ``queue`` and/or ``routing_key`` in a -``celeryroute`` section. - -An example configuration for this: - -.. code-block:: ini - - [celeryroute:otherapp.tasks.Task3] - queue = slow_tasks - routing_key = turtle - - [celeryroute:myapp.tasks.Task1] - queue = fast_tasks - -Running the worker -================== - -To run the worker, use the ``celery worker`` command, and pass an additional ``--ini`` argument. - -.. code-block:: bash - - celery worker -A pyramid_celery.celery_app --ini development.ini - -To run the celery beat task scheduler, use the ``--beat`` option (during development), or the ``celery beat`` command -(in production). - -.. code-block:: bash - - celery beat -A pyramid_celery.celery_app --ini development.ini - -To expand variables in your ``.ini`` (e.g. ``%(database_username)s``), use the ``--ini-var`` option, and pass a -comma-separated list of key-value pairs. - -.. code-block:: bash - - celery worker -A pyramid_celery.celery_app --ini development.ini --ini-var=database_username=sontek,database_password=OhYeah! - -The ``--ini-var`` values cannot contain spaces, as this will break command-line argument parsing. - -Using ``--ini-var`` multiple times is not supported, due to a bug in Celery. The issue can be tracked -`here `. - -Logging -======= - -If you use ``.ini`` configuration (rather than ``celeryconfig.py``), then the -logging configuration will be loaded from the ``.ini``, and the default Celery -loggers will not be used. - -You most likely want to add a ``[logger_celery]`` section to your ``.ini``. - -.. code-block:: ini - - [logger_celery] - level = INFO - handlers = - qualname = celery - -and then update your ``[loggers]`` section to include it. - -If you want to use the default Celery loggers, use the ``worker_hijack_root_logger`` setting. - -.. code-block:: ini - - [celery] - worker_hijack_root_logger = True - -Celery worker processes do not propagade exceptions inside tasks, swallowing them silently by default. -To fix, this, configure the ``celery.worker.job`` logger to propagate exceptions: - -.. code-block:: ini - - # Make sure Celery worker doesn't silently swallow exceptions - # See http://stackoverflow.com/a/20719461/315168 - # https://github.com/celery/celery/issues/2437 - [logger_celery_worker_job] - level = ERROR - handlers = - qualname = celery.worker.job - propagate = 1 diff --git a/setup.py b/setup.py index 8b30630..d362a99 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) -README = open(os.path.join(here, 'README.rst')).read() +README = open(os.path.join(here, 'README.md')).read() CHANGES = open(os.path.join(here, 'CHANGES.txt')).read() requires = ['pyramid', 'celery'] From da6e6d6f64e50483f3f93a305013e386b2d990de Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Mon, 15 Jul 2019 15:36:39 -0600 Subject: [PATCH 14/28] Use action='append' for --ini-var options --- README.md | 12 ++++-------- pyramid_celery/__init__.py | 8 ++++---- tests/test_celery.py | 10 +++++++++- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 8486399..6f943c0 100644 --- a/README.md +++ b/README.md @@ -186,16 +186,12 @@ the `--ini-var` option, and pass a comma-separated list of key-value pairs. ``` bash -celery worker -A pyramid_celery.celery_app --ini development.ini --ini-var=database_username=sontek,database_password=OhYeah! +celery worker -A pyramid_celery.celery_app \ + --ini development.ini \ + --ini-var database_username=sontek \ + --ini-var database_password=OhYeah! ``` -The `--ini-var` values cannot contain spaces, as this will break -command-line argument parsing. - -Using `--ini-var` multiple times is not supported, due to a bug in -Celery. The issue can be tracked here -\. - # Logging If you use `.ini` configuration (rather than `celeryconfig.py`), then diff --git a/pyramid_celery/__init__.py b/pyramid_celery/__init__.py index 324716d..1c86f40 100644 --- a/pyramid_celery/__init__.py +++ b/pyramid_celery/__init__.py @@ -12,7 +12,7 @@ def add_preload_arguments(parser): help='Paste ini configuration file.' ) parser.add_argument( - '--ini-var', default=None, + '--ini-var', default=None, action='append', help='Comma separated list of key=value to pass to ini.' ) @@ -27,7 +27,7 @@ def make_app(): help='Paste ini configuration file.' )) app.user_options['preload'].add(Option( - '--ini-var', default=None, + '--ini-var', default=None, action='append', help='Comma separated list of key=value to pass to ini.' )) return app @@ -81,8 +81,8 @@ def on_preload_parsed(options, **kwargs): exit(-1) options = {} - if ini_vars is not None: - for pairs in ini_vars.split(','): + if ini_vars: + for pairs in ini_vars: key, value = pairs.split('=') options[key] = value diff --git a/tests/test_celery.py b/tests/test_celery.py index 9e3ea84..fb790a8 100644 --- a/tests/test_celery.py +++ b/tests/test_celery.py @@ -82,6 +82,14 @@ def test_preload_options(): cmd.setup_app_from_commandline(['--ini', 'tests/configs/dev.ini']) boot.assert_called_with('tests/configs/dev.ini') + with mock.patch('pyramid_celery.bootstrap') as boot: + cmd = Command(celery_app) + cmd.setup_app_from_commandline([ + '--ini', 'tests/configs/dev.ini', + '--ini-var', 'foo=bar', + '--ini-var=bar=baz', + ]) + @pytest.mark.unit def test_celery_imports(): @@ -106,7 +114,7 @@ def test_preload_with_ini_vars(): from pyramid_celery import on_preload_parsed options = { 'ini': 'tests/configs/dev.ini', - 'ini_var': 'database=foo,password=bar', + 'ini_var': ['database=foo', 'password=bar'], } with mock.patch('pyramid_celery.bootstrap') as boot: From 4cd6aaea116d3664a238d7d3e33af5eabc9b3f5b Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Mon, 15 Jul 2019 15:42:57 -0600 Subject: [PATCH 15/28] Use shields.io badges --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6f943c0..b2a04c9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Getting Started -[![image](https://travis-ci.org/aarki/pyramid_celery.png?branch=master)](https://travis-ci.org/aarki/pyramid_celery) -[![image](https://codecov.io/gh/aarki/pyramid_celery/branch/master/graph/badge.svg)](https://codecov.io/gh/aarki/pyramid_celery) +[![image](https://img.shields.io/travis/aarki/pyramid_celery/master.svg)](https://travis-ci.org/aarki/pyramid_celery) +[![image](https://img.shields.io/codecov/c/gh/aarki/pyramid_celery/master.svg)](https://codecov.io/gh/aarki/pyramid_celery) Include `pyramid_celery`, either in your `.ini`: From 4a38449455e5843432c4ad89db55d083f74ec3c8 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Mon, 22 Jul 2019 11:18:26 -0600 Subject: [PATCH 16/28] v4.0.0 --- CHANGES.txt | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 94b043f..8206d5a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,10 @@ +4.0.0 +================ + +- Add support for new lower-case settings in Celery 4. +- Support multiple --ini-var command-line arguments. + + 3.0.0 ================ diff --git a/setup.py b/setup.py index d362a99..89e3de1 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup(name='pyramid_celery', - version='3.0.0', + version='4.0.0', description='Celery integration with pyramid', long_description=README + "\n" + CHANGES, classifiers=[ From fd4c4acb3782798acadac2b47c7ae65182581053 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Thu, 7 Nov 2019 10:13:16 -0700 Subject: [PATCH 17/28] Prevent duplicate calls to setup_app --- pyramid_celery/__init__.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pyramid_celery/__init__.py b/pyramid_celery/__init__.py index 1c86f40..55000f9 100644 --- a/pyramid_celery/__init__.py +++ b/pyramid_celery/__init__.py @@ -41,7 +41,7 @@ def configure_logging(*args, **kwargs): setup_logging(ini_file) -def setup_app(app, root, request, registry, closer, ini_location): +def setup_app(ini_location): loader = INILoader(celery_app, ini_file=ini_location) celery_config = loader.read_configuration() @@ -59,6 +59,8 @@ def setup_app(app, root, request, registry, closer, ini_location): celery_app.config_from_object(celery_config) + +def update_app(app=None, root=None, request=None, registry=None, closer=None): # include custom pyramid_* settings pyramid_conf = ( ('pyramid_app', app), @@ -95,18 +97,11 @@ def on_preload_parsed(options, **kwargs): root = env['root'] request = env['request'] closer = env['closer'] - setup_app(app, root, request, registry, closer, ini_location) + update_app(app, root, request, registry, closer) def configure(config, ini_location): - setup_app( - None, - None, - None, - config.registry, - None, - ini_location - ) + setup_app(ini_location) def includeme(config): From 62945480e3bd1e88936a7b7d3dedb97b566f973e Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Thu, 7 Nov 2019 12:16:42 -0700 Subject: [PATCH 18/28] Fix example to run on Python 3.x --- examples/long_running_with_tm/long_running_with_tm/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/long_running_with_tm/long_running_with_tm/tasks.py b/examples/long_running_with_tm/long_running_with_tm/tasks.py index 92927e9..a807c39 100644 --- a/examples/long_running_with_tm/long_running_with_tm/tasks.py +++ b/examples/long_running_with_tm/long_running_with_tm/tasks.py @@ -30,7 +30,7 @@ def delete_task(self, task_pk): @app.task def add_task(task): time.sleep(random.choice([2, 4, 6, 8, 10])) - print 'creating task %s' % task + print('creating task %s' % task) task = TaskItem(task=task) DBSession.add(task) transaction.commit() From b572769435f14925a9747fefc5d9cbbf47bb4076 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Thu, 7 Nov 2019 14:27:47 -0700 Subject: [PATCH 19/28] v4.1.0 --- CHANGES.txt | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 8206d5a..eb32ce5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,9 @@ +4.1.0 +================ + +- Prevent duplicate calls to setup_app. + + 4.0.0 ================ diff --git a/setup.py b/setup.py index 89e3de1..3e80b11 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup(name='pyramid_celery', - version='4.0.0', + version='4.1.0', description='Celery integration with pyramid', long_description=README + "\n" + CHANGES, classifiers=[ From 7cfb57829940ba567ff96587eb065eb63a420d50 Mon Sep 17 00:00:00 2001 From: Igor Raush Date: Mon, 3 Aug 2020 15:12:31 -0600 Subject: [PATCH 20/28] support broker_transport_options setting (parse dictionary) --- pyramid_celery/loaders.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pyramid_celery/loaders.py b/pyramid_celery/loaders.py index f8fc513..04e54ed 100644 --- a/pyramid_celery/loaders.py +++ b/pyramid_celery/loaders.py @@ -143,6 +143,19 @@ def read_configuration(self, fail_silently=True): tuple_settings = [tuple(item.split(',')) for item in items] config_dict[setting] = tuple_settings + dict_settings = [ + 'broker_transport_options', 'BROKER_TRANSPORT_OPTIONS', + ] + + for setting in dict_settings: + if setting in config_dict: + try: + value = json.loads(config_dict[setting]) + except ValueError: + value = config_dict[setting].strip().split() + value = dict(pair.split('=') for pair in value) + config_dict[setting] = value + beat_config = {} route_config = {} From 491d1caa6238882fad7bfa8c9b9780273d64c263 Mon Sep 17 00:00:00 2001 From: Alexander Date: Wed, 26 Jan 2022 18:12:28 +0300 Subject: [PATCH 21/28] Update loaders.py 1. refactor - move setting reading into separate methods. if there will be errors during parsing setting's value - default value will be used 2. add logger 3. fail_silently usage - if false no exceptions will be raised, they will be logged only --- pyramid_celery/loaders.py | 147 +++++++++++++++++++++++--------------- 1 file changed, 88 insertions(+), 59 deletions(-) diff --git a/pyramid_celery/loaders.py b/pyramid_celery/loaders.py index 04e54ed..2d4e1ce 100644 --- a/pyramid_celery/loaders.py +++ b/pyramid_celery/loaders.py @@ -2,9 +2,9 @@ import json import celery.loaders.base import celery.schedules +import logging from pyramid.compat import configparser from pyramid.exceptions import ConfigurationError -from pyramid.settings import asbool from functools import partial @@ -98,81 +98,110 @@ def get_route_config(parser, section): return config +logging.basicConfig() + +logger = logging.getLogger(__name__) + +#: TODO: There might be other variables requiring special handling +bool_settings = [ + 'always_eager', 'CELERY_ALWAYS_EAGER', + 'enable_utc', 'CELERY_ENABLE_UTC', + 'result_persistent', 'CELERY_RESULT_PERSISTENT', + 'worker_hijack_root_logger', 'CELERYD_HIJACK_ROOT_LOGGER', + 'use_celeryconfig', 'USE_CELERYCONFIG', +] + +list_settings = [ + 'imports', 'CELERY_IMPORTS', + 'accept_content', 'CELERY_ACCEPT_CONTENT', +] + +tuple_list_settings = [ + 'admins', 'ADMINS', +] + +dict_settings = [ + 'broker_transport_options', 'BROKER_TRANSPORT_OPTIONS', +] + + +def parse_list_setting(setting): + setting = setting.encode('ascii') + if setting.startswith('['): + setting = setting[1:] + if setting.endswith(']'): + setting = setting[:-1] + split_setting = setting.split() + return split_setting + + +def parse_tuple_list_setting(setting): + setting = setting.encode('ascii') + if setting.startswith('[') or setting.endswith(']'): + setting = setting[1:] + if setting.endswith(']'): + setting = setting[:-1] + items = setting.split('\n') + tuple_settings = [tuple(item.split(',')) for item in items] + return tuple_settings + + +def parse_dict_setting(setting): + return json.loads(setting.encode('ascii')) + + class INILoader(celery.loaders.base.BaseLoader): def __init__(self, app, **kwargs): self.celery_conf = kwargs.pop('ini_file') self.parser = configparser.SafeConfigParser() self.parser.optionxform = str - super(INILoader, self).__init__(app, **kwargs) def read_configuration(self, fail_silently=True): self.parser.read(self.celery_conf) config_dict = dict(self.parser.items('celery')) - #: TODO: There might be other variables requiring special handling - bool_settings = [ - 'always_eager', 'CELERY_ALWAYS_EAGER', - 'enable_utc', 'CELERY_ENABLE_UTC', - 'result_persistent', 'CELERY_RESULT_PERSISTENT', - 'worker_hijack_root_logger', 'CELERYD_HIJACK_ROOT_LOGGER', - 'use_celeryconfig', 'USE_CELERYCONFIG', - ] - - for setting in bool_settings: - if setting in config_dict: - config_dict[setting] = asbool(config_dict[setting]) - - list_settings = [ - 'imports', 'CELERY_IMPORTS', - 'accept_content', 'CELERY_ACCEPT_CONTENT', - ] - - for setting in list_settings: - if setting in config_dict: - split_setting = config_dict[setting].split() - config_dict[setting] = split_setting - - tuple_list_settings = [ - 'admins', 'ADMINS', - ] - - for setting in tuple_list_settings: - if setting in config_dict: - items = config_dict[setting].split() - tuple_settings = [tuple(item.split(',')) for item in items] - config_dict[setting] = tuple_settings - - dict_settings = [ - 'broker_transport_options', 'BROKER_TRANSPORT_OPTIONS', - ] - - for setting in dict_settings: - if setting in config_dict: - try: - value = json.loads(config_dict[setting]) - except ValueError: - value = config_dict[setting].strip().split() - value = dict(pair.split('=') for pair in value) - config_dict[setting] = value + for setting in config_dict.keys(): + try: + if setting in bool_settings: + config_dict[setting] = self.parser.getboolean('celery', setting) + if setting in list_settings: + config_dict[setting] = parse_list_setting(config_dict[setting]) + if setting in tuple_list_settings: + config_dict[setting] = parse_tuple_list_setting(config_dict[setting]) + if setting in dict_settings: + config_dict[setting] = parse_dict_setting(config_dict[setting]) + except Exception as exc: + if not fail_silently: + raise Exception('Can\'t parse value for {}. {}'.format(setting, exc.message)) + logger.critical('Can\'t parse value for {}. Default value will be used.'.format(setting)) + del config_dict[setting] beat_config = {} route_config = {} for section in self.parser.sections(): if section.startswith('celerybeat:'): - name = section.split(':', 1)[1] - beat_config[name] = get_beat_config(self.parser, section) + try: + name = section.split(':', 1)[1] + beat_config[name] = get_beat_config(self.parser, section) + if beat_config: + set_all(config_dict, ( + 'beat_schedule', 'CELERYBEAT_SCHEDULE'), beat_config) + except Exception as exc: + if not fail_silently: + raise Exception('Can\'t parse celerybeat config. {}'.format(exc.message)) + logger.critical('Can\'t parse celerybeat config. {}'.format(exc.message)) elif section.startswith('celeryroute:'): - name = section.split(':', 1)[1] - route_config[name] = get_route_config(self.parser, section) - - if beat_config: - set_all(config_dict, ( - 'beat_schedule', 'CELERYBEAT_SCHEDULE'), beat_config) - - if route_config: - set_all(config_dict, ( - 'task_routes', 'CELERY_ROUTES'), route_config) + try: + name = section.split(':', 1)[1] + route_config[name] = get_route_config(self.parser, section) + if route_config: + set_all(config_dict, ( + 'task_routes', 'CELERY_ROUTES'), route_config) + except Exception as exc: + if not fail_silently: + raise Exception('Can\'t parse celeryroute config', exc.message) + logger.critical('Can\'t parse celeryroute config.' + exc.message) return config_dict From 057b28cdc42dd4d879429e9eeef97470013baa86 Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jan 2022 12:17:16 +0300 Subject: [PATCH 22/28] Update loaders.py remove logging exceptions --- pyramid_celery/loaders.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/pyramid_celery/loaders.py b/pyramid_celery/loaders.py index 2d4e1ce..a4aa618 100644 --- a/pyramid_celery/loaders.py +++ b/pyramid_celery/loaders.py @@ -2,7 +2,6 @@ import json import celery.loaders.base import celery.schedules -import logging from pyramid.compat import configparser from pyramid.exceptions import ConfigurationError @@ -98,10 +97,6 @@ def get_route_config(parser, section): return config -logging.basicConfig() - -logger = logging.getLogger(__name__) - #: TODO: There might be other variables requiring special handling bool_settings = [ 'always_eager', 'CELERY_ALWAYS_EAGER', @@ -127,20 +122,16 @@ def get_route_config(parser, section): def parse_list_setting(setting): setting = setting.encode('ascii') - if setting.startswith('['): - setting = setting[1:] - if setting.endswith(']'): - setting = setting[:-1] + if setting.startswith('[') and setting.endswith(']'): + setting = setting[1:-1] split_setting = setting.split() return split_setting def parse_tuple_list_setting(setting): setting = setting.encode('ascii') - if setting.startswith('[') or setting.endswith(']'): - setting = setting[1:] - if setting.endswith(']'): - setting = setting[:-1] + if setting.startswith('[') and setting.endswith(']'): + setting = setting[1:-1] items = setting.split('\n') tuple_settings = [tuple(item.split(',')) for item in items] return tuple_settings @@ -173,8 +164,7 @@ def read_configuration(self, fail_silently=True): config_dict[setting] = parse_dict_setting(config_dict[setting]) except Exception as exc: if not fail_silently: - raise Exception('Can\'t parse value for {}. {}'.format(setting, exc.message)) - logger.critical('Can\'t parse value for {}. Default value will be used.'.format(setting)) + raise ConfigurationError('Can\'t parse value for {}. {}'.format(setting, exc.message)) del config_dict[setting] beat_config = {} @@ -190,8 +180,7 @@ def read_configuration(self, fail_silently=True): 'beat_schedule', 'CELERYBEAT_SCHEDULE'), beat_config) except Exception as exc: if not fail_silently: - raise Exception('Can\'t parse celerybeat config. {}'.format(exc.message)) - logger.critical('Can\'t parse celerybeat config. {}'.format(exc.message)) + raise ConfigurationError('Can\'t parse celerybeat config. {}'.format(exc.message)) elif section.startswith('celeryroute:'): try: name = section.split(':', 1)[1] @@ -201,7 +190,6 @@ def read_configuration(self, fail_silently=True): 'task_routes', 'CELERY_ROUTES'), route_config) except Exception as exc: if not fail_silently: - raise Exception('Can\'t parse celeryroute config', exc.message) - logger.critical('Can\'t parse celeryroute config.' + exc.message) + raise ConfigurationError('Can\'t parse celeryroute config', exc.message) return config_dict From b28300abb2030ad272c9f7302a2840bd3af7dbff Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 31 Jan 2022 12:21:21 +0300 Subject: [PATCH 23/28] Update loaders.py change fail_silently to False --- pyramid_celery/loaders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyramid_celery/loaders.py b/pyramid_celery/loaders.py index a4aa618..b731853 100644 --- a/pyramid_celery/loaders.py +++ b/pyramid_celery/loaders.py @@ -148,7 +148,7 @@ def __init__(self, app, **kwargs): self.parser.optionxform = str super(INILoader, self).__init__(app, **kwargs) - def read_configuration(self, fail_silently=True): + def read_configuration(self, fail_silently=False): self.parser.read(self.celery_conf) config_dict = dict(self.parser.items('celery')) From 534e46a04e43c940d0994fc24efa0ba3f4589cc3 Mon Sep 17 00:00:00 2001 From: Alexander Date: Thu, 3 Feb 2022 12:24:12 +0300 Subject: [PATCH 24/28] remove unnecessary checks removed unnecessary checks for list and list of tuples --- .gitignore | 1 + .idea/.gitignore | 8 +++++++ .idea/inspectionProfiles/Project_Default.xml | 18 ++++++++++++++ .../inspectionProfiles/profiles_settings.xml | 6 +++++ .idea/misc.xml | 4 ++++ .idea/pyramid_celery.iml | 24 +++++++++++++++++++ .idea/vcs.xml | 6 +++++ pyramid_celery/loaders.py | 6 ----- 8 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/pyramid_celery.iml create mode 100644 .idea/vcs.xml diff --git a/.gitignore b/.gitignore index 852263c..612726a 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,4 @@ docs/_build coverage.xml junit.xml *celerybeat-schedule* +.idea/modules.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..288ab66 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,18 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..b1f757a --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/pyramid_celery.iml b/.idea/pyramid_celery.iml new file mode 100644 index 0000000..4b379b1 --- /dev/null +++ b/.idea/pyramid_celery.iml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/pyramid_celery/loaders.py b/pyramid_celery/loaders.py index b731853..8366cc3 100644 --- a/pyramid_celery/loaders.py +++ b/pyramid_celery/loaders.py @@ -121,17 +121,11 @@ def get_route_config(parser, section): def parse_list_setting(setting): - setting = setting.encode('ascii') - if setting.startswith('[') and setting.endswith(']'): - setting = setting[1:-1] split_setting = setting.split() return split_setting def parse_tuple_list_setting(setting): - setting = setting.encode('ascii') - if setting.startswith('[') and setting.endswith(']'): - setting = setting[1:-1] items = setting.split('\n') tuple_settings = [tuple(item.split(',')) for item in items] return tuple_settings From 163f62c84ee6e848e1fb5457d22cdb828ffda14e Mon Sep 17 00:00:00 2001 From: Alexander Date: Thu, 3 Feb 2022 12:33:58 +0300 Subject: [PATCH 25/28] Revert "remove unnecessary checks" This reverts commit 534e46a04e43c940d0994fc24efa0ba3f4589cc3. --- .gitignore | 1 - .idea/.gitignore | 8 ------- .idea/inspectionProfiles/Project_Default.xml | 18 -------------- .../inspectionProfiles/profiles_settings.xml | 6 ----- .idea/misc.xml | 4 ---- .idea/pyramid_celery.iml | 24 ------------------- .idea/vcs.xml | 6 ----- pyramid_celery/loaders.py | 6 +++++ 8 files changed, 6 insertions(+), 67 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/Project_Default.xml delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/pyramid_celery.iml delete mode 100644 .idea/vcs.xml diff --git a/.gitignore b/.gitignore index 612726a..852263c 100644 --- a/.gitignore +++ b/.gitignore @@ -48,4 +48,3 @@ docs/_build coverage.xml junit.xml *celerybeat-schedule* -.idea/modules.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b8..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 288ab66..0000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2d..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index b1f757a..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/pyramid_celery.iml b/.idea/pyramid_celery.iml deleted file mode 100644 index 4b379b1..0000000 --- a/.idea/pyramid_celery.iml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/pyramid_celery/loaders.py b/pyramid_celery/loaders.py index 8366cc3..b731853 100644 --- a/pyramid_celery/loaders.py +++ b/pyramid_celery/loaders.py @@ -121,11 +121,17 @@ def get_route_config(parser, section): def parse_list_setting(setting): + setting = setting.encode('ascii') + if setting.startswith('[') and setting.endswith(']'): + setting = setting[1:-1] split_setting = setting.split() return split_setting def parse_tuple_list_setting(setting): + setting = setting.encode('ascii') + if setting.startswith('[') and setting.endswith(']'): + setting = setting[1:-1] items = setting.split('\n') tuple_settings = [tuple(item.split(',')) for item in items] return tuple_settings From 2f43d9cbb1506c50cc586e7c6cc6627b8f4ad956 Mon Sep 17 00:00:00 2001 From: Alexander Date: Thu, 3 Feb 2022 12:37:32 +0300 Subject: [PATCH 26/28] unnecessary checks removed unnecessary checks for lists and list of tuples --- pyramid_celery/loaders.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyramid_celery/loaders.py b/pyramid_celery/loaders.py index b731853..8366cc3 100644 --- a/pyramid_celery/loaders.py +++ b/pyramid_celery/loaders.py @@ -121,17 +121,11 @@ def get_route_config(parser, section): def parse_list_setting(setting): - setting = setting.encode('ascii') - if setting.startswith('[') and setting.endswith(']'): - setting = setting[1:-1] split_setting = setting.split() return split_setting def parse_tuple_list_setting(setting): - setting = setting.encode('ascii') - if setting.startswith('[') and setting.endswith(']'): - setting = setting[1:-1] items = setting.split('\n') tuple_settings = [tuple(item.split(',')) for item in items] return tuple_settings From 33d1e0ec832eec18df83fb25daf3b2a864ed53cf Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 8 Feb 2022 12:11:10 +0300 Subject: [PATCH 27/28] added DeadLock detection bootstep for worker --- pyramid_celery/__init__.py | 2 ++ pyramid_celery/bootsteps.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 pyramid_celery/bootsteps.py diff --git a/pyramid_celery/__init__.py b/pyramid_celery/__init__.py index 55000f9..6e43389 100644 --- a/pyramid_celery/__init__.py +++ b/pyramid_celery/__init__.py @@ -4,6 +4,7 @@ from celery.bin import Option from pyramid.paster import bootstrap, setup_logging from pyramid_celery.loaders import INILoader, get_any, set_all +from pyramid_celery.bootsteps import DeadlockDetection def add_preload_arguments(parser): @@ -19,6 +20,7 @@ def add_preload_arguments(parser): def make_app(): app = Celery() + app.steps['worker'].add(DeadlockDetection) if celery_version.major > 3: app.user_options['preload'].add(add_preload_arguments) else: diff --git a/pyramid_celery/bootsteps.py b/pyramid_celery/bootsteps.py new file mode 100644 index 0000000..6f08f0a --- /dev/null +++ b/pyramid_celery/bootsteps.py @@ -0,0 +1,28 @@ +from celery import bootsteps +import time + + +class DeadlockDetection(bootsteps.StartStopStep): + requires = {'celery.worker.components:Timer'} + + def __init__(self, worker, deadlock_timeout=600): + self.timeout = deadlock_timeout + self.requests = [] + self.tref = None + + def start(self, worker): + # run every 30 seconds. + self.tref = worker.timer.call_repeatedly( + 30.0, self.detect, (worker,), priority=10, + ) + + def stop(self, worker): + if self.tref: + self.tref.cancel() + self.tref = None + + def detect(self, worker): + # update active requests + for req in worker.active_requests: + if req.time_start and time() - req.time_start > self.timeout: + raise SystemExit() From 3d8edae12e6995cd57444ef8072a712b527d524b Mon Sep 17 00:00:00 2001 From: Alexander Laptev Date: Mon, 14 Feb 2022 13:25:23 +0300 Subject: [PATCH 28/28] bootstep configure update deleted DeadlockDetection bootstep by default add directive to pyramid with bootsteps configuration --- pyramid_celery/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyramid_celery/__init__.py b/pyramid_celery/__init__.py index 6e43389..377eeb7 100644 --- a/pyramid_celery/__init__.py +++ b/pyramid_celery/__init__.py @@ -20,7 +20,6 @@ def add_preload_arguments(parser): def make_app(): app = Celery() - app.steps['worker'].add(DeadlockDetection) if celery_version.major > 3: app.user_options['preload'].add(add_preload_arguments) else: @@ -106,5 +105,10 @@ def configure(config, ini_location): setup_app(ini_location) +def configure_bootsteps(config): + celery_app.steps['worker'].add(DeadlockDetection) + + def includeme(config): config.add_directive('configure_celery', configure) + config.add_directive('configure_celery_bootsteps', configure_bootsteps)