From ebecb93f5231b9a41da7858736370056ee58f436 Mon Sep 17 00:00:00 2001 From: Sam Gamble Date: Thu, 2 Feb 2023 13:24:40 +0000 Subject: [PATCH] Add retry for failed model reg tasks --- src/server/oasisapi/analyses/tasks.py | 4 ++-- src/server/oasisapi/celery_app.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/server/oasisapi/analyses/tasks.py b/src/server/oasisapi/analyses/tasks.py index d8fa402fe..5fef2b42e 100644 --- a/src/server/oasisapi/analyses/tasks.py +++ b/src/server/oasisapi/analyses/tasks.py @@ -38,7 +38,7 @@ from .models import AnalysisTaskStatus from .task_controller import get_analysis_task_controller -from ..celery_app import celery_app +from ..celery_app import celery_app, server_task_kwargs logger = get_task_logger(__name__) @@ -287,7 +287,7 @@ def log_worker_monitor(sender, **k): logger.info('AWS_IS_GZIPPED: {}'.format(settings.AWS_IS_GZIPPED)) -@celery_app.task(name='run_register_worker') +@celery_app.task(name='run_register_worker', *worker_task_kwargs) def run_register_worker(m_supplier, m_name, m_id, m_settings, m_version, m_conf): logger.info('model_supplier: {}, model_name: {}, model_id: {}'.format(m_supplier, m_name, m_id)) try: diff --git a/src/server/oasisapi/celery_app.py b/src/server/oasisapi/celery_app.py index fe5cee43a..d58889c11 100644 --- a/src/server/oasisapi/celery_app.py +++ b/src/server/oasisapi/celery_app.py @@ -10,3 +10,9 @@ celery_app = Celery('oasisapi') celery_app.config_from_object('django.conf:settings') celery_app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) + +server_task_kwargs = { + 'autoretry_for': (Exception,), + 'max_retries': 2, # The task will be run max_retries + 1 times + 'default_retry_delay': 5, # A small delay to recover from temporary bad states +}