Add retry for failed model reg tasks

OasisLMF · Feb 2, 2023 · ebecb93 · ebecb93
1 parent 82c78a5
commit ebecb93
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/src/server/oasisapi/analyses/tasks.py b/src/server/oasisapi/analyses/tasks.py
@@ -38,7 +38,7 @@
 
 from .models import AnalysisTaskStatus
 from .task_controller import get_analysis_task_controller
-from ..celery_app import celery_app
+from ..celery_app import celery_app, server_task_kwargs
 
 
 logger = get_task_logger(__name__)
@@ -287,7 +287,7 @@ def log_worker_monitor(sender, **k):
     logger.info('AWS_IS_GZIPPED: {}'.format(settings.AWS_IS_GZIPPED))
 
 
-@celery_app.task(name='run_register_worker')
+@celery_app.task(name='run_register_worker', *worker_task_kwargs)
 def run_register_worker(m_supplier, m_name, m_id, m_settings, m_version, m_conf):
     logger.info('model_supplier: {}, model_name: {}, model_id: {}'.format(m_supplier, m_name, m_id))
     try:

diff --git a/src/server/oasisapi/celery_app.py b/src/server/oasisapi/celery_app.py
@@ -10,3 +10,9 @@
 celery_app = Celery('oasisapi')
 celery_app.config_from_object('django.conf:settings')
 celery_app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+
+server_task_kwargs = {
+    'autoretry_for': (Exception,),
+    'max_retries': 2,               # The task will be run max_retries + 1 times
+    'default_retry_delay': 5,       # A small delay to recover from temporary bad states
+}