Remove caching for tasks and jobs (#4165)

* Resolved #4088

* Update CHANGELOG

* Fix linter issues

* Removed CACHEOPS settings from production.py

CHANGELOG.md

@@ -53,6 +53,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Order in an annotation file(<https://github.com/openvinotoolkit/cvat/pull/4087>)
 - Fixed task data upload progressbar (<https://github.com/openvinotoolkit/cvat/pull/4134>)
 - Email in org invitations is case sensitive (<https://github.com/openvinotoolkit/cvat/pull/4153>)
+- Caching for tasks and jobs can lead to an exception if its assignee user is removed (<https://github.com/openvinotoolkit/cvat/pull/4165>)
 - Added intelligent function when paste labels to another task (<https://github.com/openvinotoolkit/cvat/pull/4161>)
 - Uncaught TypeError: this.el.node.getScreenCTM() is null in Firefox (<https://github.com/openvinotoolkit/cvat/pull/4175>)
 - Bug: canvas is busy when start playing, start resizing a shape and do not release the mouse cursor (<https://github.com/openvinotoolkit/cvat/pull/4151>)

cvat/apps/training/apis.py

@@ -3,11 +3,10 @@
 from collections import OrderedDict
 from functools import wraps
 from typing import Callable, List, Union
+from contextlib import suppress
 
 import requests
 
-from cacheops import cache, CacheMiss
-
 from cvat.apps.engine.models import TrainingProject, ShapeType
 
 
@@ -48,11 +47,9 @@ def wrapper(*args, **kwargs):
             __amount = amount
             while __amount > 0:
                 __amount -= 1
-                try:
+                with suppress(Exception):
                     result = func(*args, **kwargs)
                     return result
-                except Exception:
-                    pass
 
         return wrapper
 
@@ -222,9 +219,6 @@ def __get_tasks(self) -> List[dict]:
         response = self.request(method='GET', url=url, headers=headers)
         return response
 
-    def __delete_token(self):
-        cache.delete(self.token_key)
-
     @retry()
     def __upload_annotation(self, project_id: str, image_id: str, annotation: List[dict]):
         url = f'{self.host}/v2/projects/{project_id}/media/images/{image_id}/annotations'
@@ -264,13 +258,9 @@ def get_token(host: str, username: str, password: str) -> dict:
             r = requests.post(url=url, files=data, verify=False)  # nosec
             return r.json()
 
-        try:
-            token = cache.get(self.token_key)
-        except CacheMiss:
-            response = get_token(self.host, self.username, self.password)
-            token = response.get('secure_token', '')
-            expires_in = response.get('expires_in', 3600)
-            cache.set(cache_key=self.token_key, data=token, timeout=expires_in)
+        response = get_token(self.host, self.username, self.password)
+        token = response.get('secure_token', '')
+
         return token
 
     @property
@@ -280,7 +270,6 @@ def token_key(self):
     def request(self, method: str, url: str, **kwargs) -> Union[list, dict, str]:
         response = requests.request(method=method, url=url, verify=False, **kwargs)
         if response.status_code == 401:
-            self.__delete_token()
             raise Exception("401")
         result = response.json()
         return result

cvat/apps/training/jobs.py

@@ -1,7 +1,6 @@
 from collections import OrderedDict
 from typing import List
 
-from cacheops import cache
 from django_rq import job
 
 from cvat.apps import dataset_manager as dm
@@ -33,7 +32,8 @@ def save_prediction_server_status_to_cache_job(cache_key,
         **status,
         'status': 'done'
     }
-    cache.set(cache_key=cache_key, data=resp, timeout=timeout)
+
+    return resp # dummy code, need to delete training app in a separate PR
 
 
 @job
@@ -44,10 +44,6 @@ def save_frame_prediction_to_cache_job(cache_key: str,
     task = Task.objects.get(pk=task_id)
     training_project_image = TrainingProjectImage.objects.filter(idx=frame, task=task).first()
     if not training_project_image:
-        cache.set(cache_key=cache_key, data={
-            'annotation': [],
-            'status': 'done'
-        }, timeout=timeout)
         return
 
     cvat_labels = Label.objects.filter(project__id=task.project_id).all()
@@ -70,7 +66,8 @@ def save_frame_prediction_to_cache_job(cache_key: str,
         'annotation': annotation,
         'status': 'done'
     }
-    cache.set(cache_key=cache_key, data=resp, timeout=timeout)
+
+    return resp # dummy code, need to delete training app in a separate PR
 
 
 @job

cvat/apps/training/views.py

@@ -1,4 +1,3 @@
-from cacheops import cache, CacheMiss
 from drf_yasg.utils import swagger_auto_schema
 from rest_framework import viewsets, status
 from rest_framework.decorators import action
@@ -25,15 +24,11 @@ def predict_image(self, request):
         if not frame:
             return Response(data='query param "frame" empty or not provided', status=status.HTTP_400_BAD_REQUEST)
         cache_key = f'predict_image_{task_id}_{frame}'
-        try:
-            resp = cache.get(cache_key)
-        except CacheMiss:
-            save_frame_prediction_to_cache_job.delay(cache_key, task_id=task_id,
-                                                     frame=frame)
-            resp = {
-                'status': 'queued',
-            }
-            cache.set(cache_key=cache_key, data=resp, timeout=60)
+        save_frame_prediction_to_cache_job.delay(cache_key, task_id=task_id,
+                                                    frame=frame)
+        resp = {
+            'status': 'queued',
+        }
 
         return Response(resp)
 
@@ -49,13 +44,9 @@ def predict_status(self, request):
             Response({'status': 'done'})
 
         cache_key = f'predict_status_{project_id}'
-        try:
-            resp = cache.get(cache_key)
-        except CacheMiss:
-            save_prediction_server_status_to_cache_job.delay(cache_key, cvat_project_id=project_id)
-            resp = {
-                'status': 'queued',
-            }
-            cache.set(cache_key=cache_key, data=resp, timeout=60)
+        save_prediction_server_status_to_cache_job.delay(cache_key, cvat_project_id=project_id)
+        resp = {
+            'status': 'queued',
+        }
 
         return Response(resp)

cvat/requirements/base.txt

@@ -3,7 +3,6 @@ click==7.1.2
 Django==3.2.11
 django-appconf==1.0.4
 django-auth-ldap==2.2.0
-django-cacheops==5.0.1
 django-compressor==2.4
 django-rq==2.3.2
 EasyProcess==0.3

cvat/settings/base.py

@@ -106,7 +106,6 @@ def add_ssh_keys():
     'django.contrib.staticfiles',
     'django_rq',
     'compressor',
-    'cacheops',
     'sendfile',
     'dj_pagination',
     'revproxy',
@@ -315,26 +314,6 @@ def add_ssh_keys():
     },
 ]
 
-# Cache DB access (e.g. for engine.task.get_frame)
-# https://github.com/Suor/django-cacheops
-CACHEOPS_REDIS = {
-    'host': 'localhost', # redis-server is on same machine
-    'port': 6379,        # default redis port
-    'db': 1,             # SELECT non-default redis database
-}
-
-CACHEOPS = {
-    # Automatically cache any Task.objects.get() calls for 15 minutes
-    # This also includes .first() and .last() calls.
-    'engine.task': {'ops': 'get', 'timeout': 60*15},
-
-    # Automatically cache any Job.objects.get() calls for 15 minutes
-    # This also includes .first() and .last() calls.
-    'engine.job': {'ops': 'get', 'timeout': 60*15},
-}
-
-CACHEOPS_DEGRADE_ON_FAILURE = True
-
 # Internationalization
 # https://docs.djangoproject.com/en/2.0/topics/i18n/
 

cvat/settings/production.py

@@ -15,8 +15,6 @@
 for key in RQ_QUEUES:
     RQ_QUEUES[key]['HOST'] = os.getenv('CVAT_REDIS_HOST', 'cvat_redis')
 
-CACHEOPS_REDIS['host'] = os.getenv('CVAT_REDIS_HOST', 'cvat_redis')
-
 # Django-sendfile:
 # https://github.com/johnsensible/django-sendfile
 SENDFILE_BACKEND = 'sendfile.backends.xsendfile'