From 7a7b4b13914261baebdbc5d44ebdc235be1d4d05 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Mon, 17 Jan 2022 12:59:09 +0300 Subject: [PATCH] Remove caching for tasks and jobs (#4165) * Resolved https://github.com/openvinotoolkit/cvat/issues/4088 * Update CHANGELOG * Fix linter issues * Removed CACHEOPS settings from production.py --- CHANGELOG.md | 1 + cvat/apps/training/apis.py | 21 +++++---------------- cvat/apps/training/jobs.py | 11 ++++------- cvat/apps/training/views.py | 27 +++++++++------------------ cvat/requirements/base.txt | 1 - cvat/settings/base.py | 21 --------------------- cvat/settings/production.py | 2 -- 7 files changed, 19 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 273b5861b7a1..8ef13b8c9e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Order in an annotation file() - Fixed task data upload progressbar () - Email in org invitations is case sensitive () +- Caching for tasks and jobs can lead to an exception if its assignee user is removed () - Added intelligent function when paste labels to another task () - Uncaught TypeError: this.el.node.getScreenCTM() is null in Firefox () - Bug: canvas is busy when start playing, start resizing a shape and do not release the mouse cursor () diff --git a/cvat/apps/training/apis.py b/cvat/apps/training/apis.py index d280f34a69f8..6946633cdd60 100644 --- a/cvat/apps/training/apis.py +++ b/cvat/apps/training/apis.py @@ -3,11 +3,10 @@ from collections import OrderedDict from functools import wraps from typing import Callable, List, Union +from contextlib import suppress import requests -from cacheops import cache, CacheMiss - from cvat.apps.engine.models import TrainingProject, ShapeType @@ -48,11 +47,9 @@ def wrapper(*args, **kwargs): __amount = amount while __amount > 0: __amount -= 1 - try: + with suppress(Exception): result = func(*args, **kwargs) return result - except Exception: - pass return wrapper @@ -222,9 +219,6 @@ def __get_tasks(self) -> List[dict]: response = self.request(method='GET', url=url, headers=headers) return response - def __delete_token(self): - cache.delete(self.token_key) - @retry() def __upload_annotation(self, project_id: str, image_id: str, annotation: List[dict]): url = f'{self.host}/v2/projects/{project_id}/media/images/{image_id}/annotations' @@ -264,13 +258,9 @@ def get_token(host: str, username: str, password: str) -> dict: r = requests.post(url=url, files=data, verify=False) # nosec return r.json() - try: - token = cache.get(self.token_key) - except CacheMiss: - response = get_token(self.host, self.username, self.password) - token = response.get('secure_token', '') - expires_in = response.get('expires_in', 3600) - cache.set(cache_key=self.token_key, data=token, timeout=expires_in) + response = get_token(self.host, self.username, self.password) + token = response.get('secure_token', '') + return token @property @@ -280,7 +270,6 @@ def token_key(self): def request(self, method: str, url: str, **kwargs) -> Union[list, dict, str]: response = requests.request(method=method, url=url, verify=False, **kwargs) if response.status_code == 401: - self.__delete_token() raise Exception("401") result = response.json() return result diff --git a/cvat/apps/training/jobs.py b/cvat/apps/training/jobs.py index 3cb50fb55c97..85b87485cb21 100644 --- a/cvat/apps/training/jobs.py +++ b/cvat/apps/training/jobs.py @@ -1,7 +1,6 @@ from collections import OrderedDict from typing import List -from cacheops import cache from django_rq import job from cvat.apps import dataset_manager as dm @@ -33,7 +32,8 @@ def save_prediction_server_status_to_cache_job(cache_key, **status, 'status': 'done' } - cache.set(cache_key=cache_key, data=resp, timeout=timeout) + + return resp # dummy code, need to delete training app in a separate PR @job @@ -44,10 +44,6 @@ def save_frame_prediction_to_cache_job(cache_key: str, task = Task.objects.get(pk=task_id) training_project_image = TrainingProjectImage.objects.filter(idx=frame, task=task).first() if not training_project_image: - cache.set(cache_key=cache_key, data={ - 'annotation': [], - 'status': 'done' - }, timeout=timeout) return cvat_labels = Label.objects.filter(project__id=task.project_id).all() @@ -70,7 +66,8 @@ def save_frame_prediction_to_cache_job(cache_key: str, 'annotation': annotation, 'status': 'done' } - cache.set(cache_key=cache_key, data=resp, timeout=timeout) + + return resp # dummy code, need to delete training app in a separate PR @job diff --git a/cvat/apps/training/views.py b/cvat/apps/training/views.py index acc75463fa1f..e3ec5fd0d6e1 100644 --- a/cvat/apps/training/views.py +++ b/cvat/apps/training/views.py @@ -1,4 +1,3 @@ -from cacheops import cache, CacheMiss from drf_yasg.utils import swagger_auto_schema from rest_framework import viewsets, status from rest_framework.decorators import action @@ -25,15 +24,11 @@ def predict_image(self, request): if not frame: return Response(data='query param "frame" empty or not provided', status=status.HTTP_400_BAD_REQUEST) cache_key = f'predict_image_{task_id}_{frame}' - try: - resp = cache.get(cache_key) - except CacheMiss: - save_frame_prediction_to_cache_job.delay(cache_key, task_id=task_id, - frame=frame) - resp = { - 'status': 'queued', - } - cache.set(cache_key=cache_key, data=resp, timeout=60) + save_frame_prediction_to_cache_job.delay(cache_key, task_id=task_id, + frame=frame) + resp = { + 'status': 'queued', + } return Response(resp) @@ -49,13 +44,9 @@ def predict_status(self, request): Response({'status': 'done'}) cache_key = f'predict_status_{project_id}' - try: - resp = cache.get(cache_key) - except CacheMiss: - save_prediction_server_status_to_cache_job.delay(cache_key, cvat_project_id=project_id) - resp = { - 'status': 'queued', - } - cache.set(cache_key=cache_key, data=resp, timeout=60) + save_prediction_server_status_to_cache_job.delay(cache_key, cvat_project_id=project_id) + resp = { + 'status': 'queued', + } return Response(resp) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 985cc86d51ce..a3adadc8b00b 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -3,7 +3,6 @@ click==7.1.2 Django==3.2.11 django-appconf==1.0.4 django-auth-ldap==2.2.0 -django-cacheops==5.0.1 django-compressor==2.4 django-rq==2.3.2 EasyProcess==0.3 diff --git a/cvat/settings/base.py b/cvat/settings/base.py index b6b70da0808d..c7593eaa3007 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -106,7 +106,6 @@ def add_ssh_keys(): 'django.contrib.staticfiles', 'django_rq', 'compressor', - 'cacheops', 'sendfile', 'dj_pagination', 'revproxy', @@ -315,26 +314,6 @@ def add_ssh_keys(): }, ] -# Cache DB access (e.g. for engine.task.get_frame) -# https://github.com/Suor/django-cacheops -CACHEOPS_REDIS = { - 'host': 'localhost', # redis-server is on same machine - 'port': 6379, # default redis port - 'db': 1, # SELECT non-default redis database -} - -CACHEOPS = { - # Automatically cache any Task.objects.get() calls for 15 minutes - # This also includes .first() and .last() calls. - 'engine.task': {'ops': 'get', 'timeout': 60*15}, - - # Automatically cache any Job.objects.get() calls for 15 minutes - # This also includes .first() and .last() calls. - 'engine.job': {'ops': 'get', 'timeout': 60*15}, -} - -CACHEOPS_DEGRADE_ON_FAILURE = True - # Internationalization # https://docs.djangoproject.com/en/2.0/topics/i18n/ diff --git a/cvat/settings/production.py b/cvat/settings/production.py index d6cea8988adc..5b6b4a921ddd 100644 --- a/cvat/settings/production.py +++ b/cvat/settings/production.py @@ -15,8 +15,6 @@ for key in RQ_QUEUES: RQ_QUEUES[key]['HOST'] = os.getenv('CVAT_REDIS_HOST', 'cvat_redis') -CACHEOPS_REDIS['host'] = os.getenv('CVAT_REDIS_HOST', 'cvat_redis') - # Django-sendfile: # https://github.com/johnsensible/django-sendfile SENDFILE_BACKEND = 'sendfile.backends.xsendfile'