Extension of cloud storage server part

cvat/apps/engine/cache.py

@@ -71,6 +71,7 @@ def prepare_chunk_buff(self, db_data, quality, chunk_number):
                 step=db_data.get_frame_step())
             if db_data.storage == StorageChoice.CLOUD_STORAGE:
                 db_cloud_storage = db_data.cloud_storage
+                assert db_cloud_storage, 'Cloud storage instance was deleted'
                 credentials = Credentials()
                 credentials.convert_from_db({
                     'type': db_cloud_storage.credentials_type,
@@ -81,26 +82,33 @@ def prepare_chunk_buff(self, db_data, quality, chunk_number):
                     'credentials': credentials,
                     'specific_attributes': db_cloud_storage.get_specific_attributes()
                 }
-                cloud_storage_instance = get_cloud_storage_instance(cloud_provider=db_cloud_storage.provider_type, **details)
-                cloud_storage_instance.initialize_content()
-                for item in reader:
-                    # full_name may be 'sub_dir/image.jpeg'
-                    full_name = f"{item['name']}{item['extension']}"
-                    if full_name not in cloud_storage_instance:
-                        raise Exception('{} file was not found on a {} storage'.format(full_name, cloud_storage_instance.name))
-                    head, file_name = os.path.split(full_name)
-                    abs_head = os.path.join(gettempdir(), head)
-                    os.makedirs(abs_head, exist_ok=True)
-                    with NamedTemporaryFile(mode='w+b', prefix='cvat', suffix=file_name, delete=False, dir=abs_head) as temp_file:
-                        source_path = temp_file.name
-                        buf = cloud_storage_instance.download_fileobj(full_name)
-                        temp_file.write(buf.getvalue())
-                        checksum = item.get('checksum', None)
-                        if not checksum:
-                            slogger.glob.warning('A manifest file does not contain checksum for image {}'.format(item.get('name')))
-                        if checksum and not md5_hash(source_path) == checksum:
-                            slogger.glob.warning('Hash sums of files {} do not match'.format(full_name))
-                        images.append((source_path, source_path, None))
+                try:
+                    cloud_storage_instance = get_cloud_storage_instance(cloud_provider=db_cloud_storage.provider_type, **details)
+                    cloud_storage_instance.initialize_content()
+                    for item in reader:
+                        # full_name may be 'sub_dir/image.jpeg'
+                        full_name = f"{item['name']}{item['extension']}"
+                        if full_name not in cloud_storage_instance:
+                            raise Exception('{} file was not found on a {} storage'.format(full_name, cloud_storage_instance.name))
+                        head, file_name = os.path.split(full_name)
+                        abs_head = os.path.join(gettempdir(), head)
+                        os.makedirs(abs_head, exist_ok=True)
+                        with NamedTemporaryFile(mode='w+b', prefix='cvat', suffix=file_name, delete=False, dir=abs_head) as temp_file:
+                            source_path = temp_file.name
+                            buf = cloud_storage_instance.download_fileobj(full_name)
+                            temp_file.write(buf.getvalue())
+                            checksum = item.get('checksum', None)
+                            if not checksum:
+                                slogger.cloud_storage[db_cloud_storage.id].warning('A manifest file does not contain checksum for image {}'.format(item.get('name')))
+                            if checksum and not md5_hash(source_path) == checksum:
+                                slogger.cloud_storage[db_cloud_storage.id].warning('Hash sums of files {} do not match'.format(full_name))
+                            images.append((source_path, source_path, None))
+                except Exception as ex:
+                    if not cloud_storage_instance.exists():
+                        msg = 'The resource {} is no longer available. It may have been deleted'.format(cloud_storage_instance.name)
+                    else:
+                        msg = str(ex)
+                    raise Exception(msg)
             else:
                 for item in reader:
                     source_path = os.path.join(upload_dir, f"{item['name']}{item['extension']}")

cvat/apps/engine/task.py

@@ -252,7 +252,6 @@ def _create_thread(tid, data, isImport=False):
                 'specific_attributes': db_cloud_storage.get_specific_attributes()
             }
             cloud_storage_instance = get_cloud_storage_instance(cloud_provider=db_cloud_storage.provider_type, **details)
-            cloud_storage_instance.download_file(manifest_file[0], db_data.get_manifest_path())
             first_sorted_media_image = sorted(media['image'])[0]
             cloud_storage_instance.download_file(first_sorted_media_image, os.path.join(upload_dir, first_sorted_media_image))
 
@@ -368,6 +367,14 @@ def update_progress(progress):
                 w, h = extractor.get_image_size(0)
             else:
                 manifest = ImageManifestManager(db_data.get_manifest_path())
+                # prepare task manifest file from cloud storage manifest file
+                cloud_storage_manifest = ImageManifestManager(
+                    os.path.join(db_data.cloud_storage.get_storage_dirname(), manifest_file[0])
+                )
+                cloud_storage_manifest.init_index()
+                media_files = sorted(media['image'])
+                content = cloud_storage_manifest.get_subset(media_files)
+                manifest.create(content)
                 manifest.init_index()
                 img_properties = manifest[0]
                 w, h = img_properties['width'], img_properties['height']

cvat/apps/engine/views.py

@@ -1306,7 +1306,12 @@ def content(self, request, pk):
             slogger.cloud_storage[pk].info(msg)
             return Response(data=msg, status=status.HTTP_404_NOT_FOUND)
         except Exception as ex:
-            return HttpResponseBadRequest(str(ex))
+            # check that cloud storage was not deleted
+            if not storage.exists():
+                msg = 'The resource {} is no longer available. It may have been deleted'.format(storage.name)
+            else:
+                msg = str(ex)
+            return HttpResponseBadRequest(msg)
 
     @swagger_auto_schema(
         method='get',
@@ -1353,7 +1358,12 @@ def preview(self, request, pk):
             slogger.glob.error(message)
             return HttpResponseNotFound(message)
         except Exception as ex:
-            return HttpResponseBadRequest(str(ex))
+            # check that cloud storage was not deleted
+            if not storage.exists():
+                msg = 'The resource {} is no longer available. It may have been deleted'.format(storage.name)
+            else:
+                msg = str(ex)
+            return HttpResponseBadRequest(msg)
 
 def rq_handler(job, exc_type, exc_value, tb):
     job.exc_info = "".join(

utils/dataset_manifest/core.py

@@ -339,6 +339,10 @@ def index(self):
     def data(self):
         pass
 
+    @abstractmethod
+    def get_subset(self, subset_names):
+        pass
+
 class VideoManifestManager(_ManifestManager):
     def __init__(self, manifest_path):
         super().__init__(manifest_path)
@@ -402,7 +406,10 @@ def video_length(self):
 
     @property
     def data(self):
-        return [self.video_name]
+        return (self.video_name)
+
+    def get_subset(self, subset_names):
+        raise NotImplementedError()
 
 #TODO: add generic manifest structure file validation
 class ManifestValidator:
@@ -484,4 +491,14 @@ def prepare_meta(sources, **kwargs):
 
     @property
     def data(self):
-        return [f"{image['name']}{image['extension']}" for _, image in self]
+        return (f"{image['name']}{image['extension']}" for _, image in self)
+
+    def get_subset(self, subset_names):
+        return ({
+            'name': f"{image['name']}",
+            'extension': f"{image['extension']}",
+            'width': image['width'],
+            'height': image['height'],
+            'meta': image['meta'],
+            'checksum': f"{image['checksum']}"
+        } for _, image in self if f"{image['name']}{image['extension']}" in subset_names)