From d91deb842090b4d2c8ffef591f9d35a64fa6e7d3 Mon Sep 17 00:00:00 2001 From: Martastain Date: Thu, 10 Oct 2024 10:11:18 +0200 Subject: [PATCH 1/3] feat: trashing project files directories --- api/projects/projects.py | 5 ++++ ayon_server/files/project_storage.py | 39 +++++++++++++++++++++++++++- ayon_server/helpers/project_list.py | 10 +++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/api/projects/projects.py b/api/projects/projects.py index 65dfceca..1ad27bc3 100644 --- a/api/projects/projects.py +++ b/api/projects/projects.py @@ -10,6 +10,7 @@ ForbiddenException, NotFoundException, ) +from ayon_server.files import Storages from ayon_server.helpers.project_list import get_project_list from ayon_server.lib.postgres import Postgres @@ -194,6 +195,10 @@ async def delete_project(user: CurrentUser, project_name: ProjectName) -> EmptyR raise ForbiddenException("You need to be a manager in order to delete projects") await project.delete() + + storage = await Storages.project(project_name) + await storage.trash() + await unassign_users_from_deleted_projects() logging.info(f"[DELETE] Deleted project {project.name}", user=user.name) diff --git a/ayon_server/files/project_storage.py b/ayon_server/files/project_storage.py index 88943e0e..e15fbfa7 100644 --- a/ayon_server/files/project_storage.py +++ b/ayon_server/files/project_storage.py @@ -1,4 +1,5 @@ import os +import time from typing import Any, Literal import aiocache @@ -21,6 +22,7 @@ ) from ayon_server.helpers.cloud import get_instance_id from ayon_server.helpers.ffprobe import extract_media_info +from ayon_server.helpers.project_list import ProjectListItem, get_project_info from ayon_server.lib.postgres import Postgres StorageType = Literal["local", "s3"] @@ -33,6 +35,7 @@ class ProjectStorage: bucket_name: str | None = None cdn_resolver: str | None = None _s3_client: Any = None + project_info: ProjectListItem | None def __init__( self, @@ -45,6 +48,7 @@ def __init__( ): self.project_name = project_name self.storage_type = storage_type + self.project_info = None self.root = root if storage_type == "s3": if not bucket_name: @@ -105,12 +109,22 @@ async def get_path( _file_id = file_id.replace("-", "") if len(_file_id) != 32: raise ValueError(f"Invalid file ID: {file_id}") + + project_dirname = self.project_name + if self.storage_type == "s3": + if self.project_info is None: + self.project_info = await get_project_info(self.project_name) + assert self.project_info # mypy + + project_timestamp = int(self.project_info.created_at.timestamp()) + project_dirname = f"{project_dirname}.{project_timestamp}" + # Take first two characters of the file ID as a subdirectory # to avoid having too many files in a single directory sub_dir = _file_id[:2] return os.path.join( root, - self.project_name, + project_dirname, file_group, sub_dir, _file_id, @@ -328,3 +342,26 @@ async def delete_thumbnail(self, thumbnail_id: str) -> None: """ logging.debug(f"Deleting thumbnail {thumbnail_id} from {self}") await self.unlink(thumbnail_id, file_group="thumbnails") + + async def trash(self) -> None: + """Mark the project storage for deletion""" + + if self.storage_type == "local": + logging.debug(f"Trashing project {self.project_name} storage") + project_dir = await self.get_root() + if not os.path.isdir(project_dir): + return + timestamp = int(time.time()) + new_dir_name = f"{self.project_name}.{timestamp}.trash" + parent_dir = os.path.dirname(project_dir) + new_dir = os.path.join(parent_dir, new_dir_name) + try: + os.rename(project_dir, new_dir) + except Exception as e: + logging.error( + f"Failed to trash project {self.project_name} storage: {e}" + ) + if self.storage_type == "s3": + # we cannot move the bucket, we'll create a new one with different timestamp + # when we re-create the project + pass diff --git a/ayon_server/helpers/project_list.py b/ayon_server/helpers/project_list.py index 85b5af13..dc8b756f 100644 --- a/ayon_server/helpers/project_list.py +++ b/ayon_server/helpers/project_list.py @@ -1,6 +1,7 @@ from datetime import datetime from typing import Any +from ayon_server.exceptions import NotFoundException from ayon_server.lib.postgres import Postgres from ayon_server.lib.redis import Redis from ayon_server.types import OPModel @@ -44,3 +45,12 @@ async def get_project_list() -> list[ProjectListItem]: else: project_list = json_loads(project_list) return [ProjectListItem(**item) for item in project_list] + + +async def get_project_info(project_name: str) -> ProjectListItem: + """Return a single project info""" + project_list = await get_project_list() + for project in project_list: + if project.name == project_name: + return project + raise NotFoundException(f"Project {project_name} not found") From dd369a1d265c0462383aa8d2afd81d81ecc19747 Mon Sep 17 00:00:00 2001 From: Martastain Date: Thu, 10 Oct 2024 11:38:03 +0200 Subject: [PATCH 2/3] fix: project root --- ayon_server/files/project_storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ayon_server/files/project_storage.py b/ayon_server/files/project_storage.py index e15fbfa7..72f9a38d 100644 --- a/ayon_server/files/project_storage.py +++ b/ayon_server/files/project_storage.py @@ -348,7 +348,8 @@ async def trash(self) -> None: if self.storage_type == "local": logging.debug(f"Trashing project {self.project_name} storage") - project_dir = await self.get_root() + projects_root = await self.get_root() + project_dir = os.path.join(projects_root, self.project_name) if not os.path.isdir(project_dir): return timestamp = int(time.time()) From 0afea99a2127036e5e91099407d1f4e7bda6739e Mon Sep 17 00:00:00 2001 From: Martastain Date: Thu, 10 Oct 2024 11:38:21 +0200 Subject: [PATCH 3/3] fix: add timeout argument to download_file helper --- ayon_server/helpers/download.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ayon_server/helpers/download.py b/ayon_server/helpers/download.py index b1cf2588..2c2ec801 100644 --- a/ayon_server/helpers/download.py +++ b/ayon_server/helpers/download.py @@ -47,6 +47,7 @@ async def download_file( target_path: str, filename: str | None = None, progress_handler: Callable[[int], Awaitable[None]] | None = None, + timeout: int | None = None, ) -> None: """Downloads a file from a url to a target path""" @@ -100,7 +101,7 @@ async def handle_progress(i: int) -> None: temp_file_path = target_path + f".{uuid.uuid1().hex}.part" i = 0 async with httpx.AsyncClient( - timeout=ayonconfig.http_timeout, follow_redirects=True + timeout=timeout or ayonconfig.http_timeout, follow_redirects=True ) as client: async with client.stream("GET", url) as response: if response.status_code != 200: