Project files: Trashing project files directories upon deletion

api/projects/projects.py

@@ -10,6 +10,7 @@
     ForbiddenException,
     NotFoundException,
 )
+from ayon_server.files import Storages
 from ayon_server.helpers.project_list import get_project_list
 from ayon_server.lib.postgres import Postgres
 
@@ -194,6 +195,10 @@ async def delete_project(user: CurrentUser, project_name: ProjectName) -> EmptyR
         raise ForbiddenException("You need to be a manager in order to delete projects")
 
     await project.delete()
+
+    storage = await Storages.project(project_name)
+    await storage.trash()
+
     await unassign_users_from_deleted_projects()
     logging.info(f"[DELETE] Deleted project {project.name}", user=user.name)
 

ayon_server/files/project_storage.py

@@ -1,4 +1,5 @@
 import os
+import time
 from typing import Any, Literal
 
 import aiocache
@@ -21,6 +22,7 @@
 )
 from ayon_server.helpers.cloud import get_instance_id
 from ayon_server.helpers.ffprobe import extract_media_info
+from ayon_server.helpers.project_list import ProjectListItem, get_project_info
 from ayon_server.lib.postgres import Postgres
 
 StorageType = Literal["local", "s3"]
@@ -33,6 +35,7 @@ class ProjectStorage:
     bucket_name: str | None = None
     cdn_resolver: str | None = None
     _s3_client: Any = None
+    project_info: ProjectListItem | None
 
     def __init__(
         self,
@@ -45,6 +48,7 @@ def __init__(
     ):
         self.project_name = project_name
         self.storage_type = storage_type
+        self.project_info = None
         self.root = root
         if storage_type == "s3":
             if not bucket_name:
@@ -105,12 +109,22 @@ async def get_path(
         _file_id = file_id.replace("-", "")
         if len(_file_id) != 32:
             raise ValueError(f"Invalid file ID: {file_id}")
+
+        project_dirname = self.project_name
+        if self.storage_type == "s3":
+            if self.project_info is None:
+                self.project_info = await get_project_info(self.project_name)
+            assert self.project_info  # mypy
+
+            project_timestamp = int(self.project_info.created_at.timestamp())
+            project_dirname = f"{project_dirname}.{project_timestamp}"
+
         # Take first two characters of the file ID as a subdirectory
         # to avoid having too many files in a single directory
         sub_dir = _file_id[:2]
         return os.path.join(
             root,
-            self.project_name,
+            project_dirname,
             file_group,
             sub_dir,
             _file_id,
@@ -328,3 +342,27 @@ async def delete_thumbnail(self, thumbnail_id: str) -> None:
         """
         logging.debug(f"Deleting thumbnail {thumbnail_id} from {self}")
         await self.unlink(thumbnail_id, file_group="thumbnails")
+
+    async def trash(self) -> None:
+        """Mark the project storage for deletion"""
+
+        if self.storage_type == "local":
+            logging.debug(f"Trashing project {self.project_name} storage")
+            projects_root = await self.get_root()
+            project_dir = os.path.join(projects_root, self.project_name)
+            if not os.path.isdir(project_dir):
+                return
+            timestamp = int(time.time())
+            new_dir_name = f"{self.project_name}.{timestamp}.trash"
+            parent_dir = os.path.dirname(project_dir)
+            new_dir = os.path.join(parent_dir, new_dir_name)
+            try:
+                os.rename(project_dir, new_dir)
+            except Exception as e:
+                logging.error(
+                    f"Failed to trash project {self.project_name} storage: {e}"
+                )
+        if self.storage_type == "s3":
+            # we cannot move the bucket, we'll create a new one with different timestamp
+            # when we re-create the project
+            pass

ayon_server/helpers/download.py

@@ -47,6 +47,7 @@ async def download_file(
     target_path: str,
     filename: str | None = None,
     progress_handler: Callable[[int], Awaitable[None]] | None = None,
+    timeout: int | None = None,
 ) -> None:
     """Downloads a file from a url to a target path"""
 
@@ -100,7 +101,7 @@ async def handle_progress(i: int) -> None:
     temp_file_path = target_path + f".{uuid.uuid1().hex}.part"
     i = 0
     async with httpx.AsyncClient(
-        timeout=ayonconfig.http_timeout, follow_redirects=True
+        timeout=timeout or ayonconfig.http_timeout, follow_redirects=True
     ) as client:
         async with client.stream("GET", url) as response:
             if response.status_code != 200:

ayon_server/helpers/project_list.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from typing import Any
 
+from ayon_server.exceptions import NotFoundException
 from ayon_server.lib.postgres import Postgres
 from ayon_server.lib.redis import Redis
 from ayon_server.types import OPModel
@@ -44,3 +45,12 @@ async def get_project_list() -> list[ProjectListItem]:
     else:
         project_list = json_loads(project_list)
         return [ProjectListItem(**item) for item in project_list]
+
+
+async def get_project_info(project_name: str) -> ProjectListItem:
+    """Return a single project info"""
+    project_list = await get_project_list()
+    for project in project_list:
+        if project.name == project_name:
+            return project
+    raise NotFoundException(f"Project {project_name} not found")