Support for S3 storage and CDN for project files

api/activities/activity.py

@@ -19,13 +19,18 @@
 )
 from ayon_server.api.responses import EmptyResponse
 from ayon_server.exceptions import BadRequestException
+from ayon_server.files import Storages
 from ayon_server.helpers.get_entity_class import get_entity_class
-from ayon_server.helpers.project_files import delete_unused_files
 from ayon_server.types import Field, OPModel
 
 from .router import router
 
 
+async def delete_unused_files(project_name: str) -> None:
+    storage = await Storages.project(project_name)
+    await storage.delete_unused_files()
+
+
 class ProjectActivityPostModel(OPModel):
     id: str | None = Field(None, description="Explicitly set the ID of the activity")
     activity_type: ActivityType = Field(..., example="comment")

api/files/files.py

@@ -1,23 +1,21 @@
-import os
-
 import aiocache
 from fastapi import Header, Request, Response
-from starlette.responses import FileResponse
+from fastapi.responses import FileResponse, RedirectResponse
 
 from ayon_server.api.dependencies import CurrentUser, ProjectName
-from ayon_server.api.files import handle_upload
 from ayon_server.api.responses import EmptyResponse
 from ayon_server.exceptions import (
     BadRequestException,
     ForbiddenException,
     NotFoundException,
 )
-from ayon_server.helpers.project_files import id_to_path
+from ayon_server.files import Storages
+from ayon_server.helpers.cdn import get_cdn_link
+from ayon_server.helpers.preview import get_file_preview
 from ayon_server.lib.postgres import Postgres
 from ayon_server.types import Field, OPModel
 from ayon_server.utils import create_uuid
 
-from .preview import get_file_preview, uncache_file_preview
 from .router import router
 from .video import serve_video
 
@@ -65,9 +63,8 @@ async def upload_project_file(
     else:
         file_id = create_uuid()
 
-    path = id_to_path(project_name, file_id)
-
-    file_size = await handle_upload(request, path)
+    storage = await Storages.project(project_name)
+    file_size = await storage.handle_upload(request, file_id)
 
     data = {
         "filename": x_file_name,
@@ -111,21 +108,8 @@ async def delete_project_file(
     if not user.is_manager and res[0]["author"] != user.name:
         raise ForbiddenException("User does not have permission to delete the file")
 
-    path = id_to_path(project_name, file_id)
-    try:
-        os.remove(path)
-    except FileNotFoundError:
-        pass
-
-    await Postgres.execute(
-        f"""
-        DELETE FROM project_{project_name}.files
-        WHERE id = $1
-        """,
-        file_id,
-    )
-
-    await uncache_file_preview(project_name, file_id)
+    storage = await Storages.project(project_name)
+    await storage.delete_file(file_id)
 
     return EmptyResponse()
 
@@ -168,7 +152,6 @@ async def get_project_file_head(
 
 @router.get("/{file_id}", response_model=None)
 async def get_project_file(
-    request: Request,
     project_name: ProjectName,
     file_id: str,
     user: CurrentUser,
@@ -181,8 +164,33 @@ async def get_project_file(
 
     check_user_access(project_name, user)
 
-    path = id_to_path(project_name, file_id)
+    storage = await Storages.project(project_name)
+
+    if storage.cdn_resolver is not None:
+        return await get_cdn_link(storage.cdn_resolver, project_name, file_id)
 
+    if storage.storage_type == "s3":
+        url = await storage.get_signed_url(file_id, ttl=3600)
+        return RedirectResponse(url=url, status_code=302)
+
+    url = f"/api/projects/{project_name}/files/{file_id}/payload"
+    return RedirectResponse(url=url, status_code=302)
+
+
+@router.get("/{file_id}/payload", response_model=None)
+async def get_project_file_payload(
+    request: Request,
+    project_name: ProjectName,
+    file_id: str,
+    user: CurrentUser,
+) -> FileResponse | Response:
+    storage = await Storages.project(project_name)
+    if storage.storage_type != "local":
+        raise BadRequestException("File storage is not local")
+
+    path = await storage.get_path(file_id)
+
+    check_user_access(project_name, user)
     headers = await get_file_headers(project_name, file_id)
 
     if headers["Content-Type"].startswith("video"):

api/review/upload.py

@@ -1,17 +1,14 @@
-import os
-
 from fastapi import Header, Query, Request
 from nxtools import logging
 
 from ayon_server.activities.create_activity import create_activity
 from ayon_server.activities.watchers.set_watchers import ensure_watching
 from ayon_server.api.dependencies import CurrentUser, ProjectName, VersionID
-from ayon_server.api.files import handle_upload
 from ayon_server.entities.version import VersionEntity
 from ayon_server.events import EventStream
 from ayon_server.exceptions import BadRequestException
-from ayon_server.helpers.ffprobe import availability_from_media_info, extract_media_info
-from ayon_server.helpers.project_files import id_to_path
+from ayon_server.files import Storages
+from ayon_server.helpers.ffprobe import availability_from_media_info
 from ayon_server.lib.postgres import Postgres
 from ayon_server.utils import create_uuid
 
@@ -52,19 +49,18 @@ async def upload_reviewable(
     await version.ensure_create_access(user)
 
     file_id = create_uuid()
-    upload_path = id_to_path(project_name, file_id)
-    file_size = await handle_upload(request, upload_path)
 
-    logging.debug(f"Uploaded file {x_file_name} ({file_size} bytes)")
+    storage = await Storages.project(project_name)
+    file_size = await storage.handle_upload(request, file_id)
 
-    # FFProbe here
+    logging.debug(f"Uploaded file {x_file_name} ({file_size} bytes)")
 
-    media_info = await extract_media_info(upload_path)
+    media_info = await storage.extract_media_info(file_id)
 
     if not media_info:
         logging.warning(f"Failed to extract media info for {x_file_name}")
         try:
-            os.remove(upload_path)
+            await storage.unlink(file_id)
         except Exception:
             pass
         raise BadRequestException("Failed to extract media info")

ayon_server/background/clean_up.py

@@ -6,7 +6,7 @@
 
 from ayon_server.background.background_worker import BackgroundWorker
 from ayon_server.config import ayonconfig
-from ayon_server.helpers.project_files import delete_unused_files
+from ayon_server.files import Storages
 from ayon_server.helpers.project_list import get_project_list
 from ayon_server.lib.postgres import Postgres
 
@@ -137,6 +137,11 @@ async def clear_events() -> None:
             break
 
 
+async def delete_unused_files(project_name: str) -> None:
+    storage = await Storages.project(project_name)
+    await storage.delete_unused_files()
+
+
 class AyonCleanUp(BackgroundWorker):
     """Background task for periodic clean-up of stuff."""
 

ayon_server/config.py

@@ -1,6 +1,7 @@
 """Server configuration object"""
 
 import os
+from typing import Literal
 
 from aiocache import caches
 from pydantic import BaseModel, Field
@@ -33,12 +34,6 @@ class AyonConfig(BaseModel):
         description="Path to the directory containing the API modules.",
     )
 
-    project_data_dir: str = Field(
-        default="/storage/server/projects",
-        description="Path to the directory containing the project files."
-        " such as comment attachments, thumbnails, etc.",
-    )
-
     avatar_dir: str = Field(
         default="/storage/server/avatars",
         description="Path to the directory containing the user avatars.",
@@ -199,6 +194,8 @@ class AyonConfig(BaseModel):
         description="Path to the log file",
     )
 
+    # Metrics settings
+
     metrics_api_key: str | None = Field(
         default=None,
         description="API key allowing access to the system metrics endpoint",
@@ -214,13 +211,38 @@ class AyonConfig(BaseModel):
         description="Send saturated metrics to Ynput Cloud",
     )
 
+    # Email settings
+
     email_from: str = Field("[email protected]", description="Email sender address")
     email_smtp_host: str | None = Field(None, description="SMTP server hostname")
     email_smtp_port: int | None = Field(None, description="SMTP server port")
     email_smtp_tls: bool = Field(False, description="Use SSL for SMTP connection")
     email_smtp_user: str | None = Field(None, description="SMTP server username")
     email_smtp_pass: str | None = Field(None, description="SMTP server password")
 
+    # Project storage
+
+    default_project_storage_type: Literal["local", "s3"] = Field(
+        "local",
+        description="Default project storage type",
+    )
+
+    default_project_storage_root: str = Field(
+        default="/storage/server/projects",
+        description="Path to the directory containing the project files."
+        " such as comment attachments, thumbnails, etc.",
+    )
+
+    default_project_storage_bucket_name: str | None = Field(
+        default=None,
+        description="Default project storage bucket name (S3)",
+    )
+
+    default_project_storage_cdn_resolver: str | None = Field(
+        default=None,
+        description="Project files CDN resolver URL",
+    )
+
 
 #
 # Load configuration from environment variables

ayon_server/files/__init__.py

@@ -0,0 +1,14 @@
+from typing import Any
+
+from ayon_server.files.project_storage import ProjectStorage
+
+
+class Storages:
+    project_storage_overrides: dict[str, Any] = {}
+
+    @classmethod
+    async def project(cls, project_name: str) -> ProjectStorage:
+        storage = cls.project_storage_overrides.get(project_name)
+        if storage:
+            return storage
+        return ProjectStorage.default(project_name)