Refactor media storage for scalability

CHANGELOG.md

@@ -21,10 +21,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 See for sample https://raw.githubusercontent.com/favoloso/conventional-changelog-emoji/master/CHANGELOG.md
 -->
 
-
 ## [0.10.0] - 2024-MM-DD
 ### 🐛 Bug Fixes
--  Fix admin % lighthouse inQueue value (#42)
+- Fix admin % lighthouse inQueue value (#42)
+- Refactor media storage for scalability (#44)
 
 ## [0.9.2] - 2024-05-20
 ### Improvements

django/core/settings.py

@@ -95,12 +95,15 @@
     AWS_S3_CUSTOM_DOMAIN = os.getenv('AWS_S3_CUSTOM_DOMAIN')
     AWS_S3_OBJECT_PARAMETERS = {'CacheControl': 'max-age=3600'}
     AWS_S3_FILE_OVERWRITE = True
+    STATIC_ROOT = os.path.join(BASE_DIR, 'collectstatic')
     # s3 static settings
-    AWS_LOCATION = 'static'
-    STATIC_URL = f'https://{AWS_S3_CUSTOM_DOMAIN}/{AWS_LOCATION}/'
-    DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
     STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
-    STATIC_ROOT = os.path.join(BASE_DIR, 'collectstatic')
+    STATIC_URL = f'https://{AWS_S3_CUSTOM_DOMAIN}/static/'
+    DEFAULT_FILE_STORAGE = 'core.storage_backends.MediaStorage'
+    MEDIA_URL = f'https://{AWS_S3_CUSTOM_DOMAIN}/media/'
+
+    if not AWS_S3_CUSTOM_DOMAIN:
+        raise ValueError("AWS_S3_CUSTOM_DOMAIN must be set when using S3 storage")
 
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 # Application definition

django/core/storage_backends.py

@@ -0,0 +1,5 @@
+from storages.backends.s3boto3 import S3Boto3Storage
+
+class MediaStorage(S3Boto3Storage):
+    location = 'media'
+    file_overwrite = False

django/performances/api.py

@@ -59,9 +59,7 @@ def save_report(request, server_id, performance_id):
     data = json.loads(request.body.decode("utf-8"))
 
     # Generate metadata used for file storage
-    user = performance.project.user
-    slug = slugify(performance.url)
-    path = f'performance/{user}/{slug}'
+    path = performance.directory_path()
     try:
         filename = f'{data["audits"]["final-screenshot"]["details"]["timestamp"]}'
     except:

django/performances/apps.py

@@ -1,6 +1,5 @@
 from django.apps import AppConfig
 
-
 class PerformancesConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'performances'

django/performances/models.py

@@ -1,7 +1,7 @@
 import os
 from django.db import models
 from projects.models import Project
-from django.conf import settings
+from django.core.files.storage import default_storage
 
 from constants import LIGHTHOUSE_FORMFACTOR_CHOICES
 
@@ -45,11 +45,24 @@ def last_score(self):
     def __str__(self):
         return f'{self.url}'
 
+    def directory_path(self):
+        return f'{self.project.directory_path()}/performances/perf_{self.pk}'
+
+    def delete(self):
+        super().delete()
+        if default_storage.exists(self.directory_path()):
+            try:
+                # Deletes the performance folder.
+                # Content has already been deleted by cascading.
+                default_storage.delete(self.directory_path())
+            except Exception as e:
+                print(f"Error deleting folder: {e}")
+
 """
     REPORT MODEL
 """
 def user_directory_path(instance, filename):
-    return 'performance/reports/{0}/{1}'.format(instance.performance.pk, filename)
+    return '{0}/{1}'.format(instance.performance.directory_path(), filename)
 
 def delete_old_reports(report):
     # Select all reports order from recent to old and if more than 4 delete all older than the 4 more recent

django/projects/models.py

@@ -119,5 +119,8 @@ def performance_score(self):
         result['last_run'] = last_run
         return result
 
+    def directory_path(self):
+        return f'user_{self.user.pk}/prjct_{self.pk}'
+
     def __str__(self):
         return self.title

django/settings/apps.py

@@ -1,6 +1,8 @@
 from django.apps import AppConfig
 
-
 class SettingsConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'settings'
+
+    def ready(self):
+        import settings.signals # Load signals

django/settings/models.py

@@ -10,3 +10,6 @@ class Profile(models.Model):
         related_name='profile'
     )
     timezone = TimeZoneField(null=True, choices_display="STANDARD")
+
+    def directory_path(self):
+        return f'user_{self.user.pk}'

django/settings/signals.py

@@ -0,0 +1,39 @@
+import logging
+import os
+from django.conf import settings
+from django.db.models.signals import post_delete
+from django.dispatch import receiver
+from django.core.files.storage import default_storage
+from .models import Profile
+
+logger = logging.getLogger(__name__)
+
+def delete_directory_contents(path):
+    if default_storage.exists(path):
+        dirs, files = default_storage.listdir(path)
+        for file in files:
+            file_path = os.path.join(path, file)
+            default_storage.delete(file_path)
+            logger.info(f"Deleted file: {file_path}")
+        for subdir in dirs:
+            subdir_path = os.path.join(path, subdir)
+            delete_directory_contents(subdir_path)
+        default_storage.delete(path)
+        logger.info(f"Deleted directory: {path}")
+
+@receiver(post_delete, sender=Profile)
+def delete_user_directory(sender, instance=None, **kwargs):
+    if instance is None:
+        logger.warning("delete_user_directory called with None instance")
+        return
+
+    dir_path = instance.directory_path()
+    try:
+        delete_directory_contents(dir_path)
+        logger.info(f"Successfully deleted user directory: {dir_path}")
+    except PermissionError as e:
+        logger.error(f"Permission error deleting directory {dir_path}: {e}")
+    except FileNotFoundError as e:
+        logger.warning(f"Directory not found {dir_path}: {e}")
+    except Exception as e:
+        logger.error(f"Unexpected error deleting directory {dir_path}: {e}")