File upload (#23)

Files can now be uploaded directly and will be stored in an Amazon S3 bucket.

.env

@@ -19,6 +19,13 @@
 #DJANGO_EMAIL_USE_SSL=  # any value but True is false
 #DJANGO_DEFAULT_FROM_EMAIL=
 
+#AWS_ACCESS_KEY_ID=
+#DJANGO_AWS_S3_REGION_NAME=
+#DJANGO_AWS_STORAGE_BUCKET_NAME=
+#DJANGO_AWS_DEFAUL_ACL=
+
+#DJANGO_MEDIA_ROOT=
+
 # CORS configuration for backend
 VIRTUAL_HOST_ROOT=localhost
 

.github/workflows/check-spec.yml

@@ -72,11 +72,12 @@ jobs:
           sudo apt-get install -y docker-compose
           mkdir -p .dev/spec
           sudo chmod 777 .dev/spec
+          touch .env.secret
 
       - name: Generate spec
         run: |
           # using x rather than a number means it appears later and gets picked up by `tail -n 1` in check_spec
-          docker-compose run --rm app bash -c "python manage.py spectacular --format openapi-json >> /spec/openapi-x.json"
+          docker-compose run --rm -e AWS_SECRET_ACCESS_KEY="not_set" app bash -c "python manage.py spectacular --format openapi-json >> /spec/openapi-x.json"
           # Copy spec for upload
           cp .dev/spec/openapi-x.json .dev/spec/openapi-${{ needs.version.outputs.version }}.json
 

.github/workflows/test.yml

@@ -49,6 +49,7 @@ jobs:
         run: |
             sudo apt-get update
             sudo apt-get install -y docker-compose
+            touch .env.secret
 
       - name: Build container
         run: docker-compose build app_test

backend_django/config/settings_base.py

@@ -20,7 +20,7 @@
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 import os
 
-API_VERSION = "2.1.18"
+API_VERSION = "2.1.19"
 
 try:
     USER_ACTIVATION_TOKEN_EXPIRY_S = int(os.environ.get("DJANGO_USER_ACTIVATION_TOKEN_EXPIRY_S"))
@@ -123,12 +123,6 @@
 DATA_UPLOAD_MAX_MEMORY_SIZE = 100000000
 
 
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/4.1/howto/static-files/
-
-STATIC_URL = 'django_static/'
-STATIC_ROOT = '/static/'
-
 # Default primary key field type
 # https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field
 
@@ -185,3 +179,41 @@
 EMAIL_USE_SSL = os.environ.get("DJANGO_EMAIL_USE_SSL") == "True"
 
 DEFAULT_FROM_EMAIL = os.environ.get("DJANGO_DEFAULT_FROM_EMAIL", "admin@galv")
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/4.1/howto/static-files/
+
+# Amazon Web Services S3 storage settings
+AWS_S3_REGION_NAME = os.environ.get("DJANGO_AWS_S3_REGION_NAME")
+AWS_STORAGE_BUCKET_NAME = os.environ.get("DJANGO_AWS_STORAGE_BUCKET_NAME")
+AWS_DEFAULT_ACL = os.environ.get("DJANGO_AWS_DEFAULT_ACL")
+AWS_S3_OBJECT_PARAMETERS = {
+    "CacheControl": "max-age=2592000",
+}
+AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com"
+
+STATICFILES_DIRS = [
+    "/static/",
+]
+
+# static files
+STATICFILES_LOCATION = "static"
+STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{STATICFILES_LOCATION}/"
+
+# media files
+MEDIAFILES_LOCATION = "media"
+MEDIA_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{MEDIAFILES_LOCATION}/"
+
+if os.environ.get("AWS_SECRET_ACCESS_KEY") is not None:
+    STORAGES = {
+        "default": {"BACKEND": "galv.storages.MediaStorage"},  # for media
+        "staticfiles": {"BACKEND": "galv.storages.StaticStorage"},
+    }
+else:
+    if AWS_S3_REGION_NAME or AWS_STORAGE_BUCKET_NAME or AWS_DEFAULT_ACL:
+        print(os.system('env'))
+        raise ValueError("AWS settings are incomplete - missing AWS_SECRET_ACCESS_KEY")
+    STORAGES = {
+        "default": {"BACKEND": "django.core.files.storage.FileSystemStorage", "LOCATION": "/media"},
+        "staticfiles": {"BACKEND": "django.core.files.storage.FileSystemStorage", "LOCATION": "/static"},
+    }

backend_django/config/urls.py

@@ -42,6 +42,7 @@
 router.register(r'schedules', views.ScheduleViewSet)
 router.register(r'cycler_tests', views.CyclerTestViewSet)
 router.register(r'experiments', views.ExperimentViewSet)
+router.register(r'arbitrary_files', views.ArbitraryFileViewSet)
 router.register(r'validation_schemas', views.ValidationSchemaViewSet)
 router.register(r'schema_validations', views.SchemaValidationViewSet)
 router.register(r'users', views.UserViewSet, basename='userproxy')

backend_django/galv/fields.py

@@ -0,0 +1,40 @@
+from django.db import models
+from django.db.models.fields.files import FieldFile
+
+from .storages import MediaStorage
+
+
+class DynamicStorageFieldFile(FieldFile):
+    def __init__(self, instance, field, name):
+        super(DynamicStorageFieldFile, self).__init__(instance, field, name)
+        self.storage = field.storage
+
+    def update_acl(self):
+        if not self:
+            return
+        # Only close the file if it's already open, which we know by
+        # the presence of self._file
+        if hasattr(self, '_file'):
+            self.close()  # This update_acl method we have already defined in UpdateACLMixin class
+        self.storage.update_acl(self.name)
+
+
+class DynamicStorageFileField(models.FileField):
+    attr_class = DynamicStorageFieldFile
+
+    def pre_save(self, model_instance, add):
+        self.storage = MediaStorage()
+        if model_instance.is_public:
+            self.storage.default_acl = "public-read"
+            self.storage.querystring_auth = False
+        else:
+            self.storage.default_acl = "private"
+            self.storage.querystring_auth = True
+
+        file = super(DynamicStorageFileField, self).pre_save(model_instance, add)
+
+        if file and file._committed:
+            # This update_acl method we have already defined
+            # in DynamicStorageFieldFile class above.
+            file.update_acl()
+        return file