From 4a3832509ae7a751132a78710cf1ca91dcb04d4f Mon Sep 17 00:00:00 2001 From: Matt Jaquiery Date: Thu, 29 Feb 2024 12:51:09 +0000 Subject: [PATCH] File upload (#23) Files can now be uploaded directly and will be stored in an Amazon S3 bucket. --- .env | 7 + .github/workflows/check-spec.yml | 3 +- .github/workflows/test.yml | 1 + backend_django/config/settings_base.py | 46 +- backend_django/config/urls.py | 1 + backend_django/galv/fields.py | 40 ++ .../galv/migrations/0001_initial.py | 556 ++++++++---------- ...2_alter_cell_custom_properties_and_more.py | 59 -- backend_django/galv/models/choices.py | 31 + backend_django/galv/models/models.py | 49 +- .../galv/serializers/serializers.py | 23 +- backend_django/galv/storages.py | 30 + backend_django/galv/tests/factories.py | 3 +- backend_django/galv/views.py | 68 ++- docker-compose.yaml | 4 + docs/source/conf.py | 2 +- docs/tags.json | 2 +- fly.toml | 3 + requirements.txt | 3 +- 19 files changed, 527 insertions(+), 404 deletions(-) create mode 100644 backend_django/galv/fields.py delete mode 100644 backend_django/galv/migrations/0002_alter_cell_custom_properties_and_more.py create mode 100644 backend_django/galv/models/choices.py create mode 100644 backend_django/galv/storages.py diff --git a/.env b/.env index 85007ab..663607c 100644 --- a/.env +++ b/.env @@ -19,6 +19,13 @@ #DJANGO_EMAIL_USE_SSL= # any value but True is false #DJANGO_DEFAULT_FROM_EMAIL= +#AWS_ACCESS_KEY_ID= +#DJANGO_AWS_S3_REGION_NAME= +#DJANGO_AWS_STORAGE_BUCKET_NAME= +#DJANGO_AWS_DEFAUL_ACL= + +#DJANGO_MEDIA_ROOT= + # CORS configuration for backend VIRTUAL_HOST_ROOT=localhost diff --git a/.github/workflows/check-spec.yml b/.github/workflows/check-spec.yml index eca74bd..7db276c 100644 --- a/.github/workflows/check-spec.yml +++ b/.github/workflows/check-spec.yml @@ -72,11 +72,12 @@ jobs: sudo apt-get install -y docker-compose mkdir -p .dev/spec sudo chmod 777 .dev/spec + touch .env.secret - name: Generate spec run: | # using x rather than a number means it appears later and gets picked up by `tail -n 1` in check_spec - docker-compose run --rm app bash -c "python manage.py spectacular --format openapi-json >> /spec/openapi-x.json" + docker-compose run --rm -e AWS_SECRET_ACCESS_KEY="not_set" app bash -c "python manage.py spectacular --format openapi-json >> /spec/openapi-x.json" # Copy spec for upload cp .dev/spec/openapi-x.json .dev/spec/openapi-${{ needs.version.outputs.version }}.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f20432e..b85a83c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,6 +49,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y docker-compose + touch .env.secret - name: Build container run: docker-compose build app_test diff --git a/backend_django/config/settings_base.py b/backend_django/config/settings_base.py index b2b91cb..0c8ca1d 100644 --- a/backend_django/config/settings_base.py +++ b/backend_django/config/settings_base.py @@ -20,7 +20,7 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. import os -API_VERSION = "2.1.18" +API_VERSION = "2.1.19" try: USER_ACTIVATION_TOKEN_EXPIRY_S = int(os.environ.get("DJANGO_USER_ACTIVATION_TOKEN_EXPIRY_S")) @@ -123,12 +123,6 @@ DATA_UPLOAD_MAX_MEMORY_SIZE = 100000000 -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/4.1/howto/static-files/ - -STATIC_URL = 'django_static/' -STATIC_ROOT = '/static/' - # Default primary key field type # https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field @@ -185,3 +179,41 @@ EMAIL_USE_SSL = os.environ.get("DJANGO_EMAIL_USE_SSL") == "True" DEFAULT_FROM_EMAIL = os.environ.get("DJANGO_DEFAULT_FROM_EMAIL", "admin@galv") + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.1/howto/static-files/ + +# Amazon Web Services S3 storage settings +AWS_S3_REGION_NAME = os.environ.get("DJANGO_AWS_S3_REGION_NAME") +AWS_STORAGE_BUCKET_NAME = os.environ.get("DJANGO_AWS_STORAGE_BUCKET_NAME") +AWS_DEFAULT_ACL = os.environ.get("DJANGO_AWS_DEFAULT_ACL") +AWS_S3_OBJECT_PARAMETERS = { + "CacheControl": "max-age=2592000", +} +AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" + +STATICFILES_DIRS = [ + "/static/", +] + +# static files +STATICFILES_LOCATION = "static" +STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{STATICFILES_LOCATION}/" + +# media files +MEDIAFILES_LOCATION = "media" +MEDIA_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{MEDIAFILES_LOCATION}/" + +if os.environ.get("AWS_SECRET_ACCESS_KEY") is not None: + STORAGES = { + "default": {"BACKEND": "galv.storages.MediaStorage"}, # for media + "staticfiles": {"BACKEND": "galv.storages.StaticStorage"}, + } +else: + if AWS_S3_REGION_NAME or AWS_STORAGE_BUCKET_NAME or AWS_DEFAULT_ACL: + print(os.system('env')) + raise ValueError("AWS settings are incomplete - missing AWS_SECRET_ACCESS_KEY") + STORAGES = { + "default": {"BACKEND": "django.core.files.storage.FileSystemStorage", "LOCATION": "/media"}, + "staticfiles": {"BACKEND": "django.core.files.storage.FileSystemStorage", "LOCATION": "/static"}, + } diff --git a/backend_django/config/urls.py b/backend_django/config/urls.py index 99f738d..d05a09a 100644 --- a/backend_django/config/urls.py +++ b/backend_django/config/urls.py @@ -42,6 +42,7 @@ router.register(r'schedules', views.ScheduleViewSet) router.register(r'cycler_tests', views.CyclerTestViewSet) router.register(r'experiments', views.ExperimentViewSet) +router.register(r'arbitrary_files', views.ArbitraryFileViewSet) router.register(r'validation_schemas', views.ValidationSchemaViewSet) router.register(r'schema_validations', views.SchemaValidationViewSet) router.register(r'users', views.UserViewSet, basename='userproxy') diff --git a/backend_django/galv/fields.py b/backend_django/galv/fields.py new file mode 100644 index 0000000..2fe09b2 --- /dev/null +++ b/backend_django/galv/fields.py @@ -0,0 +1,40 @@ +from django.db import models +from django.db.models.fields.files import FieldFile + +from .storages import MediaStorage + + +class DynamicStorageFieldFile(FieldFile): + def __init__(self, instance, field, name): + super(DynamicStorageFieldFile, self).__init__(instance, field, name) + self.storage = field.storage + + def update_acl(self): + if not self: + return + # Only close the file if it's already open, which we know by + # the presence of self._file + if hasattr(self, '_file'): + self.close() # This update_acl method we have already defined in UpdateACLMixin class + self.storage.update_acl(self.name) + + +class DynamicStorageFileField(models.FileField): + attr_class = DynamicStorageFieldFile + + def pre_save(self, model_instance, add): + self.storage = MediaStorage() + if model_instance.is_public: + self.storage.default_acl = "public-read" + self.storage.querystring_auth = False + else: + self.storage.default_acl = "private" + self.storage.querystring_auth = True + + file = super(DynamicStorageFileField, self).pre_save(model_instance, add) + + if file and file._committed: + # This update_acl method we have already defined + # in DynamicStorageFieldFile class above. + file.update_acl() + return file diff --git a/backend_django/galv/migrations/0001_initial.py b/backend_django/galv/migrations/0001_initial.py index 106a9ee..06be597 100644 --- a/backend_django/galv/migrations/0001_initial.py +++ b/backend_django/galv/migrations/0001_initial.py @@ -1,12 +1,14 @@ -# Generated by Django 4.1.4 on 2024-02-05 10:58 +# Generated by Django 5.0.2 on 2024-02-29 10:26 -from django.conf import settings import django.contrib.auth.models import django.contrib.postgres.fields -from django.db import migrations, models import django.db.models.deletion +import django_json_field_schema_validator.validators +import galv.fields import galv.models.utils import uuid +from django.conf import settings +from django.db import migrations, models class Migration(migrations.Migration): @@ -14,28 +16,12 @@ class Migration(migrations.Migration): initial = True dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('contenttypes', '0002_remove_content_type_name'), ('auth', '0012_alter_user_first_name_max_length'), + ('contenttypes', '0002_remove_content_type_name'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ - migrations.CreateModel( - name='Cell', - fields=[ - ('created', models.DateTimeField(auto_now_add=True)), - ('modified', models.DateTimeField(auto_now=True)), - ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('custom_properties', models.JSONField(default=dict)), - ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), - ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), - ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), - ('identifier', models.TextField(help_text='Unique identifier (e.g. serial number) for the cell')), - ], - options={ - 'abstract': False, - }, - ), migrations.CreateModel( name='CellChemistries', fields=[ @@ -92,32 +78,6 @@ class Migration(migrations.Migration): 'abstract': False, }, ), - migrations.CreateModel( - name='CyclerTest', - fields=[ - ('created', models.DateTimeField(auto_now_add=True)), - ('modified', models.DateTimeField(auto_now=True)), - ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('custom_properties', models.JSONField(default=dict)), - ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), - ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), - ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), - ('cell', models.ForeignKey(help_text='Cell that was tested', on_delete=django.db.models.deletion.CASCADE, related_name='cycler_tests', to='galv.cell')), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='DataColumn', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True)), - ('modified', models.DateTimeField(auto_now=True)), - ('data_type', models.TextField(help_text='Type of the data in this column')), - ('name_in_file', models.TextField(help_text='Column title e.g. in .tsv file headers')), - ], - ), migrations.CreateModel( name='DataUnit', fields=[ @@ -175,19 +135,6 @@ class Migration(migrations.Migration): 'abstract': False, }, ), - migrations.CreateModel( - name='Harvester', - fields=[ - ('created', models.DateTimeField(auto_now_add=True)), - ('modified', models.DateTimeField(auto_now=True)), - ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('name', models.TextField(help_text='Human-friendly Harvester identifier')), - ('api_key', models.TextField(help_text='API access token for the Harvester', null=True)), - ('last_check_in', models.DateTimeField(help_text='Date and time of last Harvester contact', null=True)), - ('sleep_time', models.IntegerField(default=120, help_text='Seconds to sleep between Harvester cycles')), - ('active', models.BooleanField(default=True, help_text='Whether the Harvester is active')), - ], - ), migrations.CreateModel( name='KnoxAuthToken', fields=[ @@ -214,30 +161,6 @@ class Migration(migrations.Migration): 'abstract': False, }, ), - migrations.CreateModel( - name='ObservedFile', - fields=[ - ('created', models.DateTimeField(auto_now_add=True)), - ('modified', models.DateTimeField(auto_now=True)), - ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('path', models.TextField(help_text='Absolute file path')), - ('last_observed_size', models.PositiveBigIntegerField(default=0, help_text='Size of the file as last reported by Harvester')), - ('last_observed_time', models.DateTimeField(help_text='Date and time of last Harvester report on file', null=True)), - ('state', models.TextField(choices=[('RETRY IMPORT', 'Retry Import'), ('IMPORT FAILED', 'Import Failed'), ('UNSTABLE', 'Unstable'), ('GROWING', 'Growing'), ('STABLE', 'Stable'), ('IMPORTING', 'Importing'), ('IMPORTED', 'Imported')], default='UNSTABLE', help_text='File status; autogenerated but can be manually set to RETRY IMPORT')), - ('data_generation_date', models.DateTimeField(help_text='Date and time of generated data. Time will be midnight if not specified in raw data', null=True)), - ('inferred_format', models.TextField(help_text='Format of the raw data', null=True)), - ('name', models.TextField(help_text='Name of the file', null=True)), - ('parser', models.TextField(help_text='Parser used by the harvester', null=True)), - ('num_rows', models.PositiveIntegerField(help_text='Number of rows in the file', null=True)), - ('first_sample_no', models.PositiveIntegerField(help_text='Number of the first sample in the file', null=True)), - ('last_sample_no', models.PositiveIntegerField(help_text='Number of the last sample in the file', null=True)), - ('extra_metadata', models.JSONField(help_text='Extra metadata from the harvester', null=True)), - ('harvester', models.ForeignKey(help_text='Harvester that harvested the File', on_delete=django.db.models.deletion.CASCADE, to='galv.harvester')), - ], - options={ - 'abstract': False, - }, - ), migrations.CreateModel( name='ScheduleIdentifiers', fields=[ @@ -252,16 +175,6 @@ class Migration(migrations.Migration): 'abstract': False, }, ), - migrations.CreateModel( - name='Team', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True)), - ('modified', models.DateTimeField(auto_now=True)), - ('name', models.TextField(help_text='Human-friendly Team identifier')), - ('description', models.TextField(help_text='Description of the Team', null=True)), - ], - ), migrations.CreateModel( name='GroupProxy', fields=[ @@ -291,121 +204,152 @@ class Migration(migrations.Migration): ], ), migrations.CreateModel( - name='ValidationSchema', + name='CellFamily', fields=[ ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('custom_properties', models.JSONField(default=dict)), + ('custom_properties', models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})])), ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), - ('name', models.TextField(help_text='Human-friendly identifier')), - ('schema', models.JSONField(help_text='JSON Schema')), - ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team')), + ('datasheet', models.URLField(blank=True, help_text='Link to the datasheet', null=True)), + ('nominal_voltage', models.FloatField(blank=True, help_text='Nominal voltage of the cells (in volts)', null=True)), + ('nominal_capacity', models.FloatField(blank=True, help_text='Nominal capacity of the cells (in amp hours)', null=True)), + ('initial_ac_impedance', models.FloatField(blank=True, help_text='Initial AC impedance of the cells (in ohms)', null=True)), + ('initial_dc_resistance', models.FloatField(blank=True, help_text='Initial DC resistance of the cells (in ohms)', null=True)), + ('energy_density', models.FloatField(blank=True, help_text='Energy density of the cells (in watt hours per kilogram)', null=True)), + ('power_density', models.FloatField(blank=True, help_text='Power density of the cells (in watts per kilogram)', null=True)), + ('chemistry', models.ForeignKey(blank=True, help_text='Chemistry of the cells', null=True, on_delete=django.db.models.deletion.CASCADE, to='galv.cellchemistries')), + ('form_factor', models.ForeignKey(blank=True, help_text='Physical shape of the cells', null=True, on_delete=django.db.models.deletion.CASCADE, to='galv.cellformfactors')), + ('manufacturer', models.ForeignKey(blank=True, help_text='Name of the manufacturer', null=True, on_delete=django.db.models.deletion.CASCADE, to='galv.cellmanufacturers')), + ('model', models.ForeignKey(help_text='Model number for the cells', on_delete=django.db.models.deletion.CASCADE, to='galv.cellmodels')), ], options={ 'abstract': False, }, ), migrations.CreateModel( - name='UserActivation', + name='Cell', fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('token', models.CharField(blank=True, max_length=8, null=True)), - ('token_update_date', models.DateTimeField(blank=True, null=True)), - ('redemption_date', models.DateTimeField(blank=True, null=True)), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='activation', to=settings.AUTH_USER_MODEL)), + ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('custom_properties', models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})])), + ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), + ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), + ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), + ('identifier', models.TextField(help_text='Unique identifier (e.g. serial number) for the cell')), + ('family', models.ForeignKey(help_text='Cell type', on_delete=django.db.models.deletion.CASCADE, related_name='cells', to='galv.cellfamily')), ], options={ 'abstract': False, }, ), migrations.CreateModel( - name='TimeseriesRangeLabel', + name='DataColumnType', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('label', models.TextField(help_text='Human-friendly identifier')), - ('range_start', models.PositiveBigIntegerField(help_text='Row (sample number) at which the range starts')), - ('range_end', models.PositiveBigIntegerField(help_text='Row (sample number) at which the range ends')), - ('info', models.TextField(help_text='Additional information')), - ('file', models.ForeignKey(help_text='Dataset to which the Range applies', on_delete=django.db.models.deletion.CASCADE, related_name='range_labels', to='galv.observedfile')), + ('name', models.TextField(help_text='Human-friendly identifier')), + ('description', models.TextField(help_text='Origins and purpose')), + ('is_default', models.BooleanField(default=False, help_text='Whether the Column is included in the initial list of known Column Types')), + ('is_required', models.BooleanField(default=False, help_text='Whether the Column must be present in every Dataset')), + ('override_child_name', models.TextField(blank=True, help_text='If set, this name will be used instead of the Column name in Dataframes', null=True)), + ('unit', models.ForeignKey(help_text='Unit used for measuring the values in this column', null=True, on_delete=django.db.models.deletion.SET_NULL, to='galv.dataunit')), ], options={ - 'abstract': False, + 'unique_together': {('unit', 'name')}, }, ), migrations.CreateModel( - name='TimeseriesDataStr', + name='Harvester', fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('values', django.contrib.postgres.fields.ArrayField(base_field=models.TextField(null=True), help_text='Row values (str) for Column', null=True, size=None)), - ('column', models.OneToOneField(help_text='Column whose data are listed', on_delete=django.db.models.deletion.CASCADE, to='galv.datacolumn')), + ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('name', models.TextField(help_text='Human-friendly Harvester identifier')), + ('api_key', models.TextField(help_text='API access token for the Harvester', null=True)), + ('last_check_in', models.DateTimeField(help_text='Date and time of last Harvester contact', null=True)), + ('sleep_time', models.IntegerField(default=120, help_text='Seconds to sleep between Harvester cycles')), + ('active', models.BooleanField(default=True, help_text='Whether the Harvester is active')), + ('lab', models.ForeignKey(help_text='Lab to which this Harvester belongs', on_delete=django.db.models.deletion.CASCADE, related_name='harvesters', to='galv.lab')), ], options={ - 'abstract': False, + 'unique_together': {('name', 'lab')}, }, ), migrations.CreateModel( - name='TimeseriesDataInt', + name='ObservedFile', fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('values', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(null=True), help_text='Row values (integers) for Column', null=True, size=None)), - ('column', models.OneToOneField(help_text='Column whose data are listed', on_delete=django.db.models.deletion.CASCADE, to='galv.datacolumn')), + ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('path', models.TextField(help_text='Absolute file path')), + ('last_observed_size', models.PositiveBigIntegerField(default=0, help_text='Size of the file as last reported by Harvester')), + ('last_observed_time', models.DateTimeField(help_text='Date and time of last Harvester report on file', null=True)), + ('state', models.TextField(choices=[('RETRY IMPORT', 'Retry Import'), ('IMPORT FAILED', 'Import Failed'), ('UNSTABLE', 'Unstable'), ('GROWING', 'Growing'), ('STABLE', 'Stable'), ('IMPORTING', 'Importing'), ('IMPORTED', 'Imported')], default='UNSTABLE', help_text='File status; autogenerated but can be manually set to RETRY IMPORT')), + ('data_generation_date', models.DateTimeField(help_text='Date and time of generated data. Time will be midnight if not specified in raw data', null=True)), + ('inferred_format', models.TextField(help_text='Format of the raw data', null=True)), + ('name', models.TextField(help_text='Name of the file', null=True)), + ('parser', models.TextField(help_text='Parser used by the harvester', null=True)), + ('num_rows', models.PositiveIntegerField(help_text='Number of rows in the file', null=True)), + ('first_sample_no', models.PositiveIntegerField(help_text='Number of the first sample in the file', null=True)), + ('last_sample_no', models.PositiveIntegerField(help_text='Number of the last sample in the file', null=True)), + ('extra_metadata', models.JSONField(help_text='Extra metadata from the harvester', null=True)), + ('harvester', models.ForeignKey(help_text='Harvester that harvested the File', on_delete=django.db.models.deletion.CASCADE, to='galv.harvester')), ], options={ 'abstract': False, + 'unique_together': {('path', 'harvester')}, }, ), migrations.CreateModel( - name='TimeseriesDataFloat', + name='HarvestError', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('values', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(null=True), help_text='Row values (floats) for Column', null=True, size=None)), - ('column', models.OneToOneField(help_text='Column whose data are listed', on_delete=django.db.models.deletion.CASCADE, to='galv.datacolumn')), + ('error', models.TextField(help_text='Text of the error report')), + ('timestamp', models.DateTimeField(auto_now=True, help_text='Date and time error was logged in the database', null=True)), + ('harvester', models.ForeignKey(help_text='Harvester which reported the error', on_delete=django.db.models.deletion.CASCADE, related_name='upload_errors', to='galv.harvester')), + ('file', models.ForeignKey(help_text='File where error originated', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='upload_errors', to='galv.observedfile')), ], options={ 'abstract': False, }, ), - migrations.AddField( - model_name='team', - name='admin_group', - field=models.OneToOneField(help_text='Users authorised to make changes to the Team', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='editable_team', to='galv.groupproxy'), - ), - migrations.AddField( - model_name='team', - name='lab', - field=models.ForeignKey(help_text='Lab to which this Team belongs', on_delete=django.db.models.deletion.CASCADE, related_name='teams', to='galv.lab'), - ), - migrations.AddField( - model_name='team', - name='member_group', - field=models.OneToOneField(help_text="Users authorised to view this Team's Experiments", null=True, on_delete=django.db.models.deletion.CASCADE, related_name='readable_team', to='galv.groupproxy'), + migrations.CreateModel( + name='DataColumn', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('data_type', models.TextField(help_text='Type of the data in this column')), + ('name_in_file', models.TextField(help_text='Column title e.g. in .tsv file headers')), + ('type', models.ForeignKey(help_text='Column Type which this Column instantiates', on_delete=django.db.models.deletion.CASCADE, to='galv.datacolumntype')), + ('file', models.ForeignKey(help_text='File in which this Column appears', on_delete=django.db.models.deletion.CASCADE, related_name='columns', to='galv.observedfile')), + ], + options={ + 'unique_together': {('file', 'name_in_file')}, + }, ), migrations.CreateModel( - name='SchemaValidation', + name='Team', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('object_id', models.CharField(max_length=36)), - ('status', models.TextField(choices=[('VALID', 'Valid'), ('INVALID', 'Invalid'), ('SKIPPED', 'Skipped'), ('UNCHECKED', 'Unchecked'), ('ERROR', 'Error')], help_text='Validation status')), - ('detail', models.JSONField(help_text='Validation detail', null=True)), - ('last_update', models.DateTimeField(auto_now=True, help_text='Date and time of last status update')), - ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), - ('schema', models.ForeignKey(help_text='ValidationSchema used to validate the component', on_delete=django.db.models.deletion.CASCADE, to='galv.validationschema')), + ('name', models.TextField(help_text='Human-friendly Team identifier')), + ('description', models.TextField(help_text='Description of the Team', null=True)), + ('lab', models.ForeignKey(help_text='Lab to which this Team belongs', on_delete=django.db.models.deletion.CASCADE, related_name='teams', to='galv.lab')), + ('admin_group', models.OneToOneField(help_text='Users authorised to make changes to the Team', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='editable_team', to='galv.groupproxy')), + ('member_group', models.OneToOneField(help_text="Users authorised to view this Team's Experiments", null=True, on_delete=django.db.models.deletion.CASCADE, related_name='readable_team', to='galv.groupproxy')), ], + options={ + 'unique_together': {('name', 'lab')}, + }, ), migrations.CreateModel( name='ScheduleFamily', @@ -413,7 +357,7 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('custom_properties', models.JSONField(default=dict)), + ('custom_properties', models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})])), ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), @@ -433,7 +377,7 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('custom_properties', models.JSONField(default=dict)), + ('custom_properties', models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})])), ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), @@ -447,78 +391,87 @@ class Migration(migrations.Migration): }, ), migrations.CreateModel( - name='MonitoredPath', + name='EquipmentFamily', fields=[ ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('custom_properties', models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})])), + ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), + ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), - ('path', models.TextField(help_text='Directory location on Harvester')), - ('regex', models.TextField(blank=True, help_text="\n Python.re regular expression to filter files by, \n applied to full file name starting from this Path's directory", null=True)), - ('stable_time', models.PositiveSmallIntegerField(default=60, help_text='Number of seconds files must remain stable to be processed')), - ('active', models.BooleanField(default=True)), - ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=4)), - ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=4)), - ('harvester', models.ForeignKey(help_text='Harvester with access to this directory', on_delete=django.db.models.deletion.DO_NOTHING, related_name='monitored_paths', to='galv.harvester')), - ('team', models.ForeignKey(help_text='Team with access to this Path', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='monitored_paths', to='galv.team')), + ('manufacturer', models.ForeignKey(help_text='Manufacturer of equipment', on_delete=django.db.models.deletion.CASCADE, to='galv.equipmentmanufacturers')), + ('model', models.ForeignKey(help_text='Model of equipment', on_delete=django.db.models.deletion.CASCADE, to='galv.equipmentmodels')), + ('type', models.ForeignKey(help_text='Type of equipment', on_delete=django.db.models.deletion.CASCADE, to='galv.equipmenttypes')), + ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team')), ], options={ 'abstract': False, }, ), - migrations.AddField( - model_name='lab', - name='admin_group', - field=models.OneToOneField(help_text='Users authorised to make changes to the Lab', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='editable_lab', to='galv.groupproxy'), - ), migrations.CreateModel( - name='HarvestError', + name='Equipment', fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('error', models.TextField(help_text='Text of the error report')), - ('timestamp', models.DateTimeField(auto_now=True, help_text='Date and time error was logged in the database', null=True)), - ('file', models.ForeignKey(help_text='File where error originated', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='upload_errors', to='galv.observedfile')), - ('harvester', models.ForeignKey(help_text='Harvester which reported the error', on_delete=django.db.models.deletion.CASCADE, related_name='upload_errors', to='galv.harvester')), + ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('custom_properties', models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})])), + ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), + ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), + ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), + ('identifier', models.TextField(help_text='Unique identifier (e.g. serial number) for the equipment', unique=True)), + ('calibration_date', models.DateField(blank=True, help_text='Date of last calibration', null=True)), + ('family', models.ForeignKey(help_text='Equipment type', on_delete=django.db.models.deletion.CASCADE, related_name='equipment', to='galv.equipmentfamily')), + ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team')), ], options={ 'abstract': False, }, ), migrations.CreateModel( - name='HarvesterEnvVar', + name='CyclerTest', fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('key', models.TextField(help_text='Name of the variable')), - ('value', models.TextField(help_text='Variable value')), - ('deleted', models.BooleanField(default=False, help_text='Whether this variable was deleted')), - ('harvester', models.ForeignKey(help_text='Harvester whose environment this describes', on_delete=django.db.models.deletion.CASCADE, related_name='environment_variables', to='galv.harvester')), + ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('custom_properties', models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})])), + ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), + ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), + ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), + ('cell', models.ForeignKey(help_text='Cell that was tested', on_delete=django.db.models.deletion.CASCADE, related_name='cycler_tests', to='galv.cell')), + ('equipment', models.ManyToManyField(help_text='Equipment used to test the cell', related_name='cycler_tests', to='galv.equipment')), + ('file', models.ManyToManyField(help_text='Columns of data in the test', related_name='cycler_tests', to='galv.observedfile')), + ('schedule', models.ForeignKey(blank=True, help_text='Schedule used to test the cell', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='cycler_tests', to='galv.schedule')), + ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team')), ], + options={ + 'abstract': False, + }, + ), + migrations.AddField( + model_name='cellfamily', + name='team', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team'), ), migrations.AddField( - model_name='harvester', - name='lab', - field=models.ForeignKey(help_text='Lab to which this Harvester belongs', on_delete=django.db.models.deletion.CASCADE, related_name='harvesters', to='galv.lab'), + model_name='cell', + name='team', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team'), ), migrations.CreateModel( - name='Experiment', + name='ArbitraryFile', fields=[ ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('custom_properties', models.JSONField(default=dict)), + ('custom_properties', models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})])), ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), - ('title', models.TextField(help_text='Title of the experiment')), - ('description', models.TextField(blank=True, help_text='Description of the experiment', null=True)), - ('protocol', models.JSONField(blank=True, help_text='Protocol of the experiment', null=True)), - ('protocol_file', models.FileField(blank=True, help_text='Protocol file of the experiment', null=True, upload_to='')), - ('authors', models.ManyToManyField(help_text='Authors of the experiment', to='galv.userproxy')), - ('cycler_tests', models.ManyToManyField(help_text='Cycler tests of the experiment', related_name='experiments', to='galv.cyclertest')), + ('file', galv.fields.DynamicStorageFileField(unique=True, upload_to='')), + ('is_public', models.BooleanField(default=False, help_text='Whether the file is public')), + ('name', models.TextField(help_text='The name of the file', unique=True)), + ('description', models.TextField(blank=True, help_text='The description of the file', null=True)), ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team')), ], options={ @@ -526,108 +479,87 @@ class Migration(migrations.Migration): }, ), migrations.CreateModel( - name='EquipmentFamily', + name='TimeseriesDataFloat', fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('custom_properties', models.JSONField(default=dict)), - ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), - ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), - ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), - ('manufacturer', models.ForeignKey(help_text='Manufacturer of equipment', on_delete=django.db.models.deletion.CASCADE, to='galv.equipmentmanufacturers')), - ('model', models.ForeignKey(help_text='Model of equipment', on_delete=django.db.models.deletion.CASCADE, to='galv.equipmentmodels')), - ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team')), - ('type', models.ForeignKey(help_text='Type of equipment', on_delete=django.db.models.deletion.CASCADE, to='galv.equipmenttypes')), + ('values', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(null=True), help_text='Row values (floats) for Column', null=True, size=None)), + ('column', models.OneToOneField(help_text='Column whose data are listed', on_delete=django.db.models.deletion.CASCADE, to='galv.datacolumn')), ], options={ 'abstract': False, }, ), migrations.CreateModel( - name='Equipment', + name='TimeseriesDataInt', fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('custom_properties', models.JSONField(default=dict)), - ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), - ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), - ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), - ('identifier', models.TextField(help_text='Unique identifier (e.g. serial number) for the equipment', unique=True)), - ('calibration_date', models.DateField(blank=True, help_text='Date of last calibration', null=True)), - ('family', models.ForeignKey(help_text='Equipment type', on_delete=django.db.models.deletion.CASCADE, related_name='equipment', to='galv.equipmentfamily')), - ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team')), + ('values', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(null=True), help_text='Row values (integers) for Column', null=True, size=None)), + ('column', models.OneToOneField(help_text='Column whose data are listed', on_delete=django.db.models.deletion.CASCADE, to='galv.datacolumn')), ], options={ 'abstract': False, }, ), migrations.CreateModel( - name='DataColumnType', + name='TimeseriesDataStr', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), - ('name', models.TextField(help_text='Human-friendly identifier')), - ('description', models.TextField(help_text='Origins and purpose')), - ('is_default', models.BooleanField(default=False, help_text='Whether the Column is included in the initial list of known Column Types')), - ('is_required', models.BooleanField(default=False, help_text='Whether the Column must be present in every Dataset')), - ('override_child_name', models.TextField(blank=True, help_text='If set, this name will be used instead of the Column name in Dataframes', null=True)), - ('unit', models.ForeignKey(help_text='Unit used for measuring the values in this column', null=True, on_delete=django.db.models.deletion.SET_NULL, to='galv.dataunit')), + ('values', django.contrib.postgres.fields.ArrayField(base_field=models.TextField(null=True), help_text='Row values (str) for Column', null=True, size=None)), + ('column', models.OneToOneField(help_text='Column whose data are listed', on_delete=django.db.models.deletion.CASCADE, to='galv.datacolumn')), ], + options={ + 'abstract': False, + }, ), - migrations.AddField( - model_name='datacolumn', - name='file', - field=models.ForeignKey(help_text='File in which this Column appears', on_delete=django.db.models.deletion.CASCADE, related_name='columns', to='galv.observedfile'), - ), - migrations.AddField( - model_name='datacolumn', - name='type', - field=models.ForeignKey(help_text='Column Type which this Column instantiates', on_delete=django.db.models.deletion.CASCADE, to='galv.datacolumntype'), - ), - migrations.AddField( - model_name='cyclertest', - name='equipment', - field=models.ManyToManyField(help_text='Equipment used to test the cell', related_name='cycler_tests', to='galv.equipment'), - ), - migrations.AddField( - model_name='cyclertest', - name='file', - field=models.ManyToManyField(help_text='Columns of data in the test', related_name='cycler_tests', to='galv.observedfile'), - ), - migrations.AddField( - model_name='cyclertest', - name='schedule', - field=models.ForeignKey(blank=True, help_text='Schedule used to test the cell', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='cycler_tests', to='galv.schedule'), + migrations.CreateModel( + name='TimeseriesRangeLabel', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('label', models.TextField(help_text='Human-friendly identifier')), + ('range_start', models.PositiveBigIntegerField(help_text='Row (sample number) at which the range starts')), + ('range_end', models.PositiveBigIntegerField(help_text='Row (sample number) at which the range ends')), + ('info', models.TextField(help_text='Additional information')), + ('file', models.ForeignKey(help_text='Dataset to which the Range applies', on_delete=django.db.models.deletion.CASCADE, related_name='range_labels', to='galv.observedfile')), + ], + options={ + 'abstract': False, + }, ), - migrations.AddField( - model_name='cyclertest', - name='team', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team'), + migrations.CreateModel( + name='UserActivation', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('token', models.CharField(blank=True, max_length=8, null=True)), + ('token_update_date', models.DateTimeField(blank=True, null=True)), + ('redemption_date', models.DateTimeField(blank=True, null=True)), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='activation', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, ), migrations.CreateModel( - name='CellFamily', + name='ValidationSchema', fields=[ ('created', models.DateTimeField(auto_now_add=True)), ('modified', models.DateTimeField(auto_now=True)), ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), - ('custom_properties', models.JSONField(default=dict)), + ('custom_properties', models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})])), ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), - ('datasheet', models.URLField(blank=True, help_text='Link to the datasheet', null=True)), - ('nominal_voltage', models.FloatField(blank=True, help_text='Nominal voltage of the cells (in volts)', null=True)), - ('nominal_capacity', models.FloatField(blank=True, help_text='Nominal capacity of the cells (in amp hours)', null=True)), - ('initial_ac_impedance', models.FloatField(blank=True, help_text='Initial AC impedance of the cells (in ohms)', null=True)), - ('initial_dc_resistance', models.FloatField(blank=True, help_text='Initial DC resistance of the cells (in ohms)', null=True)), - ('energy_density', models.FloatField(blank=True, help_text='Energy density of the cells (in watt hours per kilogram)', null=True)), - ('power_density', models.FloatField(blank=True, help_text='Power density of the cells (in watts per kilogram)', null=True)), - ('chemistry', models.ForeignKey(blank=True, help_text='Chemistry of the cells', null=True, on_delete=django.db.models.deletion.CASCADE, to='galv.cellchemistries')), - ('form_factor', models.ForeignKey(blank=True, help_text='Physical shape of the cells', null=True, on_delete=django.db.models.deletion.CASCADE, to='galv.cellformfactors')), - ('manufacturer', models.ForeignKey(blank=True, help_text='Name of the manufacturer', null=True, on_delete=django.db.models.deletion.CASCADE, to='galv.cellmanufacturers')), - ('model', models.ForeignKey(help_text='Model number for the cells', on_delete=django.db.models.deletion.CASCADE, to='galv.cellmodels')), + ('name', models.TextField(help_text='Human-friendly identifier')), + ('schema', models.JSONField(help_text='JSON Schema')), ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team')), ], options={ @@ -635,14 +567,31 @@ class Migration(migrations.Migration): }, ), migrations.AddField( - model_name='cell', - name='family', - field=models.ForeignKey(help_text='Cell type', on_delete=django.db.models.deletion.CASCADE, related_name='cells', to='galv.cellfamily'), + model_name='lab', + name='admin_group', + field=models.OneToOneField(help_text='Users authorised to make changes to the Lab', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='editable_lab', to='galv.groupproxy'), ), - migrations.AddField( - model_name='cell', - name='team', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team'), + migrations.CreateModel( + name='Experiment', + fields=[ + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('custom_properties', models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})])), + ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=3)), + ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User')], default=3)), + ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), + ('title', models.TextField(help_text='Title of the experiment')), + ('description', models.TextField(blank=True, help_text='Description of the experiment', null=True)), + ('protocol', models.JSONField(blank=True, help_text='Protocol of the experiment', null=True)), + ('protocol_file', models.FileField(blank=True, help_text='Protocol file of the experiment', null=True, upload_to='')), + ('cycler_tests', models.ManyToManyField(help_text='Cycler tests of the experiment', related_name='experiments', to='galv.cyclertest')), + ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_resources', to='galv.team')), + ('authors', models.ManyToManyField(help_text='Authors of the experiment', to='galv.userproxy')), + ], + options={ + 'abstract': False, + }, ), migrations.CreateModel( name='BibliographicInfo', @@ -657,45 +606,41 @@ class Migration(migrations.Migration): 'abstract': False, }, ), - migrations.AlterUniqueTogether( - name='team', - unique_together={('name', 'lab')}, - ), - migrations.AddIndex( - model_name='schemavalidation', - index=models.Index(fields=['content_type', 'object_id'], name='galv_schema_content_a0f28a_idx'), - ), - migrations.AddIndex( - model_name='schemavalidation', - index=models.Index(fields=['status'], name='galv_schema_status_b04295_idx'), - ), - migrations.AddIndex( - model_name='schemavalidation', - index=models.Index(fields=['schema'], name='galv_schema_schema__f5b262_idx'), - ), - migrations.AlterUniqueTogether( - name='observedfile', - unique_together={('path', 'harvester')}, - ), - migrations.AlterUniqueTogether( - name='monitoredpath', - unique_together={('harvester', 'path', 'regex', 'team')}, - ), - migrations.AlterUniqueTogether( - name='harvesterenvvar', - unique_together={('harvester', 'key')}, - ), - migrations.AlterUniqueTogether( - name='harvester', - unique_together={('name', 'lab')}, - ), - migrations.AlterUniqueTogether( - name='datacolumntype', - unique_together={('unit', 'name')}, + migrations.CreateModel( + name='HarvesterEnvVar', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('key', models.TextField(help_text='Name of the variable')), + ('value', models.TextField(help_text='Variable value')), + ('deleted', models.BooleanField(default=False, help_text='Whether this variable was deleted')), + ('harvester', models.ForeignKey(help_text='Harvester whose environment this describes', on_delete=django.db.models.deletion.CASCADE, related_name='environment_variables', to='galv.harvester')), + ], + options={ + 'unique_together': {('harvester', 'key')}, + }, ), - migrations.AlterUniqueTogether( - name='datacolumn', - unique_together={('file', 'name_in_file')}, + migrations.CreateModel( + name='MonitoredPath', + fields=[ + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('uuid', galv.models.utils.UUIDFieldLD(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('read_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member'), (2, 'Lab Member'), (1, 'Registered User'), (0, 'Anonymous')], default=2)), + ('path', models.TextField(help_text='Directory location on Harvester')), + ('regex', models.TextField(blank=True, help_text="\n Python.re regular expression to filter files by, \n applied to full file name starting from this Path's directory", null=True)), + ('stable_time', models.PositiveSmallIntegerField(default=60, help_text='Number of seconds files must remain stable to be processed')), + ('active', models.BooleanField(default=True)), + ('delete_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=4)), + ('edit_access_level', models.IntegerField(choices=[(4, 'Team Admin'), (3, 'Team Member')], default=4)), + ('harvester', models.ForeignKey(help_text='Harvester with access to this directory', on_delete=django.db.models.deletion.DO_NOTHING, related_name='monitored_paths', to='galv.harvester')), + ('team', models.ForeignKey(help_text='Team with access to this Path', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='monitored_paths', to='galv.team')), + ], + options={ + 'abstract': False, + 'unique_together': {('harvester', 'path', 'regex', 'team')}, + }, ), migrations.AlterUniqueTogether( name='cellfamily', @@ -705,4 +650,21 @@ class Migration(migrations.Migration): name='cell', unique_together={('identifier', 'family')}, ), + migrations.CreateModel( + name='SchemaValidation', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('object_id', models.CharField(max_length=36)), + ('status', models.TextField(choices=[('VALID', 'Valid'), ('INVALID', 'Invalid'), ('SKIPPED', 'Skipped'), ('UNCHECKED', 'Unchecked'), ('ERROR', 'Error')], help_text='Validation status')), + ('detail', models.JSONField(help_text='Validation detail', null=True)), + ('last_update', models.DateTimeField(auto_now=True, help_text='Date and time of last status update')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), + ('schema', models.ForeignKey(help_text='ValidationSchema used to validate the component', on_delete=django.db.models.deletion.CASCADE, to='galv.validationschema')), + ], + options={ + 'indexes': [models.Index(fields=['content_type', 'object_id'], name='galv_schema_content_a0f28a_idx'), models.Index(fields=['status'], name='galv_schema_status_b04295_idx'), models.Index(fields=['schema'], name='galv_schema_schema__f5b262_idx')], + }, + ), ] diff --git a/backend_django/galv/migrations/0002_alter_cell_custom_properties_and_more.py b/backend_django/galv/migrations/0002_alter_cell_custom_properties_and_more.py deleted file mode 100644 index 302b1a0..0000000 --- a/backend_django/galv/migrations/0002_alter_cell_custom_properties_and_more.py +++ /dev/null @@ -1,59 +0,0 @@ -# Generated by Django 4.1.4 on 2024-02-14 13:13 - -from django.db import migrations, models -import django_json_field_schema_validator.validators - - -class Migration(migrations.Migration): - - dependencies = [ - ('galv', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='cell', - name='custom_properties', - field=models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})]), - ), - migrations.AlterField( - model_name='cellfamily', - name='custom_properties', - field=models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})]), - ), - migrations.AlterField( - model_name='cyclertest', - name='custom_properties', - field=models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})]), - ), - migrations.AlterField( - model_name='equipment', - name='custom_properties', - field=models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})]), - ), - migrations.AlterField( - model_name='equipmentfamily', - name='custom_properties', - field=models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})]), - ), - migrations.AlterField( - model_name='experiment', - name='custom_properties', - field=models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})]), - ), - migrations.AlterField( - model_name='schedule', - name='custom_properties', - field=models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})]), - ), - migrations.AlterField( - model_name='schedulefamily', - name='custom_properties', - field=models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})]), - ), - migrations.AlterField( - model_name='validationschema', - name='custom_properties', - field=models.JSONField(default=dict, validators=[django_json_field_schema_validator.validators.JSONFieldSchemaValidator({'$defs': {'typedArray': {'additionalProperties': False, 'properties': {'_type': {'const': 'array'}, '_value': {'$comment': 'Array items are objects with _type and _value fields only, so each item in the array is individually typed.', 'items': {'$ref': '#/$defs/typedObjectProperty'}, 'type': 'array'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedBoolean': {'additionalProperties': False, 'properties': {'_type': {'const': 'boolean'}, '_value': {'type': 'boolean'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNull': {'additionalProperties': False, 'properties': {'_type': {'const': 'null'}, '_value': {'type': 'null'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedNumber': {'additionalProperties': False, 'properties': {'_type': {'const': 'number'}, '_value': {'type': 'number'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObject': {'additionalProperties': False, 'properties': {'_type': {'const': 'object'}, '_value': {'$ref': '#'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedObjectProperty': {'$comment': 'typedObjectProperty is either a known JSON type or a custom one. In either case, it is an object with _type and _value fields only. Different typed* types are used to enforce the correct _value type for each _type.', 'anyOf': [{'$ref': '#/$defs/typedString'}, {'$ref': '#/$defs/typedNumber'}, {'$ref': '#/$defs/typedBoolean'}, {'$ref': '#/$defs/typedNull'}, {'$ref': '#/$defs/typedObject'}, {'$ref': '#/$defs/typedArray'}, {'$ref': '#/$defs/typedUnknown'}]}, 'typedString': {'additionalProperties': False, 'properties': {'_type': {'const': 'string'}, '_value': {'type': 'string'}}, 'required': ['_type', '_value'], 'type': 'object'}, 'typedUnknown': {'$comment': "Custom types can be signified by using anything for _type that doesn't match the core JSON types. The _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'additionalProperties': False, 'properties': {'_type': {'type': 'string'}, '_value': {'anyOf': [{'$ref': '#/$defs/typedObjectProperty'}, {'$ref': '#'}, {'type': ['string', 'number', 'boolean', 'null']}]}}, 'required': ['_type', '_value'], 'type': 'object'}}, '$schema': 'https://json-schema.org/draft/2020-12/schema#', 'additionalProperties': {'$ref': '#/$defs/typedObjectProperty'}, 'description': "JSON schema for Galv typed JSON. All items are objects with a _type and _value field only. The _type will be one of the core JSON data types, or a custom string. If _type is a core JSON primitive, _value must have that type. If _type is 'array', then the contents must be items with _type and _value fields. If _type is 'object', then _value must be a JSON object with _type and _value fields. If _type is 'object', _value will be an object with each property value being an object with _type and _value fields. If _type is a custom string, _value can be any JSON primitive, an object with properties that are objects with _type and _value, or an array of objects with _type and _value.", 'title': 'Galv typed JSON - strict', 'type': 'object'})]), - ), - ] diff --git a/backend_django/galv/models/choices.py b/backend_django/galv/models/choices.py new file mode 100644 index 0000000..01d0df7 --- /dev/null +++ b/backend_django/galv/models/choices.py @@ -0,0 +1,31 @@ +from django.db import models + + +class FileState(models.TextChoices): + RETRY_IMPORT = "RETRY IMPORT" + IMPORT_FAILED = "IMPORT FAILED" + UNSTABLE = "UNSTABLE" + GROWING = "GROWING" + STABLE = "STABLE" + IMPORTING = "IMPORTING" + IMPORTED = "IMPORTED" + + +class UserLevel(models.Choices): + """ + User levels for access control. + Team/Lab levels only make sense in the context of a Resource. + """ + ANONYMOUS = 0 + REGISTERED_USER = 1 + LAB_MEMBER = 2 + TEAM_MEMBER = 3 + TEAM_ADMIN = 4 + + +class ValidationStatus(models.TextChoices): + VALID = "VALID" + INVALID = "INVALID" + SKIPPED = "SKIPPED" + UNCHECKED = "UNCHECKED" + ERROR = "ERROR" diff --git a/backend_django/galv/models/models.py b/backend_django/galv/models/models.py index 9987910..2ab6141 100644 --- a/backend_django/galv/models/models.py +++ b/backend_django/galv/models/models.py @@ -18,42 +18,15 @@ from django.utils.crypto import get_random_string from jsonschema.exceptions import _WrappedReferencingError from rest_framework import serializers -from rest_framework.reverse import reverse + +from .choices import FileState, UserLevel, ValidationStatus #from dry_rest_permissions.generics import allow_staff_or_superuser from .utils import CustomPropertiesModel, JSONModel, LDSources, render_pybamm_schedule, UUIDModel, \ combine_rdf_props, TimestampedModel from .autocomplete_entries import * - -class FileState(models.TextChoices): - RETRY_IMPORT = "RETRY IMPORT" - IMPORT_FAILED = "IMPORT FAILED" - UNSTABLE = "UNSTABLE" - GROWING = "GROWING" - STABLE = "STABLE" - IMPORTING = "IMPORTING" - IMPORTED = "IMPORTED" - - -class UserLevel(models.Choices): - """ - User levels for access control. - Team/Lab levels only make sense in the context of a Resource. - """ - ANONYMOUS = 0 - REGISTERED_USER = 1 - LAB_MEMBER = 2 - TEAM_MEMBER = 3 - TEAM_ADMIN = 4 - - -class ValidationStatus(models.TextChoices): - VALID = "VALID" - INVALID = "INVALID" - SKIPPED = "SKIPPED" - UNCHECKED = "UNCHECKED" - ERROR = "ERROR" +from ..fields import DynamicStorageFileField ALLOWED_USER_LEVELS_DELETE = [UserLevel(v) for v in [UserLevel.TEAM_ADMIN, UserLevel.TEAM_MEMBER]] @@ -1352,4 +1325,18 @@ class Meta: models.Index(fields=["content_type", "object_id"]), models.Index(fields=["status"]), models.Index(fields=["schema"]) - ] \ No newline at end of file + ] + + +class ArbitraryFile(JSONModel, ResourceModelPermissionsMixin): + file = DynamicStorageFileField(unique=True) + is_public = models.BooleanField(default=False, help_text="Whether the file is public") + name = models.TextField(help_text="The name of the file", null=False, blank=False, unique=True) + description = models.TextField(help_text="The description of the file", null=True, blank=True) + + def delete(self, using=None, keep_parents=False): + self.file.delete() + super(ArbitraryFile, self).delete(using, keep_parents) + + def __str__(self): + return self.name diff --git a/backend_django/galv/serializers/serializers.py b/backend_django/galv/serializers/serializers.py index 40d44f2..8a025d0 100644 --- a/backend_django/galv/serializers/serializers.py +++ b/backend_django/galv/serializers/serializers.py @@ -26,7 +26,7 @@ EquipmentManufacturers, EquipmentModels, EquipmentFamily, Schedule, ScheduleIdentifiers, CyclerTest, \ render_pybamm_schedule, ScheduleFamily, ValidationSchema, Experiment, Lab, Team, GroupProxy, UserProxy, user_labs, \ user_teams, SchemaValidation, UserActivation, UserLevel, ALLOWED_USER_LEVELS_READ, ALLOWED_USER_LEVELS_EDIT, \ - ALLOWED_USER_LEVELS_DELETE, ALLOWED_USER_LEVELS_EDIT_PATH + ALLOWED_USER_LEVELS_DELETE, ALLOWED_USER_LEVELS_EDIT_PATH, ArbitraryFile from ..models.utils import ScheduleRenderError from django.utils import timezone from django.conf.global_settings import DATA_UPLOAD_MAX_MEMORY_SIZE @@ -438,9 +438,11 @@ def validate_team(self, value): Only team members can create resources in their team. If a resource is being moved from one team to another, the user must be a member of both teams. """ - teams = user_teams(self.context['request'].user) try: + teams = user_teams(self.context['request'].user) assert value in teams + except KeyError: + raise ValidationError("No request context available to determine user's teams") except: raise ValidationError("You may only create resources in your own team(s)", code=HTTP_403_FORBIDDEN) if self.instance is not None: @@ -1791,3 +1793,20 @@ class Meta: fields = ['url', 'id', 'schema', 'validation_target', 'status', 'permissions', 'detail', 'last_update'] read_only_fields = [*fields] extra_kwargs = augment_extra_kwargs() + + +class ArbitraryFileSerializer(serializers.HyperlinkedModelSerializer, PermissionsMixin, WithTeamMixin): + + class Meta: + model = ArbitraryFile + fields = [ + 'url', 'uuid', 'name', 'description', 'file', 'team', 'is_public', 'custom_properties', + 'read_access_level', 'edit_access_level', 'delete_access_level', 'permissions' + ] + read_only_fields = ['url', 'uuid', 'file', 'permissions'] + extra_kwargs = augment_extra_kwargs() + + +class ArbitraryFileCreateSerializer(ArbitraryFileSerializer): + class Meta(ArbitraryFileSerializer.Meta): + read_only_fields = ['url', 'uuid', 'permissions'] diff --git a/backend_django/galv/storages.py b/backend_django/galv/storages.py new file mode 100644 index 0000000..19c9e0e --- /dev/null +++ b/backend_django/galv/storages.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: BSD-2-Clause +# Copyright (c) 2020-2023, The Chancellor, Masters and Scholars of the University +# of Oxford, and the 'Galv' Developers. All rights reserved. + +# adapted from https://backendengineer.io/store-django-static-and-media-files-in-aws-s3/ +from django.conf import settings +from storages.backends.s3boto3 import S3Boto3Storage +from storages.utils import clean_name + +class StaticStorage(S3Boto3Storage): + location = settings.STATICFILES_LOCATION + default_acl = "public-read" + querystring_auth = False + +class MediaStorage(S3Boto3Storage): + location = settings.MEDIAFILES_LOCATION + file_overwrite = False + custom_domain = False + default_acl = "private" + querystring_auth = True + + # adapted from https://medium.com/@hiteshgarg14/how-to-dynamically-select-storage-in-django-filefield-bc2e8f5883fd + def update_acl(self, name, acl=None, set_default=True, set_querystring_auth=True): + name = self._normalize_name(clean_name(name)) + self.bucket.Object(name).Acl().put(ACL=acl or self.default_acl) + if acl is not None: + if set_default: + self.default_acl = acl + if set_querystring_auth: + self.querystring_auth = acl != "public-read" diff --git a/backend_django/galv/tests/factories.py b/backend_django/galv/tests/factories.py index b103a6d..b45d97a 100644 --- a/backend_django/galv/tests/factories.py +++ b/backend_django/galv/tests/factories.py @@ -21,7 +21,8 @@ Equipment, ScheduleFamily, Schedule, CyclerTest, \ ScheduleIdentifiers, CellFormFactors, CellChemistries, CellManufacturers, \ CellModels, EquipmentManufacturers, EquipmentModels, EquipmentTypes, Experiment, \ - ValidationSchema, GroupProxy, UserProxy, Lab, Team, AutoCompleteEntry, UserLevel + ValidationSchema, GroupProxy, UserProxy, Lab, Team, AutoCompleteEntry +from ..models.choices import UserLevel fake = faker.Faker(django.conf.global_settings.LANGUAGE_CODE) diff --git a/backend_django/galv/views.py b/backend_django/galv/views.py index f713e0d..dd1c8e5 100644 --- a/backend_django/galv/views.py +++ b/backend_django/galv/views.py @@ -12,8 +12,10 @@ from drf_spectacular.types import OpenApiTypes from dry_rest_permissions.generics import DRYPermissions from rest_framework.mixins import ListModelMixin +from rest_framework.parsers import MultiPartParser, FormParser from rest_framework.renderers import JSONRenderer from rest_framework.reverse import reverse +from rest_framework.status import HTTP_201_CREATED, HTTP_400_BAD_REQUEST from .serializers import HarvesterSerializer, \ HarvesterCreateSerializer, \ @@ -30,7 +32,8 @@ KnoxTokenSerializer, \ KnoxTokenFullSerializer, CellFamilySerializer, EquipmentFamilySerializer, \ ScheduleSerializer, CyclerTestSerializer, ScheduleFamilySerializer, DataColumnTypeSerializer, DataColumnSerializer, \ - ExperimentSerializer, LabSerializer, TeamSerializer, ValidationSchemaSerializer, SchemaValidationSerializer + ExperimentSerializer, LabSerializer, TeamSerializer, ValidationSchemaSerializer, SchemaValidationSerializer, \ + ArbitraryFileSerializer, ArbitraryFileCreateSerializer from .models import Harvester, \ HarvestError, \ MonitoredPath, \ @@ -51,7 +54,7 @@ CellChemistries, CellFormFactors, ScheduleIdentifiers, EquipmentFamily, Schedule, CyclerTest, ScheduleFamily, \ ValidationSchema, Experiment, Lab, Team, UserProxy, GroupProxy, ValidatableBySchemaMixin, SchemaValidation, \ UserActivation, ALLOWED_USER_LEVELS_READ, ALLOWED_USER_LEVELS_EDIT, ALLOWED_USER_LEVELS_DELETE, \ - ALLOWED_USER_LEVELS_EDIT_PATH + ALLOWED_USER_LEVELS_EDIT_PATH, ArbitraryFile from .permissions import HarvesterFilterBackend, TeamFilterBackend, LabFilterBackend, GroupFilterBackend, \ ResourceFilterBackend, ObservedFileFilterBackend, UserFilterBackend, SchemaValidationFilterBackend from .serializers.utils import get_GetOrCreateTextStringSerializer @@ -1792,4 +1795,63 @@ def list(self, request, *args, **kwargs): for sv in self.queryset: sv.validate() sv.save() - return super().list(request, *args, **kwargs) \ No newline at end of file + return super().list(request, *args, **kwargs) + +@extend_schema_view( + create=extend_schema( + summary="Upload a file", + description=""" +Upload a file along with its metadata. The file will be stored in an AWS S3 bucket. + +Files with `is_public` set to `True` will be available to the public. +Files with `is_public` set to `False` will only be displayed as links which will provide access for 1 hour, +after which the link must be retrieved again. + """, + request={ + 'multipart/form-data': ArbitraryFileCreateSerializer, + 'application/x-www-form-urlencoded': ArbitraryFileCreateSerializer + }, + responses={ + 201: ArbitraryFileSerializer + } + ), + partial_update=extend_schema( + summary="Update a file's metadata", + description=""" +You can change the visibility of a file and its metadata. Files themselves cannot be updated, only deleted. + """, + request=ArbitraryFileSerializer, + responses={ + 200: ArbitraryFileSerializer + } + ), + destroy=extend_schema( + summary="Delete a file", + description="""Delete a file from the database and from the S3 bucket.""" + ) +) +class ArbitraryFileViewSet(viewsets.ModelViewSet): + """ + ArbitraryFiles are files that are not observed by the harvester, and are not + associated with any specific experiment or dataset. They are used to store + arbitrary files that are not part of the main data collection process. + + These files might include datasheets, images, or other documentation. + + Files are stored in an AWS S3 bucket. + """ + permission_classes = [DRYPermissions] + filter_backends = [ResourceFilterBackend] + queryset = ArbitraryFile.objects.all().order_by('-uuid') + search_fields = ['@name', '@description'] + http_method_names = ['get', 'post', 'patch', 'delete', 'options'] + + def get_parsers(self): + if self.request is not None and self.action_map[self.request.method.lower()] == 'create': + return [MultiPartParser(), FormParser()] + return super().get_parsers() + + def get_serializer_class(self): + if self.action is not None and self.action == 'create': + return ArbitraryFileCreateSerializer + return ArbitraryFileSerializer diff --git a/docker-compose.yaml b/docker-compose.yaml index 3b9d794..3e9437a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -25,8 +25,12 @@ services: FRONTEND_VIRTUAL_HOST: "http://${VIRTUAL_HOST_ROOT},https://${VIRTUAL_HOST_ROOT}" DJANGO_SETTINGS: "dev" DJANGO_SUPERUSER_PASSWORD: "admin" + AWS_ACCESS_KEY_ID: "AKIAZQ3DU5GQ7Y5V7TFJ" + DJANGO_AWS_STORAGE_BUCKET_NAME: "galv" + DJANGO_AWS_S3_REGION_NAME: "eu-west-2" env_file: - ./.env + - ./.env.secret restart: unless-stopped ports: - "8001:8000" diff --git a/docs/source/conf.py b/docs/source/conf.py index 53fef82..ce525fa 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,7 +11,7 @@ project = 'Galv' copyright = '2023, Oxford RSE' author = 'Oxford RSE' -release = '2.1.18' +release = '2.1.19' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/tags.json b/docs/tags.json index b009be8..49cbdb6 100644 --- a/docs/tags.json +++ b/docs/tags.json @@ -1 +1 @@ -["v2.1.18","main"] \ No newline at end of file +["v2.1.19","main"] \ No newline at end of file diff --git a/fly.toml b/fly.toml index fe9d4b2..aaa2908 100644 --- a/fly.toml +++ b/fly.toml @@ -24,6 +24,9 @@ console_command = "/code/backend_django/manage.py shell" DJANGO_EMAIL_USE_SSL = "True" DJANGO_EMAIL_HOST_USER = 'oxfordbatterymodelling@gmail.com' DJANGO_DEFAULT_FROM_EMAIL = 'oxfordbatterymodelling@gmail.com' + AWS_ACCESS_KEY_ID = "AKIAZQ3DU5GQ7Y5V7TFJ" + DJANGO_AWS_STORAGE_BUCKET_NAME = "galv" + DJANGO_AWS_S3_REGION_NAME = "eu-west-2" [http_service] internal_port = 8000 diff --git a/requirements.txt b/requirements.txt index eb72979..50f4301 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,11 @@ -Django==4.1.4 +Django==5.0.2 django-cors-headers==3.13.0 django-filter==22.1 django-dry-rest-permissions==1.2.0 django-json-field-schema-validator==0.0.2 djangorestframework==3.14.0 django-rest-knox==4.2.0 +django-storages[s3]==1.14.2 dj-database-url==2.1.0 psycopg2-binary==2.9.5 redis==4.4.0