Merge pull request #30 from Battery-Intelligence-Lab/matlab

Matlab code support.
Also changes container names to remove redundant `galvanalyser_` prefix.

.env

@@ -1,27 +1,28 @@
 # change FLASK_ENV to "production" once galvanalyser is installed and running correctly
 FLASK_ENV=development
+FLASK_DEBUG=True
 
 # this password is used for both the galvanalyser and redash databases
 POSTGRES_PASSWORD=galvanalyser
 
 # this is the directory that the galvanalyser postgres database will be located
-GALVANALYSER_DATA_PATH=/home/mrobins/git/tmp/galvanalyser_data/galvanalyser
+GALVANALYSER_DATA_PATH=./.run/data/galvanalyser
 
 # this is the directory that will be scanned for battery tester output files when
 # running the harvester test suite
-GALVANALYSER_HARVESTER_TEST_PATH=/home/mrobins/git/tmp/galvanalyser_test_datafiles
+GALVANALYSER_HARVESTER_TEST_PATH=./.run/test_datafiles
 
 # this is the base directory for the harvesters run by the server (note harvesters can
 # also be setup independently from the server if required, see documentation for
 # details). New directories added for scanning will be relative to this base directory
-GALVANALYSER_HARVESTER_BASE_PATH=/home/mrobins/git/tmp/galvanalyser_datafiles
+GALVANALYSER_HARVESTER_BASE_PATH=./.run/datafiles
 
 # username and password for default harvester account
 HARVESTER_USERNAME=harv
 HARVESTER_PASSWORD=harv
 
 # secret used to encrypt JWT for galvanalyser web app and api
-GALVANALYSER_SECRET_KEY=galvanalyser_secret_key
+GALVANALYSER_SECRET_KEY=secret_key
 
 # a non empty value ensures that the python output is sent straight to terminal (e.g. your container log) without being first buffered and that you can see the output of your application (e.g. django logs) in real time
 PYTHONUNBUFFERED=1
@@ -30,16 +31,18 @@ PYTHONUNBUFFERED=1
 # RABBITMQ_DEFAULT_USER with a password of RABBITMQ_DEFAULT_PASS
 # Make sure that the default user and password match the 
 # RABBITMQ_URL paramter, which is in the format:
-#     pyamqp://RABBITMQ_DEFAULT_USER:RABBITMQ_DEFAULT_PASS@galvanalyser_rabbitmq:5672
+#     pyamqp://RABBITMQ_DEFAULT_USER:RABBITMQ_DEFAULT_PASS@rabbitmq:5672
 RABBITMQ_DEFAULT_USER=test-user
 RABBITMQ_DEFAULT_PASS=test-user
-RABBITMQ_URL=pyamqp://test-user:test-user@galvanalyser_rabbitmq:5672
+RABBITMQ_URL=pyamqp://test-user:test-user@rabbitmq:5672
 
 # where to put logs for the celery task manager
-CELERY_LOG_DIR=/home/mrobins/git/tmp/galvanalyser_data/celery
+CELERY_LOG_DIR=./.run/data/celery
 
 # location for the redis database files
-REDIS_DATA_PATH=/home/mrobins/git/tmp/galvanalyser_data/redis
+REDIS_DATA_PATH=./.run/data/redis
 REDIS_PASSWORD=redis
-REDIS_URL=redis://:redis@galvanalyser_redis:6379
+REDIS_URL=redis://:redis@redis:6379
 
+# Required to get react running:
+NODE_OPTIONS=--openssl-legacy-provider

.github/workflows/unit-test.yml

@@ -11,7 +11,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     env:
-      GALVANALYSER_HARVESTER_TEST_PATH: /home/mrobins/git/tmp/galvanalyser_test_datafiles
+      GALVANALYSER_HARVESTER_TEST_PATH: /home/mrobins/git/tmp/test_datafiles
     steps:
       - uses: actions/checkout@v2
 
@@ -34,4 +34,4 @@ jobs:
         run: docker-compose up --detach --build
 
       - name: Run tests
-        run: docker-compose run --rm galvanalyser_app python manage.py test
+        run: docker-compose run --rm app python manage.py test

.gitignore

@@ -11,3 +11,11 @@ webstack/.env
 .pnpm-store
 *.egg-info
 env
+
+.idea/
+
+.run/
+
+.test-data/
+
+frontend/src/demo_matlab_code.m

.worker.env

@@ -1,11 +1,11 @@
 # this is the base directory for the harvesters run by the server (note harvesters can
 # also be setup independently from the server if required, see documentation for
 # details). New directories added for scanning will be relative to this base directory
-GALVANALYSER_HARVESTER_BASE_PATH=/home/mrobins/git/tmp/galvanalyser_datafiles
+GALVANALYSER_HARVESTER_BASE_PATH=/home/mrobins/git/tmp/datafiles
 
-RABBITMQ_URL=pyamqp://test-user:test-user@galvanalyser_rabbitmq:5672
-REDIS_URL=redis://:redis@galvanalyser_redis:6379
-CELERY_WORKER_LOG_DIR=/home/mrobins/git/tmp/galvanalyser_data/celery
+RABBITMQ_URL=pyamqp://test-user:test-user@rabbitmq:5672
+REDIS_URL=redis://:redis@redis:6379
+CELERY_WORKER_LOG_DIR=/home/mrobins/git/tmp/data/celery
 
 HARVESTER_USERNAME=harv
 HARVESTER_PASSWORD=harv

README.md

@@ -64,7 +64,7 @@ distributed task queue.
 The frontend is written using Javascript, the [React](https://reactjs.org/) framework 
 and using [Material-UI](https://material-ui.com/) components.
 
-<img src="https://github.com/Battery-Intelligence-Lab/galvanalyser/raw/main/docs/galvanalyser_frontend.jpg" width="600" />
+<img src="https://github.com/Battery-Intelligence-Lab/galvanalyser/raw/main/docs/frontend.jpg" width="600" />
 
 
 ### Database

backend/app.py

@@ -1,4 +1,5 @@
 from galvanalyser import app, celery
 
 if __name__ == "__main__":
+    print("Launching server")
     app.run(host='0.0.0.0', debug=True)

backend/galvanalyser/__init__.py

@@ -42,7 +42,7 @@ def create_config():
         "GALVANISER_DATABASE": {
             "NAME": "galvanalyser",
             "PORT": 5433,
-            "HOST": "galvanalyser_postgres",
+            "HOST": "postgres",
             "USER": "postgres",
             "PASSWORD": os.getenv('POSTGRES_PASSWORD'),
         },

backend/galvanalyser/database/user_data/user_row.py

@@ -7,7 +7,7 @@
 
 
 def verify_email(email):
-    return validate_email(
+    return os.environ.get('FLASK_DEBUG', False) or validate_email(
         email_address=email,
         check_blacklist=True,
         check_format=True,

backend/galvanalyser/routes.py

@@ -165,6 +165,12 @@ def hello():
     return 'Hello'
 
 
+@app.route('/', methods=['GET'])
+@cross_origin()
+def root():
+    return 'Hello'
+
+
 @app.route('/api/user/current', methods=['GET'])
 @cross_origin()
 @jwt_required()
@@ -512,7 +518,7 @@ def run_harvester_celery(machine_id):
         os.getenv('HARVESTER_USERNAME'),
         os.getenv('HARVESTER_PASSWORD'),
         machine_id,
-        'galvanalyser_postgres', '5433',
+        'postgres', '5433',
         'galvanalyser', base_path='/usr/data'
     )
 

backend/index.html

@@ -0,0 +1 @@
+Hello

backend/manage.py

@@ -21,14 +21,14 @@
 @click.option('--path', default='/usr/app/test')
 @click.option('--test', default='')
 def test(path, test):
-    if not os.path.exists(os.path.join(path, 'galvanalyser_test_case.py')):
+    if not os.path.exists(os.path.join(path, 'test_case.py')):
         raise RuntimeError((
-            'galvanalyser_test_case.py does not exist in {}. '
+            'test_case.py does not exist in {}. '
             'Is the path correct?').format(path)
         )
 
     sys.path.append(path)
-    from galvanalyser_test_case import GalvanalyserTestCase
+    from test_case import GalvanalyserTestCase
 
     test_config = copy.copy(app.config)
     test_config["GALVANISER_DATABASE"]["NAME"] = GalvanalyserTestCase.DATABASE
@@ -79,11 +79,11 @@ def test(path, test):
     runner.run(test_suite)
 
 
-@cli.command("create_galvanalyser_db")
+@cli.command("create_db")
 @click.confirmation_option(
     prompt='This will delete the current database, are you sure?'
 )
-def create_galvanalyser_db():
+def create_db():
     database.create_database(app.config)
     database.create_harvester_user(
         app.config,
@@ -207,7 +207,7 @@ def edit_machine_path(machine_id):
 def run_harvester(harvester, password, machine_id):
     harvester_main(
         harvester, password, machine_id,
-        'galvanalyser_postgres', '5433',
+        'postgres', '5433',
         'galvanalyser', base_path='/usr/data'
     )
 

backend/test/test_biologic.py

@@ -1,4 +1,4 @@
-from galvanalyser_test_case import GalvanalyserTestCase
+from test_case import GalvanalyserTestCase
 import glob
 from galvanalyser.harvester.biologic_input_file import BiologicMprInputFile
 

backend/test/galvanalyser_test_case.py → backend/test/test_case.py

@@ -29,7 +29,7 @@ def setUpClass(self):
                     'postgresql://{}:{}@{}/{}'.format(
                         self.POSTGRES_USER,
                         self.POSTGRES_USER_PWD,
-                        'galvanalyser_postgres:5433',
+                        'postgres:5433',
                         self.DATABASE,
                     ),
                 'SQLALCHEMY_ECHO': True,
@@ -38,7 +38,7 @@ def setUpClass(self):
                     'harvester': 'postgresql://{}:{}@{}/{}'.format(
                         self.HARVESTER,
                         self.HARVESTER_PWD,
-                        'galvanalyser_postgres:5433',
+                        'postgres:5433',
                         self.DATABASE,
                     ),
                     },
@@ -48,15 +48,15 @@ def setUpClass(self):
         self.Session, self.HarvesterSession = init_database(app.config)
 
         self.harvester_conn = psycopg2.connect(
-            host="galvanalyser_postgres",
+            host="postgres",
             port=5433,
             database=self.DATABASE,
             user=self.HARVESTER,
             password=self.HARVESTER_PWD,
         )
 
         self.postgres_conn = psycopg2.connect(
-            host="galvanalyser_postgres",
+            host="postgres",
             port=5433,
             database=self.DATABASE,
             user=self.POSTGRES_USER,

backend/test/test_cell_row.py

@@ -1,4 +1,4 @@
-from galvanalyser_test_case import GalvanalyserTestCase
+from test_case import GalvanalyserTestCase
 from galvanalyser.database.cell_data import (
     Cell,
 )

backend/test/test_dataset_row.py

@@ -1,4 +1,4 @@
-from galvanalyser_test_case import GalvanalyserTestCase
+from test_case import GalvanalyserTestCase
 from galvanalyser.database.experiment import (
     MetadataRow,
     AccessRow,

backend/test/test_harvester.py

@@ -1,4 +1,4 @@
-from galvanalyser_test_case import GalvanalyserTestCase
+from test_case import GalvanalyserTestCase
 from timeit import default_timer as timer
 import galvanalyser.harvester as harvester
 from galvanalyser.database.harvester import (
@@ -29,7 +29,7 @@ def test_main(self):
             database_user=self.HARVESTER,
             database_password=self.HARVESTER_PWD,
             machine_id=self.MACHINE_ID,
-            database_host="galvanalyser_postgres",
+            database_host="postgres",
             database_port=5433,
             database_name=self.DATABASE,
             base_path=None
@@ -77,7 +77,7 @@ def test_main(self):
             database_user=self.HARVESTER,
             database_password=self.HARVESTER_PWD,
             machine_id=self.MACHINE_ID,
-            database_host="galvanalyser_postgres",
+            database_host="postgres",
             database_port=5433,
             database_name=self.DATABASE,
             base_path=None

backend/test/test_input_file.py

@@ -1,4 +1,4 @@
-from galvanalyser_test_case import GalvanalyserTestCase
+from test_case import GalvanalyserTestCase
 from galvanalyser.database.experiment import (
     InputFile,
     TimeseriesDataRow,

backend/test/test_ivium.py

@@ -1,4 +1,4 @@
-from galvanalyser_test_case import GalvanalyserTestCase
+from test_case import GalvanalyserTestCase
 import glob
 from galvanalyser.harvester.ivium_input_file import IviumInputFile
 

backend/test/test_maccor.py

@@ -1,4 +1,4 @@
-from galvanalyser_test_case import GalvanalyserTestCase
+from test_case import GalvanalyserTestCase
 import os
 from galvanalyser.database.util.battery_exceptions import UnsupportedFileTypeError
 from galvanalyser.harvester.maccor_input_file import MaccorInputFile

backend/test/test_user_group.py

@@ -1,4 +1,4 @@
-from galvanalyser_test_case import GalvanalyserTestCase
+from test_case import GalvanalyserTestCase
 from galvanalyser.database.user_data import (
     Group, User
 )

docker-compose.dev.yml

@@ -1,14 +1,14 @@
 version: "2"
 services:
-  galvanalyser_app:
-    image: galvanalyser_app
+  app:
+    image: app
     build:
       dockerfile: Dockerfile
       context: ./backend
     depends_on:
-      - galvanalyser_redis
-      - galvanalyser_rabbitmq
-      - galvanalyser_postgres
+      - redis
+      - rabbitmq
+      - postgres
     ports:
       - "5001:5000"
     volumes:
@@ -22,11 +22,11 @@ services:
     restart: unless-stopped
     env_file:
      - ./.env
-  galvanalyser_worker:
-    image: galvanalyser_app
+  worker:
+    image: app
     command: ./worker.sh 
     depends_on:
-      - galvanalyser_app
+      - app
     volumes:
      - ./backend:/usr/app
      - "${GALVANALYSER_HARVESTER_BASE_PATH}:/usr/data"
@@ -36,39 +36,39 @@ services:
     restart: unless-stopped
     env_file:
      - ./.env
-  galvanalyser_frontend:
-    image: galvanalyser_frontend_dev
+  frontend:
+    image: frontend_dev
     build:
       dockerfile: Dockerfile_dev
       context: ./frontend
     depends_on:
-      - galvanalyser_app
+      - app
     ports:
       - "3000:3000"
     volumes:
      - ./frontend:/app
     restart: unless-stopped
     working_dir: /app
-    command: yarn start 
+    command: yarn start
     restart: unless-stopped
     env_file:
      - ./.env
-  galvanalyser_rabbitmq:
+  rabbitmq:
     image: "rabbitmq"
     ports:
       - "5672:5672"
     env_file:
       - ./.env
     restart: unless-stopped
-  galvanalyser_redis:
+  redis:
     image: redis:5.0-alpine
     restart: always
     command: redis-server --requirepass ${REDIS_PASSWORD}
     ports:
      - "6379:6379"
     volumes:
      - "${REDIS_DATA_PATH}:/var/lib/redis"
-  galvanalyser_postgres:
+  postgres:
     image: "postgres"
     stop_signal: SIGINT                 # Fast Shutdown mode
     ports: