Merge pull request #115 from openimis/release/v1.5.0

Release/v1.5.0

.gitignore

@@ -13,3 +13,4 @@ modules-tests.sh
 **/*.mo
 openIMIS/locale/en/LC_MESSAGES/django.mo
 **/staticfiles
+extracted_translations_fe

Dockerfile

@@ -1,19 +1,25 @@
 FROM python:3.8-buster
 ENV PYTHONUNBUFFERED 1
-RUN apt-get update && apt-get install -y apt-transport-https ca-certificates gettext
-RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
-RUN curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
-RUN apt-get update
-RUN ACCEPT_EULA=Y apt-get install -y msodbcsql17 mssql-tools 
-RUN apt-get install -y -f python3-dev unixodbc-dev
+ARG DB_ENGINE
+ENV DB_ENGINE=${DB_ENGINE:-mssql}
+RUN apt-get update && apt-get install -y apt-transport-https ca-certificates gettext unixodbc-dev && apt-get upgrade -y
+RUN apt-get install -y -f python3-dev
+
+RUN test "$DB_ENGINE" != "django.db.backends.postgresql" && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - || :
+RUN test "$DB_ENGINE" != "django.db.backends.postgresql" && curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list || :
+RUN test "$DB_ENGINE" != "django.db.backends.postgresql" && apt-get update || :
+RUN test "$DB_ENGINE" != "django.db.backends.postgresql" && ACCEPT_EULA=Y apt-get install -y msodbcsql17 mssql-tools || :
+
 RUN pip install --upgrade pip
+RUN test "$DB_ENGINE" != "django.db.backends.postgresql" && pip install mssql-cli || :
+
+
 RUN mkdir /openimis-be
 COPY . /openimis-be
 
 WORKDIR /openimis-be
 ARG OPENIMIS_CONF_JSON
 ENV OPENIMIS_CONF_JSON=${OPENIMIS_CONF_JSON}
-RUN pip install mssql-cli
 RUN pip install -r requirements.txt
 RUN python modules-requirements.py openimis.json > modules-requirements.txt
 RUN pip install -r modules-requirements.txt
@@ -22,6 +28,7 @@ ARG SENTRY_DSN
 RUN test -z "$SENTRY_DSN" || pip install -r sentry-requirements.txt && :
 
 WORKDIR /openimis-be/openIMIS
-RUN NO_DATABASE=True python manage.py compilemessages
+# For some reason, the zh_Hans (Simplified Chinese) of django-graphql-jwt fails to compile, excluding it
+RUN NO_DATABASE=True python manage.py compilemessages -x zh_Hans
 RUN NO_DATABASE=True python manage.py collectstatic --clear --noinput
 ENTRYPOINT ["/openimis-be/script/entrypoint.sh"]

README.md

@@ -34,6 +34,7 @@ In case of troubles, please consult/contact our service desk via our [ticketing
     requirements.txt`. For development workstations, one can use `pip
     install -r dev-requirements.txt` instead for more modules.
   * generate the openIMIS modules dependencies file (from openimis.json config): `python modules-requirements.py openimis.json > modules-requirements.txt`
+  * if you previously installed openIMIS on another version, it seems safe to uninstall all previous modules-requirement to be sure it match current version `pip uninstall -r modules-requirements.txt`
   * install openIMIS current modules: `pip install -r modules-requirements.txt`
   * configure the database connection (see section here below)
 * start openIMIS from within `openimis-be_py/openIMIS`: `python manage.py runserver`
@@ -127,9 +128,9 @@ When release candidate is accepted:
 	* from tarball: `https://github.com/openimis/openimis-be_py/archive/v1.1.0.tar.gz`
 * (required only once)`python -m venv ./venv`: create the python venv
 * `./venv/Script/activate[.sh/.ps1]`: Activate the venv
-* `pyhon modules-list.py openimis.json > module-list.txt`: list the module to install
+* `python modules-list.py openimis.json > module-list.txt`: list the module to install
 * `python -m pip uninstall -r module-list.txt`: uninstall the previously installed module
-* `pyhon modules-requirements.py openimis.json > modules-requirements.txt`: list the source of the module to install
+* `python modules-requirements.py openimis.json > modules-requirements.txt`: list the source of the module to install
 * `python -m pip install -r modules-requirements.txt`: Install the modules
 * Set the different required environement variables
 	* see database configuration
@@ -259,3 +260,66 @@ Notes:
   * as the option could be added `--github`. This allows to add gitignore file and workflows files so as to execute CI on every pull request (this option will execute this command `python manage.py add_github_files_to_module <module_name>`) 
   * example with using `--github` option: `python manage.py create_calcrule_module <module_name> <author> <author_email> --github`
 * from here on, your local openIMIS has a new module called `openimis-be-calcrule-<module_name>_py`, directly loaded from your directory by using single command.
+
+
+### To create release branches for all backend/frontend modules presented in openimis.json
+* from `/openimis-be_py/openIMIS`:
+  * run this command: `python manage.py create_release_branch <version> <from_branch: by default 'develop'>`. This command will execute all steps required 
+  to create release branches of all modules present in `openimis.json` (frontend json and backend json).
+
+
+### To extract all translations from frontend modules
+* from `/openimis-be_py/openIMIS`:
+  * run this command: `python manage.py extract_translations`. This command will execute all steps required 
+  to extract frontend translations of all modules present in `openimis.json`. 
+  * those translations will be copied into 'extracted_translations_fe' folder in assembly backend module
+
+
+
+## Handling errors while running openIMIS app - the most common ones
+
+### Handling error with `wheel` package
+If there are some problems with 'wheel package' after executing `pip install -r requirements.txt` (for example `error: invalid command 'bdist_wheel'`) you may need to execute two commands:
+* `pip install wheel`
+* `python setup.py bdist_wheel`
+* optionally `pip uninstall -r requirements.txt` to clean requirements and reinstall them again
+
+If those commands doesn't help you need to try with this sort of commands: 
+* `apt-get update`
+* `ACCEPT_EULA=Y apt-get install -y msodbcsql17 mssql-tools` 
+* `apt-get install -y -f python3-dev unixodbc-dev`
+* `pip install --upgrade pip`
+* `pip install mssql-cli`
+
+After executing those commands you can run `pip install -r requirements.txt` again and there shouldn't be any issues with `wheel` package.
+
+
+### Handling error with `connection_is_mariadb` after executing `python manage.py runserver`
+Another error that relates to this issue with `wheel` is such one:
+* `ImportError: cannot import name 'connection_is_mariadb' from 'django_mysql.utils'`
+This error indicates that the db client is not set up properly. But it realized that it is related to the fact that the wheel package is not working (see `### Handling error with wheel package` section).
+Therefore you need to follows steps described in this above section. 
+
+
+### Using wrong build for database docker
+Using wrong version of db docker could cause several issues both on backend and frontend for example:
+* problems with creating database schema (backend)
+* problems with filling demo dataset into database while running demo database script (backend)
+* error while running frontend (web console `Uncaught TypeError: Cannot read properties of null (reading 'health_facility_id')`) (frontend)
+
+So as to avoid those issues it is recommended to use such command to run db docker (NOTE: DO NOT USE for a production environment!):
+```
+docker build \
+  --build-arg ACCEPT_EULA=Y \
+  --build-arg SA_PASSWORD=<your secret password> \
+  . \
+  -t openimis-db
+```
+This commands will build with the latest version of database. You can specify particular version of database by adding optional parameter:
+* `SQL_SCRIPT_URL=<url to the sql script to create the database>`
+
+You can find more informations about seeting up db docker [here](https://github.com/openimis/openimis-db_dkr/tree/develop).
+
+### How to report another issues? 
+If you face another issues not described in that section you could use our [ticketing site](https://openimis.atlassian.net/servicedesk/customer/portal/1). 
+Here you can report any bugs/problems you faced during setting up openIMIS app. 

dev_launch.ps1

@@ -1,8 +1,22 @@
 # assuming venv and openimis-be_py folders are on the same level
 ..\venv\Scripts\Activate.ps1
 cd openIMIS
-[Environment]::SetEnvironmentVariable("SITE_ROOT", "api")
-[Environment]::SetEnvironmentVariable("REMOTE_USER_AUTHENTICATION", "True")
+$SITE_ROOT='iapi'
+$DB_NAME='IMISfs'
+$DB_USER='IMISuser'
+$DB_PASSWORD='IMISuser@1234'
+$DB_HOST='127.0.0.1'
+$DB_PORT='1433'
+$DJANGO_PORT='8000'
+[Environment]::SetEnvironmentVariable("SITE_ROOT", $SITE_ROOT)
+[Environment]::SetEnvironmentVariable("REMOTE_USER_AUTHENTICATION", "False")
 [Environment]::SetEnvironmentVariable("ROW_SECURITY", "False")
 [Environment]::SetEnvironmentVariable("DEBUG", "True")
+[Environment]::SetEnvironmentVariable("DB_NAME", $DB_NAME)
+[Environment]::SetEnvironmentVariable("DB_USER", $DB_USER)
+[Environment]::SetEnvironmentVariable("DB_PASSWORD", $DB_PASSWORD)
+[Environment]::SetEnvironmentVariable("DB_HOST", $DB_HOST)
+[Environment]::SetEnvironmentVariable("DB_PORT", $DB_PORT)
+[Environment]::SetEnvironmentVariable("DJANGO_PORT", $DJANGO_PORT)
+[Environment]::SetEnvironmentVariable("OPENIMIS_CONF", "../"$OPENIMIS_CONF)
 python manage.py runserver

openIMIS/apscheduler_runner/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'apscheduler_runner.apps.ApschedulerRunnerConfig'

openIMIS/apscheduler_runner/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

openIMIS/apscheduler_runner/apps.py

@@ -0,0 +1,40 @@
+import logging
+
+from apscheduler.schedulers.background import BackgroundScheduler
+# from apscheduler.executors.pool import ProcessPoolExecutor, ThreadPoolExecutor
+from django.conf import settings
+from django.apps import AppConfig
+from copy import deepcopy
+
+logger = logging.getLogger(__name__)
+
+
+class ApschedulerRunnerConfig(AppConfig):
+    name = 'apscheduler_runner'
+    scheduler = None
+
+    def ready(self):
+        self.setup_module_scheduled_tasks()
+
+    def setup_module_scheduled_tasks(self):
+        if settings.SCHEDULER_AUTOSTART:
+            self._setup_scheduler_background_task()
+
+    def _setup_scheduler_background_task(self):
+        self.scheduler = BackgroundScheduler(deepcopy(settings.SCHEDULER_CONFIG))
+        for app in settings.OPENIMIS_APPS:
+            self.__add_module_tasks_to_scheduler(app)
+        self.scheduler.start()
+
+    def __add_module_tasks_to_scheduler(self, app_):
+        try:
+            module = __import__(f"{app_}.scheduled_tasks")
+            if hasattr(module.scheduled_tasks, "schedule_tasks"):
+                module.scheduled_tasks.schedule_tasks(self.scheduler)
+                logger.debug(f"{app_} tasks scheduled")
+            else:
+                logger.debug(f"{app_} has a scheduled_tasks package but no schedule_tasks callable")
+        except ModuleNotFoundError as exc:
+            logger.debug(f"{app_} has no scheduled_tasks module, skipping")
+        except Exception as exc:
+            logger.debug(f"{app_}: unknown exception occurred during registering scheduled tasks: {exc}")

openIMIS/developer_tools/management/commands/create_release_branch.py

@@ -0,0 +1,131 @@
+import git
+import json
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from pathlib import Path
+
+from openIMIS.openimisapps import openimis_apps
+
+
+class Command(BaseCommand):
+    help = "This command will generate release branches in one command"
+
+    def add_arguments(self, parser):
+        parser.add_argument('release_tag', type=str)
+        parser.add_argument('from_brach', type=str, default='develop')
+
+    def handle(self, *args, **options):
+        release_tag = options['release_tag']
+        from_brach = options['from_brach']
+        self.__print_info(f'starting creating release branches for backend')
+        output = self.create_release_branches_backend(release_tag, from_brach)
+        self.__print_success('Creating release branches finished for backend modules')
+        self.__print_info(f'{output}')
+        self.__print_info(f'starting creating release branches for frontend')
+        output = self.create_release_branches_frontend(release_tag, from_brach)
+        self.__print_success('Creating release branches finished for frontend modules')
+        self.__print_info(f'{output}')
+
+    def create_release_branches_backend(self, version, from_branch):
+        """
+            function to create release branches 'release/<version>'
+            for all backend modules presented in openimis.json
+        """
+        output_messages = []
+        modules = openimis_apps()
+        release_branch = f'release/{version}'
+        base_path = Path(settings.BASE_DIR)
+        modules_directory = base_path.parent.parent
+        for module in modules:
+            try:
+                repo_name = f"openimis-be-{module}_py"
+                module_directory = Path(modules_directory).joinpath(repo_name)
+                self.__check_module_exist_locally(module_directory.joinpath(module), module, 'be')
+                self.__do_git_operations(module_directory, from_branch, release_branch)
+                output_messages.append({
+                    'module': module,
+                    'message': f'Operation succeded'
+                })
+            except Exception as exc:
+                output_messages.append({
+                    'module': module,
+                    'message': f'Operation failed: {exc}'
+                })
+        return output_messages
+
+    def create_release_branches_frontend(self, version, from_branch):
+        """
+           function to create release branches 'release/<version>'
+           for all frontend modules presented in openimis.json
+        """
+        output_messages = []
+        # check if main frontend module is presented locally.
+        base_path = Path(settings.BASE_DIR)
+        modules_directory = base_path.parent.parent
+        if not Path(modules_directory).joinpath('openimis-fe_js').exists():
+            raise Exception("Main assembly frontend module not presented locally")
+
+        # take the module names from fe openimis.json
+        modules = self.__read_frontend_openimis_file(modules_directory)
+        release_branch = f'release/{version}'
+        for module in modules:
+            try:
+                repo_name = f"openimis-{module}_js"
+                module_directory = Path(modules_directory).joinpath(repo_name)
+                self.__check_module_exist_locally(module_directory, module.split('-')[1], 'fe')
+                self.__do_git_operations(module_directory, from_branch, release_branch)
+                output_messages.append({
+                    'module': module,
+                    'message': f'Operation succeded'
+                })
+            except Exception as exc:
+                output_messages.append({
+                    'module': module,
+                    'message': f'Operation failed: {exc}'
+                })
+        return output_messages
+
+    def __do_git_operations(self, module_directory, from_branch, release_branch):
+        local_repo = git.Repo(module_directory)
+        current_branch = local_repo.active_branch
+        if current_branch != from_branch:
+            local_repo.git.checkout(from_branch)
+        # create release branch if not exist in local repo
+        if release_branch not in local_repo.branches:
+            local_repo.git.branch(release_branch)
+        local_repo.git.checkout(release_branch)
+        # pull changes from develop/particular branch
+        local_repo.git.pull("origin", from_branch)
+        # push branch to remote branch
+        local_repo.git.push("origin", release_branch)
+        # back to branch previously assigned
+        local_repo.git.checkout(current_branch)
+
+    def __read_frontend_openimis_file(self, modules_directory):
+        # take the module names from fe openimis.json
+        modules = []
+        openimis_json = Path(modules_directory).joinpath('openimis-fe_js').joinpath('openimis.json')
+        with open(openimis_json) as json_file:
+            json_data = json.load(json_file)
+            for module in json_data['modules']:
+                module_name = module['npm'].split('/')[1].split('@')[0]
+                modules.append(module_name)
+        return modules
+
+    def __check_module_exist_locally(self, module_directory, module, module_type):
+        # check if frontend module exists locally
+        extension = 'py' if module_type == 'be' else 'js'
+        if not Path(module_directory).exists():
+            git.Repo.clone_from(
+                f'https://github.com/openimis/openimis-{module_type}-{module}_{extension}.git',
+                module_directory
+            )
+
+    def __print_success(self, msg: str):
+        """ Print message to inform about the command progress """
+        self.stdout.write(self.style.SUCCESS(msg))
+
+    def __print_info(self, msg: str):
+        """ Print message to inform about the command progress = info """
+        self.stdout.write(self.style.WARNING(msg))