Merge pull request #368 from maykinmedia/feature/setup-configuration

add `setup_configuration` command

Dockerfile

@@ -51,10 +51,12 @@ COPY --from=backend-build /usr/local/bin/celery /usr/local/bin/celery
 
 # Stage 3.2 - Copy source code
 WORKDIR /app
+COPY ./bin/wait_for_db.sh /wait_for_db.sh
 COPY ./bin/docker_start.sh /start.sh
 COPY ./bin/celery_worker.sh /celery_worker.sh
 COPY ./bin/celery_flower.sh /celery_flower.sh
 COPY ./bin/check_celery_worker_liveness.py ./bin/
+COPY ./bin/setup_configuration.sh /setup_configuration.sh
 RUN mkdir /app/log /app/config
 
 # copy frontend build statics

bin/docker_start.sh

@@ -15,10 +15,8 @@ uwsgi_threads=${UWSGI_THREADS:-2}
 
 mountpoint=${SUBPATH:-/}
 
-until pg_isready; do
-  >&2 echo "Waiting for database connection..."
-  sleep 1
-done
+# wait for required services
+${SCRIPTPATH}/wait_for_db.sh
 
 >&2 echo "Database is up."
 

bin/setup_configuration.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# setup initial configuration using environment variables
+# Run this script from the root of the repository
+
+#set -e
+${SCRIPTPATH}/wait_for_db.sh
+
+src/manage.py migrate
+src/manage.py setup_configuration --no-selftest

bin/wait_for_db.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+
+# Wait for the database container
+# See: https://docs.docker.com/compose/startup-order/
+export PGHOST=${DB_HOST:-db}
+export PGPORT=${DB_PORT:-5432}
+
+until pg_isready; do
+  >&2 echo "Waiting for database connection..."
+  sleep 1
+done
+
+>&2 echo "Database is up."

docker-compose.yml

@@ -30,14 +30,32 @@ services:
       - CELERY_BROKER_URL=redis://redis:6379/1
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - CELERY_LOGLEVEL=DEBUG
+      # setup_configuration env vars
+      - OBJECTS_DOMAIN=web:8000
+      - OBJECTS_ORGANIZATION=Objects
+      - OBJECTTYPES_API_ROOT=https://objecttypes.example.com/api/v2/
+      - OBJECTS_OBJECTTYPES_TOKEN=some-random-string
+      - DEMO_CONFIG_ENABLE=yes
+      - DEMO_TOKEN=demo-random-string
+      - DEMO_PERSON=Demo
+      - [email protected]
     ports:
       - 8000:8000
     depends_on:
-      - db
-      - redis
+        web-init:
+          condition: service_completed_successfully
     volumes: &web_volumes
       - media:/app/media  # Shared media volume to get access to saved OAS files
 
+  web-init:
+    build: *web_build
+    environment: *web_env
+    command: /setup_configuration.sh
+    depends_on:
+      - db
+      - redis
+    volumes: *web_volumes
+
   celery:
     build: *web_build
     environment: *web_env

docs/installation/config.rst

@@ -119,6 +119,14 @@ variables. The user will only be created if it doesn't exist yet.
   which means the superuser will be created _without_ password. Only has an effect
   if ``OBJECTS_SUPERUSER_USERNAME`` is set.
 
+Initial configuration
+---------------------
+
+Both Objects API and Objecttypes API support `setup_configuration` management command,
+which allows configuration via environment variables.
+All these environment variables are described at :ref:`command line <installation_config_cli>`.
+
+
 Specifying the environment variables
 =====================================
 

docs/installation/config_cli.rst

@@ -0,0 +1,137 @@
+.. _installation_config_cli:
+
+
+===================
+Configuration (CLI)
+===================
+
+After deploying Objecttypes API and Objects API, they need to be configured to be fully functional. The
+command line tool `setup_configuration`_ assist with this configuration:
+
+* It uses environment variables for all configuration choices, therefore you can integrate this with your
+  infrastructure tooling such as init containers and/or Kubernetes Jobs.
+* The command can self-test the configuration to detect problems early on
+
+You can get the full command documentation with:
+
+.. code-block:: bash
+
+    src/manage.py setup_configuration --help
+
+.. warning:: This command is declarative - if configuration is manually changed after
+   running the command and you then run the exact same command again, the manual
+   changes will be reverted.
+
+.. _`setup_configuration`: https://github.com/maykinmedia/django-setup-configuration/
+
+Preparation
+===========
+
+The command executes the list of pluggable configuration steps, and each step
+required specific environment variables, that should be prepared.
+Here is the description of all available configuration steps and the environment variables,
+use by each step for both APIs.
+
+
+Objects API
+===========
+
+Sites configuration
+-------------------
+
+Configure the domain where Objects API is hosted
+
+* ``SITES_CONFIG_ENABLE``: enable Site configuration. Defaults to ``True``.
+* ``OBJECTS_DOMAIN``:  a ``[host]:[port]`` or ``[host]`` value. Required.
+* ``OBJECTS_ORGANIZATION``: name of Objects API organization. Required.
+
+Objecttypes configuration
+-------------------------
+
+Objects API uses Objecttypes API to validate data against JSON schemas, therefore
+it should be able to request Objecttypes API.
+
+* ``OBJECTS_OBJECTTYPES_CONFIG_ENABLE``: enable Objecttypes configuration. Defaults
+  to ``True``.
+* ``OBJECTTYPES_API_ROOT``: full URL to the Objecttypes API root, for example
+  ``https://objecttypes.gemeente.local/api/v1/``. Required.
+* ``OBJECTTYPES_API_OAS``: full URL to the Objecttypes OpenAPI specification.
+* ``OBJECTS_OBJECTTYPES_TOKEN``: authorization token. Required.
+* ``OBJECTS_OBJECTTYPES_PERSON``: Objects API contact person. Required.
+* ``OBJECTS_OBJECTTYPES_EMAIL``: Objects API contact email. Required.
+
+Demo user configuration
+-----------------------
+
+Demo user can be created to check if Objects API work. It has superuser permissions,
+so its creation is not recommended on production environment.
+
+* ``DEMO_CONFIG_ENABLE``: enable demo user configuration. Defaults to the value of the ``DEBUG`` setting.
+* ``DEMO_PERSON``: demo user contact person. Required.
+* ``DEMO_EMAIL``: demo user email. Required.
+* ``DEMO_TOKEN``: demo token. Required.
+
+
+Objecttypes API
+===============
+
+ObjectTypes API has similar configuration steps as the Objects API.
+
+Sites configuration
+-------------------
+
+Configure the domain where Objects API is hosted
+
+* ``SITES_CONFIG_ENABLE``: enable Site configuration. Defaults to ``True``.
+* ``OBJECTTYPES_DOMAIN``:  a ``[host]:[port]`` or ``[host]`` value. Required.
+* ``OBJECTTYPES_ORGANIZATION``: name of Objecttypes API organization. Required.
+
+Objects configuration
+---------------------
+
+Objects API uses Objecttypes API to validate data against JSON schemas, therefore
+it should be able to request Objecttypes API.
+
+* ``OBJECTS_OBJECTTYPES_CONFIG_ENABLE``: enable Objecttypes configuration. Defaults
+  to ``True``.
+* ``OBJECTTYPES_API_ROOT``: full URL to the Objecttypes API root, for example
+  ``https://objecttypes.gemeente.local/api/v1/``. Required.
+* ``OBJECTTYPES_API_OAS``: full URL to the Objecttypes OpenAPI specification.
+* ``OBJECTS_OBJECTTYPES_TOKEN``: authorization token. Required.
+
+Demo user configuration
+-----------------------
+
+The similar configuration as in Objects API.
+
+* ``DEMO_CONFIG_ENABLE``: enable demo user configuration. Defaults to the value of the ``DEBUG`` setting.
+* ``DEMO_PERSON``: demo user contact person. Required.
+* ``DEMO_EMAIL``: demo user email. Required.
+* ``DEMO_TOKEN``: demo token. Required.
+
+
+Execution
+=========
+
+
+With the full command invocation, everything is configured at once and immediately
+tested.
+
+.. code-block:: bash
+
+    src/manage.py setup_configuration
+
+
+You can skip the self-tests by using the ``--no-selftest`` flag.
+
+.. code-block:: bash
+
+    src/manage.py setup_configuration --no-self-test
+
+
+``setup_configuration`` command checks if the configuration already exists before changing it.
+If you want to change some of the values of the existing configuration you can use ``--overwrite`` flag.
+
+.. code-block:: bash
+
+    src/manage.py setup_configuration --overwrite

docs/installation/index.rst

@@ -16,5 +16,6 @@ this.
 
    quickstart
    config
+   config_cli
    deployment/index
    oidc

requirements/base.in

@@ -3,7 +3,11 @@ open-api-framework
 # Core python libraries
 glom # data represenation based on spec
 jsonschema
+furl
+
+# Django libraries
 django-log-outgoing-requests
+django-setup-configuration
 
 # Common ground libraries
 notifications-api-common

requirements/base.txt

@@ -85,6 +85,7 @@ django==4.2.11
     #   django-relativedelta
     #   django-rest-framework-condition
     #   django-sendfile2
+    #   django-setup-configuration
     #   django-simple-certmanager
     #   django-solo
     #   django-two-factor-auth
@@ -137,6 +138,8 @@ django-rest-framework-condition==0.1.1
     # via commonground-api-common
 django-sendfile2==0.7.0
     # via django-privates
+django-setup-configuration==0.1.0
+    # via -r requirements/base.in
 django-simple-certmanager==1.4.1
     # via zgw-consumers
 django-solo==2.2.0
@@ -182,6 +185,8 @@ faker==8.1.0
     # via zgw-consumers
 flower==2.0.1
     # via open-api-framework
+furl==2.1.3
+    # via -r requirements/base.in
 gemma-zds-client==1.0.1
     # via
     #   commonground-api-common
@@ -229,6 +234,8 @@ notifications-api-common==0.2.2
     #   commonground-api-common
 open-api-framework==0.2.0
     # via -r requirements/base.in
+orderedmultidict==1.0.1
+    # via furl
 oyaml==1.0
     # via commonground-api-common
 packaging==23.2
@@ -296,7 +303,9 @@ sentry-sdk==1.39.2
 six==1.16.0
     # via
     #   bleach
+    #   furl
     #   isodate
+    #   orderedmultidict
     #   python-dateutil
     #   qrcode
     #   requests-mock

requirements/ci.txt

@@ -125,6 +125,7 @@ django==4.2.11
     #   django-relativedelta
     #   django-rest-framework-condition
     #   django-sendfile2
+    #   django-setup-configuration
     #   django-simple-certmanager
     #   django-solo
     #   django-two-factor-auth
@@ -207,6 +208,8 @@ django-sendfile2==0.7.0
     # via
     #   -r requirements/base.txt
     #   django-privates
+django-setup-configuration==0.1.0
+    # via -r requirements/base.txt
 django-simple-certmanager==1.4.1
     # via
     #   -r requirements/base.txt
@@ -287,6 +290,8 @@ flower==2.0.1
     #   open-api-framework
 freezegun==1.1.0
     # via -r requirements/test-tools.in
+furl==2.1.3
+    # via -r requirements/base.txt
 gemma-zds-client==1.0.1
     # via
     #   -r requirements/base.txt
@@ -362,6 +367,10 @@ notifications-api-common==0.2.2
     #   commonground-api-common
 open-api-framework==0.2.0
     # via -r requirements/base.txt
+orderedmultidict==1.0.1
+    # via
+    #   -r requirements/base.txt
+    #   furl
 oyaml==1.0
     # via
     #   -r requirements/base.txt
@@ -468,7 +477,9 @@ six==1.16.0
     # via
     #   -r requirements/base.txt
     #   bleach
+    #   furl
     #   isodate
+    #   orderedmultidict
     #   python-dateutil
     #   qrcode
     #   requests-mock

requirements/dev.txt

@@ -147,6 +147,7 @@ django==4.2.11
     #   django-relativedelta
     #   django-rest-framework-condition
     #   django-sendfile2
+    #   django-setup-configuration
     #   django-simple-certmanager
     #   django-solo
     #   django-two-factor-auth
@@ -233,6 +234,8 @@ django-sendfile2==0.7.0
     # via
     #   -r requirements/ci.txt
     #   django-privates
+django-setup-configuration==0.1.0
+    # via -r requirements/ci.txt
 django-simple-certmanager==1.4.1
     # via
     #   -r requirements/ci.txt
@@ -321,6 +324,8 @@ flower==2.0.1
     #   open-api-framework
 freezegun==1.1.0
     # via -r requirements/ci.txt
+furl==2.1.3
+    # via -r requirements/ci.txt
 gemma-zds-client==1.0.1
     # via
     #   -r requirements/ci.txt
@@ -407,6 +412,10 @@ notifications-api-common==0.2.2
     #   commonground-api-common
 open-api-framework==0.2.0
     # via -r requirements/ci.txt
+orderedmultidict==1.0.1
+    # via
+    #   -r requirements/ci.txt
+    #   furl
 oyaml==1.0
     # via
     #   -r requirements/ci.txt
@@ -535,7 +544,9 @@ six==1.16.0
     # via
     #   -r requirements/ci.txt
     #   bleach
+    #   furl
     #   isodate
+    #   orderedmultidict
     #   python-dateutil
     #   qrcode
     #   requests-mock