Merge pull request #696 from sgfost/build/config-cleanup

build refactor: replace config.ini with combination of environment variables + docker secrets

- use environment variables for basic configuration and docker secrets for anything that should not be exposed
- adding new configuration variables or secrets should now be independent of each other
- closes comses/planning#145

Makefile

@@ -1,22 +1,21 @@
+# set DEPLOY_ENVIRONMENT in config.mk
 DEPLOY_ENVIRONMENT := dev
-DB_USER=comsesnet
+
 DOCKER_SHARED_DIR=docker/shared
 DOCKER_DB_DATA_DIR=docker/pgdata
+
 BUILD_DIR=build
 SECRETS_DIR=${BUILD_DIR}/secrets
 DB_PASSWORD_PATH=${SECRETS_DIR}/db_password
 PGPASS_PATH=${SECRETS_DIR}/.pgpass
-SECRET_KEY_PATH=${SECRETS_DIR}/secret_key
-SENTRY_DSN_PATH=${SECRETS_DIR}/sentry_dsn
+SECRET_KEY_PATH=${SECRETS_DIR}/django_secret_key
+EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key
+GENERATED_SECRETS=$(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SECRET_KEY_PATH)
+
+ENVREPLACE := deploy/scripts/envreplace
 DEPLOY_CONF_DIR=deploy/conf
-PGPASS_TEMPLATE=${DEPLOY_CONF_DIR}/pgpass.template
-CONFIG_INI_TEMPLATE=${DEPLOY_CONF_DIR}/config.ini.template
-DOCKER_ENV_PATH=${DEPLOY_CONF_DIR}/docker.env
-CONFIG_INI_PATH=${SECRETS_DIR}/config.ini
-MAIL_API_KEY_PATH=${SECRETS_DIR}/mail_api_key
-SECRETS=$(MAIL_API_KEY_PATH) $(DB_PASSWORD_PATH) $(CONFIG_INI_PATH) $(PGPASS_PATH) $(SENTRY_DSN_PATH) $(SECRET_KEY_PATH) .env
-BUILD_ID=$(shell git describe --tags --abbrev=1)
-BUILD_ID_PATH=${SECRETS_DIR}/.build-id.txt
+ENV_TEMPLATE=${DEPLOY_CONF_DIR}/.env.template
+
 # assumes a .tar.xz file
 BORG_REPO_URL := https://example.com/repo.tar.xz
 BORG_REPO_PATH=${BUILD_DIR}/sparse-repo.tar.xz
@@ -31,16 +30,12 @@ include .env
 .EXPORT_ALL_VARIABLES:
 
 .PHONY: build
-build: docker-compose.yml secrets $(DOCKER_SHARED_DIR) $(BUILD_ID_PATH)
+build: docker-compose.yml secrets $(DOCKER_SHARED_DIR)
 	docker compose build --pull
 
 $(BORG_REPO_PATH):
 	wget ${BORG_REPO_URL} -P ${BUILD_DIR}
 
-$(BUILD_ID_PATH):
-	BUILD_ID=${BUILD_ID} \
-	echo "$${BUILD_ID}" > ${BUILD_ID_PATH}
-
 config.mk:
 	DEPLOY_ENVIRONMENT=${DEPLOY_ENVIRONMENT} envsubst < ${DEPLOY_CONF_DIR}/config.mk.template > config.mk
 
@@ -54,6 +49,10 @@ $(DOCKER_SHARED_DIR):
 ${SECRETS_DIR}:
 	mkdir -p ${SECRETS_DIR}
 
+$(SECRET_KEY_PATH): | ${SECRETS_DIR}
+	SECRET_KEY=$$(openssl rand -base64 48); \
+	echo "$${SECRET_KEY}" > $(SECRET_KEY_PATH)
+
 $(DB_PASSWORD_PATH): | ${SECRETS_DIR}
 	DB_PASSWORD=$$(openssl rand -base64 48); \
 	TODAY=$$(date +%Y-%m-%d-%H:%M:%S); \
@@ -65,49 +64,41 @@ $(DB_PASSWORD_PATH): | ${SECRETS_DIR}
 	@echo "db password at $(DB_PASSWORD_PATH) was reset, may need to manually update existing db password"
 
 $(PGPASS_PATH): $(DB_PASSWORD_PATH) $(PGPASS_TEMPLATE) | ${SECRETS_DIR}
-	DB_PASSWORD=$$(cat $(DB_PASSWORD_PATH)); \
-	sed -e "s|DB_PASSWORD|$$DB_PASSWORD|" -e "s|DB_HOST|${DB_HOST}|" \
-			-e "s|DB_USER|${DB_USER}|" $(PGPASS_TEMPLATE) > $(PGPASS_PATH)
+	echo "${DB_HOST}:5432:*:${DB_USER}:$$(cat $(DB_PASSWORD_PATH))" > $(PGPASS_PATH)
 	chmod 0600 $(PGPASS_PATH)
 
-$(MAIL_API_KEY_PATH): | ${SECRETS_DIR}
-	touch "$(MAIL_API_KEY_PATH)"
+.PHONY: release-version
+release-version: .env
+	$(ENVREPLACE) RELEASE_VERSION $$(git describe --tags --abbrev=1) .env
 
-$(SENTRY_DSN_PATH): | ${SECRETS_DIR}
-	touch "$(SENTRY_DSN_PATH)"
-
-.env: $(DOCKER_ENV_PATH)
-	cp ${DOCKER_ENV_PATH} .env
-
-$(CONFIG_INI_PATH): .env $(DB_PASSWORD_PATH) $(CONFIG_INI_TEMPLATE) $(SECRET_KEY_PATH) $(BUILD_ID_PATH)
-	DB_HOST=${DB_HOST} DB_NAME=${DB_NAME} DB_PASSWORD=$$(cat ${DB_PASSWORD_PATH}) \
-	DB_USER=${DB_USER} DB_PORT=${DB_PORT} DJANGO_SECRET_KEY=$$(cat ${SECRET_KEY_PATH}) \
-	TEST_USERNAME=___test_user___ TEST_BASIC_AUTH_PASSWORD=$$(openssl rand -base64 42) \
-	SENTRY_DSN=$(shell cat $(SENTRY_DSN_PATH)) \
-	TEST_USER_ID=1111111 BUILD_ID=${BUILD_ID} \
-	envsubst < ${CONFIG_INI_TEMPLATE} > ${CONFIG_INI_PATH}
-
-$(SECRET_KEY_PATH): | ${SECRETS_DIR}
-	SECRET_KEY=$$(openssl rand -base64 48); \
-	echo "$${SECRET_KEY}" > $(SECRET_KEY_PATH)
+.env: $(DB_PASSWORD_PATH) $(SECRET_KEY_PATH)
+	if [ ! -f .env ]; then \
+		cp $(ENV_TEMPLATE) .env; \
+	fi; \
+	# $(ENVREPLACE) DB_PASSWORD $$(cat $(DB_PASSWORD_PATH)) .env; \
+	# $(ENVREPLACE) SECRET_KEY $$(cat $(SECRET_KEY_PATH)) .env; \
+	$(ENVREPLACE) TEST_BASIC_AUTH_PASSWORD $$(openssl rand -base64 42) .env
 
 .PHONY: docker-compose.yml
-docker-compose.yml: base.yml dev.yml staging.yml prod.yml config.mk $(PGPASS_PATH) .env
+docker-compose.yml: base.yml dev.yml staging.yml prod.yml config.mk $(PGPASS_PATH) release-version
 	case "$(DEPLOY_ENVIRONMENT)" in \
 	  dev|staging|e2e) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \
 	  prod) docker compose -f base.yml -f staging.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \
 	  *) echo "invalid environment. must be either dev, staging or prod" 1>&2; exit 1;; \
 	esac
 
 .PHONY: set-db-password
-set-db-password: $(DB_PASSWORD_PATH) $(CONFIG_INI_PATH)
+set-db-password: $(DB_PASSWORD_PATH) .env
 	docker compose run --rm server psql -h db -U comsesnet comsesnet -c "ALTER USER ${DB_USER} with password '$(shell cat ${DB_PASSWORD_PATH})';"
 
 .PHONY: secrets
-secrets: $(SECRETS_DIR) $(SECRETS)
+secrets: $(SECRETS_DIR) $(GENERATED_SECRETS)
+	for secret_path in $(EXT_SECRETS); do \
+		touch ${SECRETS_DIR}/$$secret_path; \
+	done
 
 .PHONY: deploy
-deploy: build
+deploy: build .env
 	docker compose pull db redis elasticsearch
 ifneq ($(DEPLOY_ENVIRONMENT),dev)
 	docker compose pull nginx
@@ -137,7 +128,11 @@ restore: build $(BORG_REPO_PATH) | $(REPO_BACKUPS_PATH)
 .PHONY: clean
 clean:
 	@echo "Backing up generated files to /tmp directory"
-	mv .env config.mk docker-compose.yml $(CONFIG_INI_PATH) $(shell mktemp -d)
+	mv .env config.mk docker-compose.yml $(shell mktemp -d)
+
+.PHONY: clean_deploy
+clean_deploy: clean
+	+@$(MAKE) deploy
 
 .PHONY: test
 test: build

base.yml

@@ -33,7 +33,9 @@ services:
       - ./deploy/elasticsearch.conf.d/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties
       - esdata:/usr/share/elasticsearch/data
   db:
-    image: postgis/postgis:15-3.3
+    image: postgis/postgis:15-3.4
+    secrets:
+      - db_password
     volumes:
       - ./docker/pgdata:/var/lib/postgresql/data
       - ./build/secrets/db_password:/run/secrets/db_password
@@ -56,8 +58,16 @@ services:
   server:
     build: django
     image: comses/server
+    secrets:
+      - db_password
+      - discourse_api_key
+      - discourse_sso_secret
+      - django_secret_key
+      - github_client_secret
+      - orcid_client_secret
+      - hcaptcha_secret
+      - mail_api_key
     volumes:
-      - ./build/secrets:/run/secrets
       - ./deploy/elasticsearch.conf.d:/etc/elasticsearch
       - ./docker/shared:/shared
     depends_on:
@@ -69,8 +79,27 @@ services:
         condition: service_started
       vite:
         condition: service_started
-    environment:
-      CLEAN_DATABASE: "false"
+    env_file:
+      - .env
+
+secrets:
+  db_password:
+    file: ./build/secrets/db_password
+  discourse_api_key:
+    file: ./build/secrets/discourse_api_key
+  discourse_sso_secret:
+    file: ./build/secrets/discourse_sso_secret
+  django_secret_key:
+    file: ./build/secrets/django_secret_key
+  github_client_secret:
+    file: ./build/secrets/github_client_secret
+  hcaptcha_secret:
+    file: ./build/secrets/hcaptcha_secret
+  mail_api_key:
+    file: ./build/secrets/mail_api_key
+  orcid_client_secret:
+    file: ./build/secrets/orcid_client_secret
+
 volumes:
   esdata:
     driver: local

deploy/conf/.env.template

@@ -0,0 +1,53 @@
+# app
+RELEASE_VERSION=
+EXPIRED_JOB_DAYS_THRESHOLD=180
+EXPIRED_EVENT_DAYS_THRESHOLD=2
+
+# database
+DB_NAME=comsesnet
+DB_USER=comsesnet
+DB_HOST=db
+DB_PORT=5432
+DB_PASSWORD=
+CLEAN_DATABASE="false"  # allowed values: "true" or "false"
+
+# elastic search
+ES_VERSION=7.17.18
+
+# captcha
+HCAPTCHA_SITEKEY=
+# HCAPTCHA_SECRET=
+
+# discourse
+DISCOURSE_BASE_URL=
+DISCOURSE_API_USERNAME=
+
+# email
+EMAIL_SUBJECT_PREFIX=[CoMSES Net]
+# MAILGUN_API_KEY=
+MAILGUN_SENDER_DOMAIN=
+
+# logging
+LOG_DIRECTORY=/shared/logs
+SENTRY_DSN=
+
+# secrets
+# DISCOURSE_SSO_SECRET=
+# DISCOURSE_API_KEY=
+# django secret key
+# SECRET_KEY=
+
+GITHUB_CLIENT_ID=
+# GITHUB_CLIENT_SECRET=
+ORCID_CLIENT_ID=
+# ORCID_CLIENT_SECRET=
+
+# storage
+DATA_ROOT=/shared
+LIBRARY_ROOT=/shared/library
+REPOSITORY_ROOT=/shared/repository
+
+# test
+TEST_USER_ID=10000000
+TEST_USERNAME=__test_user__
+TEST_BASIC_AUTH_PASSWORD=

deploy/scripts/envreplace

@@ -0,0 +1,26 @@
+#!/usr/bin/bash
+
+# envreplace: replace environment variables in a file
+# usage: envreplace <VAR_NAME> <new_value> [file (default: .env)]
+
+VAR_NAME=$1
+NEW_VALUE=$2
+ENV_FILE=${3:-".env"}
+
+if [ -z "$VAR_NAME" ] || [ -z "$NEW_VALUE" ]; then
+  echo "usage: envreplace <VAR_NAME> <new_value> [file (default: .env)]"
+  exit 1
+fi
+
+if sed --version 2>&1 | grep -q "GNU"; then
+    SED_CMD="sed -i" # GNU sed
+else
+    SED_CMD="sed -i ''" # BSD sed
+fi
+
+if grep -q "^$VAR_NAME=" "$ENV_FILE"; then
+    $SED_CMD "s|^$VAR_NAME=.*|$VAR_NAME=$NEW_VALUE|" "$ENV_FILE"
+else
+    echo "$VAR_NAME not found in $ENV_FILE"
+    # echo "$VAR_NAME=$NEW_VALUE" >> $ENV_FILE
+fi