feat: implement monitoring and observability (#70)

.env.example

@@ -0,0 +1,15 @@
+# API
+FRONTEND_URL=https://domain.com
+POSTGRES_URL=postgresql://postgres:5432/db
+POSTGRES_USER=
+POSTGRES_PASSWORD=
+KEYCLOAK_ISSUER_URL=https://keycloak.domain.com/realms/realm_name
+
+# Keycloak
+KEYCLOAK_URL=https://keycloak.domain.com
+KEYCLOAK_ADMIN=
+KEYCLOAK_ADMIN_PASSWORD=
+
+# Grafana
+GRAFANA_USER=
+GRAFANA_PASSWORD=

.github/workflows/cd.yml

@@ -18,6 +18,10 @@ jobs:
                     - infra/postgres
                     - infra/keycloak
                     - infra/nginx
+                    - infra/promtail
+                    - infra/loki
+                    - infra/prometheus
+                    - infra/grafana
                     - apps/api
                     - apps/ui
         steps:
@@ -53,5 +57,4 @@ jobs:
             cd $HOME/mally && 
             docker compose -f docker-compose.prod.yml down && 
             docker compose -f docker-compose.prod.yml pull && 
-            source .env && 
             docker compose -f docker-compose.prod.yml up -d

.gitignore

@@ -7,4 +7,6 @@
 .angular
 dist/
 node_modules/
-screenshots/
+screenshots/
+logs/
+.env

apps/api/pom.xml

@@ -117,6 +117,19 @@
 			<groupId>org.springframework</groupId>
 			<artifactId>spring-webflux</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>com.github.loki4j</groupId>
+			<artifactId>loki-logback-appender</artifactId>
+			<version>1.5.2</version>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-actuator</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>io.micrometer</groupId>
+			<artifactId>micrometer-registry-prometheus</artifactId>
+		</dependency>
 	</dependencies>
 	<dependencyManagement>
 		<dependencies>

apps/api/src/main/java/com/mally/api/configuration/WebSecurityConfig.java

@@ -40,6 +40,7 @@ public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Excepti
                         .requestMatchers("/pastebin/paste/**").permitAll()
                         .requestMatchers("/health/**").permitAll()
                         .requestMatchers("/auth/**").permitAll()
+                        .requestMatchers("/actuator/**").permitAll()
                         .anyRequest().authenticated()
                 )
                 .oauth2ResourceServer(

apps/api/src/main/resources/application.prod.yml

@@ -20,6 +20,15 @@ spring:
                 jwt:
                     issuer-uri: ${KEYCLOAK_ISSUER_URL}
 
+management:
+  endpoints:
+    web:
+      exposure:
+        include: "metrics,prometheus"
+  metrics:
+    tags:
+      application: 'Mally'
+
 bucket4j:
     enabled: true
     filter-config-caching-enabled: true

apps/api/src/main/resources/logback-spring.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration debug="false">
+    <include resource="org/springframework/boot/logging/logback/defaults.xml" />
+    <include resource="org/springframework/boot/logging/logback/console-appender.xml" />
+    <property name="LOG_PATH" value="logs/api" />
+    <property name="CONSOLE_LOG_PATTERN" value="%highlight([%level]) %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %X{id} %c{1} - %msg%n" />
+    <property name="FILE_LOG_PATTERN" value="[%level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %X{id} %c{1} - %msg%n" />
+
+    <appender name="CONSOLE_APPENDER" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>${CONSOLE_LOG_PATTERN}</pattern>
+            <charset>utf8</charset>
+        </encoder>
+    </appender>
+
+    <appender name="ROLLING_TEXT_FILE_APPENDER" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${LOG_PATH}/api.log</file>
+        <encoder>
+            <Pattern>${FILE_LOG_PATTERN}</Pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <!-- rollover daily -->
+            <fileNamePattern>${LOG_PATH}/spring-with-grafana-loki-text.%d{yyyy-MM-dd}.%i.gz</fileNamePattern>
+            <maxFileSize>5GB</maxFileSize>
+            <!-- keep 30 days worth of history -->
+            <maxHistory>30</maxHistory>
+            <totalSizeCap>20GB</totalSizeCap>
+        </rollingPolicy>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="CONSOLE_APPENDER" />
+        <appender-ref ref="ROLLING_TEXT_FILE_APPENDER" />
+    </root>
+</configuration>

docker-compose.dev.yml

@@ -0,0 +1,174 @@
+services:
+    postgres:
+        container_name: mally-postgres
+        build:
+            context: .
+            dockerfile: ./infra/postgres/Dockerfile
+        restart: unless-stopped
+        healthcheck:
+            test: [ "CMD", "pg_isready", "-q", "-d", "keycloak", "-U", "postgres" ]
+            timeout: 45s
+            interval: 10s
+            retries: 5
+        environment:
+            POSTGRES_DBS: 'mally,keycloak'
+            POSTGRES_USER: ${POSTGRES_USER}
+            POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+        env_file:
+            - .env
+        networks:
+            - mally-network
+        volumes:
+            - postgres:/var/lib/postgresql/data
+
+    keycloak:
+        container_name: mally-keycloak
+        build:
+            context: .
+            dockerfile: ./infra/keycloak/Dockerfile
+        restart: unless-stopped
+        healthcheck:
+            test: ["CMD", "curl", "-f", "http://0.0.0.0:9000/health/ready"]
+            timeout: 45s
+            interval: 10s
+            retries: 15
+        environment:
+            JAVA_OPTS_APPEND: -Dkeycloak.profile.feature.upload_scripts=enabled
+            KC_DB: postgres
+            KC_DB_URL: jdbc:postgresql://postgres/keycloak
+            KC_DB_USERNAME: ${POSTGRES_USER}
+            KC_DB_PASSWORD: ${POSTGRES_PASSWORD}
+            KC_HEALTH_ENABLED: 'true'
+            KC_HTTP_ENABLED: 'true'
+            KC_METRICS_ENABLED: 'true'
+            KC_HOSTNAME_STRICT_HTTPS: 'false'
+            KC_HOSTNAME_URL: ${KEYCLOAK_URL}
+            KC_PROXY: edge
+            KC_PROXY_HEADERS: xforwarded
+            KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN}
+            KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
+        env_file:
+            - .env
+        depends_on:
+            postgres:
+                condition: service_healthy
+        networks:
+            - mally-network
+        command: start --hostname ${KEYCLOAK_URL} --import-realm
+
+    api:
+        container_name: mally-api
+        build:
+            context: .
+            dockerfile: ./apps/api/Dockerfile
+        restart: unless-stopped
+        healthcheck:
+            test: [ "CMD", "curl", "-f", "http://0.0.0.0:8080/health/" ]
+            timeout: 45s
+            interval: 10s
+            retries: 15
+        environment:
+            DATABASE_URL: ${POSTGRES_URL}
+            DATABASE_USERNAME: ${POSTGRES_USER}
+            DATABASE_PASSWORD: ${POSTGRES_PASSWORD}
+            KEYCLOAK_ISSUER_URL: ${KEYCLOAK_ISSUER_URL}
+            FRONTEND_URL: ${FRONTEND_URL}
+        env_file:
+            -  .env
+        volumes:
+            - ./logs/api:/app/logs/api
+        networks:
+            - mally-network
+        depends_on:
+            postgres:
+                condition: service_healthy
+            keycloak:
+                condition: service_healthy
+
+    ui:
+        container_name: mally-ui
+        build:
+            context: .
+            dockerfile: ./apps/ui/Dockerfile
+        restart: unless-stopped
+        networks:
+            - mally-network
+        depends_on:
+            api:
+                condition: service_healthy
+            keycloak:
+                condition: service_healthy
+
+    nginx:
+        container_name: mally-nginx
+        build:
+            context: .
+            dockerfile: ./infra/nginx/Dockerfile
+        restart: unless-stopped
+        networks:
+            - mally-network
+        depends_on:
+            - api
+            - ui
+        ports:
+            - '80:80'
+            - '443:443'
+        volumes:
+            - ./certbot/www/:/var/www/certbot/:rw
+            - ./certbot/conf/:/etc/letsencrypt/:rw
+
+    loki:
+        container_name: mally-loki
+        build:
+            context: .
+            dockerfile: ./infra/loki/Dockerfile
+        restart: unless-stopped
+        command: -config.file=/etc/loki/loki.yml
+        networks:
+            - mally-network
+
+    promtail:
+        container_name: mally-promtail
+        build:
+            context: .
+            dockerfile: ./infra/promtail/Dockerfile
+        restart: unless-stopped
+        volumes:
+            - ./logs/api/:/var/log/
+        command: -config.file=/etc/promtail/promtail.yml
+        networks:
+            - mally-network
+
+    prometheus:
+        container_name: mally-prometheus
+        build:
+            context: .
+            dockerfile: ./infra/prometheus/Dockerfile
+        restart: unless-stopped
+        command: '--config.file=/etc/prometheus/config.yml'
+        networks:
+            - mally-network
+
+    grafana:
+        container_name: mally-grafana
+        build:
+            context: .
+            dockerfile: ./infra/grafana/Dockerfile
+        restart: unless-stopped
+        environment:
+            GF_SECURITY_ADMIN_USER: ${GRAFANA_USER}
+            GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD}
+        env_file:
+            - .env
+        volumes:
+            - grafana:/var/lib/grafana
+        networks:
+            - mally-network
+
+volumes:
+    postgres:
+    grafana:
+
+networks:
+    mally-network:
+        name: mally-network

docker-compose.prod.yml

@@ -12,6 +12,8 @@ services:
             POSTGRES_DBS: 'mally,keycloak'
             POSTGRES_USER: ${POSTGRES_USER}
             POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+        env_file:
+            - .env
         networks:
             - mally-network
         volumes:
@@ -41,6 +43,8 @@ services:
             KC_PROXY_HEADERS: xforwarded
             KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN}
             KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
+        env_file:
+            - .env
         depends_on:
             postgres:
                 condition: service_healthy
@@ -63,6 +67,10 @@ services:
             DATABASE_PASSWORD: ${POSTGRES_PASSWORD}
             KEYCLOAK_ISSUER_URL: ${KEYCLOAK_ISSUER_URL}
             FRONTEND_URL: ${FRONTEND_URL}
+        env_file:
+            - .env
+        volumes:
+            - ./logs/api:/app/logs/api
         networks:
             - mally-network
         depends_on:
@@ -99,8 +107,49 @@ services:
             - ./certbot/www/:/var/www/certbot/:rw
             - ./certbot/conf/:/etc/letsencrypt/:rw
 
+    loki:
+        container_name: mally-loki
+        image: ghcr.io/neumanf/mally-loki
+        restart: unless-stopped
+        command: -config.file=/etc/loki/loki.yml
+        networks:
+            - mally-network
+
+    promtail:
+        container_name: mally-promtail
+        image: ghcr.io/neumanf/mally-promtail
+        restart: unless-stopped
+        volumes:
+            - ./logs/api/:/var/log/
+        command: -config.file=/etc/promtail/promtail.yml
+        networks:
+            - mally-network
+
+    prometheus:
+        container_name: mally-prometheus
+        image: ghcr.io/neumanf/mally-prometheus
+        restart: unless-stopped
+        command: '--config.file=/etc/prometheus/config.yml'
+        networks:
+            - mally-network
+
+    grafana:
+        container_name: mally-grafana
+        image: ghcr.io/neumanf/mally-grafana
+        restart: unless-stopped
+        environment:
+            GF_SECURITY_ADMIN_USER: ${GRAFANA_USER}
+            GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD}
+        env_file:
+            - .env
+        volumes:
+            - grafana:/var/lib/grafana
+        networks:
+            - mally-network
+
 volumes:
     postgres:
+    grafana:
 
 networks:
     mally-network:

infra/grafana/Dockerfile

@@ -0,0 +1,3 @@
+FROM grafana/grafana:11.2.1
+
+COPY ./infra/grafana/conf /etc/grafana/provisioning/datasources