From 4150d67cd7c0de3ee200cfb1a884a3e6f9020e5b Mon Sep 17 00:00:00 2001 From: "Neuman F." <61904986+neumanf@users.noreply.github.com> Date: Sun, 29 Sep 2024 12:20:01 -0300 Subject: [PATCH] feat: implement monitoring and observability (#70) --- .env.example | 15 ++ .github/workflows/cd.yml | 5 +- .gitignore | 4 +- apps/api/pom.xml | 13 ++ .../api/configuration/WebSecurityConfig.java | 1 + .../src/main/resources/application.prod.yml | 9 + .../api/src/main/resources/logback-spring.xml | 35 ++++ docker-compose.dev.yml | 174 ++++++++++++++++++ docker-compose.prod.yml | 49 +++++ infra/grafana/Dockerfile | 3 + infra/grafana/conf/grafana.yml | 18 ++ infra/loki/Dockerfile | 3 + infra/loki/conf/loki.yml | 30 +++ infra/nginx/conf/certbot.conf | 13 ++ infra/nginx/conf/nginx.conf | 26 ++- infra/nginx/scripts/entrypoint.sh | 2 +- infra/prometheus/Dockerfile | 3 + infra/prometheus/conf/prometheus.yml | 7 + infra/promtail/Dockerfile | 3 + infra/promtail/conf/promtail.yml | 22 +++ 20 files changed, 430 insertions(+), 5 deletions(-) create mode 100644 .env.example create mode 100644 apps/api/src/main/resources/logback-spring.xml create mode 100644 docker-compose.dev.yml create mode 100644 infra/grafana/Dockerfile create mode 100644 infra/grafana/conf/grafana.yml create mode 100644 infra/loki/Dockerfile create mode 100644 infra/loki/conf/loki.yml create mode 100644 infra/prometheus/Dockerfile create mode 100644 infra/prometheus/conf/prometheus.yml create mode 100644 infra/promtail/Dockerfile create mode 100644 infra/promtail/conf/promtail.yml diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ffa7133 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# API +FRONTEND_URL=https://domain.com +POSTGRES_URL=postgresql://postgres:5432/db +POSTGRES_USER= +POSTGRES_PASSWORD= +KEYCLOAK_ISSUER_URL=https://keycloak.domain.com/realms/realm_name + +# Keycloak +KEYCLOAK_URL=https://keycloak.domain.com +KEYCLOAK_ADMIN= +KEYCLOAK_ADMIN_PASSWORD= + +# Grafana +GRAFANA_USER= +GRAFANA_PASSWORD= \ No newline at end of file diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index b5ceaed..23d6c64 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -18,6 +18,10 @@ jobs: - infra/postgres - infra/keycloak - infra/nginx + - infra/promtail + - infra/loki + - infra/prometheus + - infra/grafana - apps/api - apps/ui steps: @@ -53,5 +57,4 @@ jobs: cd $HOME/mally && docker compose -f docker-compose.prod.yml down && docker compose -f docker-compose.prod.yml pull && - source .env && docker compose -f docker-compose.prod.yml up -d diff --git a/.gitignore b/.gitignore index 063261d..e6898d3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ .angular dist/ node_modules/ -screenshots/ \ No newline at end of file +screenshots/ +logs/ +.env \ No newline at end of file diff --git a/apps/api/pom.xml b/apps/api/pom.xml index a8a9848..4b42855 100644 --- a/apps/api/pom.xml +++ b/apps/api/pom.xml @@ -117,6 +117,19 @@ org.springframework spring-webflux + + com.github.loki4j + loki-logback-appender + 1.5.2 + + + org.springframework.boot + spring-boot-starter-actuator + + + io.micrometer + micrometer-registry-prometheus + diff --git a/apps/api/src/main/java/com/mally/api/configuration/WebSecurityConfig.java b/apps/api/src/main/java/com/mally/api/configuration/WebSecurityConfig.java index 8ce50d8..e94902f 100644 --- a/apps/api/src/main/java/com/mally/api/configuration/WebSecurityConfig.java +++ b/apps/api/src/main/java/com/mally/api/configuration/WebSecurityConfig.java @@ -40,6 +40,7 @@ public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Excepti .requestMatchers("/pastebin/paste/**").permitAll() .requestMatchers("/health/**").permitAll() .requestMatchers("/auth/**").permitAll() + .requestMatchers("/actuator/**").permitAll() .anyRequest().authenticated() ) .oauth2ResourceServer( diff --git a/apps/api/src/main/resources/application.prod.yml b/apps/api/src/main/resources/application.prod.yml index fc2179d..103bf0e 100644 --- a/apps/api/src/main/resources/application.prod.yml +++ b/apps/api/src/main/resources/application.prod.yml @@ -20,6 +20,15 @@ spring: jwt: issuer-uri: ${KEYCLOAK_ISSUER_URL} +management: + endpoints: + web: + exposure: + include: "metrics,prometheus" + metrics: + tags: + application: 'Mally' + bucket4j: enabled: true filter-config-caching-enabled: true diff --git a/apps/api/src/main/resources/logback-spring.xml b/apps/api/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..a8a0860 --- /dev/null +++ b/apps/api/src/main/resources/logback-spring.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + ${CONSOLE_LOG_PATTERN} + utf8 + + + + + ${LOG_PATH}/api.log + + ${FILE_LOG_PATTERN} + + + + ${LOG_PATH}/spring-with-grafana-loki-text.%d{yyyy-MM-dd}.%i.gz + 5GB + + 30 + 20GB + + + + + + + + \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..8dfdfd8 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,174 @@ +services: + postgres: + container_name: mally-postgres + build: + context: . + dockerfile: ./infra/postgres/Dockerfile + restart: unless-stopped + healthcheck: + test: [ "CMD", "pg_isready", "-q", "-d", "keycloak", "-U", "postgres" ] + timeout: 45s + interval: 10s + retries: 5 + environment: + POSTGRES_DBS: 'mally,keycloak' + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + env_file: + - .env + networks: + - mally-network + volumes: + - postgres:/var/lib/postgresql/data + + keycloak: + container_name: mally-keycloak + build: + context: . + dockerfile: ./infra/keycloak/Dockerfile + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://0.0.0.0:9000/health/ready"] + timeout: 45s + interval: 10s + retries: 15 + environment: + JAVA_OPTS_APPEND: -Dkeycloak.profile.feature.upload_scripts=enabled + KC_DB: postgres + KC_DB_URL: jdbc:postgresql://postgres/keycloak + KC_DB_USERNAME: ${POSTGRES_USER} + KC_DB_PASSWORD: ${POSTGRES_PASSWORD} + KC_HEALTH_ENABLED: 'true' + KC_HTTP_ENABLED: 'true' + KC_METRICS_ENABLED: 'true' + KC_HOSTNAME_STRICT_HTTPS: 'false' + KC_HOSTNAME_URL: ${KEYCLOAK_URL} + KC_PROXY: edge + KC_PROXY_HEADERS: xforwarded + KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN} + KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} + env_file: + - .env + depends_on: + postgres: + condition: service_healthy + networks: + - mally-network + command: start --hostname ${KEYCLOAK_URL} --import-realm + + api: + container_name: mally-api + build: + context: . + dockerfile: ./apps/api/Dockerfile + restart: unless-stopped + healthcheck: + test: [ "CMD", "curl", "-f", "http://0.0.0.0:8080/health/" ] + timeout: 45s + interval: 10s + retries: 15 + environment: + DATABASE_URL: ${POSTGRES_URL} + DATABASE_USERNAME: ${POSTGRES_USER} + DATABASE_PASSWORD: ${POSTGRES_PASSWORD} + KEYCLOAK_ISSUER_URL: ${KEYCLOAK_ISSUER_URL} + FRONTEND_URL: ${FRONTEND_URL} + env_file: + - .env + volumes: + - ./logs/api:/app/logs/api + networks: + - mally-network + depends_on: + postgres: + condition: service_healthy + keycloak: + condition: service_healthy + + ui: + container_name: mally-ui + build: + context: . + dockerfile: ./apps/ui/Dockerfile + restart: unless-stopped + networks: + - mally-network + depends_on: + api: + condition: service_healthy + keycloak: + condition: service_healthy + + nginx: + container_name: mally-nginx + build: + context: . + dockerfile: ./infra/nginx/Dockerfile + restart: unless-stopped + networks: + - mally-network + depends_on: + - api + - ui + ports: + - '80:80' + - '443:443' + volumes: + - ./certbot/www/:/var/www/certbot/:rw + - ./certbot/conf/:/etc/letsencrypt/:rw + + loki: + container_name: mally-loki + build: + context: . + dockerfile: ./infra/loki/Dockerfile + restart: unless-stopped + command: -config.file=/etc/loki/loki.yml + networks: + - mally-network + + promtail: + container_name: mally-promtail + build: + context: . + dockerfile: ./infra/promtail/Dockerfile + restart: unless-stopped + volumes: + - ./logs/api/:/var/log/ + command: -config.file=/etc/promtail/promtail.yml + networks: + - mally-network + + prometheus: + container_name: mally-prometheus + build: + context: . + dockerfile: ./infra/prometheus/Dockerfile + restart: unless-stopped + command: '--config.file=/etc/prometheus/config.yml' + networks: + - mally-network + + grafana: + container_name: mally-grafana + build: + context: . + dockerfile: ./infra/grafana/Dockerfile + restart: unless-stopped + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_USER} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD} + env_file: + - .env + volumes: + - grafana:/var/lib/grafana + networks: + - mally-network + +volumes: + postgres: + grafana: + +networks: + mally-network: + name: mally-network diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 1082acf..1bfb8ac 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -12,6 +12,8 @@ services: POSTGRES_DBS: 'mally,keycloak' POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + env_file: + - .env networks: - mally-network volumes: @@ -41,6 +43,8 @@ services: KC_PROXY_HEADERS: xforwarded KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN} KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} + env_file: + - .env depends_on: postgres: condition: service_healthy @@ -63,6 +67,10 @@ services: DATABASE_PASSWORD: ${POSTGRES_PASSWORD} KEYCLOAK_ISSUER_URL: ${KEYCLOAK_ISSUER_URL} FRONTEND_URL: ${FRONTEND_URL} + env_file: + - .env + volumes: + - ./logs/api:/app/logs/api networks: - mally-network depends_on: @@ -99,8 +107,49 @@ services: - ./certbot/www/:/var/www/certbot/:rw - ./certbot/conf/:/etc/letsencrypt/:rw + loki: + container_name: mally-loki + image: ghcr.io/neumanf/mally-loki + restart: unless-stopped + command: -config.file=/etc/loki/loki.yml + networks: + - mally-network + + promtail: + container_name: mally-promtail + image: ghcr.io/neumanf/mally-promtail + restart: unless-stopped + volumes: + - ./logs/api/:/var/log/ + command: -config.file=/etc/promtail/promtail.yml + networks: + - mally-network + + prometheus: + container_name: mally-prometheus + image: ghcr.io/neumanf/mally-prometheus + restart: unless-stopped + command: '--config.file=/etc/prometheus/config.yml' + networks: + - mally-network + + grafana: + container_name: mally-grafana + image: ghcr.io/neumanf/mally-grafana + restart: unless-stopped + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_USER} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD} + env_file: + - .env + volumes: + - grafana:/var/lib/grafana + networks: + - mally-network + volumes: postgres: + grafana: networks: mally-network: diff --git a/infra/grafana/Dockerfile b/infra/grafana/Dockerfile new file mode 100644 index 0000000..1d7e838 --- /dev/null +++ b/infra/grafana/Dockerfile @@ -0,0 +1,3 @@ +FROM grafana/grafana:11.2.1 + +COPY ./infra/grafana/conf /etc/grafana/provisioning/datasources \ No newline at end of file diff --git a/infra/grafana/conf/grafana.yml b/infra/grafana/conf/grafana.yml new file mode 100644 index 0000000..b96e3b4 --- /dev/null +++ b/infra/grafana/conf/grafana.yml @@ -0,0 +1,18 @@ +apiVersion: 1 +datasources: + - name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + basicAuth: false + version: 1 + editable: true + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + version: 1 + editable: true diff --git a/infra/loki/Dockerfile b/infra/loki/Dockerfile new file mode 100644 index 0000000..012d5d7 --- /dev/null +++ b/infra/loki/Dockerfile @@ -0,0 +1,3 @@ +FROM grafana/loki:3.2.0 + +COPY ./infra/loki/conf /etc/loki \ No newline at end of file diff --git a/infra/loki/conf/loki.yml b/infra/loki/conf/loki.yml new file mode 100644 index 0000000..379e899 --- /dev/null +++ b/infra/loki/conf/loki.yml @@ -0,0 +1,30 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2023-12-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: false + diff --git a/infra/nginx/conf/certbot.conf b/infra/nginx/conf/certbot.conf index df7bade..b7e28ee 100644 --- a/infra/nginx/conf/certbot.conf +++ b/infra/nginx/conf/certbot.conf @@ -32,6 +32,19 @@ server { root /var/www/certbot; } + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 80; + server_name g.mally.neumanf.com; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + location / { return 301 https://$host$request_uri; } diff --git a/infra/nginx/conf/nginx.conf b/infra/nginx/conf/nginx.conf index cb0e9e4..9510d31 100644 --- a/infra/nginx/conf/nginx.conf +++ b/infra/nginx/conf/nginx.conf @@ -87,11 +87,33 @@ server { } location / { - proxy_pass http://keycloak:8080; - proxy_set_header X-Forwarded-For $proxy_protocol_addr; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Port 443; + + proxy_pass http://keycloak:8080; + } +} + +# Server block for g.mally.neumanf.com +server { + listen 443 ssl; + http2 on; + server_name g.mally.neumanf.com; + charset utf-8; + + ssl_certificate /etc/letsencrypt/live/mally.neumanf.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/mally.neumanf.com/privkey.pem; + + location /.well-known/acme-challenge/ { + allow all; + root /var/www/certbot; + } + + location / { + proxy_set_header Host $http_host; + + proxy_pass http://grafana:3000; } } diff --git a/infra/nginx/scripts/entrypoint.sh b/infra/nginx/scripts/entrypoint.sh index d0851d5..21c6f1d 100755 --- a/infra/nginx/scripts/entrypoint.sh +++ b/infra/nginx/scripts/entrypoint.sh @@ -17,7 +17,7 @@ if [ ! -f /etc/letsencrypt/live/mally.neumanf.com/fullchain.pem ]; then # Request a certificate for the domain certbot certonly --webroot --webroot-path=/var/www/certbot \ --email fabricionewman@gmail.com --agree-tos --no-eff-email \ - -d mally.neumanf.com -d api.mally.neumanf.com -d auth.mally.neumanf.com + -d mally.neumanf.com -d api.mally.neumanf.com -d auth.mally.neumanf.com -d g.mally.neumanf.com # Replace Nginx configuration with the SSL version cp /conf/nginx.conf /etc/nginx/conf.d/default.conf diff --git a/infra/prometheus/Dockerfile b/infra/prometheus/Dockerfile new file mode 100644 index 0000000..c952119 --- /dev/null +++ b/infra/prometheus/Dockerfile @@ -0,0 +1,3 @@ +FROM prom/prometheus:v2.54.1 + +COPY ./infra/prometheus/conf/prometheus.yml /etc/prometheus/config.yml \ No newline at end of file diff --git a/infra/prometheus/conf/prometheus.yml b/infra/prometheus/conf/prometheus.yml new file mode 100644 index 0000000..fd57953 --- /dev/null +++ b/infra/prometheus/conf/prometheus.yml @@ -0,0 +1,7 @@ +scrape_configs: + - job_name: 'Spring' + scrape_interval: 15s + scrape_timeout: 10s + metrics_path: '/actuator/prometheus' + static_configs: + - targets: ['api:8080'] diff --git a/infra/promtail/Dockerfile b/infra/promtail/Dockerfile new file mode 100644 index 0000000..4413cda --- /dev/null +++ b/infra/promtail/Dockerfile @@ -0,0 +1,3 @@ +FROM grafana/promtail:3.2.0 + +COPY ./infra/promtail/conf /etc/promtail \ No newline at end of file diff --git a/infra/promtail/conf/promtail.yml b/infra/promtail/conf/promtail.yml new file mode 100644 index 0000000..ea5c7f5 --- /dev/null +++ b/infra/promtail/conf/promtail.yml @@ -0,0 +1,22 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: spring-boot-app + static_configs: + - targets: + - api:8080 + labels: + job: spring-boot-app + __path__: /var/log/*log + pipeline_stages: + - multiline: + firstline: '^\[\w+]' + max_wait_time: 3s