Add monitoring setup production environment

docker-compose-production.yml

@@ -77,3 +77,80 @@ services:
       context: ./
       dockerfile: docker/prod/code-upload-worker/Dockerfile
 
+  prometheus:
+    hostname: prometheus
+    image: prom/prometheus:latest
+    user: "1000"
+    volumes:
+      - ./monitoring/prometheus/prometheus_production.yml:/etc/prometheus/prometheus.yml
+      - ./monitoring/prometheus/rules.yml:/etc/rules/rules.yml
+      - ./monitoring/prometheus/prometheus_db:/var/lib/prometheus
+      - ./monitoring/prometheus/prometheus_db:/prometheus
+      - ./monitoring/prometheus/prometheus_db:/etc/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.external-url=http://localhost:9090/prometheus'
+    ports:
+      - '9090:9090'
+
+  grafana:
+    hostname: grafana
+    image: grafana/grafana:latest
+    user: "1000"
+    env_file:
+      - docker/prod/docker_production.env
+    volumes:
+      - ./monitoring/grafana/grafana_db:/var/lib/grafana
+    depends_on:
+      - prometheus
+    ports:
+      - '3000:3000'
+
+  statsd-exporter:
+    hostname: statsd
+    image: prom/statsd-exporter:latest
+    command:
+      - '--log.level=info'
+      - '--web.telemetry-path=/statsd/metrics'
+    ports:
+      - '9125:9125/udp'
+      - '9125:9125/tcp'
+      - '9102:9102'
+
+  node_exporter:
+    hostname: node_exporter
+    image: prom/node-exporter
+    ports:
+      - '9100:9100'
+
+  nginx-ingress:
+    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-nginx-ingress:${COMMIT_ID}
+    build:
+      context: ./
+      dockerfile: docker/prod/nginx-ingress/Dockerfile
+      args:
+        MONITORING_ENV: production
+    depends_on: 
+      - prometheus
+      - grafana
+      - statsd-exporter
+      - alert-manager
+    ports:
+      - '80:80'
+      - '443:443'
+
+  alert-manager:
+    hostname: alert_manager
+    image: prom/alertmanager
+    user: "1000"
+    volumes:
+      - ./monitoring/prometheus:/prometheus
+      - ./monitoring/alertmanager/data:/data
+      - ./monitoring/alertmanager/templates:/etc/alertmanager/templates
+    command:
+      - '--config.file=/prometheus/alert_manager.yml'
+      - '--storage.path=/data'
+      - '--web.external-url=http://localhost:9093/alert_manager'
+    ports:
+      - '9093:9093'
+

docker/prod/nginx-ingress/nginx_production.conf

@@ -0,0 +1,68 @@
+upstream prometheus {
+  server prometheus:9090 fail_timeout=0;
+}
+
+upstream grafana {
+  server grafana:3000 fail_timeout=0;
+}
+
+upstream statsd_exporter {
+  server statsd:9102 fail_timeout=0;
+}
+
+upstream alert_manager {
+  server alert_manager:9093 fail_timeout=0;
+}
+
+server {
+  server_name monitoring.eval.ai;
+  listen 80;
+  return 301 https://monitoring.eval.ai$request_uri;
+}
+
+server {
+  server_name monitoring.eval.ai;
+  listen 443 ssl;
+  location / {
+    root /usr/share/nginx/html;
+    index index.html index.htm;
+  }
+
+  ssl on;
+  ssl_certificate /etc/ssl/eval_ai.crt;
+  ssl_certificate_key /etc/ssl/eval_ai.key;
+  ssl_prefer_server_ciphers on;
+  # enables all versions of TLS, but not SSLv2 or 3 which are weak and now deprecated.
+  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+
+  access_log /var/log/nginx/access.log;
+  error_log /var/log/nginx/error.log;
+
+  location /prometheus {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_pass http://prometheus;
+  }
+
+  location /grafana {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_pass http://grafana;
+  }
+
+  location /statsd {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_pass http://statsd_exporter;
+  }
+
+  location /alert_manager {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_pass http://alert_manager;
+  }
+}