Enable submission worker metrics and fix node exporter metrics (Cloud…

…-CV#3618)
vahetag · Oct 19, 2021 · d19d7ca · d19d7ca
1 parent ba8ca06
commit d19d7ca
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 2 deletions.
diff --git a/apps/challenges/aws_utils.py b/apps/challenges/aws_utils.py
@@ -84,6 +84,8 @@
     "RDS_PORT": settings.DATABASES["default"]["PORT"],
     "SECRET_KEY": settings.SECRET_KEY,
     "SENTRY_URL": os.environ.get("SENTRY_URL"),
+    "STATSD_ENDPOINT": os.environ.get("STATSD_ENDPOINT"),
+    "STATSD_PORT": os.environ.get("STATSD_PORT"),
 }
 
 VPC_DICT = {

diff --git a/apps/challenges/task_definitions.py b/apps/challenges/task_definitions.py
@@ -112,6 +112,14 @@
                 {{
                     "name": "AWS_SES_REGION_ENDPOINT",
                     "value": "{AWS_SES_REGION_ENDPOINT}"
+                }},
+                {{
+                    "name": "STATSD_ENDPOINT",
+                    "value: "{STATSD_ENDPOINT}"
+                }},
+                {{
+                    "name": "STATSD_PORT",
+                    "value: "{STATSD_PORT}"
                 }}
             ],
             "workingDirectory": "/code",
@@ -346,6 +354,14 @@
         {{
             "name": "AWS_SES_REGION_ENDPOINT",
             "value": "{AWS_SES_REGION_ENDPOINT}"
+        }},
+        {{
+            "name": "STATSD_ENDPOINT",
+            "value: "{STATSD_ENDPOINT}"
+        }},
+        {{
+            "name": "STATSD_PORT",
+            "value: "{STATSD_PORT}"
         }}
     ],
     "workingDirectory": "/code",
@@ -414,10 +430,17 @@
             "name": "EVALAI_DNS",
             "value": "{EVALAI_DNS}"
         }},
-
         {{
             "name": "EFS_ID",
             "value": "{EFS_ID}"
+        }},
+        {{
+            "name": "STATSD_ENDPOINT",
+            "value: "{STATSD_ENDPOINT}"
+        }},
+        {{
+            "name": "STATSD_PORT",
+            "value: "{STATSD_PORT}"
         }}
 
     ],

diff --git a/docker/prod/nodejs/nginx_production.conf b/docker/prod/nodejs/nginx_production.conf
@@ -2,6 +2,10 @@ upstream django_app {
   server django:8000 fail_timeout=0;
 }
 
+upstream node_exporter {
+  server node_exporter:9100 fail_timeout=0;
+}
+
 server {
   server_name evalapi.cloudcv.org evalai.cloudcv.org;
   listen 80;

diff --git a/monitoring/prometheus/rules.yml b/monitoring/prometheus/rules.yml
@@ -14,7 +14,7 @@ groups:
 - name: Instance-Status
   rules:
   - alert: InstanceDown
-    expr: up == 0
+    expr: up{job="node_exporter"} == 0
     for: 5m
     annotations:
       title: "Instance(s) Down"