Merge branch 'master' into edit-guidelines

.gitignore

@@ -71,3 +71,5 @@ out/
 monitoring/prometheus/prometheus_db/*
 !monitoring/prometheus/prometheus_db/.gitkeep
 monitoring/grafana/grafana_db/grafana.db
+monitoring/alertmanager/data/*
+!monitoring/alertmanager/data/.gitkeep

apps/challenges/migrations/0086_add_is_multi_metric_leaderboard_field.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.20 on 2021-08-07 14:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("challenges", "0085_challenge_submission_time_limit"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="challengephasesplit",
+            name="is_multi_metric_leaderboard",
+            field=models.BooleanField(default=True),
+        ),
+    ]

apps/challenges/models.py

@@ -402,6 +402,8 @@ class ChallengePhaseSplit(TimeStampedModel):
     is_leaderboard_order_descending = models.BooleanField(default=True)
     show_leaderboard_by_latest_submission = models.BooleanField(default=False)
     show_execution_time = models.BooleanField(default=False)
+    # Allow ordering leaderboard by all metrics
+    is_multi_metric_leaderboard = models.BooleanField(default=True)
 
     def __str__(self):
         return "{0} : {1}".format(

apps/challenges/serializers.py

@@ -144,6 +144,7 @@ class ChallengePhaseSplitSerializer(serializers.ModelSerializer):
 
     dataset_split_name = serializers.SerializerMethodField()
     challenge_phase_name = serializers.SerializerMethodField()
+    leaderboard_schema = serializers.SerializerMethodField()
 
     class Meta:
         model = ChallengePhaseSplit
@@ -156,8 +157,13 @@ class Meta:
             "visibility",
             "show_leaderboard_by_latest_submission",
             "show_execution_time",
+            "leaderboard_schema",
+            "is_multi_metric_leaderboard",
         )
 
+    def get_leaderboard_schema(self, obj):
+        return obj.leaderboard.schema
+
     def get_dataset_split_name(self, obj):
         return obj.dataset_split.name
 

apps/challenges/views.py

@@ -336,16 +336,18 @@ def add_participant_team_to_challenge(
     # Check if user is in allowed list.
     user_email = request.user.email
     if len(challenge.allowed_email_domains) > 0:
-        if not is_user_in_allowed_email_domains(user_email, challenge_pk):
-            message = "Sorry, users with {} email domain(s) are only allowed to participate in this challenge."
-            domains = ""
-            for domain in challenge.allowed_email_domains:
-                domains = "{}{}{}".format(domains, "/", domain)
-            domains = domains[1:]
-            response_data = {"error": message.format(domains)}
-            return Response(
-                response_data, status=status.HTTP_406_NOT_ACCEPTABLE
-            )
+        domains = ""
+        for domain in challenge.allowed_email_domains:
+            domains = "{}{}{}".format(domains, "/", domain)
+        domains = domains[1:]
+        for participant_email in participant_team.get_all_participants_email():
+            if not is_user_in_allowed_email_domains(participant_email, challenge_pk):
+                message = "Sorry, team consisting of users with non-{} email domain(s) are not allowed \
+                    to participate in this challenge."
+                response_data = {"error": message.format(domains)}
+                return Response(
+                    response_data, status=status.HTTP_406_NOT_ACCEPTABLE
+                )
 
     # Check if user is in blocked list.
     if is_user_in_blocked_email_domains(user_email, challenge_pk):
@@ -1246,7 +1248,6 @@ def create_challenge_using_zip_file(request, challenge_host_team_pk):
                 challenge_phase_data[
                     field
                 ] = challenge_phase_data_from_hosts.get(field)
-
     try:
         with transaction.atomic():
             serializer = ZipChallengeSerializer(

apps/jobs/utils.py

@@ -234,7 +234,7 @@ def handle_submission_rerun(submission, updated_status):
 
 
 def calculate_distinct_sorted_leaderboard_data(
-    user, challenge_obj, challenge_phase_split, only_public_entries
+    user, challenge_obj, challenge_phase_split, only_public_entries, order_by
 ):
     """
     Function to calculate and return the sorted leaderboard data
@@ -253,13 +253,39 @@ def calculate_distinct_sorted_leaderboard_data(
     leaderboard = challenge_phase_split.leaderboard
 
     # Get the default order by key to rank the entries on the leaderboard
+    default_order_by = None
+    is_leaderboard_order_descending = (
+        challenge_phase_split.is_leaderboard_order_descending
+    )
     try:
         default_order_by = leaderboard.schema["default_order_by"]
     except KeyError:
         response_data = {
             "error": "Sorry, default_order_by key is missing in leaderboard schema!"
         }
         return response_data, status.HTTP_400_BAD_REQUEST
+    # Use order by field from request only if it is valid
+    try:
+        if order_by in leaderboard.schema["labels"]:
+            default_order_by = order_by
+    except KeyError:
+        response_data = {
+            "error": "Sorry, labels key is missing in leaderboard schema!"
+        }
+        return response_data, status.HTTP_400_BAD_REQUEST
+
+    leaderboard_schema = leaderboard.schema
+    if (
+        leaderboard_schema.get("metadata") is not None
+        and leaderboard_schema.get("metadata").get(default_order_by)
+        is not None
+    ):
+        is_leaderboard_order_descending = (
+            leaderboard_schema["metadata"][default_order_by].get(
+                "sort_ascending"
+            )
+            is False
+        )
 
     # Exclude the submissions done by members of the host team
     # while populating leaderboard
@@ -398,9 +424,7 @@ def calculate_distinct_sorted_leaderboard_data(
                 float(k["filtering_score"]),
                 float(-k["filtering_error"]),
             ),
-            reverse=True
-            if challenge_phase_split.is_leaderboard_order_descending
-            else False,
+            reverse=True if is_leaderboard_order_descending else False,
         )
     distinct_sorted_leaderboard_data = []
     team_list = []

apps/jobs/views.py

@@ -581,6 +581,7 @@ def leaderboard(request, challenge_phase_split_id):
         challenge_phase_split_id
     )
     challenge_obj = challenge_phase_split.challenge_phase.challenge
+    order_by = request.GET.get("order_by")
     (
         response_data,
         http_status_code,
@@ -589,6 +590,7 @@ def leaderboard(request, challenge_phase_split_id):
         challenge_obj,
         challenge_phase_split,
         only_public_entries=True,
+        order_by=order_by,
     )
     # The response 400 will be returned if the leaderboard isn't public or `default_order_by` key is missing in leaderboard.
     if http_status_code == status.HTTP_400_BAD_REQUEST:
@@ -792,7 +794,7 @@ def get_all_entries_on_public_leaderboard(request, challenge_phase_split_pk):
             "error": "Sorry, you are not authorized to make this request!"
         }
         return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
-
+    order_by = request.GET.get("order_by")
     (
         response_data,
         http_status_code,
@@ -801,6 +803,7 @@ def get_all_entries_on_public_leaderboard(request, challenge_phase_split_pk):
         challenge_obj,
         challenge_phase_split,
         only_public_entries=False,
+        order_by=order_by,
     )
     # The response 400 will be returned if the leaderboard isn't public or `default_order_by` key is missing in leaderboard.
     if http_status_code == status.HTTP_400_BAD_REQUEST:
@@ -1109,8 +1112,8 @@ def update_submission(request, challenge_pk):
         challenge_phase_pk = request.data.get("challenge_phase")
         submission_pk = request.data.get("submission")
         submission_status = request.data.get("submission_status", "").lower()
-        stdout_content = request.data.get("stdout", "")
-        stderr_content = request.data.get("stderr", "")
+        stdout_content = request.data.get("stdout", "").encode("utf-8")
+        stderr_content = request.data.get("stderr", "").encode("utf-8")
         submission_result = request.data.get("result", "")
         metadata = request.data.get("metadata", "")
         submission = get_submission_model(submission_pk)
@@ -2333,6 +2336,7 @@ def get_github_badge_data(
         challenge_obj,
         challenge_phase_split,
         only_public_entries=True,
+        order_by=None,
     )
     if http_status_code == status.HTTP_400_BAD_REQUEST:
         return Response(response_data, status=http_status_code)

docker-compose-monitoring.yml

@@ -5,6 +5,7 @@ services:
     user: "1000"
     volumes:
       - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./monitoring/prometheus/rules.yml:/etc/rules/rules.yml
       - ./monitoring/prometheus/prometheus_db:/var/lib/prometheus
       - ./monitoring/prometheus/prometheus_db:/prometheus
       - ./monitoring/prometheus/prometheus_db:/etc/prometheus
@@ -30,3 +31,18 @@ services:
     image: prom/node-exporter
     ports:
       - "9100:9100"
+
+  alert-manager:
+    hostname: alert_manager
+    image: prom/alertmanager
+    user: "1000"
+    volumes:
+      - ./monitoring/prometheus:/prometheus
+      - ./monitoring/alertmanager/data:/data
+      - ./monitoring/alertmanager/templates:/etc/alertmanager/templates
+    command:
+      - '--config.file=/prometheus/alert_manager.yml'
+      - '--storage.path=/data'
+      - '--web.external-url=http://localhost:9093/alert_manager'
+    ports:
+      - 9093:9093

docker-compose-staging.yml

@@ -60,7 +60,7 @@ services:
         awslogs-create-group: 'true'
 
   nodejs_v2:
-    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-production-frontend-v2:${COMMIT_ID}
+    image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-staging-frontend-v2:${COMMIT_ID}
     build:
       context: ./
       dockerfile: docker/prod/nodejs_v2/Dockerfile
@@ -75,7 +75,7 @@ services:
       driver: awslogs
       options:
         awslogs-region: us-east-1
-        awslogs-group: nodejs_production_v2
+        awslogs-group: nodejs_staging_v2
         awslogs-create-group: "true"
 
   remote-worker:
@@ -101,6 +101,7 @@ services:
     user: "1000"
     volumes:
       - ./monitoring/prometheus/prometheus_staging.yml:/etc/prometheus/prometheus.yml
+      - ./monitoring/prometheus/rules.yml:/etc/rules/rules.yml
       - ./monitoring/prometheus/prometheus_db:/var/lib/prometheus
       - ./monitoring/prometheus/prometheus_db:/prometheus
       - ./monitoring/prometheus/prometheus_db:/etc/prometheus
@@ -151,6 +152,22 @@ services:
       - prometheus
       - grafana
       - statsd-exporter
+      - alert-manager
     ports:
       - '80:80'
       - '443:443'
+
+  alert-manager:
+    hostname: alert_manager
+    image: prom/alertmanager
+    user: "1000"
+    volumes:
+      - ./monitoring/prometheus:/prometheus
+      - ./monitoring/alertmanager/data:/data
+      - ./monitoring/alertmanager/templates:/etc/alertmanager/templates
+    command:
+      - '--config.file=/prometheus/alert_manager.yml'
+      - '--storage.path=/data'
+      - '--web.external-url=http://localhost:9093/alert_manager'
+    ports:
+      - '9093:9093'

docker/prod/nginx-ingress/nginx_staging.conf

@@ -10,6 +10,10 @@ upstream statsd_exporter {
   server statsd:9102 fail_timeout=0;
 }
 
+upstream alert_manager {
+  server alert_manager:9093 fail_timeout=0;
+}
+
 server {
   server_name monitoring-staging.eval.ai;
   listen 80;
@@ -54,4 +58,11 @@ server {
     proxy_set_header X-Forwarded-Proto $scheme;
     proxy_pass http://statsd_exporter;
   }
+
+  location /alert_manager {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_pass http://alert_manager;
+  }
 }

docker/prod/nodejs_v2/nginx_staging.conf

@@ -2,9 +2,9 @@ upstream django_app {
   server django:8000 fail_timeout=0;
 }
 
-upstream node_exporter {
-  server node_exporter:9100 fail_timeout=0;
-}
+# upstream node_exporter {
+#   server node_exporter:9100 fail_timeout=0;
+# }
 
 server {
   server_name staging.eval.ai;

frontend/src/css/modules/landing.scss

@@ -561,7 +561,7 @@ a.view-more {
 
 .evalai-sponsor-logo {
     max-width: 100%;
-    height: 120px;
+    height: 300px;
     text-align: center;
     img {
         height: 55%;
@@ -584,5 +584,10 @@ a.view-more {
 }
 
 #gsoc {
-    height: 45%;
+    height: 55%;
+    margin-left: 16.667%;
+}
+
+#aws {
+    margin-top: 2%;
 }