Backend+Frontend: Add feature for multi metric leaderboard(Cloud-CV#3487

) * Add multi metric leaderboard * Fix build * Refactor codebase * Fix tests * Highlight order_by column Co-authored-by: Rishabh Jain <[email protected]>
vinceli1004 · Aug 18, 2021 · 20ab744 · 20ab744
1 parent 528d187
commit 20ab744
Show file tree

Hide file tree

Showing 10 changed files with 112 additions and 16 deletions.
diff --git a/apps/challenges/migrations/0086_add_is_multi_metric_leaderboard_field.py b/apps/challenges/migrations/0086_add_is_multi_metric_leaderboard_field.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2.20 on 2021-08-07 14:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("challenges", "0085_challenge_submission_time_limit"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="challengephasesplit",
+            name="is_multi_metric_leaderboard",
+            field=models.BooleanField(default=True),
+        ),
+    ]
diff --git a/apps/challenges/models.py b/apps/challenges/models.py
@@ -402,6 +402,8 @@ class ChallengePhaseSplit(TimeStampedModel):
     is_leaderboard_order_descending = models.BooleanField(default=True)
     show_leaderboard_by_latest_submission = models.BooleanField(default=False)
     show_execution_time = models.BooleanField(default=False)
+    # Allow ordering leaderboard by all metrics
+    is_multi_metric_leaderboard = models.BooleanField(default=True)
 
     def __str__(self):
         return "{0} : {1}".format(

diff --git a/apps/challenges/serializers.py b/apps/challenges/serializers.py
@@ -144,6 +144,7 @@ class ChallengePhaseSplitSerializer(serializers.ModelSerializer):
 
     dataset_split_name = serializers.SerializerMethodField()
     challenge_phase_name = serializers.SerializerMethodField()
+    leaderboard_schema = serializers.SerializerMethodField()
 
     class Meta:
         model = ChallengePhaseSplit
@@ -156,8 +157,13 @@ class Meta:
             "visibility",
             "show_leaderboard_by_latest_submission",
             "show_execution_time",
+            "leaderboard_schema",
+            "is_multi_metric_leaderboard",
         )
 
+    def get_leaderboard_schema(self, obj):
+        return obj.leaderboard.schema
+
     def get_dataset_split_name(self, obj):
         return obj.dataset_split.name
 

diff --git a/apps/challenges/views.py b/apps/challenges/views.py
@@ -1246,7 +1246,6 @@ def create_challenge_using_zip_file(request, challenge_host_team_pk):
                 challenge_phase_data[
                     field
                 ] = challenge_phase_data_from_hosts.get(field)
-
     try:
         with transaction.atomic():
             serializer = ZipChallengeSerializer(

diff --git a/apps/jobs/utils.py b/apps/jobs/utils.py
@@ -234,7 +234,7 @@ def handle_submission_rerun(submission, updated_status):
 
 
 def calculate_distinct_sorted_leaderboard_data(
-    user, challenge_obj, challenge_phase_split, only_public_entries
+    user, challenge_obj, challenge_phase_split, only_public_entries, order_by
 ):
     """
     Function to calculate and return the sorted leaderboard data
@@ -253,13 +253,39 @@ def calculate_distinct_sorted_leaderboard_data(
     leaderboard = challenge_phase_split.leaderboard
 
     # Get the default order by key to rank the entries on the leaderboard
+    default_order_by = None
+    is_leaderboard_order_descending = (
+        challenge_phase_split.is_leaderboard_order_descending
+    )
     try:
         default_order_by = leaderboard.schema["default_order_by"]
     except KeyError:
         response_data = {
             "error": "Sorry, default_order_by key is missing in leaderboard schema!"
         }
         return response_data, status.HTTP_400_BAD_REQUEST
+    # Use order by field from request only if it is valid
+    try:
+        if order_by in leaderboard.schema["labels"]:
+            default_order_by = order_by
+    except KeyError:
+        response_data = {
+            "error": "Sorry, labels key is missing in leaderboard schema!"
+        }
+        return response_data, status.HTTP_400_BAD_REQUEST
+
+    leaderboard_schema = leaderboard.schema
+    if (
+        leaderboard_schema.get("metadata") is not None
+        and leaderboard_schema.get("metadata").get(default_order_by)
+        is not None
+    ):
+        is_leaderboard_order_descending = (
+            leaderboard_schema["metadata"][default_order_by].get(
+                "sort_ascending"
+            )
+            is False
+        )
 
     # Exclude the submissions done by members of the host team
     # while populating leaderboard
@@ -398,9 +424,7 @@ def calculate_distinct_sorted_leaderboard_data(
                 float(k["filtering_score"]),
                 float(-k["filtering_error"]),
             ),
-            reverse=True
-            if challenge_phase_split.is_leaderboard_order_descending
-            else False,
+            reverse=True if is_leaderboard_order_descending else False,
         )
     distinct_sorted_leaderboard_data = []
     team_list = []

diff --git a/apps/jobs/views.py b/apps/jobs/views.py
@@ -493,7 +493,7 @@ def change_submission_data_and_visibility(
 
 
 @swagger_auto_schema(
-    methods=["get"],
+    methods=["get", "post"],
     manual_parameters=[
         openapi.Parameter(
             name="challenge_phase_split_id",
@@ -563,7 +563,7 @@ def change_submission_data_and_visibility(
         )
     },
 )
-@api_view(["GET"])
+@api_view(["GET", "POST"])
 @throttle_classes([AnonRateThrottle])
 def leaderboard(request, challenge_phase_split_id):
     """
@@ -581,6 +581,7 @@ def leaderboard(request, challenge_phase_split_id):
         challenge_phase_split_id
     )
     challenge_obj = challenge_phase_split.challenge_phase.challenge
+    order_by = request.data.get("order_by")
     (
         response_data,
         http_status_code,
@@ -589,6 +590,7 @@ def leaderboard(request, challenge_phase_split_id):
         challenge_obj,
         challenge_phase_split,
         only_public_entries=True,
+        order_by=order_by,
     )
     # The response 400 will be returned if the leaderboard isn't public or `default_order_by` key is missing in leaderboard.
     if http_status_code == status.HTTP_400_BAD_REQUEST:

diff --git a/frontend/src/js/controllers/challengeCtrl.js b/frontend/src/js/controllers/challengeCtrl.js
@@ -24,12 +24,15 @@
         vm.projectUrl = "";
         vm.publicationUrl = "";
         vm.isPublicSubmission = null;
+        vm.isMultiMetricLeaderboardEnabled = {};
         vm.wrnMsg = {};
         vm.page = {};
         vm.isParticipated = false;
         vm.isActive = false;
         vm.phases = {};
         vm.phaseSplits = {};
+        vm.orderLeaderboardBy = decodeURIComponent($stateParams.metric);
+        vm.phaseSplitLeaderboardSchema = {};
         vm.submissionMetaAttributes = []; // Stores the attributes format and phase ID for all the phases of a challenge.
         vm.metaAttributesforCurrentSubmission = null; // Stores the attributes while making a submission for a selected phase.
         vm.selectedPhaseSplit = {};
@@ -829,6 +832,8 @@
                         vm.phaseSplits[i].showPrivate = true;
                         vm.showPrivateIds.push(vm.phaseSplits[i].id);
                     }
+                    vm.isMultiMetricLeaderboardEnabled[vm.phaseSplits[i].id] = vm.phaseSplits[i].is_multi_metric_leaderboard;
+                    vm.phaseSplitLeaderboardSchema[vm.phaseSplits[i].id] = vm.phaseSplits[i].leaderboard_schema;
                 }
                 utilities.hideLoader();
             },
@@ -909,8 +914,10 @@
             vm.stopLeaderboard();
             vm.poller = $interval(function() {
                 parameters.url = "jobs/" + "challenge_phase_split/" + vm.phaseSplitId + "/leaderboard/?page_size=1000";
-                parameters.method = 'GET';
-                parameters.data = {};
+                parameters.method = 'POST';
+                parameters.data = {
+                    "order_by": vm.orderLeaderboardBy
+                };
                 parameters.callback = {
                     onSuccess: function(response) {
                         var details = response.data;
@@ -967,8 +974,10 @@
             // Show leaderboard
             vm.leaderboard = {};
             parameters.url = "jobs/" + "challenge_phase_split/" + vm.phaseSplitId + "/leaderboard/?page_size=1000";
-            parameters.method = 'GET';
-            parameters.data = {};
+            parameters.method = 'POST';
+            parameters.data = {
+                "order_by": vm.orderLeaderboardBy
+            };
             parameters.callback = {
                 onSuccess: function(response) {
                     var details = response.data;
@@ -1389,8 +1398,10 @@
             vm.startLoader("Loading Leaderboard Items");
             vm.leaderboard = {};
             parameters.url = "jobs/" + "challenge_phase_split/" + vm.phaseSplitId + "/leaderboard/?page_size=1000";
-            parameters.method = 'GET';
-            parameters.data = {};
+            parameters.method = 'POST';
+            parameters.data = {
+                "order_by": vm.orderLeaderboardBy
+            };
             parameters.callback = {
                 onSuccess: function(response) {
                     var details = response.data;
@@ -2731,6 +2742,10 @@
             }
         };
 
+        vm.encodeMetricURI = function(metric) {
+            return encodeURIComponent(metric);
+        };
+
     }
 
 })();
diff --git a/frontend/src/js/route-config/route-config.js b/frontend/src/js/route-config/route-config.js
@@ -286,6 +286,15 @@
             title: 'Leaderboard'
         };
 
+        var challenge_phase_metric_leaderboard = {
+            name: "web.challenge-main.challenge-page.phase-metric-leaderboard",
+            url: "/leaderboard/:phaseSplitId/:metric",
+            controller: 'ChallengeCtrl',
+            controllerAs: 'challenge',
+            templateUrl: baseUrl + "/web/challenge/leaderboard.html",
+            title: 'Leaderboard'
+        };
+
         var profile = {
             name: "web.profile",
             parent: "web",
@@ -503,6 +512,7 @@
         $stateProvider.state(my_challenge_all_submission);
         $stateProvider.state(leaderboard);
         $stateProvider.state(challenge_phase_leaderboard);
+        $stateProvider.state(challenge_phase_metric_leaderboard);
 
         // featured challenge details
         $stateProvider.state(featured_challenge_page);

diff --git a/frontend/src/views/web/challenge/leaderboard.html b/frontend/src/views/web/challenge/leaderboard.html
@@ -40,6 +40,17 @@ <h5 class="w-300">Leaderboard</h5>
                 </span>
             </div>
         </div>
+        <div class="row" ng-if="challenge.isMultiMetricLeaderboardEnabled[challenge.phaseSplitId]">
+            <div class="col xs12 s6">
+                <span>
+                    <md-select ng-model="challenge.orderLeaderboardBy" placeholder="Order by metric" class="rm-margin">
+                        <md-option value="{{key}}" ui-sref="web.challenge-main.challenge-page.phase-metric-leaderboard({phaseSplitId:challenge.phaseSplitId, metric:challenge.encodeMetricURI(key)})"
+                         ng-repeat="key in challenge.phaseSplitLeaderboardSchema[challenge.phaseSplitId].labels">
+                            <span class="fs-16 w-300">{{key}}</span>
+                    </md-select>
+                </span>
+            </div>
+        </div>
         <div class="row">
             <div class="horizontal-scroll">
                 <div class="col s12">
@@ -93,8 +104,10 @@ <h5 class="w-300">Leaderboard</h5>
                             <td ng-repeat="key in challenge.leaderboard[0].leaderboard__schema.labels">
                                 <a href="#"
                                     ng-click="$parent.challenge.sortLeaderboard($parent.challenge, 'number', $index);">
-                                    <span class="w-300 fs-20 leaderboard-label" ng-if="challenge.isMetricOrderedAscending(key)">{{key}} (&#x2193;) <span class="description" ng-if="challenge.getLabelDescription(key).length != 0">{{challenge.getLabelDescription(key)}}</span></span>
-                                    <span class="w-300 fs-20 leaderboard-label" ng-if="!challenge.isMetricOrderedAscending(key)">{{key}} (&#x2191;) <span class="description" ng-if="challenge.getLabelDescription(key).length != 0">{{challenge.getLabelDescription(key)}}</span></span>
+                                    <span class="w-300 fs-20 leaderboard-label" ng-if="challenge.isMetricOrderedAscending(key) && key != challenge.orderLeaderboardBy ">{{key}} (&#x2193;) <span class="description" ng-if="challenge.getLabelDescription(key).length != 0">{{challenge.getLabelDescription(key)}}</span></span>
+                                    <span class="w-300 fs-20 leaderboard-label" ng-if="!challenge.isMetricOrderedAscending(key) && key != challenge.orderLeaderboardBy">{{key}} (&#x2191;) <span class="description" ng-if="challenge.getLabelDescription(key).length != 0">{{challenge.getLabelDescription(key)}}</span></span>
+                                    <span class="w-400 fs-20 leaderboard-label" ng-if="challenge.isMetricOrderedAscending(key) && key == challenge.orderLeaderboardBy ">{{key}} (&#x2193;) <span class="description" ng-if="challenge.getLabelDescription(key).length != 0">{{challenge.getLabelDescription(key)}}</span></span>
+                                    <span class="w-400 fs-20 leaderboard-label" ng-if="!challenge.isMetricOrderedAscending(key) && key == challenge.orderLeaderboardBy">{{key}} (&#x2191;) <span class="description" ng-if="challenge.getLabelDescription(key).length != 0">{{challenge.getLabelDescription(key)}}</span></span>
                                     <span class="fa-stack fa-1x">
                                         <i class="fa fa-sort-asc fa-stack-1x"
                                             ng-class="$parent.challenge.reverseSort && $parent.challenge.sortColumn == 'number' && $parent.challenge.columnIndexSort == $index ? 'text-dark-black' : 'text-light-black w-300'"></i>
@@ -150,7 +163,8 @@ <h5 class="w-300">Leaderboard</h5>
                                     ng-if="key.submission__is_verified_by_host"></span>
                             </td>
                             <td ng-repeat="(i, score) in key.result track by $index">
-                                {{score | number : challenge.selectedPhaseSplit.leaderboard_decimal_precision}}
+                                <span class="w-400 fs-16" ng-if="challenge.leaderboard[0].leaderboard__schema.labels[i] == challenge.orderLeaderboardBy">{{score | number : challenge.selectedPhaseSplit.leaderboard_decimal_precision}}</span>
+                                <span class="w-300 fs-16" ng-if="challenge.leaderboard[0].leaderboard__schema.labels[i] != challenge.orderLeaderboardBy">{{score | number : challenge.selectedPhaseSplit.leaderboard_decimal_precision}}</span>
                                 <span class="new badge orange-background fs-16 w-300 partial-evaluation"
                                       data-badge-caption="E" ng-if="score === '#'"></span>
                                 <span ng-if="key.error != nil"> ±

diff --git a/tests/unit/challenges/test_views.py b/tests/unit/challenges/test_views.py
@@ -2962,6 +2962,8 @@ def test_get_challenge_phase_split(self):
                 "visibility": self.challenge_phase_split.visibility,
                 "show_leaderboard_by_latest_submission": self.challenge_phase_split.show_leaderboard_by_latest_submission,
                 "show_execution_time": False,
+                "leaderboard_schema": self.challenge_phase_split.leaderboard.schema,
+                "is_multi_metric_leaderboard": self.challenge_phase_split.is_multi_metric_leaderboard,
             }
         ]
         self.client.force_authenticate(user=self.participant_user)
@@ -2999,6 +3001,8 @@ def test_get_challenge_phase_split_when_user_is_challenge_host(self):
                 "visibility": self.challenge_phase_split.visibility,
                 "show_leaderboard_by_latest_submission": self.challenge_phase_split.show_leaderboard_by_latest_submission,
                 "show_execution_time": False,
+                "leaderboard_schema": self.challenge_phase_split.leaderboard.schema,
+                "is_multi_metric_leaderboard": self.challenge_phase_split.is_multi_metric_leaderboard,
             },
             {
                 "id": self.challenge_phase_split_host.id,
@@ -3009,6 +3013,8 @@ def test_get_challenge_phase_split_when_user_is_challenge_host(self):
                 "visibility": self.challenge_phase_split_host.visibility,
                 "show_leaderboard_by_latest_submission": self.challenge_phase_split_host.show_leaderboard_by_latest_submission,
                 "show_execution_time": False,
+                "leaderboard_schema": self.challenge_phase_split_host.leaderboard.schema,
+                "is_multi_metric_leaderboard": self.challenge_phase_split_host.is_multi_metric_leaderboard,
             },
         ]
         self.client.force_authenticate(user=self.user)