From 1e81627ef001ad90d6f7b89432b523c5f2b673f4 Mon Sep 17 00:00:00 2001 From: Ram81 Date: Sun, 20 Jun 2021 20:20:12 +0530 Subject: [PATCH 1/5] Add multi metric leaderboard --- ...d_enable_multi_metric_leaderboard_field.py | 18 +++++++++++++ apps/challenges/models.py | 2 ++ apps/challenges/serializers.py | 7 +++++ apps/jobs/utils.py | 12 ++++++++- apps/jobs/views.py | 6 +++-- frontend/src/js/controllers/challengeCtrl.js | 27 ++++++++++++++----- frontend/src/js/route-config/route-config.js | 10 +++++++ .../src/views/web/challenge/leaderboard.html | 11 ++++++++ 8 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 apps/challenges/migrations/0085_add_enable_multi_metric_leaderboard_field.py diff --git a/apps/challenges/migrations/0085_add_enable_multi_metric_leaderboard_field.py b/apps/challenges/migrations/0085_add_enable_multi_metric_leaderboard_field.py new file mode 100644 index 0000000000..fd46efe4e2 --- /dev/null +++ b/apps/challenges/migrations/0085_add_enable_multi_metric_leaderboard_field.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.13 on 2021-06-20 14:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("challenges", "0084_challenge_is_static_dataset_code_upload"), + ] + + operations = [ + migrations.AddField( + model_name="challenge", + name="is_multi_metric_leaderboard", + field=models.BooleanField(default=False), + ), + ] diff --git a/apps/challenges/models.py b/apps/challenges/models.py index 7c5442c3c5..1fe124a710 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -173,6 +173,8 @@ def __init__(self, *args, **kwargs): desired_worker_instance = models.IntegerField( null=True, blank=True, default=1 ) + # Allow ordering leaderboard by all metrics + is_multi_metric_leaderboard = models.BooleanField(default=False) class Meta: app_label = "challenges" diff --git a/apps/challenges/serializers.py b/apps/challenges/serializers.py index e93250a399..2d7d394ff0 100644 --- a/apps/challenges/serializers.py +++ b/apps/challenges/serializers.py @@ -144,6 +144,7 @@ class ChallengePhaseSplitSerializer(serializers.ModelSerializer): dataset_split_name = serializers.SerializerMethodField() challenge_phase_name = serializers.SerializerMethodField() + leaderboard_schema = serializers.SerializerMethodField() class Meta: model = ChallengePhaseSplit @@ -156,8 +157,13 @@ class Meta: "visibility", "show_leaderboard_by_latest_submission", "show_execution_time", + "leaderboard_schema", ) + def get_leaderboard_schema(self, obj): + print(obj.leaderboard.schema) + return obj.leaderboard.schema + def get_dataset_split_name(self, obj): return obj.dataset_split.name @@ -262,6 +268,7 @@ class Meta: "max_worker_instance", "min_worker_instance", "desired_worker_instance", + "is_multi_metric_leaderboard", ) diff --git a/apps/jobs/utils.py b/apps/jobs/utils.py index 9b63101039..d82e04a187 100644 --- a/apps/jobs/utils.py +++ b/apps/jobs/utils.py @@ -234,7 +234,7 @@ def handle_submission_rerun(submission, updated_status): def calculate_distinct_sorted_leaderboard_data( - user, challenge_obj, challenge_phase_split, only_public_entries + user, challenge_obj, challenge_phase_split, only_public_entries, order_by ): """ Function to calculate and return the sorted leaderboard data @@ -253,6 +253,7 @@ def calculate_distinct_sorted_leaderboard_data( leaderboard = challenge_phase_split.leaderboard # Get the default order by key to rank the entries on the leaderboard + default_order_by = None try: default_order_by = leaderboard.schema["default_order_by"] except KeyError: @@ -260,6 +261,15 @@ def calculate_distinct_sorted_leaderboard_data( "error": "Sorry, default_order_by key is missing in leaderboard schema!" } return response_data, status.HTTP_400_BAD_REQUEST + # Use order by field from request only if it is valid + try: + if order_by in leaderboard.schema["labels"]: + default_order_by = order_by + except KeyError: + response_data = { + "error": "Sorry, labels key is missing in leaderboard schema!" + } + return response_data, status.HTTP_400_BAD_REQUEST # Exclude the submissions done by members of the host team # while populating leaderboard diff --git a/apps/jobs/views.py b/apps/jobs/views.py index 29b4850da0..76ac72b71b 100644 --- a/apps/jobs/views.py +++ b/apps/jobs/views.py @@ -492,7 +492,7 @@ def change_submission_data_and_visibility( @swagger_auto_schema( - methods=["get"], + methods=["get", "post"], manual_parameters=[ openapi.Parameter( name="challenge_phase_split_id", @@ -562,7 +562,7 @@ def change_submission_data_and_visibility( ) }, ) -@api_view(["GET"]) +@api_view(["GET", "POST"]) @throttle_classes([AnonRateThrottle]) def leaderboard(request, challenge_phase_split_id): """ @@ -580,6 +580,7 @@ def leaderboard(request, challenge_phase_split_id): challenge_phase_split_id ) challenge_obj = challenge_phase_split.challenge_phase.challenge + order_by = request.data.get("order_by") ( response_data, http_status_code, @@ -588,6 +589,7 @@ def leaderboard(request, challenge_phase_split_id): challenge_obj, challenge_phase_split, only_public_entries=True, + order_by=order_by, ) # The response 400 will be returned if the leaderboard isn't public or `default_order_by` key is missing in leaderboard. if http_status_code == status.HTTP_400_BAD_REQUEST: diff --git a/frontend/src/js/controllers/challengeCtrl.js b/frontend/src/js/controllers/challengeCtrl.js index ff58821393..6755fc714b 100644 --- a/frontend/src/js/controllers/challengeCtrl.js +++ b/frontend/src/js/controllers/challengeCtrl.js @@ -24,12 +24,15 @@ vm.projectUrl = ""; vm.publicationUrl = ""; vm.isPublicSubmission = null; + vm.isMultiMetricLeaderboardEnabled = false; vm.wrnMsg = {}; vm.page = {}; vm.isParticipated = false; vm.isActive = false; vm.phases = {}; vm.phaseSplits = {}; + vm.orderLeaderboardBy = decodeURIComponent($stateParams.metric); + vm.phaseSplitLeaderboardSchema = {}; vm.submissionMetaAttributes = []; // Stores the attributes format and phase ID for all the phases of a challenge. vm.metaAttributesforCurrentSubmission = null; // Stores the attributes while making a submission for a selected phase. vm.selectedPhaseSplit = {}; @@ -313,6 +316,7 @@ vm.isRegistrationOpen = details.is_registration_open; vm.approved_by_admin = details.approved_by_admin; vm.isRemoteChallenge = details.remote_evaluation; + vm.isMultiMetricLeaderboardEnabled = details.is_multi_metric_leaderboard; vm.getTeamName(vm.challengeId); if (vm.page.image === null) { @@ -829,6 +833,7 @@ vm.phaseSplits[i].showPrivate = true; vm.showPrivateIds.push(vm.phaseSplits[i].id); } + vm.phaseSplitLeaderboardSchema[vm.phaseSplits[i].id] = vm.phaseSplits[i].leaderboard_schema; } utilities.hideLoader(); }, @@ -909,8 +914,10 @@ vm.stopLeaderboard(); vm.poller = $interval(function() { parameters.url = "jobs/" + "challenge_phase_split/" + vm.phaseSplitId + "/leaderboard/?page_size=1000"; - parameters.method = 'GET'; - parameters.data = {}; + parameters.method = 'POST'; + parameters.data = { + "order_by": vm.orderLeaderboardBy + }; parameters.callback = { onSuccess: function(response) { var details = response.data; @@ -967,8 +974,10 @@ // Show leaderboard vm.leaderboard = {}; parameters.url = "jobs/" + "challenge_phase_split/" + vm.phaseSplitId + "/leaderboard/?page_size=1000"; - parameters.method = 'GET'; - parameters.data = {}; + parameters.method = 'POST'; + parameters.data = { + "order_by": vm.orderLeaderboardBy + }; parameters.callback = { onSuccess: function(response) { var details = response.data; @@ -1389,8 +1398,10 @@ vm.startLoader("Loading Leaderboard Items"); vm.leaderboard = {}; parameters.url = "jobs/" + "challenge_phase_split/" + vm.phaseSplitId + "/leaderboard/?page_size=1000"; - parameters.method = 'GET'; - parameters.data = {}; + parameters.method = 'POST'; + parameters.data = { + "order_by": vm.orderLeaderboardBy + }; parameters.callback = { onSuccess: function(response) { var details = response.data; @@ -2694,6 +2705,10 @@ } }; + vm.encodeMetricURI = function(metric) { + return encodeURIComponent(metric); + } + } })(); diff --git a/frontend/src/js/route-config/route-config.js b/frontend/src/js/route-config/route-config.js index abddc5432a..acbcebb256 100644 --- a/frontend/src/js/route-config/route-config.js +++ b/frontend/src/js/route-config/route-config.js @@ -286,6 +286,15 @@ title: 'Leaderboard' }; + var challenge_phase_metric_leaderboard = { + name: "web.challenge-main.challenge-page.phase-metric-leaderboard", + url: "/leaderboard/:phaseSplitId/:metric", + controller: 'ChallengeCtrl', + controllerAs: 'challenge', + templateUrl: baseUrl + "/web/challenge/leaderboard.html", + title: 'Leaderboard' + }; + var profile = { name: "web.profile", parent: "web", @@ -503,6 +512,7 @@ $stateProvider.state(my_challenge_all_submission); $stateProvider.state(leaderboard); $stateProvider.state(challenge_phase_leaderboard); + $stateProvider.state(challenge_phase_metric_leaderboard); // featured challenge details $stateProvider.state(featured_challenge_page); diff --git a/frontend/src/views/web/challenge/leaderboard.html b/frontend/src/views/web/challenge/leaderboard.html index 6e7f5b6923..bd477757ca 100644 --- a/frontend/src/views/web/challenge/leaderboard.html +++ b/frontend/src/views/web/challenge/leaderboard.html @@ -40,6 +40,17 @@
Leaderboard
+
+
+ + + + {{key}} + + +
+
From 8764a4296e2d1b15897404e919f6ae4aa238e99b Mon Sep 17 00:00:00 2001 From: Ram81 Date: Mon, 21 Jun 2021 17:45:19 +0530 Subject: [PATCH 2/5] Fix build --- frontend/src/js/controllers/challengeCtrl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/js/controllers/challengeCtrl.js b/frontend/src/js/controllers/challengeCtrl.js index 6755fc714b..0e49456b9d 100644 --- a/frontend/src/js/controllers/challengeCtrl.js +++ b/frontend/src/js/controllers/challengeCtrl.js @@ -2707,7 +2707,7 @@ vm.encodeMetricURI = function(metric) { return encodeURIComponent(metric); - } + }; } From d8f271c080fcec2240d991de42df5ed5efc5a1ab Mon Sep 17 00:00:00 2001 From: Ram81 Date: Sat, 7 Aug 2021 22:49:34 +0530 Subject: [PATCH 3/5] Refactor codebase --- ..._add_is_multi_metric_leaderboard_field.py} | 8 ++++---- apps/challenges/models.py | 4 ++-- apps/challenges/serializers.py | 3 +-- apps/challenges/views.py | 1 - apps/jobs/utils.py | 20 ++++++++++++++++--- frontend/src/js/controllers/challengeCtrl.js | 4 ++-- .../src/views/web/challenge/leaderboard.html | 2 +- 7 files changed, 27 insertions(+), 15 deletions(-) rename apps/challenges/migrations/{0085_add_enable_multi_metric_leaderboard_field.py => 0086_add_is_multi_metric_leaderboard_field.py} (51%) diff --git a/apps/challenges/migrations/0085_add_enable_multi_metric_leaderboard_field.py b/apps/challenges/migrations/0086_add_is_multi_metric_leaderboard_field.py similarity index 51% rename from apps/challenges/migrations/0085_add_enable_multi_metric_leaderboard_field.py rename to apps/challenges/migrations/0086_add_is_multi_metric_leaderboard_field.py index fd46efe4e2..61d0e6e228 100644 --- a/apps/challenges/migrations/0085_add_enable_multi_metric_leaderboard_field.py +++ b/apps/challenges/migrations/0086_add_is_multi_metric_leaderboard_field.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.13 on 2021-06-20 14:33 +# Generated by Django 2.2.20 on 2021-08-07 14:19 from django.db import migrations, models @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ("challenges", "0084_challenge_is_static_dataset_code_upload"), + ("challenges", "0085_challenge_submission_time_limit"), ] operations = [ migrations.AddField( - model_name="challenge", + model_name="challengephasesplit", name="is_multi_metric_leaderboard", - field=models.BooleanField(default=False), + field=models.BooleanField(default=True), ), ] diff --git a/apps/challenges/models.py b/apps/challenges/models.py index 1fe124a710..3732aa7ec2 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -173,8 +173,6 @@ def __init__(self, *args, **kwargs): desired_worker_instance = models.IntegerField( null=True, blank=True, default=1 ) - # Allow ordering leaderboard by all metrics - is_multi_metric_leaderboard = models.BooleanField(default=False) class Meta: app_label = "challenges" @@ -404,6 +402,8 @@ class ChallengePhaseSplit(TimeStampedModel): is_leaderboard_order_descending = models.BooleanField(default=True) show_leaderboard_by_latest_submission = models.BooleanField(default=False) show_execution_time = models.BooleanField(default=False) + # Allow ordering leaderboard by all metrics + is_multi_metric_leaderboard = models.BooleanField(default=True) def __str__(self): return "{0} : {1}".format( diff --git a/apps/challenges/serializers.py b/apps/challenges/serializers.py index 2d7d394ff0..d6047606d7 100644 --- a/apps/challenges/serializers.py +++ b/apps/challenges/serializers.py @@ -158,10 +158,10 @@ class Meta: "show_leaderboard_by_latest_submission", "show_execution_time", "leaderboard_schema", + "is_multi_metric_leaderboard", ) def get_leaderboard_schema(self, obj): - print(obj.leaderboard.schema) return obj.leaderboard.schema def get_dataset_split_name(self, obj): @@ -268,7 +268,6 @@ class Meta: "max_worker_instance", "min_worker_instance", "desired_worker_instance", - "is_multi_metric_leaderboard", ) diff --git a/apps/challenges/views.py b/apps/challenges/views.py index 1397f6142b..89d670f4d8 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -1246,7 +1246,6 @@ def create_challenge_using_zip_file(request, challenge_host_team_pk): challenge_phase_data[ field ] = challenge_phase_data_from_hosts.get(field) - try: with transaction.atomic(): serializer = ZipChallengeSerializer( diff --git a/apps/jobs/utils.py b/apps/jobs/utils.py index d82e04a187..306a56deb8 100644 --- a/apps/jobs/utils.py +++ b/apps/jobs/utils.py @@ -254,6 +254,9 @@ def calculate_distinct_sorted_leaderboard_data( # Get the default order by key to rank the entries on the leaderboard default_order_by = None + is_leaderboard_order_descending = ( + challenge_phase_split.is_leaderboard_order_descending + ) try: default_order_by = leaderboard.schema["default_order_by"] except KeyError: @@ -271,6 +274,19 @@ def calculate_distinct_sorted_leaderboard_data( } return response_data, status.HTTP_400_BAD_REQUEST + leaderboard_schema = leaderboard.schema + if ( + leaderboard_schema.get("metadata") is not None + and leaderboard_schema.get("metadata").get(default_order_by) + is not None + ): + is_leaderboard_order_descending = ( + leaderboard_schema["metadata"][default_order_by].get( + "sort_ascending" + ) + is False + ) + # Exclude the submissions done by members of the host team # while populating leaderboard challenge_hosts_emails = ( @@ -408,9 +424,7 @@ def calculate_distinct_sorted_leaderboard_data( float(k["filtering_score"]), float(-k["filtering_error"]), ), - reverse=True - if challenge_phase_split.is_leaderboard_order_descending - else False, + reverse=True if is_leaderboard_order_descending else False, ) distinct_sorted_leaderboard_data = [] team_list = [] diff --git a/frontend/src/js/controllers/challengeCtrl.js b/frontend/src/js/controllers/challengeCtrl.js index 0e49456b9d..85a4691bd0 100644 --- a/frontend/src/js/controllers/challengeCtrl.js +++ b/frontend/src/js/controllers/challengeCtrl.js @@ -24,7 +24,7 @@ vm.projectUrl = ""; vm.publicationUrl = ""; vm.isPublicSubmission = null; - vm.isMultiMetricLeaderboardEnabled = false; + vm.isMultiMetricLeaderboardEnabled = {}; vm.wrnMsg = {}; vm.page = {}; vm.isParticipated = false; @@ -316,7 +316,6 @@ vm.isRegistrationOpen = details.is_registration_open; vm.approved_by_admin = details.approved_by_admin; vm.isRemoteChallenge = details.remote_evaluation; - vm.isMultiMetricLeaderboardEnabled = details.is_multi_metric_leaderboard; vm.getTeamName(vm.challengeId); if (vm.page.image === null) { @@ -833,6 +832,7 @@ vm.phaseSplits[i].showPrivate = true; vm.showPrivateIds.push(vm.phaseSplits[i].id); } + vm.isMultiMetricLeaderboardEnabled[vm.phaseSplits[i].id] = vm.phaseSplits[i].is_multi_metric_leaderboard; vm.phaseSplitLeaderboardSchema[vm.phaseSplits[i].id] = vm.phaseSplits[i].leaderboard_schema; } utilities.hideLoader(); diff --git a/frontend/src/views/web/challenge/leaderboard.html b/frontend/src/views/web/challenge/leaderboard.html index bd477757ca..8657a9d25f 100644 --- a/frontend/src/views/web/challenge/leaderboard.html +++ b/frontend/src/views/web/challenge/leaderboard.html @@ -40,7 +40,7 @@
Leaderboard
-
+
From f5688db4fabf1f547552d5188acadb7683125a64 Mon Sep 17 00:00:00 2001 From: Ram81 Date: Sat, 7 Aug 2021 23:02:27 +0530 Subject: [PATCH 4/5] Fix tests --- tests/unit/challenges/test_views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/challenges/test_views.py b/tests/unit/challenges/test_views.py index 32210fd59d..b199bddd0b 100644 --- a/tests/unit/challenges/test_views.py +++ b/tests/unit/challenges/test_views.py @@ -2962,6 +2962,8 @@ def test_get_challenge_phase_split(self): "visibility": self.challenge_phase_split.visibility, "show_leaderboard_by_latest_submission": self.challenge_phase_split.show_leaderboard_by_latest_submission, "show_execution_time": False, + "leaderboard_schema": self.challenge_phase_split.leaderboard.schema, + "is_multi_metric_leaderboard": self.challenge_phase_split.is_multi_metric_leaderboard, } ] self.client.force_authenticate(user=self.participant_user) @@ -2999,6 +3001,8 @@ def test_get_challenge_phase_split_when_user_is_challenge_host(self): "visibility": self.challenge_phase_split.visibility, "show_leaderboard_by_latest_submission": self.challenge_phase_split.show_leaderboard_by_latest_submission, "show_execution_time": False, + "leaderboard_schema": self.challenge_phase_split.leaderboard.schema, + "is_multi_metric_leaderboard": self.challenge_phase_split.is_multi_metric_leaderboard, }, { "id": self.challenge_phase_split_host.id, @@ -3009,6 +3013,8 @@ def test_get_challenge_phase_split_when_user_is_challenge_host(self): "visibility": self.challenge_phase_split_host.visibility, "show_leaderboard_by_latest_submission": self.challenge_phase_split_host.show_leaderboard_by_latest_submission, "show_execution_time": False, + "leaderboard_schema": self.challenge_phase_split_host.leaderboard.schema, + "is_multi_metric_leaderboard": self.challenge_phase_split_host.is_multi_metric_leaderboard, }, ] self.client.force_authenticate(user=self.user) From ec882e0b51273fcfd1c61c515432918cf4d77ad4 Mon Sep 17 00:00:00 2001 From: Ram81 Date: Tue, 17 Aug 2021 21:59:19 -0400 Subject: [PATCH 5/5] Highlight order_by column --- frontend/src/views/web/challenge/leaderboard.html | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/web/challenge/leaderboard.html b/frontend/src/views/web/challenge/leaderboard.html index 8657a9d25f..af1c13cb54 100644 --- a/frontend/src/views/web/challenge/leaderboard.html +++ b/frontend/src/views/web/challenge/leaderboard.html @@ -104,8 +104,10 @@
Leaderboard
- {{key}} (↓) {{challenge.getLabelDescription(key)}} - {{key}} (↑) {{challenge.getLabelDescription(key)}} + {{key}} (↓) {{challenge.getLabelDescription(key)}} + {{key}} (↑) {{challenge.getLabelDescription(key)}} + {{key}} (↓) {{challenge.getLabelDescription(key)}} + {{key}} (↑) {{challenge.getLabelDescription(key)}} @@ -161,7 +163,8 @@
Leaderboard
ng-if="key.submission__is_verified_by_host">
- {{score | number : challenge.selectedPhaseSplit.leaderboard_decimal_precision}} + {{score | number : challenge.selectedPhaseSplit.leaderboard_decimal_precision}} + {{score | number : challenge.selectedPhaseSplit.leaderboard_decimal_precision}} ±