From d62e9e3d271561ed305f8d2a364b927a0ff344be Mon Sep 17 00:00:00 2001 From: Kaustubh Maske Patil <37668193+nikochiko@users.noreply.github.com> Date: Sun, 8 Dec 2019 14:08:19 +0530 Subject: [PATCH 01/34] Worker: Add tests for load_challenge_and_return_max_submissions in worker(#2497) * Added test for load_challenge_and_return_max_submissions * Added test for submission worker [`load_challenge_and_return_max_submissions`](https://github.com/Cloud-CV/EvalAI/blob/master/scripts/workers/submission_worker.py#L648) * According to [GCI-task](https://codein.withgoogle.com/dashboard/task-instances/5189522401263616/) * Update test_submission_worker.py * Use mock * Removed attempt to access database which caused Travis build to fail. * Fix Travis CI errors * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py * Revert "Update test_submission_worker.py" This reverts commit c602515ac0479f97b4f8427bfbafec926d114a20. * Update test_submission_worker.py * Update test_submission_worker.py * Update test_submission_worker.py --- tests/unit/worker/test_submission_worker.py | 54 ++++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/tests/unit/worker/test_submission_worker.py b/tests/unit/worker/test_submission_worker.py index 05ebf3615b..7b5597b033 100644 --- a/tests/unit/worker/test_submission_worker.py +++ b/tests/unit/worker/test_submission_worker.py @@ -1,21 +1,31 @@ import boto3 +import mock import os import shutil import tempfile +from datetime import timedelta from moto import mock_sqs from os.path import join -from unittest import TestCase +from django.contrib.auth.models import User +from django.core.files.uploadedfile import SimpleUploadedFile +from django.utils import timezone + +from rest_framework.test import APITestCase + +from challenges.models import Challenge +from hosts.models import ChallengeHostTeam from scripts.workers.submission_worker import ( create_dir, create_dir_as_python_package, + load_challenge_and_return_max_submissions, return_file_url_per_environment, get_or_create_sqs_queue ) -class BaseAPITestClass(TestCase): +class BaseAPITestClass(APITestCase): def setUp(self): self.BASE_TEMP_DIR = tempfile.mkdtemp() self.temp_directory = join(self.BASE_TEMP_DIR, "temp_dir") @@ -30,6 +40,32 @@ def setUp(self): aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"), aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"), ) + self.user = User.objects.create( + username="someuser", + email="user@test.com", + password="secret_password", + ) + self.challenge_host_team = ChallengeHostTeam.objects.create( + team_name="Test Challenge Host Team", created_by=self.user + ) + self.challenge = Challenge.objects.create( + title="Test Challenge", + description="Description for test challenge", + terms_and_conditions="Terms and conditions for test challenge", + submission_guidelines="Submission guidelines for test challenge", + creator=self.challenge_host_team, + start_date=timezone.now() - timedelta(days=2), + end_date=timezone.now() + timedelta(days=1), + published=False, + enable_forum=True, + anonymous_leaderboard=False, + max_concurrent_submission_evaluation=100, + evaluation_script=SimpleUploadedFile( + "test_sample_file.txt", + b"Dummy file content", + content_type="text/plain", + ), + ) def test_create_dir(self): create_dir(self.temp_directory) @@ -45,6 +81,20 @@ def test_return_file_url_per_environment(self): returned_url = return_file_url_per_environment(self.url) self.assertEqual(returned_url, "http://testserver/test/url") + @mock.patch("scripts.workers.submission_worker.load_challenge") + def test_load_challenge_and_return_max_submissions(self, mocked_load_challenge): + q_params = {"pk": self.challenge.pk} + response = load_challenge_and_return_max_submissions(q_params) + mocked_load_challenge.assert_called_with(self.challenge) + self.assertEqual(response, (self.challenge.max_concurrent_submission_evaluation, self.challenge)) + + @mock.patch("scripts.workers.submission_worker.logger.exception") + def test_load_challenge_and_return_max_submissions_when_challenge_does_not_exist(self, mock_logger): + non_existing_challenge_pk = self.challenge.pk + 1 + with self.assertRaises(Challenge.DoesNotExist): + load_challenge_and_return_max_submissions({"pk": non_existing_challenge_pk}) + mock_logger.assert_called_with("Challenge with pk {} doesn't exist".format(non_existing_challenge_pk)) + @mock_sqs() def test_get_or_create_sqs_queue_for_existing_queue(self): self.sqs_client.create_queue(QueueName="test_queue") From 0f52da4654040baf77d98938d0d4684783884d01 Mon Sep 17 00:00:00 2001 From: Sanjeev Singh Date: Mon, 16 Dec 2019 03:05:50 +0530 Subject: [PATCH 02/34] Backend: Fix sending participant invite to a user if user's email is in blocked/banned emails(#2493) * Add check to stop sending invite to participant team if they user's email is in blocked email domains - Added checks in `challenges/utils.py` for `blocked email domains` and `allowed email domains`. - Updated `add_participant_to_team` with these common checks. - Added checks in `invite_participant_to_team` api with these checks. * Renamed the checks name * Minor change * Add check for banned email ids * minor change * Minor changes * Added initial tests * Updated test_views * Updated test * Updated tests * Updated tests * Updated tests and api * Updated tests --- apps/challenges/utils.py | 17 +++ apps/challenges/views.py | 31 ++--- apps/participants/views.py | 54 +++++++++ tests/unit/participants/test_views.py | 165 +++++++++++++++++++++++++- 4 files changed, 248 insertions(+), 19 deletions(-) diff --git a/apps/challenges/utils.py b/apps/challenges/utils.py index 3294deb0fc..9e6e777228 100644 --- a/apps/challenges/utils.py +++ b/apps/challenges/utils.py @@ -189,3 +189,20 @@ def create_federated_user(name, repository, aws_keys): DurationSeconds=43200, ) return response + + +def is_user_in_allowed_email_domains(email, challenge_pk): + challenge = get_challenge_model(challenge_pk) + for domain in challenge.allowed_email_domains: + if domain.lower() in email.lower(): + return True + return False + + +def is_user_in_blocked_email_domains(email, challenge_pk): + challenge = get_challenge_model(challenge_pk) + for domain in challenge.blocked_email_domains: + domain = "@" + domain + if domain.lower() in email.lower(): + return True + return False diff --git a/apps/challenges/views.py b/apps/challenges/views.py index 4c3b4bfea3..7bfac4665e 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -52,6 +52,8 @@ get_challenge_phase_split_model, get_dataset_split_model, get_leaderboard_model, + is_user_in_allowed_email_domains, + is_user_in_blocked_email_domains ) from hosts.models import ChallengeHost, ChallengeHostTeam from hosts.utils import ( @@ -273,12 +275,7 @@ def add_participant_team_to_challenge( # Check if user is in allowed list. user_email = request.user.email if len(challenge.allowed_email_domains) > 0: - present = False - for domain in challenge.allowed_email_domains: - if domain.lower() in user_email.lower(): - present = True - break - if not present: + if not is_user_in_allowed_email_domains(user_email, challenge_pk): message = "Sorry, users with {} email domain(s) are only allowed to participate in this challenge." domains = "" for domain in challenge.allowed_email_domains: @@ -290,18 +287,16 @@ def add_participant_team_to_challenge( ) # Check if user is in blocked list. - for domain in challenge.blocked_email_domains: - domain = "@" + domain - if domain.lower() in user_email.lower(): - message = "Sorry, users with {} email domain(s) are not allowed to participate in this challenge." - domains = "" - for domain in challenge.blocked_email_domains: - domains = "{}{}{}".format(domains, "/", domain) - domains = domains[1:] - response_data = {"error": message.format(domains)} - return Response( - response_data, status=status.HTTP_406_NOT_ACCEPTABLE - ) + if is_user_in_blocked_email_domains(user_email, challenge_pk): + message = "Sorry, users with {} email domain(s) are not allowed to participate in this challenge." + domains = "" + for domain in challenge.blocked_email_domains: + domains = "{}{}{}".format(domains, "/", domain) + domains = domains[1:] + response_data = {"error": message.format(domains)} + return Response( + response_data, status=status.HTTP_406_NOT_ACCEPTABLE + ) # check to disallow the user if he is a Challenge Host for this challenge participant_team_user_ids = set( diff --git a/apps/participants/views.py b/apps/participants/views.py index 9629490563..43a2e16a58 100644 --- a/apps/participants/views.py +++ b/apps/participants/views.py @@ -17,6 +17,11 @@ from base.utils import paginated_queryset from challenges.models import Challenge from challenges.serializers import ChallengeSerializer +from challenges.utils import ( + get_challenge_model, + is_user_in_allowed_email_domains, + is_user_in_blocked_email_domains +) from hosts.utils import is_user_a_host_of_challenge from .models import Participant, ParticipantTeam @@ -197,6 +202,55 @@ def invite_participant_to_team(request, pk): } return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) + if len(team_participated_challenges) > 0: + for challenge_pk in team_participated_challenges: + challenge = get_challenge_model(challenge_pk) + + if len(challenge.banned_email_ids) > 0: + # Check if team participants emails are banned + for participant_email in participant_team.get_all_participants_email(): + if participant_email in challenge.banned_email_ids: + message = "You cannot invite as you're a part of {} team and it has been banned " + "from this challenge. Please contact the challenge host." + response_data = {"error": message.format(participant_team.team_name)} + return Response( + response_data, status=status.HTTP_406_NOT_ACCEPTABLE + ) + + # Check if invited user is banned + if email in challenge.banned_email_ids: + message = "You cannot invite as the invited user has been banned " + "from this challenge. Please contact the challenge host." + response_data = {"error": message} + return Response( + response_data, status=status.HTTP_406_NOT_ACCEPTABLE + ) + + # Check if user is in allowed list. + if len(challenge.allowed_email_domains) > 0: + if not is_user_in_allowed_email_domains(email, challenge_pk): + message = "Sorry, users with {} email domain(s) are only allowed to participate in this challenge." + domains = "" + for domain in challenge.allowed_email_domains: + domains = "{}{}{}".format(domains, "/", domain) + domains = domains[1:] + response_data = {"error": message.format(domains)} + return Response( + response_data, status=status.HTTP_406_NOT_ACCEPTABLE + ) + + # Check if user is in blocked list. + if is_user_in_blocked_email_domains(email, challenge_pk): + message = "Sorry, users with {} email domain(s) are not allowed to participate in this challenge." + domains = "" + for domain in challenge.blocked_email_domains: + domains = "{}{}{}".format(domains, "/", domain) + domains = domains[1:] + response_data = {"error": message.format(domains)} + return Response( + response_data, status=status.HTTP_406_NOT_ACCEPTABLE + ) + serializer = InviteParticipantToTeamSerializer( data=request.data, context={"participant_team": participant_team, "request": request}, diff --git a/tests/unit/participants/test_views.py b/tests/unit/participants/test_views.py index f7028f58d5..ce6aaec5cf 100644 --- a/tests/unit/participants/test_views.py +++ b/tests/unit/participants/test_views.py @@ -255,6 +255,34 @@ def test_particular_participant_team_delete(self): class InviteParticipantToTeamTest(BaseAPITestClass): def setUp(self): super(InviteParticipantToTeamTest, self).setUp() + + self.user1 = User.objects.create( + username="user1", + email="user1@platform.com", + password="user1_password", + ) + + EmailAddress.objects.create( + user=self.user1, + email="user1@platform.com", + primary=True, + verified=True, + ) + + self.participant_team1 = ParticipantTeam.objects.create( + team_name="Team A", created_by=self.user1 + ) + + self.participant1 = Participant.objects.create( + user=self.user1, + status=Participant.ACCEPTED, + team=self.participant_team1, + ) + + self.challenge_host_team = ChallengeHostTeam.objects.create( + team_name="Host Team 1", created_by=self.user1 + ) + self.data = {"email": self.invite_user.email} self.url = reverse_lazy( "participants:invite_participant_to_team", @@ -308,13 +336,148 @@ def test_invite_user_which_does_not_exist_to_team(self): def test_particular_participant_team_for_invite_does_not_exist(self): self.url = reverse_lazy( "participants:invite_participant_to_team", - kwargs={"pk": self.participant_team.pk + 1}, + kwargs={"pk": self.participant_team.pk + 2}, ) expected = {"error": "Participant Team does not exist"} response = self.client.post(self.url, {}) self.assertEqual(response.data, expected) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + def test_invite_when_team_participants_emails_are_banned(self): + self.challenge1 = Challenge.objects.create( + title="Test Challenge 1", + short_description="Short description for test challenge 1", + description="Description for test challenge 1", + terms_and_conditions="Terms and conditions for test challenge 1", + submission_guidelines="Submission guidelines for test challenge 1", + creator=self.challenge_host_team, + published=False, + is_registration_open=True, + enable_forum=True, + banned_email_ids=["user1@platform.com"], + leaderboard_description="Lorem ipsum dolor sit amet, consectetur adipiscing elit", + anonymous_leaderboard=False, + start_date=timezone.now() - timedelta(days=2), + end_date=timezone.now() + timedelta(days=1), + ) + + self.challenge1.participant_teams.add(self.participant_team1) + self.data = {"email": self.invite_user.email} + self.client.force_authenticate(user=self.user1) + self.url = reverse_lazy( + "participants:invite_participant_to_team", + kwargs={ + "pk": self.participant_team1.pk + }, + ) + + response = self.client.post(self.url, self.data) + message = "You cannot invite as you're a part of {} team and it has been banned " + "from this challenge. Please contact the challenge host." + expected = {"error": message.format(self.participant_team1.team_name)} + self.assertEqual(response.data, expected) + self.assertEqual(response.status_code, status.HTTP_406_NOT_ACCEPTABLE) + + def test_invite_when_invited_user_is_banned(self): + self.challenge1 = Challenge.objects.create( + title="Test Challenge 1", + short_description="Short description for test challenge 1", + description="Description for test challenge 1", + terms_and_conditions="Terms and conditions for test challenge 1", + submission_guidelines="Submission guidelines for test challenge 1", + creator=self.challenge_host_team, + published=False, + is_registration_open=True, + enable_forum=True, + banned_email_ids=["other@platform.com"], + leaderboard_description="Lorem ipsum dolor sit amet, consectetur adipiscing elit", + anonymous_leaderboard=False, + start_date=timezone.now() - timedelta(days=2), + end_date=timezone.now() + timedelta(days=1), + ) + self.challenge1.participant_teams.add(self.participant_team1) + self.data = {"email": self.invite_user.email} + self.client.force_authenticate(user=self.user1) + self.url = reverse_lazy( + "participants:invite_participant_to_team", + kwargs={ + "pk": self.participant_team1.pk + }, + ) + response = self.client.post(self.url, self.data) + message = "You cannot invite as the invited user has been banned " + "from this challenge. Please contact the challenge host." + expected = {"error": message} + self.assertEqual(response.data, expected) + self.assertEqual(response.status_code, status.HTTP_406_NOT_ACCEPTABLE) + + def test_invite_when_invited_user_is_in_blocked_domains(self): + self.challenge1 = Challenge.objects.create( + title="Test Challenge 1", + short_description="Short description for test challenge 1", + description="Description for test challenge 1", + terms_and_conditions="Terms and conditions for test challenge 1", + submission_guidelines="Submission guidelines for test challenge 1", + creator=self.challenge_host_team, + published=False, + is_registration_open=True, + enable_forum=True, + blocked_email_domains=["platform"], + leaderboard_description="Lorem ipsum dolor sit amet, consectetur adipiscing elit", + anonymous_leaderboard=False, + start_date=timezone.now() - timedelta(days=2), + end_date=timezone.now() + timedelta(days=1), + ) + self.challenge1.participant_teams.add(self.participant_team1) + self.data = {"email": self.invite_user.email} + self.client.force_authenticate(user=self.user1) + self.url = reverse_lazy( + "participants:invite_participant_to_team", + kwargs={ + "pk": self.participant_team1.pk + }, + ) + + response = self.client.post(self.url, self.data) + message = "Sorry, users with {} email domain(s) are not allowed to participate in this challenge." + expected = {"error": message.format("platform")} + self.assertEqual(response.data, expected) + self.assertEqual(response.status_code, status.HTTP_406_NOT_ACCEPTABLE) + + def test_invite_when_invited_user_is_not_in_allowed_domains(self): + self.challenge1 = Challenge.objects.create( + title="Test Challenge 1", + short_description="Short description for test challenge 1", + description="Description for test challenge 1", + terms_and_conditions="Terms and conditions for test challenge 1", + submission_guidelines="Submission guidelines for test challenge 1", + creator=self.challenge_host_team, + published=False, + is_registration_open=True, + enable_forum=True, + allowed_email_domains=["example1"], + leaderboard_description="Lorem ipsum dolor sit amet, consectetur adipiscing elit", + anonymous_leaderboard=False, + start_date=timezone.now() - timedelta(days=2), + end_date=timezone.now() + timedelta(days=1), + ) + self.challenge1.participant_teams.add(self.participant_team1) + self.data = {"email": self.invite_user.email} + self.client.force_authenticate(user=self.user1) + self.url = reverse_lazy( + "participants:invite_participant_to_team", + kwargs={ + "pk": self.participant_team1.pk + }, + ) + + response = self.client.post(self.url, self.data) + message = "Sorry, users with {} email domain(s) are only allowed to participate in this challenge." + expected = {"error": message.format("example1")} + + self.assertEqual(response.data, expected) + self.assertEqual(response.status_code, status.HTTP_406_NOT_ACCEPTABLE) + def test_invite_participant_to_team_when_user_cannot_be_invited(self): """ NOTE From cc11f7ac8d5180cf8ecb8604e205954cea70721e Mon Sep 17 00:00:00 2001 From: Andre Christoga Pramaditya Date: Mon, 16 Dec 2019 05:03:28 +0700 Subject: [PATCH 03/34] Docs: Fix GitHub name in the docs(#2514) --- docs/source/faq(developers).md | 2 +- frontend/src/views/web/get-involved.html | 2 +- frontend/src/views/web/profile.html | 2 +- frontend/src/views/web/update-profile.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/faq(developers).md b/docs/source/faq(developers).md index 0c3ef62934..e3d006a4ce 100644 --- a/docs/source/faq(developers).md +++ b/docs/source/faq(developers).md @@ -9,7 +9,7 @@ Alternatively, if you come across a new bug on the site, please file a new issue Please refer to [Technologies Used](https://evalai.readthedocs.io/en/latest/architecture.html) -#### Q. Where could I learn Github Commands? +#### Q. Where could I learn GitHub Commands? Refer to [GitHub Guide](https://help.github.com/articles/git-and-github-learning-resources/). diff --git a/frontend/src/views/web/get-involved.html b/frontend/src/views/web/get-involved.html index 2748dabe8f..c3d4c288c7 100644 --- a/frontend/src/views/web/get-involved.html +++ b/frontend/src/views/web/get-involved.html @@ -16,7 +16,7 @@
Report issues
EvalAI Google Group, or contact us at team@cloudcv.org.

Improving and maintaining the site
-

The EvalAI project is fully open source, and is maintained by a large community of volunteers on Github. +

The EvalAI project is fully open source, and is maintained by a large community of volunteers on GitHub. We are in need of coders and designers so if you would like to help out, please drop us a line! The best way to get started is to write us at team@cloudcv.org or ping us on our Gitter Channel.

diff --git a/frontend/src/views/web/profile.html b/frontend/src/views/web/profile.html index e810b8fb9f..9859d223cd 100644 --- a/frontend/src/views/web/profile.html +++ b/frontend/src/views/web/profile.html @@ -43,7 +43,7 @@
diff --git a/frontend/src/views/web/challenge/edit-challenge/edit-challenge-date.html b/frontend/src/views/web/challenge/edit-challenge/edit-challenge-date.html new file mode 100644 index 0000000000..57c1a35bb6 --- /dev/null +++ b/frontend/src/views/web/challenge/edit-challenge/edit-challenge-date.html @@ -0,0 +1,43 @@ +
+
+
+
+
Edit Challenge Start and End Date
+
+ Start date and time + + +
+
+ End date and time + + +
+
+
    +
  • + Cancel +
    • +
  • + +
    • +
+
+
+
+
+
From f21866f8fa8ef86967f254f4fdaad44e476f9c92 Mon Sep 17 00:00:00 2001 From: Kartik Verma Date: Tue, 24 Dec 2019 01:48:43 +0530 Subject: [PATCH 13/34] Remote Worker: Fix API endpoint to delete message from SQS queue(#2429) * Fix delete_submission_message_from_queue API * Changed behaviour of remote submission worker to delete queue message using updated contract * Resolve test for delete_message_from_sqs_queue_url * Refactored code to fix unit tests Co-authored-by: Rishabh Jain --- apps/jobs/urls.py | 2 +- apps/jobs/views.py | 7 ++--- scripts/workers/remote_submission_worker.py | 27 ++++++++++++++++--- tests/unit/remoteworker/test_remote_worker.py | 13 ++++++--- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/apps/jobs/urls.py b/apps/jobs/urls.py index 42f6dac14a..82418c0a2c 100644 --- a/apps/jobs/urls.py +++ b/apps/jobs/urls.py @@ -46,7 +46,7 @@ name="get_submissions_for_challenge", ), url( - r"^queues/(?P[\w-]+)/receipt/(?P[\w-]+)/$", + r"^queues/(?P[\w-]+)/$", views.delete_submission_message_from_queue, name="delete_submission_message_from_queue", ), diff --git a/apps/jobs/views.py b/apps/jobs/views.py index cc91fcd473..a2c7f99999 100644 --- a/apps/jobs/views.py +++ b/apps/jobs/views.py @@ -1129,11 +1129,11 @@ def get_submission_message_from_queue(request, queue_name): return Response(response_data, status=status.HTTP_400_BAD_REQUEST) -@api_view(["GET"]) +@api_view(["POST"]) @throttle_classes([UserRateThrottle]) @permission_classes((permissions.IsAuthenticated, HasVerifiedEmail)) @authentication_classes((ExpiringTokenAuthentication,)) -def delete_submission_message_from_queue(request, queue_name, receipt_handle): +def delete_submission_message_from_queue(request, queue_name): """ API to delete submission message from AWS SQS queue Arguments: @@ -1151,13 +1151,14 @@ def delete_submission_message_from_queue(request, queue_name, receipt_handle): return Response(response_data, status=status.HTTP_400_BAD_REQUEST) challenge_pk = challenge.pk + receipt_handle = request.data["receipt_handle"] if not is_user_a_host_of_challenge(request.user, challenge_pk): response_data = { "error": "Sorry, you are not authorized to access this resource" } return Response(response_data, status=status.HTTP_401_UNAUTHORIZED) - queue = get_sqs_queue_object() + queue = get_sqs_queue_object(queue_name) try: message = queue.Message(receipt_handle) message.delete() diff --git a/scripts/workers/remote_submission_worker.py b/scripts/workers/remote_submission_worker.py index c2ec074b7c..8a0789d8ca 100644 --- a/scripts/workers/remote_submission_worker.py +++ b/scripts/workers/remote_submission_worker.py @@ -44,7 +44,7 @@ EVALUATION_SCRIPTS = {} URLS = { "get_message_from_sqs_queue": "/api/jobs/challenge/queues/{}/", - "delete_message_from_sqs_queue": "/api/jobs/queues/{}/receipt/{}/", + "delete_message_from_sqs_queue": "/api/jobs/queues/{}/", "get_submission_by_pk": "/api/jobs/submission/{}", "get_challenge_phases_by_challenge_pk": "/api/challenges/{}/phases/", "get_challenge_by_queue_name": "/api/challenges/challenge/queues/{}/", @@ -378,6 +378,23 @@ def make_request(url, method, data=None): raise return response.json() + elif method == "POST": + try: + response = requests.post(url=url, headers=headers, data=data) + response.raise_for_status() + except requests.exceptions.RequestException: + logger.info( + "The worker is not able to establish connection with EvalAI" + ) + raise + except requests.exceptions.HTTPError: + logger.info( + "The request to URL {} is failed due to {}" + % (url, response.json()) + ) + raise + return response.json() + def get_message_from_sqs_queue(): url = URLS.get("get_message_from_sqs_queue").format(QUEUE_NAME) @@ -388,11 +405,13 @@ def get_message_from_sqs_queue(): def delete_message_from_sqs_queue(receipt_handle): url = URLS.get("delete_message_from_sqs_queue").format( - QUEUE_NAME, receipt_handle + QUEUE_NAME ) url = return_url_per_environment(url) - response = make_request(url, "GET") # noqa - return + response = make_request(url, "POST", data={ + "receipt_handle": receipt_handle + }) # noqa + return response def get_submission_by_pk(submission_pk): diff --git a/tests/unit/remoteworker/test_remote_worker.py b/tests/unit/remoteworker/test_remote_worker.py index c3e1de0dd2..c2f89fe02d 100644 --- a/tests/unit/remoteworker/test_remote_worker.py +++ b/tests/unit/remoteworker/test_remote_worker.py @@ -36,8 +36,8 @@ def make_request_url(self): def get_message_from_sqs_queue_url(self, queue_name): return "/api/jobs/challenge/queues/{}/".format(queue_name) - def delete_message_from_sqs_queue_url(self, queue_name, receipt_handle): - return "/api/jobs/queues/{}/receipt/{}/".format(queue_name, receipt_handle) + def delete_message_from_sqs_queue_url(self, queue_name): + return "/api/jobs/queues/{}/".format(queue_name) def get_submission_by_pk_url(self, submission_pk): return "/api/jobs/submission/{}".format(submission_pk) @@ -74,6 +74,10 @@ def test_make_request_patch(self, mock_make_request): make_request(self.url, "PATCH", data=self.data) mock_make_request.patch.assert_called_with(url=self.url, headers=self.headers, data=self.data) + def test_make_request_post(self, mock_make_request): + make_request(self.url, "POST", data=self.data) + mock_make_request.post.assert_called_with(url=self.url, headers=self.headers, data=self.data) + @mock.patch("scripts.workers.remote_submission_worker.QUEUE_NAME", "evalai_submission_queue") @mock.patch("scripts.workers.remote_submission_worker.return_url_per_environment") @@ -89,11 +93,12 @@ def test_get_message_from_sqs_queue(self, mock_make_request, mock_url): def test_delete_message_from_sqs_queue(self, mock_make_request, mock_url): test_receipt_handle = "MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw" - url = self.delete_message_from_sqs_queue_url("evalai_submission_queue", test_receipt_handle) + url = self.delete_message_from_sqs_queue_url("evalai_submission_queue") delete_message_from_sqs_queue(test_receipt_handle) mock_url.assert_called_with(url) url = mock_url(url) - mock_make_request.assert_called_with(url, "GET") + expected_data = {"receipt_handle": "MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw"} + mock_make_request.assert_called_with(url, "POST", data=expected_data) def test_get_challenge_by_queue_name(self, mock_make_request, mock_url): url = self.get_challenge_by_queue_name_url("evalai_submission_queue") From dcbc4f4077568d8e5d3b2e048f5817ff1cbde68b Mon Sep 17 00:00:00 2001 From: Kartik Verma Date: Tue, 24 Dec 2019 02:01:25 +0530 Subject: [PATCH 14/34] Backend: Add test environment URL field in phase model(#2392) * Added environment_url field in Phase Signed-off-by: vkartik97 <3920286+vkartik97@users.noreply.github.com> * Resolving Migrations conflicts Signed-off-by: vkartik97 <3920286+vkartik97@users.noreply.github.com> * Removing changes in seed script * Resolve conflicts in migrations * update error message when challenge isn't docker based Co-authored-by: Rishabh Jain --- ...dding_environment_url_for_rl_challenges.py | 21 ++++++++++++ apps/challenges/models.py | 2 ++ apps/challenges/urls.py | 5 +++ apps/challenges/views.py | 32 +++++++++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 apps/challenges/migrations/0058_adding_environment_url_for_rl_challenges.py diff --git a/apps/challenges/migrations/0058_adding_environment_url_for_rl_challenges.py b/apps/challenges/migrations/0058_adding_environment_url_for_rl_challenges.py new file mode 100644 index 0000000000..5b2872a00b --- /dev/null +++ b/apps/challenges/migrations/0058_adding_environment_url_for_rl_challenges.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-07-17 09:25 +from __future__ import unicode_literals + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('challenges', '0058_add_show_leaderboard_by_latest_submission_field_in_challenge_phase_split_model'), + ] + + operations = [ + migrations.AddField( + model_name='challengephase', + name='environment_url', + field=models.CharField(max_length=2128, null=True, validators=[django.core.validators.URLValidator()]), + ), + ] diff --git a/apps/challenges/models.py b/apps/challenges/models.py index cfa3a4e22a..e1f952eea7 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from django.contrib.auth.models import User +from django.core.validators import URLValidator from django.db.models.signals import pre_save from django.dispatch import receiver from django.utils import timezone @@ -232,6 +233,7 @@ def __init__(self, *args, **kwargs): null=True, ) slug = models.SlugField(max_length=200, null=True, unique=True) + environment_url = models.CharField(validators=[URLValidator()], null=True, max_length=2128) # Max length of URL and tag is 2000 and 128 respectively class Meta: app_label = "challenges" diff --git a/apps/challenges/urls.py b/apps/challenges/urls.py index 61d9865bc1..3f44c5b8dd 100644 --- a/apps/challenges/urls.py +++ b/apps/challenges/urls.py @@ -151,4 +151,9 @@ views.get_challenge_phase_by_slug, name="get_challenge_phase_by_slug", ), + url( + r"^phase/environment/(?P[\w-]+)/$", + views.get_challenge_phase_environment_url, + name="get_challenge_phase_environment_url", + ), ] diff --git a/apps/challenges/views.py b/apps/challenges/views.py index 2fb695c462..c75d064d47 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -2162,3 +2162,35 @@ def get_challenge_phase_by_slug(request, slug): serializer = ChallengePhaseSerializer(challenge_phase) response_data = serializer.data return Response(response_data, status=status.HTTP_200_OK) + + +@api_view(["GET"]) +@throttle_classes([UserRateThrottle]) +@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail)) +@authentication_classes((ExpiringTokenAuthentication,)) +def get_challenge_phase_environment_url(request, slug): + """ + Returns environment image url and tag required for RL challenge evaluation + """ + try: + challenge_phase = ChallengePhase.objects.get(slug=slug) + except ChallengePhase.DoesNotExist: + response_data = { + "error": "Challenge phase with slug {} does not exist".format(slug) + } + return Response(response_data, status=status.HTTP_400_BAD_REQUEST) + challenge = get_challenge_model(challenge_phase.challenge.pk) + if not is_user_a_host_of_challenge(request.user, challenge.pk): + response_data = { + "error": "Sorry, you are not authorized to access test environment URL." + } + return Response(response_data, status=status.HTTP_400_BAD_REQUEST) + if not challenge.is_docker_based: + response_data = { + "error": "The challenge doesn't require uploading Docker images, hence no test environment URL." + } + return Response(response_data, status=status.HTTP_400_BAD_REQUEST) + response_data = { + "environment_url": challenge_phase.environment_url + } + return Response(response_data, status=status.HTTP_200_OK) From f5adee083d939cb31940685bbe396786983dbccf Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Tue, 24 Dec 2019 09:20:44 -0500 Subject: [PATCH 15/34] Requirements: Add kubernetes python-client(#2535) --- requirements/common.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/common.txt b/requirements/common.txt index 78e039587c..14b88606f9 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -15,6 +15,7 @@ django-ses==0.8.5 docker-compose==1.21.0 drfdocs==0.0.11 drf-yasg==1.11.0 +kubernetes==10.0.1 moto==1.3.8 pika==0.10.0 pickleshare==0.7.4 From 4d495c51457c281b7009cbad5f93cbe495c987a4 Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Tue, 24 Dec 2019 12:34:01 -0500 Subject: [PATCH 16/34] Backend: Add blank=True in challenge phase environment URL (#2536) * Add blank=True for phase environment URL * Add migrations file --- ...0059_add_blank_in_phase_environment_url.py | 26 +++++++++++++++++++ apps/challenges/models.py | 22 ++++++++++------ 2 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 apps/challenges/migrations/0059_add_blank_in_phase_environment_url.py diff --git a/apps/challenges/migrations/0059_add_blank_in_phase_environment_url.py b/apps/challenges/migrations/0059_add_blank_in_phase_environment_url.py new file mode 100644 index 0000000000..2c111bf743 --- /dev/null +++ b/apps/challenges/migrations/0059_add_blank_in_phase_environment_url.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2019-12-24 17:28 +from __future__ import unicode_literals + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("challenges", "0058_adding_environment_url_for_rl_challenges") + ] + + operations = [ + migrations.AlterField( + model_name="challengephase", + name="environment_url", + field=models.CharField( + blank=True, + max_length=2128, + null=True, + validators=[django.core.validators.URLValidator()], + ), + ) + ] diff --git a/apps/challenges/models.py b/apps/challenges/models.py index e1f952eea7..ff975755c3 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -85,7 +85,7 @@ def __init__(self, *args, **kwargs): models.TextField(null=True, blank=True), default=[], blank=True, - null=True + null=True, ) remote_evaluation = models.BooleanField( default=False, verbose_name="Remote Evaluation", db_index=True @@ -123,11 +123,11 @@ def __init__(self, *args, **kwargs): max_length=20, verbose_name="evalai-cli version", null=True, blank=True ) # The number of active workers on Fargate of the challenge. - workers = models.IntegerField( - null=True, blank=True, default=None - ) + workers = models.IntegerField(null=True, blank=True, default=None) # The task definition ARN for the challenge, used for updating and creating service. - task_def_arn = models.CharField(null=True, blank=True, max_length=2048, default="") + task_def_arn = models.CharField( + null=True, blank=True, max_length=2048, default="" + ) class Meta: app_label = "challenges" @@ -171,7 +171,9 @@ def is_active(self): weak=False, ) signals.post_save.connect( - model_field_name(field_name="evaluation_script")(restart_workers_signal_callback), + model_field_name(field_name="evaluation_script")( + restart_workers_signal_callback + ), sender=Challenge, weak=False, ) @@ -233,7 +235,9 @@ def __init__(self, *args, **kwargs): null=True, ) slug = models.SlugField(max_length=200, null=True, unique=True) - environment_url = models.CharField(validators=[URLValidator()], null=True, max_length=2128) # Max length of URL and tag is 2000 and 128 respectively + environment_url = models.CharField( + validators=[URLValidator()], null=True, blank=True, max_length=2128 + ) # Max length of URL and tag is 2000 and 128 respectively class Meta: app_label = "challenges" @@ -282,7 +286,9 @@ def save(self, *args, **kwargs): weak=False, ) signals.post_save.connect( - model_field_name(field_name="test_annotation")(restart_workers_signal_callback), + model_field_name(field_name="test_annotation")( + restart_workers_signal_callback + ), sender=ChallengePhase, weak=False, ) From 9fd7cba86ad95b9c80d318a8e9a2fc410bf229bd Mon Sep 17 00:00:00 2001 From: Kaustubh Maske Patil <37668193+nikochiko@users.noreply.github.com> Date: Thu, 26 Dec 2019 23:19:57 +0530 Subject: [PATCH 17/34] Worker: Refactor code and add tests for download_and_extract_zip_file function(#2532) * Refactored download_and_extract_zip_file * Correct names * Fix flake8 issues * Add tests for extract_zip_file and delete_zip_file * Fix import issue * Remove unused imports * Revert "Remove unused imports" This reverts commit 0bb5423d7d5b87cc6b04a03b9feb2bac0578000b. * Merge with PR#2531 * Remove unrequired changes --- scripts/workers/submission_worker.py | 43 ++++++++--- tests/unit/worker/test_submission_worker.py | 84 +++++++++++++++++++++ 2 files changed, 115 insertions(+), 12 deletions(-) diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py index 7b0f501014..09001203f8 100644 --- a/scripts/workers/submission_worker.py +++ b/scripts/workers/submission_worker.py @@ -130,6 +130,35 @@ def download_and_extract_file(url, download_location): f.write(chunk) +def extract_zip_file(download_location, extract_location): + """ + Helper function to extract zip file + Params: + * `download_location`: Location of zip file + * `extract_location`: Location of directory for extracted file + """ + zip_ref = zipfile.ZipFile(download_location, "r") + zip_ref.extractall(extract_location) + zip_ref.close() + + +def delete_zip_file(download_location): + """ + Helper function to remove zip file from location `download_location` + Params: + * `download_location`: Location of file to be removed. + """ + try: + os.remove(download_location) + except Exception as e: + logger.error( + "Failed to remove zip file {}, error {}".format( + download_location, e + ) + ) + traceback.print_exc() + + def download_and_extract_zip_file(url, download_location, extract_location): """ * Function to extract download a zip file, extract it and then removes the zip file. @@ -147,19 +176,9 @@ def download_and_extract_zip_file(url, download_location, extract_location): if chunk: f.write(chunk) # extract zip file - zip_ref = zipfile.ZipFile(download_location, "r") - zip_ref.extractall(extract_location) - zip_ref.close() + extract_zip_file(download_location, extract_location) # delete zip file - try: - os.remove(download_location) - except Exception as e: - logger.error( - "Failed to remove zip file {}, error {}".format( - download_location, e - ) - ) - traceback.print_exc() + delete_zip_file(download_location) def create_dir(directory): diff --git a/tests/unit/worker/test_submission_worker.py b/tests/unit/worker/test_submission_worker.py index 166f3a1e7f..ef499420f7 100644 --- a/tests/unit/worker/test_submission_worker.py +++ b/tests/unit/worker/test_submission_worker.py @@ -1,11 +1,14 @@ import boto3 import mock import os +import responses import shutil import tempfile +import zipfile from datetime import timedelta from moto import mock_sqs +from io import BytesIO from os.path import join from django.contrib.auth.models import User @@ -24,6 +27,9 @@ from scripts.workers.submission_worker import ( create_dir, create_dir_as_python_package, + delete_zip_file, + download_and_extract_zip_file, + extract_zip_file, extract_submission_data, load_challenge_and_return_max_submissions, return_file_url_per_environment, @@ -220,3 +226,81 @@ def test_get_or_create_sqs_queue_for_non_existing_queue(self): queue_url = self.sqs_client.get_queue_url(QueueName='test_queue_2')['QueueUrl'] self.assertTrue(queue_url) self.sqs_client.delete_queue(QueueUrl=queue_url) + + +class DownloadAndExtractZipFileTest(BaseAPITestClass): + def setUp(self): + super(DownloadAndExtractZipFileTest, self).setUp() + self.zip_name = "test" + self.req_url = "{}/{}".format(self.testserver, self.zip_name) + self.extract_location = join(self.BASE_TEMP_DIR, "test-dir") + self.download_location = join(self.extract_location, "{}.zip".format(self.zip_name)) + create_dir(self.extract_location) + + self.file_name = "test_file.txt" + self.file_content = b"file_content" + + self.zip_file = BytesIO() + with zipfile.ZipFile(self.zip_file, mode="w", compression=zipfile.ZIP_DEFLATED) as zipper: + zipper.writestr(self.file_name, self.file_content) + + def tearDown(self): + if os.path.exists(self.extract_location): + shutil.rmtree(self.extract_location) + + @responses.activate + @mock.patch("scripts.workers.submission_worker.delete_zip_file") + @mock.patch("scripts.workers.submission_worker.extract_zip_file") + def test_download_and_extract_zip_file_success(self, mock_extract_zip, mock_delete_zip): + responses.add( + responses.GET, self.req_url, + content_type="application/zip", + body=self.zip_file.getvalue(), status=200) + + download_and_extract_zip_file(self.req_url, self.download_location, self.extract_location) + + with open(self.download_location, "rb") as downloaded: + self.assertEqual(downloaded.read(), self.zip_file.getvalue()) + mock_extract_zip.assert_called_with(self.download_location, self.extract_location) + mock_delete_zip.assert_called_with(self.download_location) + + @responses.activate + @mock.patch("scripts.workers.submission_worker.logger.error") + def test_download_and_extract_zip_file_when_download_fails(self, mock_logger): + e = "Error description" + responses.add( + responses.GET, self.req_url, + body=Exception(e)) + error_message = "Failed to fetch file from {}, error {}".format(self.req_url, e) + + download_and_extract_zip_file(self.req_url, self.download_location, self.extract_location) + + mock_logger.assert_called_with(error_message) + + def test_extract_zip_file(self): + with open(self.download_location, "wb") as zf: + zf.write(self.zip_file.getvalue()) + + extract_zip_file(self.download_location, self.extract_location) + extracted_path = join(self.extract_location, self.file_name) + self.assertTrue(os.path.exists(extracted_path)) + with open(extracted_path, "rb") as extracted: + self.assertEqual(extracted.read(), self.file_content) + + def test_delete_zip_file(self): + with open(self.download_location, "wb") as zf: + zf.write(self.zip_file.getvalue()) + + delete_zip_file(self.download_location) + + self.assertFalse(os.path.exists(self.download_location)) + + @mock.patch("scripts.workers.submission_worker.logger.error") + @mock.patch("scripts.workers.submission_worker.os.remove") + def test_delete_zip_file_error(self, mock_remove, mock_logger): + e = "Error description" + mock_remove.side_effect = Exception(e) + error_message = "Failed to remove zip file {}, error {}".format(self.download_location, e) + + delete_zip_file(self.download_location) + mock_logger.assert_called_with(error_message) From 7961addc9e6717ce962ea74e275b6d1070f6a601 Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Thu, 26 Dec 2019 17:30:47 -0500 Subject: [PATCH 18/34] RL Worker: Minor code and indentation fixes(#2548) --- scripts/workers/rl_submission_worker.py | 77 ++++++++++++------------- scripts/workers/worker_util.py | 25 ++++---- 2 files changed, 48 insertions(+), 54 deletions(-) diff --git a/scripts/workers/rl_submission_worker.py b/scripts/workers/rl_submission_worker.py index 70203c899c..6089c1ab8b 100644 --- a/scripts/workers/rl_submission_worker.py +++ b/scripts/workers/rl_submission_worker.py @@ -3,9 +3,7 @@ import signal import time -from .worker_util import ( - EvalAI_Interface -) +from worker_util import EvalAI_Interface from kubernetes import client, config @@ -23,59 +21,57 @@ def exit_gracefully(self, signum, frame): logger = logging.getLogger(__name__) -AUTH_TOKEN = os.environ.get("AUTH_TOKEN", "x") -DJANGO_SERVER = os.environ.get("DJANGO_SERVER", "http://localhost") -DJANGO_SERVER_PORT = os.environ.get("DJANGO_SERVER_PORT", "8000") +AUTH_TOKEN = os.environ.get("AUTH_TOKEN", "auth_token") +EVALAI_API_SERVER = os.environ.get( + "EVALAI_API_SERVER", "http://localhost:8000" +) QUEUE_NAME = os.environ.get("QUEUE_NAME", "evalai_submission_queue") -ENVIRONMENT_IMAGE = os.environ.get("ENVIRONMENT_IMAGE", "x:tag") +ENVIRONMENT_IMAGE = os.environ.get("ENVIRONMENT_IMAGE", "image_name:tag") MESSAGE_FETCH_DEPLAY = int(os.environ.get("MESSAGE_FETCH_DEPLAY", "5")) def create_deployment_object(image, submission, message): - PYTHONUNBUFFERED_ENV = client.V1EnvVar( - name="PYTHONUNBUFFERED", - value="1", - ) - AUTH_TOKEN_ENV = client.V1EnvVar( - name="AUTH_TOKEN", - value=AUTH_TOKEN - ) - DJANGO_SERVER_ENV = client.V1EnvVar( - name="DJANGO_SERVER", - value=DJANGO_SERVER - ) - MESSAGE_BODY_ENV = client.V1EnvVar( - name="BODY", - value=str(message) + PYTHONUNBUFFERED_ENV = client.V1EnvVar(name="PYTHONUNBUFFERED", value="1") + AUTH_TOKEN_ENV = client.V1EnvVar(name="AUTH_TOKEN", value=AUTH_TOKEN) + EVALAI_API_SERVER_ENV = client.V1EnvVar( + name="EVALAI_API_SERVER", value=EVALAI_API_SERVER ) + MESSAGE_BODY_ENV = client.V1EnvVar(name="BODY", value=str(message)) agent_container = client.V1Container( - name="agent", - image=image, - env=[PYTHONUNBUFFERED_ENV] + name="agent", image=image, env=[PYTHONUNBUFFERED_ENV] ) environment_container = client.V1Container( name="environment", image=ENVIRONMENT_IMAGE, - env=[PYTHONUNBUFFERED_ENV, AUTH_TOKEN_ENV, DJANGO_SERVER_ENV, MESSAGE_BODY_ENV] + env=[ + PYTHONUNBUFFERED_ENV, + AUTH_TOKEN_ENV, + EVALAI_API_SERVER_ENV, + MESSAGE_BODY_ENV, + ], ) template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={"app": "evaluation"}), - spec=client.V1PodSpec(containers=[environment_container, agent_container])) + spec=client.V1PodSpec( + containers=[environment_container, agent_container] + ), + ) spec = client.ExtensionsV1beta1DeploymentSpec( - replicas=1, - template=template) + replicas=1, template=template + ) deployment = client.ExtensionsV1beta1Deployment( api_version="extensions/v1beta1", kind="Deployment", metadata=client.V1ObjectMeta(name="submission-{0}".format(submission)), - spec=spec) + spec=spec, + ) return deployment def create_deployment(api_instance, deployment): api_response = api_instance.create_namespaced_deployment( - body=deployment, - namespace="default") + body=deployment, namespace="default" + ) logger.info("Deployment created. status='%s'" % str(api_response.status)) @@ -90,9 +86,7 @@ def process_submission_callback(message, api): logger.info(submission_data) api.update_submission_status(submission_data, message["challenge_pk"]) dep = create_deployment_object( - message["submitted_image_uri"], - message["submission_pk"], - message + message["submitted_image_uri"], message["submission_pk"], message ) create_deployment(extensions_v1beta1, dep) @@ -100,11 +94,14 @@ def process_submission_callback(message, api): def main(): api = EvalAI_Interface( AUTH_TOKEN=AUTH_TOKEN, - DJANGO_SERVER=DJANGO_SERVER, - DJANGO_SERVER_PORT=DJANGO_SERVER_PORT, + EVALAI_API_SERVER=EVALAI_API_SERVER, QUEUE_NAME=QUEUE_NAME, ) - logger.info("String RL Worker for {}".format(api.get_challenge_by_queue_name()["title"])) + logger.info( + "String RL Worker for {}".format( + api.get_challenge_by_queue_name()["title"] + ) + ) killer = GracefulKiller() while True: logger.info( @@ -128,7 +125,9 @@ def main(): "Processing message body: {}".format(message_body) ) process_submission_callback(message_body, api) - api.delete_message_from_sqs_queue(message.get("receipt_handle")) + api.delete_message_from_sqs_queue( + message.get("receipt_handle") + ) time.sleep(MESSAGE_FETCH_DEPLAY) if killer.kill_now: break diff --git a/scripts/workers/worker_util.py b/scripts/workers/worker_util.py index 1facc50e2a..d31846e9a1 100644 --- a/scripts/workers/worker_util.py +++ b/scripts/workers/worker_util.py @@ -16,18 +16,9 @@ class EvalAI_Interface: - - def __init__( - self, - AUTH_TOKEN, - DJANGO_SERVER, - DJANGO_SERVER_PORT, - QUEUE_NAME, - - ): + def __init__(self, AUTH_TOKEN, EVALAI_API_SERVER, QUEUE_NAME): self.AUTH_TOKEN = AUTH_TOKEN - self.DJANGO_SERVER = DJANGO_SERVER - self.DJANGO_SERVER_PORT = DJANGO_SERVER_PORT + self.EVALAI_API_SERVER = EVALAI_API_SERVER self.QUEUE_NAME = QUEUE_NAME def get_request_headers(self): @@ -37,7 +28,9 @@ def get_request_headers(self): def make_request(self, url, method, data=None): headers = self.get_request_headers() try: - response = requests.request(method=method, url=url, headers=headers) + response = requests.request( + method=method, url=url, headers=headers, data=data + ) response.raise_for_status() except requests.exceptions.RequestException: logger.info( @@ -47,7 +40,7 @@ def make_request(self, url, method, data=None): return response.json() def return_url_per_environment(self, url): - base_url = "{0}:{1}".format(self.DJANGO_SERVER, self.DJANGO_SERVER_PORT) + base_url = "{0}".format(self.EVALAI_API_SERVER) url = "{0}{1}".format(base_url, url) return url @@ -63,7 +56,7 @@ def delete_message_from_sqs_queue(self, receipt_handle): ) url = self.return_url_per_environment(url) response = self.make_request(url, "GET") # noqa - return + return response.status_code def get_submission_by_pk(self, submission_pk): url = URLS.get("get_submission_by_pk").format(submission_pk) @@ -72,7 +65,9 @@ def get_submission_by_pk(self, submission_pk): return response def get_challenge_phases_by_challenge_pk(self, challenge_pk): - url = URLS.get("get_challenge_phases_by_challenge_pk").format(challenge_pk) + url = URLS.get("get_challenge_phases_by_challenge_pk").format( + challenge_pk + ) url = self.return_url_per_environment(url) response = self.make_request(url, "GET") return response From b95877c37eb93a1671c476eb9f4d86fb3c657739 Mon Sep 17 00:00:00 2001 From: Kaustubh Maske Patil <37668193+nikochiko@users.noreply.github.com> Date: Fri, 27 Dec 2019 22:46:39 +0530 Subject: [PATCH 19/34] Worker: Add tests for download_and_extract_file function(#2545) * Worker: Add test for download_and_extract_file * Fix order of responses args --- tests/unit/worker/test_submission_worker.py | 40 +++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/unit/worker/test_submission_worker.py b/tests/unit/worker/test_submission_worker.py index ef499420f7..a9dfe8b5d7 100644 --- a/tests/unit/worker/test_submission_worker.py +++ b/tests/unit/worker/test_submission_worker.py @@ -27,6 +27,7 @@ from scripts.workers.submission_worker import ( create_dir, create_dir_as_python_package, + download_and_extract_file, delete_zip_file, download_and_extract_zip_file, extract_zip_file, @@ -228,6 +229,45 @@ def test_get_or_create_sqs_queue_for_non_existing_queue(self): self.sqs_client.delete_queue(QueueUrl=queue_url) +class DownloadAndExtractFileTest(BaseAPITestClass): + def setUp(self): + super(DownloadAndExtractFileTest, self).setUp() + self.req_url = "{}{}".format(self.testserver, self.url) + self.file_content = b"file content" + + create_dir(self.temp_directory) + self.download_location = join(self.temp_directory, "dummy_file") + + def tearDown(self): + if os.path.exists(self.temp_directory): + shutil.rmtree(self.temp_directory) + + @responses.activate + def test_download_and_extract_file_success(self): + responses.add(responses.GET, self.req_url, + body=self.file_content, + content_type='application/octet-stream', + status=200) + + download_and_extract_file(self.req_url, self.download_location) + + self.assertTrue(os.path.exists(self.download_location)) + with open(self.download_location, "rb") as f: + self.assertEqual(f.read(), self.file_content) + + @responses.activate + @mock.patch("scripts.workers.submission_worker.logger.error") + def test_download_and_extract_file_when_download_fails(self, mock_logger): + error = "ExampleError: Example Error description" + responses.add(responses.GET, self.req_url, body=Exception(error)) + expected = "Failed to fetch file from {}, error {}".format(self.req_url, error) + + download_and_extract_file(self.req_url, self.download_location) + + mock_logger.assert_called_with(expected) + self.assertFalse(os.path.exists(self.download_location)) + + class DownloadAndExtractZipFileTest(BaseAPITestClass): def setUp(self): super(DownloadAndExtractZipFileTest, self).setUp() From 8de2555259a2ee6a71efb9d7cbec03bdfa9f8bbb Mon Sep 17 00:00:00 2001 From: Kaustubh Maske Patil <37668193+nikochiko@users.noreply.github.com> Date: Fri, 27 Dec 2019 23:36:28 +0530 Subject: [PATCH 20/34] Remote Worker: Add tests for download_and_extract_file function(#2550) * Add test for download_and_extract_file for remote worker * Fix missing import * Fix undefined variable --- tests/unit/remoteworker/test_remote_worker.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/unit/remoteworker/test_remote_worker.py b/tests/unit/remoteworker/test_remote_worker.py index c2f89fe02d..98296478e2 100644 --- a/tests/unit/remoteworker/test_remote_worker.py +++ b/tests/unit/remoteworker/test_remote_worker.py @@ -1,5 +1,6 @@ import mock import os +import responses import shutil import tempfile @@ -12,6 +13,7 @@ make_request, get_message_from_sqs_queue, delete_message_from_sqs_queue, + download_and_extract_file, get_submission_by_pk, get_challenge_phases_by_challenge_pk, get_challenge_by_queue_name, @@ -29,6 +31,7 @@ def setUp(self): self.challenge_phase_pk = 1 self.data = {"test": "data"} self.headers = {"Authorization": "Token test_token"} + self.testserver = "http://testserver" def make_request_url(self): return "/test/url" @@ -170,3 +173,42 @@ def test_create_dir_as_python_package(self): shutil.rmtree(self.temp_directory) self.assertFalse(os.path.exists(self.temp_directory)) + + +class DownloadAndExtractFileTest(BaseTestClass): + def setUp(self): + super(DownloadAndExtractFileTest, self).setUp() + self.req_url = "{}{}".format(self.testserver, self.make_request_url()) + self.file_content = b'file content' + + self.temp_directory = tempfile.mkdtemp() + self.download_location = join(self.temp_directory, "dummy_file") + + def tearDown(self): + if os.path.exists(self.temp_directory): + shutil.rmtree(self.temp_directory) + + @responses.activate + def test_download_and_extract_file_success(self): + responses.add(responses.GET, self.req_url, + body=self.file_content, + content_type='application/octet-stream', + status=200) + + download_and_extract_file(self.req_url, self.download_location) + + self.assertTrue(os.path.exists(self.download_location)) + with open(self.download_location, "rb") as f: + self.assertEqual(f.read(), self.file_content) + + @responses.activate + @mock.patch("scripts.workers.remote_submission_worker.logger.error") + def test_download_and_extract_file_when_download_fails(self, mock_logger): + error = "ExampleError: Example Error description" + responses.add(responses.GET, self.req_url, body=Exception(error)) + expected = "Failed to fetch file from {}, error {}".format(self.req_url, error) + + download_and_extract_file(self.req_url, self.download_location) + + mock_logger.assert_called_with(expected) + self.assertFalse(os.path.exists(self.download_location)) From 8680bcf102411a576bb891f15151b98bed58374b Mon Sep 17 00:00:00 2001 From: jayy <35180217+nsjcorps@users.noreply.github.com> Date: Sat, 28 Dec 2019 18:44:28 +0100 Subject: [PATCH 21/34] Frontend: Add check for editing challenge start/end date by challenge host(#2552) --- frontend/src/views/web/challenge/challenge-page.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/web/challenge/challenge-page.html b/frontend/src/views/web/challenge/challenge-page.html index af6d857c6d..8e6d025e92 100644 --- a/frontend/src/views/web/challenge/challenge-page.html +++ b/frontend/src/views/web/challenge/challenge-page.html @@ -48,7 +48,7 @@ {{ challenge.page.start_date | date:'medium' }}   - + @@ -59,7 +59,7 @@ {{ challenge.page.end_date | date:'medium' }}   - + From 3169c7fb3cecbdca17a3a92b8d7286d84a462769 Mon Sep 17 00:00:00 2001 From: Kartik Verma Date: Sun, 29 Dec 2019 01:26:43 +0530 Subject: [PATCH 22/34] RL Submission Worker: Add submitted_image_uri as a SQS message parameter for docker based challenges(#2376) * Script to configure Kube Config for EKS. Signed-off-by: Kartik Verma * API to handle Docker Based Submissions Signed-off-by: Kartik Verma * Modified tests for modified endpoint Signed-off-by: Kartik Verma * Fixed Tests Signed-off-by: Kartik Verma * Fix admin job submission test Signed-off-by: Kartik Verma * Fixing Flake8 Error Signed-off-by: Kartik Verma * EvalAI API Interface for Workers Signed-off-by: Kartik Verma * Delete redendent script * Delete redundent util * Add dertails for message dict in comments * Add details for submitted_image_uri key in message * Initialize message * Express intent clearly for usage for submitted_image_uri * Fix wrong assignment of variable * Add task to run submission for docker based challenge * Fix linting errors * Add behavior to re run docker based submission --- apps/jobs/admin.py | 24 +++++++++++++++---- apps/jobs/sender.py | 19 +++++++--------- apps/jobs/tasks.py | 6 ++++- apps/jobs/views.py | 43 +++++++++++++++++++++++++++++++---- tests/unit/jobs/test_views.py | 6 ++++- 5 files changed, 76 insertions(+), 22 deletions(-) diff --git a/apps/jobs/admin.py b/apps/jobs/admin.py index 3a8e046d31..7bc5c7fcac 100644 --- a/apps/jobs/admin.py +++ b/apps/jobs/admin.py @@ -1,13 +1,13 @@ import logging +import requests -from django.contrib import admin +from django.contrib import admin, messages from base.admin import ImportExportTimeStampedAdmin from .models import Submission from .sender import publish_submission_message - logger = logging.getLogger(__name__) @@ -69,9 +69,23 @@ def submit_job_to_worker(self, request, queryset): challenge_id, challenge_phase_id, submission_id ) ) - publish_submission_message( - challenge_id, challenge_phase_id, submission.id - ) + message = { + "challenge_pk": challenge_id, + "phase_pk": challenge_phase_id, + "submission_pk": submission.id + } + + if submission.challenge_phase.challenge.is_docker_based: + try: + response = requests.get(submission.input_file) + except Exception as e: + messages.error(request, "Failed to get input_file with exception: {0}".format(e)) + return + + if response and response.status_code == 200: + message["submitted_image_uri"] = response.json()["submitted_image_uri"] + + publish_submission_message(message) queryset.update(status=Submission.SUBMITTED) submit_job_to_worker.short_description = "Run selected submissions" diff --git a/apps/jobs/sender.py b/apps/jobs/sender.py index 2a1aaf12b1..0747b4a4f7 100644 --- a/apps/jobs/sender.py +++ b/apps/jobs/sender.py @@ -55,27 +55,24 @@ def get_or_create_sqs_queue(queue_name): return queue -def publish_submission_message(challenge_pk, phase_pk, submission_pk): +def publish_submission_message(message): """ Args: - challenge_pk: Challenge Id - phase_pk: Challenge Phase Id - submission_pk: Submission Id + message: A Dict with following keys + - "challenge_pk": int + - "phase_pk": int + - "submission_pk": int + - "submitted_image_uri": str, (only available when the challenge is a code upload challenge) Returns: Returns SQS response """ - message = { - "challenge_pk": challenge_pk, - "phase_pk": phase_pk, - "submission_pk": submission_pk, - } try: - challenge = Challenge.objects.get(pk=challenge_pk) + challenge = Challenge.objects.get(pk=message["challenge_pk"]) except Challenge.DoesNotExist: logger.exception( - "Challenge does not exist for the given id {}".format(challenge_pk) + "Challenge does not exist for the given id {}".format(message["challenge_pk"]) ) return queue_name = challenge.queue diff --git a/apps/jobs/tasks.py b/apps/jobs/tasks.py index f57fd4d23f..6f3db2ea93 100644 --- a/apps/jobs/tasks.py +++ b/apps/jobs/tasks.py @@ -73,7 +73,11 @@ def download_file_and_publish_submission_message( submission = serializer.instance # publish messages in the submission worker queue - publish_submission_message(challenge_phase.challenge.pk, challenge_phase.pk, submission.pk) + publish_submission_message({ + "challenge_pk": challenge_phase.challenge.pk, + "phase_pk": challenge_phase.pk, + "submission_pk": submission.pk + }) logger.info("Message published to submission worker successfully!") shutil.rmtree(downloaded_file['temp_dir_path']) except Exception as e: diff --git a/apps/jobs/views.py b/apps/jobs/views.py index a2c7f99999..d862796dfa 100644 --- a/apps/jobs/views.py +++ b/apps/jobs/views.py @@ -3,6 +3,7 @@ import json import logging +import requests from rest_framework import permissions, status from rest_framework.decorators import ( api_view, @@ -276,14 +277,30 @@ def challenge_submission(request, challenge_id, challenge_phase_id): "request": request, }, ) + message = { + "challenge_pk": challenge_id, + "phase_pk": challenge_phase_id + } + if challenge.is_docker_based: + try: + file_content = json.loads( + request.FILES['input_file'].read() + ) + message["submitted_image_uri"] = file_content["submitted_image_uri"] + except: + response_data = { + "error": "Error in deserializing submitted_image_uri from submission file" + } + return Response( + response_data, status=status.HTTP_400_BAD_REQUEST + ) if serializer.is_valid(): serializer.save() response_data = serializer.data submission = serializer.instance + message["submission_pk"] = submission.id # publish message in the queue - publish_submission_message( - challenge_id, challenge_phase_id, submission.id - ) + publish_submission_message(message) return Response(response_data, status=status.HTTP_201_CREATED) return Response( serializer.errors, status=status.HTTP_406_NOT_ACCEPTABLE @@ -1027,7 +1044,25 @@ def re_run_submission(request, submission_pk): } return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) - publish_submission_message(challenge.pk, challenge_phase.pk, submission.pk) + message = { + "challenge_pk": challenge.pk, + "phase_pk": challenge_phase.pk, + "submission_pk": submission.pk + } + + if submission.challenge_phase.challenge.is_docker_based: + try: + response = requests.get(submission.input_file) + except Exception as e: + response_data = { + "error": "Failed to get submission input file with error: {0}".format(e) + } + return Response(response_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + if response and response.status_code == 200: + message["submitted_image_uri"] = response.json()["submitted_image_uri"] + + publish_submission_message(message) response_data = { "success": "Submission is successfully submitted for re-running" } diff --git a/tests/unit/jobs/test_views.py b/tests/unit/jobs/test_views.py index bd47fd3f45..9645c0bd46 100644 --- a/tests/unit/jobs/test_views.py +++ b/tests/unit/jobs/test_views.py @@ -171,6 +171,10 @@ def setUp(self): "dummy_input.txt", b"file_content", content_type="text/plain" ) + self.rl_submission_file = SimpleUploadedFile( + "submission.json", b'{"submitted_image_uri": "evalai-repo.com"}', + ) + def tearDown(self): shutil.rmtree("/tmp/evalai") @@ -500,7 +504,7 @@ def test_challenge_submission_for_docker_based_challenges(self): response = self.client.post( self.url, - {"status": "submitting", "input_file": self.input_file}, + {"status": "submitting", "input_file": self.rl_submission_file}, format="multipart", ) From 0dcc0026fb606cc41fbdab00b683c250e2380050 Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Sun, 29 Dec 2019 19:31:29 -0500 Subject: [PATCH 23/34] RL submission worker: Fix API's, get or create SQS queue and re-running submission from admin (#2558) * API fixes for RL submission worker * Change name for SQS queue * Fix submission re-run from admin for docker based challenges --- apps/base/utils.py | 16 +++++++-------- apps/jobs/admin.py | 15 ++++++++++---- apps/jobs/views.py | 36 +++++++++++++++++++++------------- scripts/workers/worker_util.py | 9 ++++----- 4 files changed, 44 insertions(+), 32 deletions(-) diff --git a/apps/base/utils.py b/apps/base/utils.py index f54c1de14f..14bad1b0d1 100644 --- a/apps/base/utils.py +++ b/apps/base/utils.py @@ -154,7 +154,7 @@ def get_boto3_client(resource, aws_keys): logger.exception(e) -def get_sqs_queue_object(): +def get_or_create_sqs_queue_object(queue_name): if settings.DEBUG or settings.TEST: queue_name = "evalai_submission_queue" sqs = boto3.resource( @@ -212,21 +212,18 @@ def send_slack_notification(webhook=settings.SLACK_WEB_HOOK_URL, message=""): try: data = { "text": message["text"], - "attachments": [ - { - "color": "ffaf4b", - "fields": message["fields"] - } - ] + "attachments": [{"color": "ffaf4b", "fields": message["fields"]}], } return requests.post( webhook, data=json.dumps(data), - headers={"Content-Type": "application/json"} + headers={"Content-Type": "application/json"}, ) except Exception as e: logger.exception( - "Exception raised while sending slack notification. \n Exception message: {}".format(e) + "Exception raised while sending slack notification. \n Exception message: {}".format( + e + ) ) @@ -235,4 +232,5 @@ def decorator(func): if not (settings.DEBUG or settings.TEST): return func return aws_mocker(func) + return decorator diff --git a/apps/jobs/admin.py b/apps/jobs/admin.py index 7bc5c7fcac..ef1d28c946 100644 --- a/apps/jobs/admin.py +++ b/apps/jobs/admin.py @@ -72,18 +72,25 @@ def submit_job_to_worker(self, request, queryset): message = { "challenge_pk": challenge_id, "phase_pk": challenge_phase_id, - "submission_pk": submission.id + "submission_pk": submission.id, } if submission.challenge_phase.challenge.is_docker_based: try: - response = requests.get(submission.input_file) + response = requests.get(submission.input_file.url) except Exception as e: - messages.error(request, "Failed to get input_file with exception: {0}".format(e)) + messages.error( + request, + "Failed to get input_file with exception: {0}".format( + e + ), + ) return if response and response.status_code == 200: - message["submitted_image_uri"] = response.json()["submitted_image_uri"] + message["submitted_image_uri"] = response.json()[ + "submitted_image_uri" + ] publish_submission_message(message) queryset.update(status=Submission.SUBMITTED) diff --git a/apps/jobs/views.py b/apps/jobs/views.py index d862796dfa..b6184023ee 100644 --- a/apps/jobs/views.py +++ b/apps/jobs/views.py @@ -30,7 +30,7 @@ from base.utils import ( paginated_queryset, StandardResultSetPagination, - get_sqs_queue_object, + get_or_create_sqs_queue_object, get_boto3_client, ) from challenges.models import ( @@ -279,17 +279,19 @@ def challenge_submission(request, challenge_id, challenge_phase_id): ) message = { "challenge_pk": challenge_id, - "phase_pk": challenge_phase_id + "phase_pk": challenge_phase_id, } if challenge.is_docker_based: try: - file_content = json.loads( - request.FILES['input_file'].read() - ) - message["submitted_image_uri"] = file_content["submitted_image_uri"] - except: + file_content = json.loads(request.FILES["input_file"].read()) + message["submitted_image_uri"] = file_content[ + "submitted_image_uri" + ] + except Exception as ex: response_data = { - "error": "Error in deserializing submitted_image_uri from submission file" + "error": "Error {} in submitted_image_uri from submission file".format( + ex + ) } return Response( response_data, status=status.HTTP_400_BAD_REQUEST @@ -1047,7 +1049,7 @@ def re_run_submission(request, submission_pk): message = { "challenge_pk": challenge.pk, "phase_pk": challenge_phase.pk, - "submission_pk": submission.pk + "submission_pk": submission.pk, } if submission.challenge_phase.challenge.is_docker_based: @@ -1055,12 +1057,18 @@ def re_run_submission(request, submission_pk): response = requests.get(submission.input_file) except Exception as e: response_data = { - "error": "Failed to get submission input file with error: {0}".format(e) + "error": "Failed to get submission input file with error: {0}".format( + e + ) } - return Response(response_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + return Response( + response_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR + ) if response and response.status_code == 200: - message["submitted_image_uri"] = response.json()["submitted_image_uri"] + message["submitted_image_uri"] = response.json()[ + "submitted_image_uri" + ] publish_submission_message(message) response_data = { @@ -1137,7 +1145,7 @@ def get_submission_message_from_queue(request, queue_name): } return Response(response_data, status=status.HTTP_401_UNAUTHORIZED) - queue = get_sqs_queue_object() + queue = get_or_create_sqs_queue_object(queue_name) try: messages = queue.receive_messages() if len(messages): @@ -1193,7 +1201,7 @@ def delete_submission_message_from_queue(request, queue_name): } return Response(response_data, status=status.HTTP_401_UNAUTHORIZED) - queue = get_sqs_queue_object(queue_name) + queue = get_or_create_sqs_queue_object(queue_name) try: message = queue.Message(receipt_handle) message.delete() diff --git a/scripts/workers/worker_util.py b/scripts/workers/worker_util.py index d31846e9a1..035f69aafb 100644 --- a/scripts/workers/worker_util.py +++ b/scripts/workers/worker_util.py @@ -6,7 +6,7 @@ URLS = { "get_message_from_sqs_queue": "/api/jobs/challenge/queues/{}/", - "delete_message_from_sqs_queue": "/api/jobs/queues/{}/receipt/{}/", + "delete_message_from_sqs_queue": "/api/jobs/queues/{}/", "get_submission_by_pk": "/api/jobs/submission/{}", "get_challenge_phases_by_challenge_pk": "/api/challenges/{}/phases/", "get_challenge_by_queue_name": "/api/challenges/challenge/queues/{}/", @@ -51,11 +51,10 @@ def get_message_from_sqs_queue(self): return response def delete_message_from_sqs_queue(self, receipt_handle): - url = URLS.get("delete_message_from_sqs_queue").format( - self.QUEUE_NAME, receipt_handle - ) + url = URLS.get("delete_message_from_sqs_queue").format(self.QUEUE_NAME) url = self.return_url_per_environment(url) - response = self.make_request(url, "GET") # noqa + data = {"receipt_handle": receipt_handle} + response = self.make_request(url, "POST", data) # noqa return response.status_code def get_submission_by_pk(self, submission_pk): From bf36a9d666ec1aa22238dd8a0793bb284adc77a1 Mon Sep 17 00:00:00 2001 From: Sanjeev Singh Date: Wed, 1 Jan 2020 23:22:58 +0530 Subject: [PATCH 24/34] Backend: Add show_leaderboard_by_latest_submission field in challenge phase split serializer(#2568) * Add `show_leaderboard_by_latest_submission` field in challenge phase split serializer - Added field `show_leaderboard_by_latest_submission` in `ChallengePhaseSplitSerializer` in order to get this fields through `challenge_phase_split_list` from a particular object of phase splits. * Updated tests --- apps/challenges/serializers.py | 1 + tests/unit/challenges/test_views.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/apps/challenges/serializers.py b/apps/challenges/serializers.py index c2a88bd438..ac14d8512b 100644 --- a/apps/challenges/serializers.py +++ b/apps/challenges/serializers.py @@ -113,6 +113,7 @@ class Meta: "challenge_phase_name", "dataset_split_name", "visibility", + "show_leaderboard_by_latest_submission" ) def get_dataset_split_name(self, obj): diff --git a/tests/unit/challenges/test_views.py b/tests/unit/challenges/test_views.py index f87819718a..7055737788 100644 --- a/tests/unit/challenges/test_views.py +++ b/tests/unit/challenges/test_views.py @@ -2543,6 +2543,7 @@ def setUp(self): visibility=ChallengePhaseSplit.PUBLIC, leaderboard_decimal_precision=2, is_leaderboard_order_descending=True, + show_leaderboard_by_latest_submission=False ) self.challenge_phase_split_host = ChallengePhaseSplit.objects.create( @@ -2550,6 +2551,7 @@ def setUp(self): challenge_phase=self.challenge_phase, leaderboard=self.leaderboard, visibility=ChallengePhaseSplit.HOST, + show_leaderboard_by_latest_submission=False ) def tearDown(self): @@ -2573,6 +2575,8 @@ def test_get_challenge_phase_split(self): "dataset_split": self.dataset_split.id, "dataset_split_name": self.dataset_split.name, "visibility": self.challenge_phase_split.visibility, + "show_leaderboard_by_latest_submission": + self.challenge_phase_split.show_leaderboard_by_latest_submission } ] self.client.force_authenticate(user=self.participant_user) @@ -2608,6 +2612,8 @@ def test_get_challenge_phase_split_when_user_is_challenge_host(self): "dataset_split": self.dataset_split.id, "dataset_split_name": self.dataset_split.name, "visibility": self.challenge_phase_split.visibility, + "show_leaderboard_by_latest_submission": + self.challenge_phase_split.show_leaderboard_by_latest_submission }, { "id": self.challenge_phase_split_host.id, @@ -2616,6 +2622,8 @@ def test_get_challenge_phase_split_when_user_is_challenge_host(self): "dataset_split": self.dataset_split_host.id, "dataset_split_name": self.dataset_split_host.name, "visibility": self.challenge_phase_split_host.visibility, + "show_leaderboard_by_latest_submission": + self.challenge_phase_split_host.show_leaderboard_by_latest_submission }, ] self.client.force_authenticate(user=self.user) From ddae77748de73d67dc0555a9f55e74abcee0cece Mon Sep 17 00:00:00 2001 From: Kaustubh Maske Patil <37668193+nikochiko@users.noreply.github.com> Date: Tue, 7 Jan 2020 20:28:10 +0530 Subject: [PATCH 25/34] Docs: Update django version in architecture.md(#2579) --- docs/source/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/architecture.md b/docs/source/architecture.md index d3b14df209..abfda434de 100644 --- a/docs/source/architecture.md +++ b/docs/source/architecture.md @@ -6,7 +6,7 @@ EvalAI helps researchers, students, and data scientists to create, collaborate, #### Django -Django is the heart of the application, which powers our backend. We use Django version 1.11.18. +Django is the heart of the application, which powers our backend. We use Django version 1.11.23. #### Django Rest Framework From 7942f07af0d1b729c46e9d490aba3f945ee2e391 Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Wed, 15 Jan 2020 13:05:56 -0500 Subject: [PATCH 26/34] RL submission worker: Fix http request returning response in worker_utils(#2575) --- scripts/workers/worker_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/workers/worker_util.py b/scripts/workers/worker_util.py index 035f69aafb..e9f1c7605a 100644 --- a/scripts/workers/worker_util.py +++ b/scripts/workers/worker_util.py @@ -55,7 +55,7 @@ def delete_message_from_sqs_queue(self, receipt_handle): url = self.return_url_per_environment(url) data = {"receipt_handle": receipt_handle} response = self.make_request(url, "POST", data) # noqa - return response.status_code + return response def get_submission_by_pk(self, submission_pk): url = URLS.get("get_submission_by_pk").format(submission_pk) From 81869ab86eb97baa0afdbbfdb6e66620d59328f0 Mon Sep 17 00:00:00 2001 From: Takitsuse Nagisa <57856193+takitsuse@users.noreply.github.com> Date: Sun, 19 Jan 2020 12:11:23 +0900 Subject: [PATCH 27/34] Frontend: Add feature to update maximum_submissions_per_month using UI(#2540) Co-authored-by: Rishabh Jain --- frontend/src/js/controllers/challengeCtrl.js | 2 ++ frontend/tests/controllers-test/challengeCtrl.test.js | 1 + 2 files changed, 3 insertions(+) diff --git a/frontend/src/js/controllers/challengeCtrl.js b/frontend/src/js/controllers/challengeCtrl.js index 9bcb711561..3b19106b8d 100644 --- a/frontend/src/js/controllers/challengeCtrl.js +++ b/frontend/src/js/controllers/challengeCtrl.js @@ -1891,6 +1891,7 @@ vm.challengePhaseDialog = function(ev, phase) { vm.page.challenge_phase = phase; vm.page.max_submissions_per_day = phase.max_submissions_per_day; + vm.page.max_submissions_per_month = phase.max_submissions_per_month; vm.phaseStartDate = moment(phase.start_date); vm.phaseEndDate = moment(phase.end_date); vm.testAnnotationFile = null; @@ -1916,6 +1917,7 @@ formData.append("start_date", vm.phaseStartDate.toISOString()); formData.append("end_date", vm.phaseEndDate.toISOString()); formData.append("max_submissions_per_day", vm.page.challenge_phase.max_submissions_per_day); + formData.append("max_submissions_per_month", vm.page.challenge_phase.max_submissions_per_month); formData.append("max_submissions", vm.page.challenge_phase.max_submissions); if (vm.testAnnotationFile) { formData.append("test_annotation", vm.testAnnotationFile); diff --git a/frontend/tests/controllers-test/challengeCtrl.test.js b/frontend/tests/controllers-test/challengeCtrl.test.js index 6682b80191..d516d19746 100644 --- a/frontend/tests/controllers-test/challengeCtrl.test.js +++ b/frontend/tests/controllers-test/challengeCtrl.test.js @@ -2280,6 +2280,7 @@ describe('Unit tests for challenge controller', function () { vm.challengePhaseDialog(ev, phase); expect(vm.page.challenge_phase).toEqual(phase); expect(vm.page.max_submissions_per_day).toEqual(phase.max_submissions_per_day); + expect(vm.page.max_submissions_per_month).toEqual(phase.max_submissions_per_month); expect(vm.phaseStartDate).toEqual(moment(phase.start_date)); expect(vm.phaseEndDate).toEqual(moment(phase.end_date)); expect(vm.testAnnotationFile).toEqual(null); From 963a989dbef2b1ad0156dc33b1df4ecb4802ae63 Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Mon, 20 Jan 2020 00:05:15 -0500 Subject: [PATCH 28/34] Backend: Add Job ID in submission model for code upload submission evaluation(#2631) --- ...13_add_job_id_field_in_submission_model.py | 25 +++++++++++++++++++ apps/jobs/models.py | 7 ++++++ 2 files changed, 32 insertions(+) create mode 100644 apps/jobs/migrations/0013_add_job_id_field_in_submission_model.py diff --git a/apps/jobs/migrations/0013_add_job_id_field_in_submission_model.py b/apps/jobs/migrations/0013_add_job_id_field_in_submission_model.py new file mode 100644 index 0000000000..eac39a7d20 --- /dev/null +++ b/apps/jobs/migrations/0013_add_job_id_field_in_submission_model.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-01-20 04:55 +from __future__ import unicode_literals + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("jobs", "0012_add_baseline_submission")] + + operations = [ + migrations.AddField( + model_name="submission", + name="job_id", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.TextField(blank=True, null=True), + blank=True, + default=[], + null=True, + size=None, + ), + ) + ] diff --git a/apps/jobs/models.py b/apps/jobs/models.py index 7da628f6ad..eb5535c8ae 100644 --- a/apps/jobs/models.py +++ b/apps/jobs/models.py @@ -3,6 +3,7 @@ import logging from django.contrib.auth.models import User +from django.contrib.postgres.fields import ArrayField from django.db import models from django.db.models import Max from rest_framework.exceptions import PermissionDenied @@ -107,6 +108,12 @@ class Submission(TimeStampedModel): publication_url = models.CharField(max_length=1000, default="", blank=True) project_url = models.CharField(max_length=1000, default="", blank=True) is_baseline = models.BooleanField(default=False) + job_id = ArrayField( + models.TextField(null=True, blank=True), + default=[], + blank=True, + null=True, + ) def __str__(self): return "{}".format(self.id) From 033299e11fbaebfc1a905b6699fc3768e8e74f14 Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Mon, 20 Jan 2020 00:30:30 -0500 Subject: [PATCH 29/34] Backend: Rename job_id to job_name in submission model to be consistent with kubernetes(#2632) --- ...b_id_field_in_submission_model_to_job_name.py | 16 ++++++++++++++++ apps/jobs/models.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 apps/jobs/migrations/0014_rename_job_id_field_in_submission_model_to_job_name.py diff --git a/apps/jobs/migrations/0014_rename_job_id_field_in_submission_model_to_job_name.py b/apps/jobs/migrations/0014_rename_job_id_field_in_submission_model_to_job_name.py new file mode 100644 index 0000000000..159e4d49c2 --- /dev/null +++ b/apps/jobs/migrations/0014_rename_job_id_field_in_submission_model_to_job_name.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-01-20 05:23 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("jobs", "0013_add_job_id_field_in_submission_model")] + + operations = [ + migrations.RenameField( + model_name="submission", old_name="job_id", new_name="job_name" + ) + ] diff --git a/apps/jobs/models.py b/apps/jobs/models.py index eb5535c8ae..ba53431466 100644 --- a/apps/jobs/models.py +++ b/apps/jobs/models.py @@ -108,7 +108,7 @@ class Submission(TimeStampedModel): publication_url = models.CharField(max_length=1000, default="", blank=True) project_url = models.CharField(max_length=1000, default="", blank=True) is_baseline = models.BooleanField(default=False) - job_id = ArrayField( + job_name = ArrayField( models.TextField(null=True, blank=True), default=[], blank=True, From 334c7d274b5c7c2c0ad6a089941f23424a3c411b Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Wed, 22 Jan 2020 01:47:50 -0500 Subject: [PATCH 30/34] Backend: Add feature to update job_name in update_submission API(#2637) * Add feature to update job name in submission * Fix test cases --- apps/jobs/admin.py | 1 + apps/jobs/serializers.py | 1 + apps/jobs/views.py | 11 +++++++- tests/unit/challenges/test_views.py | 24 +++++++---------- tests/unit/jobs/test_views.py | 41 +++++++++++++++++++---------- 5 files changed, 48 insertions(+), 30 deletions(-) diff --git a/apps/jobs/admin.py b/apps/jobs/admin.py index ef1d28c946..34e250779c 100644 --- a/apps/jobs/admin.py +++ b/apps/jobs/admin.py @@ -34,6 +34,7 @@ class SubmissionAdmin(ImportExportTimeStampedAdmin): "stderr_file", "submission_result_file", "submission_metadata_file", + "job_name", ) list_filter = ( "challenge_phase__challenge", diff --git a/apps/jobs/serializers.py b/apps/jobs/serializers.py index ab848a3950..c6deaea9eb 100644 --- a/apps/jobs/serializers.py +++ b/apps/jobs/serializers.py @@ -50,6 +50,7 @@ class Meta: "submission_result_file", "when_made_public", "is_baseline", + "job_name", ) def get_participant_team_name(self, obj): diff --git a/apps/jobs/views.py b/apps/jobs/views.py index b6184023ee..db88387aaf 100644 --- a/apps/jobs/views.py +++ b/apps/jobs/views.py @@ -999,11 +999,20 @@ def update_submission(request, challenge_pk): if request.method == "PATCH": submission_pk = request.data.get("submission") submission_status = request.data.get("submission_status", "").lower() + job_name = request.data.get("job_name", "").lower() submission = get_submission_model(submission_pk) + jobs = submission.job_name + if job_name: + jobs.append(job_name) if submission_status not in [Submission.RUNNING]: response_data = {"error": "Sorry, submission status is invalid"} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) - data = {"status": submission_status, "started_at": timezone.now()} + + data = { + "status": submission_status, + "started_at": timezone.now(), + "job_name": jobs, + } serializer = SubmissionSerializer( submission, data=data, partial=True, context={"request": request} ) diff --git a/tests/unit/challenges/test_views.py b/tests/unit/challenges/test_views.py index 7055737788..9eaf29801f 100644 --- a/tests/unit/challenges/test_views.py +++ b/tests/unit/challenges/test_views.py @@ -2543,7 +2543,7 @@ def setUp(self): visibility=ChallengePhaseSplit.PUBLIC, leaderboard_decimal_precision=2, is_leaderboard_order_descending=True, - show_leaderboard_by_latest_submission=False + show_leaderboard_by_latest_submission=False, ) self.challenge_phase_split_host = ChallengePhaseSplit.objects.create( @@ -2551,7 +2551,7 @@ def setUp(self): challenge_phase=self.challenge_phase, leaderboard=self.leaderboard, visibility=ChallengePhaseSplit.HOST, - show_leaderboard_by_latest_submission=False + show_leaderboard_by_latest_submission=False, ) def tearDown(self): @@ -2575,8 +2575,7 @@ def test_get_challenge_phase_split(self): "dataset_split": self.dataset_split.id, "dataset_split_name": self.dataset_split.name, "visibility": self.challenge_phase_split.visibility, - "show_leaderboard_by_latest_submission": - self.challenge_phase_split.show_leaderboard_by_latest_submission + "show_leaderboard_by_latest_submission": self.challenge_phase_split.show_leaderboard_by_latest_submission, } ] self.client.force_authenticate(user=self.participant_user) @@ -2612,8 +2611,7 @@ def test_get_challenge_phase_split_when_user_is_challenge_host(self): "dataset_split": self.dataset_split.id, "dataset_split_name": self.dataset_split.name, "visibility": self.challenge_phase_split.visibility, - "show_leaderboard_by_latest_submission": - self.challenge_phase_split.show_leaderboard_by_latest_submission + "show_leaderboard_by_latest_submission": self.challenge_phase_split.show_leaderboard_by_latest_submission, }, { "id": self.challenge_phase_split_host.id, @@ -2622,8 +2620,7 @@ def test_get_challenge_phase_split_when_user_is_challenge_host(self): "dataset_split": self.dataset_split_host.id, "dataset_split_name": self.dataset_split_host.name, "visibility": self.challenge_phase_split_host.visibility, - "show_leaderboard_by_latest_submission": - self.challenge_phase_split_host.show_leaderboard_by_latest_submission + "show_leaderboard_by_latest_submission": self.challenge_phase_split_host.show_leaderboard_by_latest_submission, }, ] self.client.force_authenticate(user=self.user) @@ -3206,6 +3203,7 @@ def test_get_all_submissions_when_user_is_participant_of_challenge(self): "is_flagged": self.submission1.is_flagged, "when_made_public": self.submission1.when_made_public, "is_baseline": self.submission1.is_baseline, + "job_name": self.submission1.job_name, } ] self.challenge5.participant_teams.add(self.participant_team6) @@ -3720,7 +3718,7 @@ def test_get_dataset_split(self): "visibility": self.challenge_phase_split.visibility, "leaderboard_decimal_precision": self.challenge_phase_split.leaderboard_decimal_precision, "is_leaderboard_order_descending": self.challenge_phase_split.is_leaderboard_order_descending, - "show_leaderboard_by_latest_submission": self.challenge_phase_split.show_leaderboard_by_latest_submission + "show_leaderboard_by_latest_submission": self.challenge_phase_split.show_leaderboard_by_latest_submission, } response = self.client.get(self.url) self.assertEqual(response.data, expected) @@ -4078,9 +4076,7 @@ def test_get_aws_credentials_when_challenge_is_not_docker_based(self): "challenges:get_aws_credentials_for_participant_team", kwargs={"phase_pk": self.challenge_phase.pk}, ) - expected = { - "error": "Sorry, this is not a docker based challenge." - } + expected = {"error": "Sorry, this is not a docker based challenge."} response = self.client.get(self.url, {}) self.assertEqual(response.data, expected) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) @@ -4090,9 +4086,7 @@ def test_get_aws_credentials_when_not_participated(self): "challenges:get_aws_credentials_for_participant_team", kwargs={"phase_pk": self.challenge_phase.pk}, ) - expected = { - "error": "You have not participated in this challenge." - } + expected = {"error": "You have not participated in this challenge."} self.client.force_authenticate(user=self.user2) response = self.client.get(self.url, {}) self.assertEqual(response.data, expected) diff --git a/tests/unit/jobs/test_views.py b/tests/unit/jobs/test_views.py index 9645c0bd46..878614e2b9 100644 --- a/tests/unit/jobs/test_views.py +++ b/tests/unit/jobs/test_views.py @@ -172,7 +172,7 @@ def setUp(self): ) self.rl_submission_file = SimpleUploadedFile( - "submission.json", b'{"submitted_image_uri": "evalai-repo.com"}', + "submission.json", b'{"submitted_image_uri": "evalai-repo.com"}' ) def tearDown(self): @@ -638,6 +638,7 @@ def test_get_challenge_submissions(self): "is_flagged": self.submission.is_flagged, "when_made_public": self.submission.when_made_public, "is_baseline": self.submission.is_baseline, + "job_name": self.submission.job_name, } ] self.challenge.participant_teams.add(self.participant_team) @@ -1309,6 +1310,7 @@ def test_change_submission_data_and_visibility_when_submission_exist(self): self.submission.when_made_public.isoformat(), "Z" ).replace("+00:00", ""), "is_baseline": self.submission.is_baseline, + "job_name": self.submission.job_name, } self.challenge.participant_teams.add(self.participant_team) response = self.client.patch(self.url, self.data) @@ -1354,6 +1356,7 @@ def test_change_submission_data_and_visibility_when_challenge_phase_is_private_a self.private_submission.when_made_public.isoformat(), "Z" ).replace("+00:00", ""), "is_baseline": self.submission.is_baseline, + "job_name": self.submission.job_name, } self.client.force_authenticate(user=self.user) @@ -1417,15 +1420,14 @@ def test_change_submission_data_and_visibility_when_is_public_is_false( self.submission.when_made_public.isoformat(), "Z" ).replace("+00:00", ""), "is_baseline": self.submission.is_baseline, + "job_name": self.submission.job_name, } self.challenge.participant_teams.add(self.participant_team) response = self.client.patch(self.url, self.data) self.assertEqual(response.data, expected) self.assertEqual(response.status_code, status.HTTP_200_OK) - def test_toggle_baseline_when_user_is_not_a_host( - self - ): + def test_toggle_baseline_when_user_is_not_a_host(self): self.url = reverse_lazy( "jobs:change_submission_data_and_visibility", kwargs={ @@ -1437,14 +1439,14 @@ def test_toggle_baseline_when_user_is_not_a_host( self.data = {"is_baseline": True} self.challenge.save() self.client.force_authenticate(user=self.user1) - expected = {"error": "Sorry, you are not authorized to make this request"} + expected = { + "error": "Sorry, you are not authorized to make this request" + } response = self.client.patch(self.url, self.data) self.assertEqual(response.data, expected) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - def test_toggle_baseline_when_user_is_host( - self - ): + def test_toggle_baseline_when_user_is_host(self): self.url = reverse_lazy( "jobs:change_submission_data_and_visibility", kwargs={ @@ -1474,14 +1476,17 @@ def test_toggle_baseline_when_user_is_host( "stderr_file": None, "submission_result_file": None, "submitted_at": "{0}{1}".format( - self.host_participant_team_submission.submitted_at.isoformat(), "Z" + self.host_participant_team_submission.submitted_at.isoformat(), + "Z", ).replace("+00:00", ""), "is_public": self.host_participant_team_submission.is_public, "is_flagged": self.host_participant_team_submission.is_flagged, "when_made_public": "{0}{1}".format( - self.host_participant_team_submission.when_made_public.isoformat(), "Z" + self.host_participant_team_submission.when_made_public.isoformat(), + "Z", ).replace("+00:00", ""), "is_baseline": True, + "job_name": self.host_participant_team_submission.job_name, } response = self.client.patch(self.url, self.data) self.assertEqual(response.data, expected) @@ -1555,6 +1560,7 @@ def test_get_submission_by_pk_when_user_created_the_submission(self): self.submission.when_made_public.isoformat(), "Z" ).replace("+00:00", ""), "is_baseline": self.submission.is_baseline, + "job_name": self.submission.job_name, } self.client.force_authenticate(user=self.submission.created_by) @@ -1595,6 +1601,7 @@ def test_get_submission_by_pk_when_user_is_challenge_host(self): self.submission.when_made_public.isoformat(), "Z" ).replace("+00:00", ""), "is_baseline": self.submission.is_baseline, + "job_name": self.submission.job_name, } self.client.force_authenticate(user=self.user) @@ -1734,9 +1741,15 @@ def setUp(self): self.result_json_2 = {"score": 10.0, "test-score": 20.0} - self.result_json_host_participant_team = {"score": 52.0, "test-score": 80.0} + self.result_json_host_participant_team = { + "score": 52.0, + "test-score": 80.0, + } - self.result_json_host_participant_team_2 = {"score": 20.0, "test-score": 60.0} + self.result_json_host_participant_team_2 = { + "score": 20.0, + "test-score": 60.0, + } self.expected_results = [ self.result_json["score"], @@ -1889,7 +1902,7 @@ def test_get_leaderboard_with_baseline_entry(self): "submission__submitted_at": self.submission.submitted_at, "submission__is_baseline": False, "submission__method_name": self.submission.method_name, - } + }, ], } expected = collections.OrderedDict(expected) @@ -1975,7 +1988,7 @@ def test_get_leaderboard_with_multiple_baseline_entries(self): "submission__submitted_at": self.host_participant_team_submission_2.submitted_at, "submission__is_baseline": True, "submission__method_name": self.host_participant_team_submission_2.method_name, - } + }, ], } expected = collections.OrderedDict(expected) From 8ca5848006c6b23d9a5482d94aa1a74e438df403 Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Wed, 22 Jan 2020 01:55:38 -0500 Subject: [PATCH 31/34] Code upload worker: Update script with deployment type as Job & add feature to terminate job(#2638) --- scripts/workers/rl_submission_worker.py | 148 ++++++++++++------ .../{worker_util.py => worker_utils.py} | 0 2 files changed, 104 insertions(+), 44 deletions(-) rename scripts/workers/{worker_util.py => worker_utils.py} (100%) diff --git a/scripts/workers/rl_submission_worker.py b/scripts/workers/rl_submission_worker.py index 6089c1ab8b..ca62357fca 100644 --- a/scripts/workers/rl_submission_worker.py +++ b/scripts/workers/rl_submission_worker.py @@ -1,12 +1,14 @@ import logging import os import signal -import time -from worker_util import EvalAI_Interface +from worker_utils import EvalAI_Interface from kubernetes import client, config +# TODO: Add exception in all the commands +# from kubernetes.client.rest import ApiException + class GracefulKiller: kill_now = False @@ -20,6 +22,8 @@ def exit_gracefully(self, signum, frame): logger = logging.getLogger(__name__) +config.load_kube_config() +batch_v1 = client.BatchV1Api() AUTH_TOKEN = os.environ.get("AUTH_TOKEN", "auth_token") EVALAI_API_SERVER = os.environ.get( @@ -27,19 +31,31 @@ def exit_gracefully(self, signum, frame): ) QUEUE_NAME = os.environ.get("QUEUE_NAME", "evalai_submission_queue") ENVIRONMENT_IMAGE = os.environ.get("ENVIRONMENT_IMAGE", "image_name:tag") -MESSAGE_FETCH_DEPLAY = int(os.environ.get("MESSAGE_FETCH_DEPLAY", "5")) -def create_deployment_object(image, submission, message): +def create_job_object(message): + """Function to create the AWS EKS Job object + + Arguments: + message {[dict]} -- Submission message from AWS SQS queue + + Returns: + [AWS EKS Job class object] -- AWS EKS Job class object + """ + PYTHONUNBUFFERED_ENV = client.V1EnvVar(name="PYTHONUNBUFFERED", value="1") AUTH_TOKEN_ENV = client.V1EnvVar(name="AUTH_TOKEN", value=AUTH_TOKEN) EVALAI_API_SERVER_ENV = client.V1EnvVar( name="EVALAI_API_SERVER", value=EVALAI_API_SERVER ) MESSAGE_BODY_ENV = client.V1EnvVar(name="BODY", value=str(message)) + submission_pk = message["submission_pk"] + image = message["submitted_image_uri"] + # Configureate Pod agent container agent_container = client.V1Container( name="agent", image=image, env=[PYTHONUNBUFFERED_ENV] ) + # Configureate Pod environment container environment_container = client.V1Container( name="environment", image=ENVIRONMENT_IMAGE, @@ -50,85 +66,129 @@ def create_deployment_object(image, submission, message): MESSAGE_BODY_ENV, ], ) + # Create and configurate a spec section template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels={"app": "evaluation"}), spec=client.V1PodSpec( - containers=[environment_container, agent_container] + containers=[environment_container, agent_container], + restart_policy="Never", ), ) - spec = client.ExtensionsV1beta1DeploymentSpec( - replicas=1, template=template - ) - deployment = client.ExtensionsV1beta1Deployment( - api_version="extensions/v1beta1", - kind="Deployment", - metadata=client.V1ObjectMeta(name="submission-{0}".format(submission)), + # Create the specification of deployment + spec = client.V1JobSpec(backoff_limit=1, template=template) + # Instantiate the job object + job = client.V1Job( + api_version="batch/v1", + kind="Job", + metadata=client.V1ObjectMeta( + name="submission-{0}".format(submission_pk) + ), spec=spec, ) - return deployment + return job + + +def create_job(api_instance, job): + """Function to create a job on AWS EKS cluster + Arguments: + api_instance {[AWS EKS API object]} -- API object for creating job + job {[AWS EKS job object]} -- Job object returned after running create_job_object fucntion -def create_deployment(api_instance, deployment): - api_response = api_instance.create_namespaced_deployment( - body=deployment, namespace="default" + Returns: + [V1Job object] -- [AWS EKS V1Job] + For reference: https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1Job.md + """ + api_response = api_instance.create_namespaced_job( + body=job, namespace="default", pretty=True ) logger.info("Deployment created. status='%s'" % str(api_response.status)) + return api_response -def process_submission_callback(message, api): - config.load_kube_config() - extensions_v1beta1 = client.ExtensionsV1beta1Api() - logger.info(message) - submission_data = { - "submission_status": "running", - "submission": message["submission_pk"], - } - logger.info(submission_data) - api.update_submission_status(submission_data, message["challenge_pk"]) - dep = create_deployment_object( - message["submitted_image_uri"], message["submission_pk"], message +def delete_job(api_instance, job_name): + """Function to delete a job on AWS EKS cluster + + Arguments: + api_instance {[AWS EKS API object]} -- API object for deleting job + job_name {[string]} -- Name of the job to be terminated + """ + api_response = api_instance.delete_namespaced_job( + name=job_name, + namespace="default", + body=client.V1DeleteOptions( + propagation_policy="Foreground", grace_period_seconds=5 + ), ) - create_deployment(extensions_v1beta1, dep) + logger.info("Job deleted. status='%s'" % str(api_response.status)) + + +def process_submission_callback(body, evalai): + """Function to process submission message from SQS Queue + + Arguments: + body {[dict]} -- Submission message body from AWS SQS Queue + evalai {[EvalAI class object]} -- EvalAI class object imported from worker_utils + """ + try: + logger.info("[x] Received submission message %s" % body) + job = create_job_object(body) + response = create_job(batch_v1, job) + submission_data = { + "submission_status": "running", + "submission": body["submission_pk"], + "job_name": response.metadata.name, + } + evalai.update_submission_status(submission_data, body["challenge_pk"]) + except Exception as e: + logger.exception( + "Exception while receiving message from submission queue with error {}".format( + e + ) + ) def main(): - api = EvalAI_Interface( + killer = GracefulKiller() + evalai = EvalAI_Interface( AUTH_TOKEN=AUTH_TOKEN, EVALAI_API_SERVER=EVALAI_API_SERVER, QUEUE_NAME=QUEUE_NAME, ) logger.info( - "String RL Worker for {}".format( - api.get_challenge_by_queue_name()["title"] + "Deploying Worker for {}".format( + evalai.get_challenge_by_queue_name()["title"] ) ) - killer = GracefulKiller() while True: logger.info( "Fetching new messages from the queue {}".format(QUEUE_NAME) ) - message = api.get_message_from_sqs_queue() - logger.info(message) + message = evalai.get_message_from_sqs_queue() message_body = message.get("body") if message_body: submission_pk = message_body.get("submission_pk") - submission = api.get_submission_by_pk(submission_pk) + submission = evalai.get_submission_by_pk(submission_pk) if submission: - if submission.get("status") == "finished": + if ( + submission.get("status") == "finished" + or submission.get("status") == "failed" + ): + # Fetch the last job name from the list as it is the latest running job + job_name = submission.get("job_name")[-1] + delete_job(batch_v1, job_name) message_receipt_handle = message.get("receipt_handle") - api.delete_message_from_sqs_queue(message_receipt_handle) + evalai.delete_message_from_sqs_queue( + message_receipt_handle + ) elif submission.get("status") == "running": continue else: message_receipt_handle = message.get("receipt_handle") logger.info( - "Processing message body: {}".format(message_body) - ) - process_submission_callback(message_body, api) - api.delete_message_from_sqs_queue( - message.get("receipt_handle") + "Processing message body: {0}".format(message_body) ) - time.sleep(MESSAGE_FETCH_DEPLAY) + process_submission_callback(message_body, evalai) if killer.kill_now: break diff --git a/scripts/workers/worker_util.py b/scripts/workers/worker_utils.py similarity index 100% rename from scripts/workers/worker_util.py rename to scripts/workers/worker_utils.py From 2fb0656e6f4fac1a76eed3b0d8bb0a859863c0aa Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Mon, 3 Feb 2020 00:39:24 -0500 Subject: [PATCH 32/34] Frontend: Add stats on the landing page(#2649) --- frontend/src/views/web/landing.html | 80 +++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 5 deletions(-) diff --git a/frontend/src/views/web/landing.html b/frontend/src/views/web/landing.html index f5d523216b..ac9a1ab604 100644 --- a/frontend/src/views/web/landing.html +++ b/frontend/src/views/web/landing.html @@ -8,7 +8,7 @@

Evaluating state-of-the-art in AI

EvalAI is an open source platform for evaluating and - comparing machine learning (ML) and artificial intelligence algorithms (AI) at scale.

+ comparing machine learning (ML) and artificial intelligence (AI) algorithms at scale.

  • @@ -26,6 +26,76 @@

    Evaluating state-of-the-art in AI

+
+
+
+
+
+
+

45+

+
+
+
+
+
+ AI Hosted Challenges +
+
+
+
+
+
+
+
+
+

5500+

+
+
+
+
+
+ Users +
+
+
+
+
+
+
+
+
+

51000+

+
+
+
+
+
+ Submissions +
+
+
+
+
+
+
+
+
+

20+

+
+
+
+
+
+ Organizations +
+
+
+
+
+
+
+
+

Features

@@ -97,7 +167,7 @@
Faster evaluation
-
+

Popular challenges

@@ -139,7 +209,7 @@

Popular challenges

-
+
@@ -255,7 +325,7 @@

Partner Organizations

@@ -264,7 +334,7 @@

Partner Organizations

-
+

Cite our work

From 1b2da22d4c90900be94b7fee145ff23940593b43 Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Mon, 3 Feb 2020 14:18:59 -0500 Subject: [PATCH 33/34] Backend: Add filtering by past, present & future challenges in challenge admin(#2650) --- apps/challenges/admin.py | 4 ++++ apps/challenges/admin_filters.py | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 apps/challenges/admin_filters.py diff --git a/apps/challenges/admin.py b/apps/challenges/admin.py index 44eca234eb..c79ea49a08 100644 --- a/apps/challenges/admin.py +++ b/apps/challenges/admin.py @@ -1,5 +1,6 @@ from django import forms from django.contrib import admin, messages + from django.contrib.admin.helpers import ActionForm from base.admin import ImportExportTimeStampedAdmin @@ -12,6 +13,8 @@ stop_workers, ) +from .admin_filters import ChallengeFilter + from .models import ( Challenge, ChallengeConfiguration, @@ -52,6 +55,7 @@ class ChallengeAdmin(ImportExportTimeStampedAdmin): "task_def_arn", ) list_filter = ( + ChallengeFilter, "published", "is_registration_open", "enable_forum", diff --git a/apps/challenges/admin_filters.py b/apps/challenges/admin_filters.py new file mode 100644 index 0000000000..120ba37dd6 --- /dev/null +++ b/apps/challenges/admin_filters.py @@ -0,0 +1,38 @@ +from django.contrib.admin import SimpleListFilter +from django.utils import timezone + + +class ChallengeFilter(SimpleListFilter): + + title = "Challenges" + parameter_name = "challenge" + + def lookups(self, request, model_admin): + options = [ + ("past", "Past"), + ("present", "Ongoing"), + ("future", "Upcoming"), + ] + return options + + def queryset(self, request, queryset): + q_params = { + "published": True, + "approved_by_admin": True, + "is_disabled": False, + } + if self.value() == "past": + q_params["end_date__lt"] = timezone.now() + challenges = queryset.filter(**q_params) + return challenges + + elif self.value() == "present": + q_params["start_date__lt"] = timezone.now() + q_params["end_date__gt"] = timezone.now() + challenges = queryset.filter(**q_params) + return challenges + + elif self.value() == "future": + q_params["start_date__gt"] = timezone.now() + challenges = queryset.filter(**q_params) + return challenges From c84316acc6ab1dad001c52037feb2f10c7c050da Mon Sep 17 00:00:00 2001 From: Rishabh Jain Date: Mon, 3 Feb 2020 17:01:20 -0500 Subject: [PATCH 34/34] Frontend: Aesthetic changes for stats on landing page (#2652) --- frontend/src/views/web/landing.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/views/web/landing.html b/frontend/src/views/web/landing.html index ac9a1ab604..e9c4786675 100644 --- a/frontend/src/views/web/landing.html +++ b/frontend/src/views/web/landing.html @@ -32,13 +32,13 @@

Evaluating state-of-the-art in AI
-

45+

+

50+

- AI Hosted Challenges + Hosted AI Challenges
@@ -48,7 +48,7 @@

45+

-

5500+

+

5,500+

@@ -64,7 +64,7 @@

5500+

-

51000+

+

51,000+

@@ -80,7 +80,7 @@

51000+

-

20+

+

20+