From 97e75c1d95cf9d0baeddeb00c94a68adf7f1d3eb Mon Sep 17 00:00:00 2001 From: Vince Li Date: Thu, 12 Aug 2021 03:29:16 -0400 Subject: [PATCH 1/9] completed requirements task --- .../migrations/0086_challenge_requirements.py | 19 +++++++++ .../migrations/0087_auto_20210811_0825.py | 19 +++++++++ .../migrations/0088_auto_20210811_0833.py | 19 +++++++++ .../migrations/0089_auto_20210811_0842.py | 19 +++++++++ apps/challenges/models.py | 4 ++ apps/challenges/serializers.py | 5 +++ apps/challenges/urls.py | 5 +++ apps/challenges/views.py | 39 +++++++++++++++++++ frontend/src/js/controllers/challengeCtrl.js | 22 +++++++++++ frontend/src/views/web/challenge/manage.html | 9 +++++ scripts/workers/submission_worker.py | 34 +++++++++++++++- 11 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 apps/challenges/migrations/0086_challenge_requirements.py create mode 100644 apps/challenges/migrations/0087_auto_20210811_0825.py create mode 100644 apps/challenges/migrations/0088_auto_20210811_0833.py create mode 100644 apps/challenges/migrations/0089_auto_20210811_0842.py diff --git a/apps/challenges/migrations/0086_challenge_requirements.py b/apps/challenges/migrations/0086_challenge_requirements.py new file mode 100644 index 0000000000..a26310f2f8 --- /dev/null +++ b/apps/challenges/migrations/0086_challenge_requirements.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.20 on 2021-08-11 06:51 + +import base.utils +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('challenges', '0085_challenge_submission_time_limit'), + ] + + operations = [ + migrations.AddField( + model_name='challenge', + name='requirements', + field=models.FileField(blank=True, upload_to=base.utils.RandomFileName('requirements')), + ), + ] diff --git a/apps/challenges/migrations/0087_auto_20210811_0825.py b/apps/challenges/migrations/0087_auto_20210811_0825.py new file mode 100644 index 0000000000..52d1333aa3 --- /dev/null +++ b/apps/challenges/migrations/0087_auto_20210811_0825.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.20 on 2021-08-11 08:25 + +import base.utils +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('challenges', '0086_challenge_requirements'), + ] + + operations = [ + migrations.AlterField( + model_name='challenge', + name='requirements', + field=models.FileField(blank=True, null=True, upload_to=base.utils.RandomFileName('requirements')), + ), + ] diff --git a/apps/challenges/migrations/0088_auto_20210811_0833.py b/apps/challenges/migrations/0088_auto_20210811_0833.py new file mode 100644 index 0000000000..13fbaf33a4 --- /dev/null +++ b/apps/challenges/migrations/0088_auto_20210811_0833.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.20 on 2021-08-11 08:33 + +import base.utils +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('challenges', '0087_auto_20210811_0825'), + ] + + operations = [ + migrations.AlterField( + model_name='challenge', + name='requirements', + field=models.FileField(default=False, upload_to=base.utils.RandomFileName('requirements')), + ), + ] diff --git a/apps/challenges/migrations/0089_auto_20210811_0842.py b/apps/challenges/migrations/0089_auto_20210811_0842.py new file mode 100644 index 0000000000..6328109d69 --- /dev/null +++ b/apps/challenges/migrations/0089_auto_20210811_0842.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.20 on 2021-08-11 08:42 + +import base.utils +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('challenges', '0088_auto_20210811_0833'), + ] + + operations = [ + migrations.AlterField( + model_name='challenge', + name='requirements', + field=models.FileField(blank=True, upload_to=base.utils.RandomFileName('requirements')), + ), + ] diff --git a/apps/challenges/models.py b/apps/challenges/models.py index 7c5442c3c5..3f49c9be99 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -73,6 +73,10 @@ def __init__(self, *args, **kwargs): evaluation_script = models.FileField( default=False, upload_to=RandomFileName("evaluation_scripts") ) # should be zip format + requirements = models.FileField( + upload_to=RandomFileName("requirements"), blank=True + ) + approved_by_admin = models.BooleanField( default=False, verbose_name="Approved By Admin", db_index=True ) diff --git a/apps/challenges/serializers.py b/apps/challenges/serializers.py index e93250a399..4290465b43 100644 --- a/apps/challenges/serializers.py +++ b/apps/challenges/serializers.py @@ -65,6 +65,7 @@ class Meta: "remote_evaluation", "workers", "created_at", + "requirements" ) @@ -219,6 +220,9 @@ def __init__(self, *args, **kwargs): github_repository = context.get("github_repository") if github_repository: kwargs["data"]["github_repository"] = github_repository + requirements = context.get("requirements") + if requirements: + kwargs["data"]["requirements"] = requirements class Meta: model = Challenge @@ -262,6 +266,7 @@ class Meta: "max_worker_instance", "min_worker_instance", "desired_worker_instance", + "requirements", ) diff --git a/apps/challenges/urls.py b/apps/challenges/urls.py index 73f19e8af0..dc1d777fe0 100644 --- a/apps/challenges/urls.py +++ b/apps/challenges/urls.py @@ -151,6 +151,11 @@ views.get_challenge_phases_by_challenge_pk, name="get_challenge_phases_by_challenge_pk", ), + url( + r"^challenge/requirements/(?P[0-9]+)/$", + views.get_challenge_requirements_by_challenge_pk, + name="get_challenge_requirements_by_challenge_pk", + ), url( r"^challenge/phase/(?P[0-9]+)/$", views.get_challenge_phase_by_pk, diff --git a/apps/challenges/views.py b/apps/challenges/views.py index 1397f6142b..ffc3898443 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -1078,6 +1078,27 @@ def create_challenge_using_zip_file(request, challenge_host_team_pk): challenge_image_file = None else: challenge_image_file = None + + # Check for requirements in yaml file. + try: + requirements = yaml_file_data["requirements"] + if requirements.endswith(".txt"): + requirements_path = join( + BASE_LOCATION, + unique_folder_name, + extracted_folder_name, + requirements, + ) + if isfile(requirements_path): + requirements_file = ContentFile( + get_file_content(requirements_path, "rb"), 'requirements.txt' + ) + else: + requirements_file = "" + except KeyError: + requirements_file = "" + + # check for challenge description file try: @@ -1256,6 +1277,7 @@ def create_challenge_using_zip_file(request, challenge_host_team_pk): "challenge_host_team": challenge_host_team, "image": challenge_image_file, "evaluation_script": challenge_evaluation_script_file, + "requirements" : requirements_file, }, ) if serializer.is_valid(): @@ -2602,6 +2624,23 @@ def get_challenge_phases_by_challenge_pk(request, challenge_pk): response_data = serializer.data return Response(response_data, status=status.HTTP_200_OK) +@api_view(["GET"]) +@throttle_classes([AnonRateThrottle]) +def get_challenge_requirements_by_challenge_pk(request, challenge_pk): + """ + API endpoint to fetch all the requirements corresponding to a challenge using challenge pk + Arguments: + challenge_pk -- Challenge Id for which the requirements are to be fetched + Returns: + Response Object -- An object containing all requirements + """ + challenge = get_challenge_model(challenge_pk) + requirement_data = [] + if (challenge.requirements and challenge.requirements.name.endswith(".txt")): + requirement_data = challenge.requirements.read() + requirement_data = requirement_data.splitlines() + response_data = {"requirements": requirement_data} + return Response(response_data, status=status.HTTP_200_OK) @api_view(["GET"]) @throttle_classes([AnonRateThrottle]) diff --git a/frontend/src/js/controllers/challengeCtrl.js b/frontend/src/js/controllers/challengeCtrl.js index 8fbce15eb1..f0f0ff08fc 100644 --- a/frontend/src/js/controllers/challengeCtrl.js +++ b/frontend/src/js/controllers/challengeCtrl.js @@ -93,6 +93,7 @@ vm.previousPublicSubmissionId = null; vm.workerLogs = []; + vm.requirements = []; utilities.showLoader(); @@ -500,7 +501,28 @@ utilities.hideLoader(); } }; + + utilities.sendRequest(parameters); + + parameters.method = 'GET'; + parameters.url = 'challenges/challenge/requirements/' + vm.challengeId + '/'; + parameters.data = {}; + parameters.callback = { + onSuccess: function(response) { + var details = response.data; + vm.requirements = []; + for (var i = 0; iSubmission worker actions +
+
+
Requirements
+
+
+ +
+
{{requirement}}
+
Submission worker logs
diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py index f3992d101a..e086809499 100644 --- a/scripts/workers/submission_worker.py +++ b/scripts/workers/submission_worker.py @@ -14,6 +14,7 @@ import requests import signal import shutil +import subprocess import sys import tempfile import time @@ -260,12 +261,41 @@ def extract_challenge_data(challenge, phases): challenge_data_directory = CHALLENGE_DATA_DIR.format( challenge_id=challenge.id ) + # create challenge directory as package + create_dir_as_python_package(challenge_data_directory) + + if (challenge.requirements): + challenge_requirements_url = return_file_url_per_environment(challenge.requirements.url) + if (challenge_requirements_url and challenge_requirements_url.endswith(".txt")): + try: + response = requests.get(challenge_requirements_url, stream=True) + save_location = join(challenge_data_directory, "requirements_{}.txt".format(challenge.id)) + if response and response.status_code == 200: + with open(save_location, "wb") as f: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + logger.info(save_location) + try: + output = subprocess.check_output([sys.executable, "-m", "pip", "install", "-r", save_location]) + logger.info(output) + except Exception as e: + logger.error(e) + os.remove(save_location) + except Exception as e: + logger.error( + "{} Failed to fetch file from {}, error {}".format( + WORKER_LOGS_PREFIX, challenge_requirements_url, e + ) + ) + response = None + + evaluation_script_url = challenge.evaluation_script.url evaluation_script_url = return_file_url_per_environment( evaluation_script_url ) - # create challenge directory as package - create_dir_as_python_package(challenge_data_directory) + # set entry in map PHASE_ANNOTATION_FILE_NAME_MAP[challenge.id] = {} From 80e404f8fce2298ded962968a158e01c886f7681 Mon Sep 17 00:00:00 2001 From: Vince Li Date: Thu, 12 Aug 2021 22:46:40 -0400 Subject: [PATCH 2/9] make default for requirements empty --- .../migrations/0090_auto_20210813_0245.py | 19 +++++++++++++++++++ apps/challenges/models.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 apps/challenges/migrations/0090_auto_20210813_0245.py diff --git a/apps/challenges/migrations/0090_auto_20210813_0245.py b/apps/challenges/migrations/0090_auto_20210813_0245.py new file mode 100644 index 0000000000..113d4ba41d --- /dev/null +++ b/apps/challenges/migrations/0090_auto_20210813_0245.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.20 on 2021-08-13 02:45 + +import base.utils +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('challenges', '0089_auto_20210811_0842'), + ] + + operations = [ + migrations.AlterField( + model_name='challenge', + name='requirements', + field=models.FileField(blank=True, default='', upload_to=base.utils.RandomFileName('requirements')), + ), + ] diff --git a/apps/challenges/models.py b/apps/challenges/models.py index 3f49c9be99..f0c1b0f388 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -74,7 +74,7 @@ def __init__(self, *args, **kwargs): default=False, upload_to=RandomFileName("evaluation_scripts") ) # should be zip format requirements = models.FileField( - upload_to=RandomFileName("requirements"), blank=True + upload_to=RandomFileName("requirements"), blank=True, default="" ) approved_by_admin = models.BooleanField( From 4ee201357a39e25d7f55f0eedb2907cdc4968e5f Mon Sep 17 00:00:00 2001 From: Vince Li Date: Sat, 4 Sep 2021 21:05:16 -0400 Subject: [PATCH 3/9] made requirements.txt part of the evaluation_script folder --- .../0091_remove_challenge_requirements.py | 17 +++++ apps/challenges/models.py | 5 +- apps/challenges/serializers.py | 5 -- apps/challenges/views.py | 65 ++++++++++++------- scripts/workers/submission_worker.py | 39 ++++------- 5 files changed, 70 insertions(+), 61 deletions(-) create mode 100644 apps/challenges/migrations/0091_remove_challenge_requirements.py diff --git a/apps/challenges/migrations/0091_remove_challenge_requirements.py b/apps/challenges/migrations/0091_remove_challenge_requirements.py new file mode 100644 index 0000000000..9c297e4ded --- /dev/null +++ b/apps/challenges/migrations/0091_remove_challenge_requirements.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.20 on 2021-09-04 23:30 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('challenges', '0090_auto_20210813_0245'), + ] + + operations = [ + migrations.RemoveField( + model_name='challenge', + name='requirements', + ), + ] diff --git a/apps/challenges/models.py b/apps/challenges/models.py index f0c1b0f388..36c1a839f3 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -73,10 +73,7 @@ def __init__(self, *args, **kwargs): evaluation_script = models.FileField( default=False, upload_to=RandomFileName("evaluation_scripts") ) # should be zip format - requirements = models.FileField( - upload_to=RandomFileName("requirements"), blank=True, default="" - ) - + approved_by_admin = models.BooleanField( default=False, verbose_name="Approved By Admin", db_index=True ) diff --git a/apps/challenges/serializers.py b/apps/challenges/serializers.py index 4290465b43..e93250a399 100644 --- a/apps/challenges/serializers.py +++ b/apps/challenges/serializers.py @@ -65,7 +65,6 @@ class Meta: "remote_evaluation", "workers", "created_at", - "requirements" ) @@ -220,9 +219,6 @@ def __init__(self, *args, **kwargs): github_repository = context.get("github_repository") if github_repository: kwargs["data"]["github_repository"] = github_repository - requirements = context.get("requirements") - if requirements: - kwargs["data"]["requirements"] = requirements class Meta: model = Challenge @@ -266,7 +262,6 @@ class Meta: "max_worker_instance", "min_worker_instance", "desired_worker_instance", - "requirements", ) diff --git a/apps/challenges/views.py b/apps/challenges/views.py index ffc3898443..4697d35adf 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -1078,27 +1078,6 @@ def create_challenge_using_zip_file(request, challenge_host_team_pk): challenge_image_file = None else: challenge_image_file = None - - # Check for requirements in yaml file. - try: - requirements = yaml_file_data["requirements"] - if requirements.endswith(".txt"): - requirements_path = join( - BASE_LOCATION, - unique_folder_name, - extracted_folder_name, - requirements, - ) - if isfile(requirements_path): - requirements_file = ContentFile( - get_file_content(requirements_path, "rb"), 'requirements.txt' - ) - else: - requirements_file = "" - except KeyError: - requirements_file = "" - - # check for challenge description file try: @@ -1277,7 +1256,6 @@ def create_challenge_using_zip_file(request, challenge_host_team_pk): "challenge_host_team": challenge_host_team, "image": challenge_image_file, "evaluation_script": challenge_evaluation_script_file, - "requirements" : requirements_file, }, ) if serializer.is_valid(): @@ -2634,11 +2612,48 @@ def get_challenge_requirements_by_challenge_pk(request, challenge_pk): Returns: Response Object -- An object containing all requirements """ - challenge = get_challenge_model(challenge_pk) + # TODO: MODIFY requirement_data = [] - if (challenge.requirements and challenge.requirements.name.endswith(".txt")): - requirement_data = challenge.requirements.read() + + challenge = get_challenge_model(challenge_pk) + evaluation_script_url = challenge.evaluation_script.url + base_location = tempfile.mkdtemp() + zip_location = join( + base_location, "{}.zip".format(challenge_pk) + ) + extract_location = join( + base_location, "data{}".format(challenge_pk) + ) + """ + with open(zip_location, "wb") as f: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + """ + zip_ref = zipfile.ZipFile(challenge.evaluation_script, "r") + zip_ref.extractall(extract_location) + zip_ref.close() + try: + os.remove(zip_location) + except Exception as e: + logger.error("Failed to remove zip file {}".format(zip_location)) + requirements_location = join(extract_location, "requirements.txt"); + + if os.path.isfile(requirements_location): + f = open(requirements_location, "r") + requirement_data = f.read() requirement_data = requirement_data.splitlines() + f.close() + + try: + shutil.rmtree(base_location) + except: # noqa: E722 + logger.exception( + "Temporary directory {} for challenge {} not removed".format( + base_location, challenge_pk + ) + ) + response_data = {"requirements": requirement_data} return Response(response_data, status=status.HTTP_200_OK) diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py index e086809499..de78e5ff70 100644 --- a/scripts/workers/submission_worker.py +++ b/scripts/workers/submission_worker.py @@ -264,32 +264,6 @@ def extract_challenge_data(challenge, phases): # create challenge directory as package create_dir_as_python_package(challenge_data_directory) - if (challenge.requirements): - challenge_requirements_url = return_file_url_per_environment(challenge.requirements.url) - if (challenge_requirements_url and challenge_requirements_url.endswith(".txt")): - try: - response = requests.get(challenge_requirements_url, stream=True) - save_location = join(challenge_data_directory, "requirements_{}.txt".format(challenge.id)) - if response and response.status_code == 200: - with open(save_location, "wb") as f: - for chunk in response.iter_content(chunk_size=1024): - if chunk: - f.write(chunk) - logger.info(save_location) - try: - output = subprocess.check_output([sys.executable, "-m", "pip", "install", "-r", save_location]) - logger.info(output) - except Exception as e: - logger.error(e) - os.remove(save_location) - except Exception as e: - logger.error( - "{} Failed to fetch file from {}, error {}".format( - WORKER_LOGS_PREFIX, challenge_requirements_url, e - ) - ) - response = None - evaluation_script_url = challenge.evaluation_script.url evaluation_script_url = return_file_url_per_environment( @@ -307,6 +281,17 @@ def extract_challenge_data(challenge, phases): evaluation_script_url, challenge_zip_file, challenge_data_directory ) + + try: + requirements_location = join(challenge_data_directory, "requirements.txt"); + if os.path.isfile(requirements_location): + subprocess_output = subprocess.check_output([sys.executable, "-m", "pip", "install", "-r", requirements_location]) + print(subprocess_output) + else: + logger.info("No requirements for challenge {}".format(challenge.id)) + except Exception as e: + logger.error(e) + phase_data_base_directory = PHASE_DATA_BASE_DIR.format( challenge_id=challenge.id ) @@ -448,7 +433,7 @@ def run_submission( # create a temporary run directory under submission directory, so that # main directory does not gets polluted temp_run_dir = join(submission_data_dir, "run") - create_dir(temp_run_dir) + create_dir(temp_run_dir) stdout_file = join(temp_run_dir, "temp_stdout.txt") stderr_file = join(temp_run_dir, "temp_stderr.txt") From c6a6121796f72b66e9d5e30394e710ca42dec0f8 Mon Sep 17 00:00:00 2001 From: Vince Li Date: Wed, 8 Sep 2021 17:06:08 -0400 Subject: [PATCH 4/9] style changes --- apps/challenges/models.py | 2 +- apps/challenges/views.py | 22 +++++++++++----------- scripts/workers/submission_worker.py | 7 ++----- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/apps/challenges/models.py b/apps/challenges/models.py index 36c1a839f3..91dc5b7121 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -73,7 +73,7 @@ def __init__(self, *args, **kwargs): evaluation_script = models.FileField( default=False, upload_to=RandomFileName("evaluation_scripts") ) # should be zip format - + approved_by_admin = models.BooleanField( default=False, verbose_name="Approved By Admin", db_index=True ) diff --git a/apps/challenges/views.py b/apps/challenges/views.py index 4697d35adf..7145aaf9b3 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -2602,6 +2602,7 @@ def get_challenge_phases_by_challenge_pk(request, challenge_pk): response_data = serializer.data return Response(response_data, status=status.HTTP_200_OK) + @api_view(["GET"]) @throttle_classes([AnonRateThrottle]) def get_challenge_requirements_by_challenge_pk(request, challenge_pk): @@ -2612,11 +2613,9 @@ def get_challenge_requirements_by_challenge_pk(request, challenge_pk): Returns: Response Object -- An object containing all requirements """ - # TODO: MODIFY requirement_data = [] challenge = get_challenge_model(challenge_pk) - evaluation_script_url = challenge.evaluation_script.url base_location = tempfile.mkdtemp() zip_location = join( base_location, "{}.zip".format(challenge_pk) @@ -2624,20 +2623,20 @@ def get_challenge_requirements_by_challenge_pk(request, challenge_pk): extract_location = join( base_location, "data{}".format(challenge_pk) ) - """ - with open(zip_location, "wb") as f: - for chunk in response.iter_content(chunk_size=1024): - if chunk: - f.write(chunk) - """ + zip_ref = zipfile.ZipFile(challenge.evaluation_script, "r") zip_ref.extractall(extract_location) zip_ref.close() try: os.remove(zip_location) - except Exception as e: - logger.error("Failed to remove zip file {}".format(zip_location)) - requirements_location = join(extract_location, "requirements.txt"); + except: + logger.exception( + "Temporary directory {} for challenge {} not removed".format( + zip_location, challenge_pk + ) + ) + + requirements_location = join(extract_location, "requirements.txt") if os.path.isfile(requirements_location): f = open(requirements_location, "r") @@ -2657,6 +2656,7 @@ def get_challenge_requirements_by_challenge_pk(request, challenge_pk): response_data = {"requirements": requirement_data} return Response(response_data, status=status.HTTP_200_OK) + @api_view(["GET"]) @throttle_classes([AnonRateThrottle]) def get_challenge_phase_by_pk(request, pk): diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py index de78e5ff70..b04780f455 100644 --- a/scripts/workers/submission_worker.py +++ b/scripts/workers/submission_worker.py @@ -263,14 +263,12 @@ def extract_challenge_data(challenge, phases): ) # create challenge directory as package create_dir_as_python_package(challenge_data_directory) - evaluation_script_url = challenge.evaluation_script.url evaluation_script_url = return_file_url_per_environment( evaluation_script_url ) - # set entry in map PHASE_ANNOTATION_FILE_NAME_MAP[challenge.id] = {} @@ -281,9 +279,8 @@ def extract_challenge_data(challenge, phases): evaluation_script_url, challenge_zip_file, challenge_data_directory ) - try: - requirements_location = join(challenge_data_directory, "requirements.txt"); + requirements_location = join(challenge_data_directory, "requirements.txt") if os.path.isfile(requirements_location): subprocess_output = subprocess.check_output([sys.executable, "-m", "pip", "install", "-r", requirements_location]) print(subprocess_output) @@ -433,7 +430,7 @@ def run_submission( # create a temporary run directory under submission directory, so that # main directory does not gets polluted temp_run_dir = join(submission_data_dir, "run") - create_dir(temp_run_dir) + create_dir(temp_run_dir) stdout_file = join(temp_run_dir, "temp_stdout.txt") stderr_file = join(temp_run_dir, "temp_stderr.txt") From 4547f5b7c38ecc5d3f546f1324ed23205f78dadc Mon Sep 17 00:00:00 2001 From: Vince Li Date: Wed, 8 Sep 2021 22:54:36 -0400 Subject: [PATCH 5/9] responding to comments --- .../migrations/0086_challenge_requirements.py | 19 ------------------- .../migrations/0087_auto_20210811_0825.py | 19 ------------------- .../migrations/0088_auto_20210811_0833.py | 19 ------------------- .../migrations/0089_auto_20210811_0842.py | 19 ------------------- .../migrations/0090_auto_20210813_0245.py | 19 ------------------- .../0091_remove_challenge_requirements.py | 17 ----------------- apps/challenges/models.py | 1 - apps/challenges/views.py | 10 +++++----- scripts/workers/submission_worker.py | 1 - 9 files changed, 5 insertions(+), 119 deletions(-) delete mode 100644 apps/challenges/migrations/0086_challenge_requirements.py delete mode 100644 apps/challenges/migrations/0087_auto_20210811_0825.py delete mode 100644 apps/challenges/migrations/0088_auto_20210811_0833.py delete mode 100644 apps/challenges/migrations/0089_auto_20210811_0842.py delete mode 100644 apps/challenges/migrations/0090_auto_20210813_0245.py delete mode 100644 apps/challenges/migrations/0091_remove_challenge_requirements.py diff --git a/apps/challenges/migrations/0086_challenge_requirements.py b/apps/challenges/migrations/0086_challenge_requirements.py deleted file mode 100644 index a26310f2f8..0000000000 --- a/apps/challenges/migrations/0086_challenge_requirements.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 2.2.20 on 2021-08-11 06:51 - -import base.utils -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('challenges', '0085_challenge_submission_time_limit'), - ] - - operations = [ - migrations.AddField( - model_name='challenge', - name='requirements', - field=models.FileField(blank=True, upload_to=base.utils.RandomFileName('requirements')), - ), - ] diff --git a/apps/challenges/migrations/0087_auto_20210811_0825.py b/apps/challenges/migrations/0087_auto_20210811_0825.py deleted file mode 100644 index 52d1333aa3..0000000000 --- a/apps/challenges/migrations/0087_auto_20210811_0825.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 2.2.20 on 2021-08-11 08:25 - -import base.utils -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('challenges', '0086_challenge_requirements'), - ] - - operations = [ - migrations.AlterField( - model_name='challenge', - name='requirements', - field=models.FileField(blank=True, null=True, upload_to=base.utils.RandomFileName('requirements')), - ), - ] diff --git a/apps/challenges/migrations/0088_auto_20210811_0833.py b/apps/challenges/migrations/0088_auto_20210811_0833.py deleted file mode 100644 index 13fbaf33a4..0000000000 --- a/apps/challenges/migrations/0088_auto_20210811_0833.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 2.2.20 on 2021-08-11 08:33 - -import base.utils -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('challenges', '0087_auto_20210811_0825'), - ] - - operations = [ - migrations.AlterField( - model_name='challenge', - name='requirements', - field=models.FileField(default=False, upload_to=base.utils.RandomFileName('requirements')), - ), - ] diff --git a/apps/challenges/migrations/0089_auto_20210811_0842.py b/apps/challenges/migrations/0089_auto_20210811_0842.py deleted file mode 100644 index 6328109d69..0000000000 --- a/apps/challenges/migrations/0089_auto_20210811_0842.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 2.2.20 on 2021-08-11 08:42 - -import base.utils -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('challenges', '0088_auto_20210811_0833'), - ] - - operations = [ - migrations.AlterField( - model_name='challenge', - name='requirements', - field=models.FileField(blank=True, upload_to=base.utils.RandomFileName('requirements')), - ), - ] diff --git a/apps/challenges/migrations/0090_auto_20210813_0245.py b/apps/challenges/migrations/0090_auto_20210813_0245.py deleted file mode 100644 index 113d4ba41d..0000000000 --- a/apps/challenges/migrations/0090_auto_20210813_0245.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 2.2.20 on 2021-08-13 02:45 - -import base.utils -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('challenges', '0089_auto_20210811_0842'), - ] - - operations = [ - migrations.AlterField( - model_name='challenge', - name='requirements', - field=models.FileField(blank=True, default='', upload_to=base.utils.RandomFileName('requirements')), - ), - ] diff --git a/apps/challenges/migrations/0091_remove_challenge_requirements.py b/apps/challenges/migrations/0091_remove_challenge_requirements.py deleted file mode 100644 index 9c297e4ded..0000000000 --- a/apps/challenges/migrations/0091_remove_challenge_requirements.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 2.2.20 on 2021-09-04 23:30 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('challenges', '0090_auto_20210813_0245'), - ] - - operations = [ - migrations.RemoveField( - model_name='challenge', - name='requirements', - ), - ] diff --git a/apps/challenges/models.py b/apps/challenges/models.py index 91dc5b7121..7c5442c3c5 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -73,7 +73,6 @@ def __init__(self, *args, **kwargs): evaluation_script = models.FileField( default=False, upload_to=RandomFileName("evaluation_scripts") ) # should be zip format - approved_by_admin = models.BooleanField( default=False, verbose_name="Approved By Admin", db_index=True ) diff --git a/apps/challenges/views.py b/apps/challenges/views.py index 7145aaf9b3..c710457c83 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -2631,8 +2631,8 @@ def get_challenge_requirements_by_challenge_pk(request, challenge_pk): os.remove(zip_location) except: logger.exception( - "Temporary directory {} for challenge {} not removed".format( - zip_location, challenge_pk + "Temporary directory {} for challenge {} not removed. Error: {}".format( + zip_location, challenge_pk, e ) ) @@ -2646,10 +2646,10 @@ def get_challenge_requirements_by_challenge_pk(request, challenge_pk): try: shutil.rmtree(base_location) - except: # noqa: E722 + except Exception as e: logger.exception( - "Temporary directory {} for challenge {} not removed".format( - base_location, challenge_pk + "Temporary directory {} for challenge {} not removed. Error: {}".format( + base_location, challenge_pk, e ) ) diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py index b04780f455..79f3ea2fa7 100644 --- a/scripts/workers/submission_worker.py +++ b/scripts/workers/submission_worker.py @@ -283,7 +283,6 @@ def extract_challenge_data(challenge, phases): requirements_location = join(challenge_data_directory, "requirements.txt") if os.path.isfile(requirements_location): subprocess_output = subprocess.check_output([sys.executable, "-m", "pip", "install", "-r", requirements_location]) - print(subprocess_output) else: logger.info("No requirements for challenge {}".format(challenge.id)) except Exception as e: From 22559ad323d6f29ea89acd16f3291e7eec1dbb6e Mon Sep 17 00:00:00 2001 From: Vince Li Date: Wed, 8 Sep 2021 23:04:03 -0400 Subject: [PATCH 6/9] added fixes for missed comments --- apps/challenges/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/challenges/views.py b/apps/challenges/views.py index c710457c83..3f9f9c4748 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -2629,7 +2629,7 @@ def get_challenge_requirements_by_challenge_pk(request, challenge_pk): zip_ref.close() try: os.remove(zip_location) - except: + except Exception as e: logger.exception( "Temporary directory {} for challenge {} not removed. Error: {}".format( zip_location, challenge_pk, e From 0164a59b9c8b1dbfa3b1e3a72b5cb8ef6e34b183 Mon Sep 17 00:00:00 2001 From: Vince Li Date: Thu, 9 Sep 2021 16:03:21 -0400 Subject: [PATCH 7/9] removed unused variable --- scripts/workers/submission_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py index 79f3ea2fa7..8a03df4465 100644 --- a/scripts/workers/submission_worker.py +++ b/scripts/workers/submission_worker.py @@ -282,7 +282,7 @@ def extract_challenge_data(challenge, phases): try: requirements_location = join(challenge_data_directory, "requirements.txt") if os.path.isfile(requirements_location): - subprocess_output = subprocess.check_output([sys.executable, "-m", "pip", "install", "-r", requirements_location]) + subprocess.check_output([sys.executable, "-m", "pip", "install", "-r", requirements_location]) else: logger.info("No requirements for challenge {}".format(challenge.id)) except Exception as e: From 3769ec96d75d3ae27db49d70453887c6f17b8114 Mon Sep 17 00:00:00 2001 From: Vince Li Date: Fri, 10 Sep 2021 11:35:17 -0400 Subject: [PATCH 8/9] frontend changes removed --- frontend/src/js/controllers/challengeCtrl.js | 22 -------------------- frontend/src/views/web/challenge/manage.html | 9 -------- 2 files changed, 31 deletions(-) diff --git a/frontend/src/js/controllers/challengeCtrl.js b/frontend/src/js/controllers/challengeCtrl.js index f0f0ff08fc..8fbce15eb1 100644 --- a/frontend/src/js/controllers/challengeCtrl.js +++ b/frontend/src/js/controllers/challengeCtrl.js @@ -93,7 +93,6 @@ vm.previousPublicSubmissionId = null; vm.workerLogs = []; - vm.requirements = []; utilities.showLoader(); @@ -501,28 +500,7 @@ utilities.hideLoader(); } }; - - utilities.sendRequest(parameters); - - parameters.method = 'GET'; - parameters.url = 'challenges/challenge/requirements/' + vm.challengeId + '/'; - parameters.data = {}; - parameters.callback = { - onSuccess: function(response) { - var details = response.data; - vm.requirements = []; - for (var i = 0; iSubmission worker actions
-
-
-
Requirements
-
-
- -
-
{{requirement}}
-
Submission worker logs
From 39c13d761148f2d80e7e79b5a06d66d10672717a Mon Sep 17 00:00:00 2001 From: Vince Li Date: Thu, 16 Sep 2021 15:57:36 -0400 Subject: [PATCH 9/9] removed requirements api, clarified worker error message --- apps/challenges/urls.py | 5 --- apps/challenges/views.py | 54 ---------------------------- scripts/workers/submission_worker.py | 2 +- 3 files changed, 1 insertion(+), 60 deletions(-) diff --git a/apps/challenges/urls.py b/apps/challenges/urls.py index dc1d777fe0..73f19e8af0 100644 --- a/apps/challenges/urls.py +++ b/apps/challenges/urls.py @@ -151,11 +151,6 @@ views.get_challenge_phases_by_challenge_pk, name="get_challenge_phases_by_challenge_pk", ), - url( - r"^challenge/requirements/(?P[0-9]+)/$", - views.get_challenge_requirements_by_challenge_pk, - name="get_challenge_requirements_by_challenge_pk", - ), url( r"^challenge/phase/(?P[0-9]+)/$", views.get_challenge_phase_by_pk, diff --git a/apps/challenges/views.py b/apps/challenges/views.py index 3f9f9c4748..1397f6142b 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -2603,60 +2603,6 @@ def get_challenge_phases_by_challenge_pk(request, challenge_pk): return Response(response_data, status=status.HTTP_200_OK) -@api_view(["GET"]) -@throttle_classes([AnonRateThrottle]) -def get_challenge_requirements_by_challenge_pk(request, challenge_pk): - """ - API endpoint to fetch all the requirements corresponding to a challenge using challenge pk - Arguments: - challenge_pk -- Challenge Id for which the requirements are to be fetched - Returns: - Response Object -- An object containing all requirements - """ - requirement_data = [] - - challenge = get_challenge_model(challenge_pk) - base_location = tempfile.mkdtemp() - zip_location = join( - base_location, "{}.zip".format(challenge_pk) - ) - extract_location = join( - base_location, "data{}".format(challenge_pk) - ) - - zip_ref = zipfile.ZipFile(challenge.evaluation_script, "r") - zip_ref.extractall(extract_location) - zip_ref.close() - try: - os.remove(zip_location) - except Exception as e: - logger.exception( - "Temporary directory {} for challenge {} not removed. Error: {}".format( - zip_location, challenge_pk, e - ) - ) - - requirements_location = join(extract_location, "requirements.txt") - - if os.path.isfile(requirements_location): - f = open(requirements_location, "r") - requirement_data = f.read() - requirement_data = requirement_data.splitlines() - f.close() - - try: - shutil.rmtree(base_location) - except Exception as e: - logger.exception( - "Temporary directory {} for challenge {} not removed. Error: {}".format( - base_location, challenge_pk, e - ) - ) - - response_data = {"requirements": requirement_data} - return Response(response_data, status=status.HTTP_200_OK) - - @api_view(["GET"]) @throttle_classes([AnonRateThrottle]) def get_challenge_phase_by_pk(request, pk): diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py index 8a03df4465..46ce4abaac 100644 --- a/scripts/workers/submission_worker.py +++ b/scripts/workers/submission_worker.py @@ -284,7 +284,7 @@ def extract_challenge_data(challenge, phases): if os.path.isfile(requirements_location): subprocess.check_output([sys.executable, "-m", "pip", "install", "-r", requirements_location]) else: - logger.info("No requirements for challenge {}".format(challenge.id)) + logger.info("No custom requirements for challenge {}".format(challenge.id)) except Exception as e: logger.error(e)