From 978367bcdfda3d31d800bff71723abaaf4937e6d Mon Sep 17 00:00:00 2001 From: savish28 <32800267+savish28@users.noreply.github.com> Date: Sat, 5 Jun 2021 20:34:12 +0530 Subject: [PATCH] Backend: Add static dataset code upload worker service & task defination with start/stop/delete worker action (#3436) * Add is_static_dataset_docker_based_challenge field in Challenge model * [Tests] is_static_dataset_docker_based_challenge field for Challenge model * Add: docker_input_file filed in Submission model * Tests: docker_input_file filed in Submission model * Update: Message Queue body, adding is_static_code_upload_submission * Add: submission_input_file filed in Submission model * Tests: submission_input_file field in Submission model * Update: String Builder to simple string challenge title * Change verbose name static dataset challenge * is_static_dataset challenge verbose name change * Use Common name code upload instead of docker * Add: Static Code Upload challenge workers defn and connecting to django admin pipeline Co-authored-by: Rishabh Jain --- apps/challenges/aws_utils.py | 288 ++++++++++++++++++++++++++++++----- 1 file changed, 252 insertions(+), 36 deletions(-) diff --git a/apps/challenges/aws_utils.py b/apps/challenges/aws_utils.py index 304bb8dcfe..7945774893 100644 --- a/apps/challenges/aws_utils.py +++ b/apps/challenges/aws_utils.py @@ -227,7 +227,7 @@ "networkMode":"awsvpc", "containerDefinitions":[ {{ - "name": "{container_name}", + "name": "{code_upload_container_name}", "image": "{CODE_UPLOAD_WORKER_IMAGE}", "essential": True, "environment": [ @@ -305,6 +305,214 @@ }} """ +task_definition_static_code_upload_worker = """ +{{ + "family":"{queue_name}", + "executionRoleArn":"{EXECUTION_ROLE_ARN}", + "networkMode":"awsvpc", + "containerDefinitions":[ + {{ + "name": "{code_upload_container_name}", + "image": "{CODE_UPLOAD_WORKER_IMAGE}", + "essential": True, + "environment": [ + {{ + "name": "AWS_DEFAULT_REGION", + "value": "{AWS_REGION}" + }}, + {{ + "name": "AWS_ACCESS_KEY_ID", + "value": "{AWS_ACCESS_KEY_ID}" + }}, + {{ + "name": "AWS_SECRET_ACCESS_KEY", + "value": "{AWS_SECRET_ACCESS_KEY}" + }}, + {{ + "name": "CLUSTER_NAME", + "value": "{cluster_name}" + }}, + {{ + "name": "CLUSTER_ENDPOINT", + "value": "{cluster_endpoint}" + }}, + {{ + "name": "CERTIFICATE", + "value": "{certificate}" + }}, + {{ + "name": "CIDR", + "value": "{CIDR}" + }}, + {{ + "name": "QUEUE_NAME", + "value": "{queue_name}" + }}, + {{ + "name": "EVALAI_API_SERVER", + "value": "{EVALAI_API_SERVER}" + }}, + + {{ + "name": "AUTH_TOKEN", + "value": "{auth_token}" + }}, + + {{ + "name": "EVALAI_DNS", + "value": "{EVALAI_DNS}" + }}, + + {{ + "name": "EFS_ID", + "value": "{EFS_ID}" + }} + + ], + "workingDirectory": "/code", + "readonlyRootFilesystem": False, + "logConfiguration": {{ + "logDriver": "awslogs", + "options": {{ + "awslogs-group": "{log_group_name}", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "{queue_name}", + "awslogs-create-group": "true", + }}, + }}, + }}, + {{ + "name": "{container_name}", + "image": "{WORKER_IMAGE}", + "essential": True, + "environment": [ + {{ + "name": "AWS_DEFAULT_REGION", + "value": "{AWS_REGION}" + }}, + {{ + "name": "AWS_ACCOUNT_ID", + "value": "{AWS_ACCOUNT_ID}" + }}, + {{ + "name": "AWS_ACCESS_KEY_ID", + "value": "{AWS_ACCESS_KEY_ID}" + }}, + {{ + "name": "AWS_SECRET_ACCESS_KEY", + "value": "{AWS_SECRET_ACCESS_KEY}" + }}, + {{ + "name": "AWS_STORAGE_BUCKET_NAME", + "value": "{AWS_STORAGE_BUCKET_NAME}" + }}, + {{ + "name": "CHALLENGE_PK", + "value": "{challenge_pk}" + }}, + {{ + "name": "CHALLENGE_QUEUE", + "value": "{queue_name}" + }}, + {{ + "name": "DJANGO_SERVER", + "value": "{DJANGO_SERVER}" + }}, + {{ + "name": "DJANGO_SETTINGS_MODULE", + "value": "settings.{ENV}" + }}, + {{ + "name": "DEBUG", + "value": "{DEBUG}" + }}, + {{ + "name": "EMAIL_HOST", + "value": "{EMAIL_HOST}" + }}, + {{ + "name": "EMAIL_HOST_PASSWORD", + "value": "{EMAIL_HOST_PASSWORD}" + }}, + {{ + "name": "EMAIL_HOST_USER", + "value": "{EMAIL_HOST_USER}" + }}, + {{ + "name": "EMAIL_PORT", + "value": "{EMAIL_PORT}" + }}, + {{ + "name": "EMAIL_USE_TLS", + "value": "{EMAIL_USE_TLS}" + }}, + {{ + "name": "MEMCACHED_LOCATION", + "value": "{MEMCACHED_LOCATION}" + }}, + {{ + "name": "PYTHONUNBUFFERED", + "value": "1" + }}, + {{ + "name": "RDS_DB_NAME", + "value": "{RDS_DB_NAME}" + }}, + {{ + "name": "RDS_HOSTNAME", + "value": "{RDS_HOSTNAME}" + }}, + {{ + "name": "RDS_PASSWORD", + "value": "{RDS_PASSWORD}" + }}, + {{ + "name": "RDS_USERNAME", + "value": "{RDS_USERNAME}" + }}, + {{ + "name": "RDS_PORT", + "value": "{RDS_PORT}" + }}, + {{ + "name": "SECRET_KEY", + "value": "{SECRET_KEY}" + }}, + {{ + "name": "SENTRY_URL", + "value": "{SENTRY_URL}" + }}, + {{ + "name": "AWS_SES_REGION_NAME", + "value": "{AWS_SES_REGION_NAME}" + }}, + {{ + "name": "AWS_SES_REGION_ENDPOINT", + "value": "{AWS_SES_REGION_ENDPOINT}" + }} + ], + "workingDirectory": "/code", + "readonlyRootFilesystem": False, + "logConfiguration": {{ + "logDriver": "awslogs", + "options": {{ + "awslogs-group": "{log_group_name}", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "{queue_name}", + "awslogs-create-group": "true", + }}, + }}, + }} + + ], + "requiresCompatibilities":[ + "FARGATE" + ], + "cpu": "{CPU}", + "memory": "{MEMORY}", +}} +""" + service_definition = """ {{ "cluster":"{CLUSTER}", @@ -424,6 +632,7 @@ def register_task_def_by_challenge_pk(client, queue_name, challenge): dict: A dict of the task definition and it's ARN if succesful, and an error dictionary if not """ container_name = "worker_{}".format(queue_name) + code_upload_container_name = "code_upload_worker_{}".format(queue_name) worker_cpu_cores = challenge.worker_cpu_cores worker_memory = challenge.worker_memory log_group_name = get_log_group_name(challenge.pk) @@ -452,23 +661,45 @@ def register_task_def_by_challenge_pk(client, queue_name, challenge): return e.response # challenge host auth token to be used by code-upload-worker token = JwtToken.objects.get(user=challenge.creator.created_by) - definition = task_definition_code_upload_worker.format( - queue_name=queue_name, - container_name=container_name, - ENV=ENV, - challenge_pk=challenge.pk, - auth_token=token.refresh_token, - cluster_name=cluster_name, - cluster_endpoint=cluster_endpoint, - certificate=cluster_certificate, - CPU=worker_cpu_cores, - MEMORY=worker_memory, - log_group_name=log_group_name, - EVALAI_DNS=EVALAI_DNS, - EFS_ID=efs_id, - **COMMON_SETTINGS_DICT, - **challenge_aws_keys, - ) + if challenge.is_static_dataset_docker_based_challenge: + definition = task_definition_static_code_upload_worker.format( + queue_name=queue_name, + container_name=container_name, + code_upload_container_name=code_upload_container_name, + ENV=ENV, + challenge_pk=challenge.pk, + auth_token=token.refresh_token, + cluster_name=cluster_name, + cluster_endpoint=cluster_endpoint, + certificate=cluster_certificate, + CPU=worker_cpu_cores, + MEMORY=worker_memory, + log_group_name=log_group_name, + EVALAI_DNS=EVALAI_DNS, + EFS_ID=efs_id, + AWS_SES_REGION_NAME=AWS_SES_REGION_NAME, + AWS_SES_REGION_ENDPOINT=AWS_SES_REGION_ENDPOINT, + **COMMON_SETTINGS_DICT, + **challenge_aws_keys, + ) + else: + definition = task_definition_code_upload_worker.format( + queue_name=queue_name, + code_upload_container_name=code_upload_container_name, + ENV=ENV, + challenge_pk=challenge.pk, + auth_token=token.refresh_token, + cluster_name=cluster_name, + cluster_endpoint=cluster_endpoint, + certificate=cluster_certificate, + CPU=worker_cpu_cores, + MEMORY=worker_memory, + log_group_name=log_group_name, + EVALAI_DNS=EVALAI_DNS, + EFS_ID=efs_id, + **COMMON_SETTINGS_DICT, + **challenge_aws_keys, + ) else: definition = task_definition.format( queue_name=queue_name, @@ -709,12 +940,7 @@ def start_workers(queryset): count = 0 failures = [] for challenge in queryset: - if challenge.is_docker_based: - response = "Sorry. This feature is not available for code upload/docker based challenges." - failures.append( - {"message": response, "challenge_pk": challenge.pk} - ) - elif (challenge.workers == 0) or (challenge.workers is None): + if (challenge.workers == 0) or (challenge.workers is None): response = service_manager( client, challenge=challenge, num_of_tasks=1 ) @@ -763,12 +989,7 @@ def stop_workers(queryset): count = 0 failures = [] for challenge in queryset: - if challenge.is_docker_based: - response = "Sorry. This feature is not available for code upload/docker based challenges." - failures.append( - {"message": response, "challenge_pk": challenge.pk} - ) - elif (challenge.workers is not None) and (challenge.workers > 0): + if (challenge.workers is not None) and (challenge.workers > 0): response = service_manager( client, challenge=challenge, num_of_tasks=0 ) @@ -870,12 +1091,7 @@ def delete_workers(queryset): count = 0 failures = [] for challenge in queryset: - if challenge.is_docker_based: - response = "Sorry. This feature is not available for code upload/docker based challenges." - failures.append( - {"message": response, "challenge_pk": challenge.pk} - ) - elif challenge.workers is not None: + if challenge.workers is not None: response = delete_service_by_challenge_pk(challenge=challenge) if response["ResponseMetadata"]["HTTPStatusCode"] != HTTPStatus.OK: failures.append(