Skip to content

Commit

Permalink
Backend: Add static dataset code upload worker service & task definat…
Browse files Browse the repository at this point in the history 
…ion with start/stop/delete worker action (#3436)

* Add is_static_dataset_docker_based_challenge field in Challenge model

* [Tests] is_static_dataset_docker_based_challenge field for Challenge model

* Add: docker_input_file filed in Submission model

* Tests: docker_input_file filed in Submission model

* Update: Message Queue body, adding is_static_code_upload_submission

* Add: submission_input_file filed in Submission model

* Tests: submission_input_file  field in Submission model

* Update: String Builder to simple string challenge title

* Change verbose name static dataset challenge

* is_static_dataset challenge verbose name change

* Use Common name code upload instead of docker

* Add: Static Code Upload challenge workers defn and connecting to django admin pipeline

Co-authored-by: Rishabh Jain <[email protected]>
  • Loading branch information
@savish28 @RishabhJain2018
savish28 and RishabhJain2018 authored Jun 5, 2021
1 parent 7001bc8 commit 978367b
Showing 1 changed file with 252 additions and 36 deletions.
288 changes: 252 additions & 36 deletions apps/challenges/aws_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@
"networkMode":"awsvpc",
"containerDefinitions":[
{{
"name": "{container_name}",
"name": "{code_upload_container_name}",
"image": "{CODE_UPLOAD_WORKER_IMAGE}",
"essential": True,
"environment": [
Expand Down Expand Up @@ -305,6 +305,214 @@
}}
"""

task_definition_static_code_upload_worker = """
{{
"family":"{queue_name}",
"executionRoleArn":"{EXECUTION_ROLE_ARN}",
"networkMode":"awsvpc",
"containerDefinitions":[
{{
"name": "{code_upload_container_name}",
"image": "{CODE_UPLOAD_WORKER_IMAGE}",
"essential": True,
"environment": [
{{
"name": "AWS_DEFAULT_REGION",
"value": "{AWS_REGION}"
}},
{{
"name": "AWS_ACCESS_KEY_ID",
"value": "{AWS_ACCESS_KEY_ID}"
}},
{{
"name": "AWS_SECRET_ACCESS_KEY",
"value": "{AWS_SECRET_ACCESS_KEY}"
}},
{{
"name": "CLUSTER_NAME",
"value": "{cluster_name}"
}},
{{
"name": "CLUSTER_ENDPOINT",
"value": "{cluster_endpoint}"
}},
{{
"name": "CERTIFICATE",
"value": "{certificate}"
}},
{{
"name": "CIDR",
"value": "{CIDR}"
}},
{{
"name": "QUEUE_NAME",
"value": "{queue_name}"
}},
{{
"name": "EVALAI_API_SERVER",
"value": "{EVALAI_API_SERVER}"
}},
{{
"name": "AUTH_TOKEN",
"value": "{auth_token}"
}},
{{
"name": "EVALAI_DNS",
"value": "{EVALAI_DNS}"
}},
{{
"name": "EFS_ID",
"value": "{EFS_ID}"
}}
],
"workingDirectory": "/code",
"readonlyRootFilesystem": False,
"logConfiguration": {{
"logDriver": "awslogs",
"options": {{
"awslogs-group": "{log_group_name}",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "{queue_name}",
"awslogs-create-group": "true",
}},
}},
}},
{{
"name": "{container_name}",
"image": "{WORKER_IMAGE}",
"essential": True,
"environment": [
{{
"name": "AWS_DEFAULT_REGION",
"value": "{AWS_REGION}"
}},
{{
"name": "AWS_ACCOUNT_ID",
"value": "{AWS_ACCOUNT_ID}"
}},
{{
"name": "AWS_ACCESS_KEY_ID",
"value": "{AWS_ACCESS_KEY_ID}"
}},
{{
"name": "AWS_SECRET_ACCESS_KEY",
"value": "{AWS_SECRET_ACCESS_KEY}"
}},
{{
"name": "AWS_STORAGE_BUCKET_NAME",
"value": "{AWS_STORAGE_BUCKET_NAME}"
}},
{{
"name": "CHALLENGE_PK",
"value": "{challenge_pk}"
}},
{{
"name": "CHALLENGE_QUEUE",
"value": "{queue_name}"
}},
{{
"name": "DJANGO_SERVER",
"value": "{DJANGO_SERVER}"
}},
{{
"name": "DJANGO_SETTINGS_MODULE",
"value": "settings.{ENV}"
}},
{{
"name": "DEBUG",
"value": "{DEBUG}"
}},
{{
"name": "EMAIL_HOST",
"value": "{EMAIL_HOST}"
}},
{{
"name": "EMAIL_HOST_PASSWORD",
"value": "{EMAIL_HOST_PASSWORD}"
}},
{{
"name": "EMAIL_HOST_USER",
"value": "{EMAIL_HOST_USER}"
}},
{{
"name": "EMAIL_PORT",
"value": "{EMAIL_PORT}"
}},
{{
"name": "EMAIL_USE_TLS",
"value": "{EMAIL_USE_TLS}"
}},
{{
"name": "MEMCACHED_LOCATION",
"value": "{MEMCACHED_LOCATION}"
}},
{{
"name": "PYTHONUNBUFFERED",
"value": "1"
}},
{{
"name": "RDS_DB_NAME",
"value": "{RDS_DB_NAME}"
}},
{{
"name": "RDS_HOSTNAME",
"value": "{RDS_HOSTNAME}"
}},
{{
"name": "RDS_PASSWORD",
"value": "{RDS_PASSWORD}"
}},
{{
"name": "RDS_USERNAME",
"value": "{RDS_USERNAME}"
}},
{{
"name": "RDS_PORT",
"value": "{RDS_PORT}"
}},
{{
"name": "SECRET_KEY",
"value": "{SECRET_KEY}"
}},
{{
"name": "SENTRY_URL",
"value": "{SENTRY_URL}"
}},
{{
"name": "AWS_SES_REGION_NAME",
"value": "{AWS_SES_REGION_NAME}"
}},
{{
"name": "AWS_SES_REGION_ENDPOINT",
"value": "{AWS_SES_REGION_ENDPOINT}"
}}
],
"workingDirectory": "/code",
"readonlyRootFilesystem": False,
"logConfiguration": {{
"logDriver": "awslogs",
"options": {{
"awslogs-group": "{log_group_name}",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "{queue_name}",
"awslogs-create-group": "true",
}},
}},
}}
],
"requiresCompatibilities":[
"FARGATE"
],
"cpu": "{CPU}",
"memory": "{MEMORY}",
}}
"""

service_definition = """
{{
"cluster":"{CLUSTER}",
Expand Down Expand Up @@ -424,6 +632,7 @@ def register_task_def_by_challenge_pk(client, queue_name, challenge):
dict: A dict of the task definition and it's ARN if succesful, and an error dictionary if not
"""
container_name = "worker_{}".format(queue_name)
code_upload_container_name = "code_upload_worker_{}".format(queue_name)
worker_cpu_cores = challenge.worker_cpu_cores
worker_memory = challenge.worker_memory
log_group_name = get_log_group_name(challenge.pk)
Expand Down Expand Up @@ -452,23 +661,45 @@ def register_task_def_by_challenge_pk(client, queue_name, challenge):
return e.response
# challenge host auth token to be used by code-upload-worker
token = JwtToken.objects.get(user=challenge.creator.created_by)
definition = task_definition_code_upload_worker.format(
queue_name=queue_name,
container_name=container_name,
ENV=ENV,
challenge_pk=challenge.pk,
auth_token=token.refresh_token,
cluster_name=cluster_name,
cluster_endpoint=cluster_endpoint,
certificate=cluster_certificate,
CPU=worker_cpu_cores,
MEMORY=worker_memory,
log_group_name=log_group_name,
EVALAI_DNS=EVALAI_DNS,
EFS_ID=efs_id,
**COMMON_SETTINGS_DICT,
**challenge_aws_keys,
)
if challenge.is_static_dataset_docker_based_challenge:
definition = task_definition_static_code_upload_worker.format(
queue_name=queue_name,
container_name=container_name,
code_upload_container_name=code_upload_container_name,
ENV=ENV,
challenge_pk=challenge.pk,
auth_token=token.refresh_token,
cluster_name=cluster_name,
cluster_endpoint=cluster_endpoint,
certificate=cluster_certificate,
CPU=worker_cpu_cores,
MEMORY=worker_memory,
log_group_name=log_group_name,
EVALAI_DNS=EVALAI_DNS,
EFS_ID=efs_id,
AWS_SES_REGION_NAME=AWS_SES_REGION_NAME,
AWS_SES_REGION_ENDPOINT=AWS_SES_REGION_ENDPOINT,
**COMMON_SETTINGS_DICT,
**challenge_aws_keys,
)
else:
definition = task_definition_code_upload_worker.format(
queue_name=queue_name,
code_upload_container_name=code_upload_container_name,
ENV=ENV,
challenge_pk=challenge.pk,
auth_token=token.refresh_token,
cluster_name=cluster_name,
cluster_endpoint=cluster_endpoint,
certificate=cluster_certificate,
CPU=worker_cpu_cores,
MEMORY=worker_memory,
log_group_name=log_group_name,
EVALAI_DNS=EVALAI_DNS,
EFS_ID=efs_id,
**COMMON_SETTINGS_DICT,
**challenge_aws_keys,
)
else:
definition = task_definition.format(
queue_name=queue_name,
Expand Down Expand Up @@ -709,12 +940,7 @@ def start_workers(queryset):
count = 0
failures = []
for challenge in queryset:
if challenge.is_docker_based:
response = "Sorry. This feature is not available for code upload/docker based challenges."
failures.append(
{"message": response, "challenge_pk": challenge.pk}
)
elif (challenge.workers == 0) or (challenge.workers is None):
if (challenge.workers == 0) or (challenge.workers is None):
response = service_manager(
client, challenge=challenge, num_of_tasks=1
)
Expand Down Expand Up @@ -763,12 +989,7 @@ def stop_workers(queryset):
count = 0
failures = []
for challenge in queryset:
if challenge.is_docker_based:
response = "Sorry. This feature is not available for code upload/docker based challenges."
failures.append(
{"message": response, "challenge_pk": challenge.pk}
)
elif (challenge.workers is not None) and (challenge.workers > 0):
if (challenge.workers is not None) and (challenge.workers > 0):
response = service_manager(
client, challenge=challenge, num_of_tasks=0
)
Expand Down Expand Up @@ -870,12 +1091,7 @@ def delete_workers(queryset):
count = 0
failures = []
for challenge in queryset:
if challenge.is_docker_based:
response = "Sorry. This feature is not available for code upload/docker based challenges."
failures.append(
{"message": response, "challenge_pk": challenge.pk}
)
elif challenge.workers is not None:
if challenge.workers is not None:
response = delete_service_by_challenge_pk(challenge=challenge)
if response["ResponseMetadata"]["HTTPStatusCode"] != HTTPStatus.OK:
failures.append(
Expand Down

0 comments on commit 978367b

Please sign in to comment.