From afd863ece9766d5b8cefe4cec31e14af5b1f2044 Mon Sep 17 00:00:00 2001 From: Aviral Jain Date: Tue, 29 Jun 2021 01:40:54 +0530 Subject: [PATCH 01/31] Deployment: Update travis auto deployment script for prometheus and grafana(#3447) * update travis auto deployment scripts to integrate prometheus and grafana for staging * change prometheus config file for staging env * add prometheus staging file * change endpoint for prometheus django metrics * change endpoint for django prometheus metrics * revert to previous prometheus django endpoint * add prometheus support for multiple processes running on django on staging Co-authored-by: Ram Ramrakhya Co-authored-by: Rishabh Jain --- docker-compose-staging.yml | 26 ++++++++++++++++++++ docker/prod/django/uwsgi.ini | 1 + docker/prod/docker_staging.env | 5 ++++ monitoring/prometheus/prometheus_staging.yml | 14 +++++++++++ scripts/deployment/deploy.sh | 14 +++++++++++ 5 files changed, 60 insertions(+) create mode 100644 monitoring/prometheus/prometheus_staging.yml diff --git a/docker-compose-staging.yml b/docker-compose-staging.yml index 27583d886f..b97e615eca 100644 --- a/docker-compose-staging.yml +++ b/docker-compose-staging.yml @@ -94,3 +94,29 @@ services: build: context: ./ dockerfile: docker/prod/code-upload-worker/Dockerfile + + prometheus: + image: prom/prometheus:latest + user: "1000" + volumes: + - ./monitoring/prometheus/prometheus_staging.yml:/etc/prometheus/prometheus.yml + - ./monitoring/prometheus/prometheus_db:/var/lib/prometheus + - ./monitoring/prometheus/prometheus_db:/prometheus + - ./monitoring/prometheus/prometheus_db:/etc/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + restart: unless-stopped + ports: + - '9090:9090' + + grafana: + image: grafana/grafana:latest + user: "1000" + env_file: + - docker/prod/docker_staging.env + volumes: + - ./monitoring/grafana/grafana_db:/var/lib/grafana + depends_on: + - prometheus + ports: + - '3000:3000' diff --git a/docker/prod/django/uwsgi.ini b/docker/prod/django/uwsgi.ini index bf5d31768c..f113e1d5e4 100644 --- a/docker/prod/django/uwsgi.ini +++ b/docker/prod/django/uwsgi.ini @@ -8,3 +8,4 @@ vaccum = true python-autoreload = 1 buffer-size=32768 chmod-socket = 777 +lazy-apps = true diff --git a/docker/prod/docker_staging.env b/docker/prod/docker_staging.env index 1a5273bbff..fa459d7471 100644 --- a/docker/prod/docker_staging.env +++ b/docker/prod/docker_staging.env @@ -55,3 +55,8 @@ EKS_NODEGROUP_ROLE_ARN=x SLACK_WEB_HOOK_URL=x ENVIRONMENT=staging SERVICE_DNS=x + +GF_SECURITY_ADMIN_USER=x +GF_SECURITY_ADMIN_PASSWORD=x + +prometheus_multiproc_dir=/code/monitoring/prometheus/django diff --git a/monitoring/prometheus/prometheus_staging.yml b/monitoring/prometheus/prometheus_staging.yml new file mode 100644 index 0000000000..66870e94b6 --- /dev/null +++ b/monitoring/prometheus/prometheus_staging.yml @@ -0,0 +1,14 @@ +global: + scrape_interval: 30s + external_labels: + monitor: 'evalai-monitor' + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'django-prometheus' + metrics_path: '/api/prometheus/metrics' + static_configs: + - targets: ['https://staging.eval.ai'] diff --git a/scripts/deployment/deploy.sh b/scripts/deployment/deploy.sh index 08f348135f..205f0a4274 100755 --- a/scripts/deployment/deploy.sh +++ b/scripts/deployment/deploy.sh @@ -135,6 +135,16 @@ case $opt in echo "Deployed worker docker container for queue: " $queue done ;; + deploy-prometheus) + echo "Deploying prometheus docker container..." + docker-compose -f docker-compose-${env}.yml up -d prometheus + echo "Completed deploy operation." + ;; + deploy-grafana) + echo "Deploying grafana docker container..." + docker-compose -f docker-compose-${env}.yml up -d grafana + echo "Completed deploy operation." + ;; scale) service=${3} instances=${4} @@ -172,6 +182,10 @@ case $opt in echo " Eg. ./scripts/deployment/deploy.sh deploy-remote-worker production " echo " deploy-workers : Deploy worker containers in the respective environment." echo " Eg. ./scripts/deployment/deploy.sh deploy production " + echo " deploy-prometheus : Deploy prometheus container in the respective environment." + echo " Eg. ./scripts/deployment/deploy.sh deploy-prometheus production" + echo " deploy-grafana : Deploy grafana container in the respective environment." + echo " Eg. ./scripts/deployment/deploy.sh deploy-grafana production" echo " scale : Scale particular docker service in an environment." echo " Eg. ./scripts/deployment/deploy.sh scale production django 5" echo " clean : Remove all docker containers and images." From 6629e5149f3104563fb3ffafe6c0490d1aaa46ce Mon Sep 17 00:00:00 2001 From: savish28 <32800267+savish28@users.noreply.github.com> Date: Tue, 29 Jun 2021 22:20:31 +0530 Subject: [PATCH 02/31] Backend: bug fixes in static code upload worker integrations(#3506) * Bug fix static code upload worker challenge * Challenge object dict bug 2 * Create config map * Refactor code to optimize code * Update Api endpoint for submission_input_file * Change variable name api_instance -> api_instance_client * Add submission_pk in request make_submission * Updated comments submission_input_file api --- apps/jobs/views.py | 43 +++++++++++----- .../workers/code_upload_submission_worker.py | 51 ++++++++++++------- .../make_submission.sh | 4 +- 3 files changed, 65 insertions(+), 33 deletions(-) diff --git a/apps/jobs/views.py b/apps/jobs/views.py index 11eb837268..29b4850da0 100644 --- a/apps/jobs/views.py +++ b/apps/jobs/views.py @@ -486,18 +486,6 @@ def change_submission_data_and_visibility( if serializer.is_valid(): serializer.save() response_data = serializer.data - if ( - request.FILES.get("submission_input_file") - and challenge.is_static_dataset_code_upload - ): - message = { - "challenge_pk": challenge_pk, - "phase_pk": challenge_phase_pk, - "submission_pk": submission_pk, - "is_static_dataset_code_upload_submission": False, - } - # publish message in the queue - publish_submission_message(message) return Response(response_data, status=status.HTTP_200_OK) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @@ -1260,9 +1248,38 @@ def update_submission(request, challenge_pk): if request.method == "PATCH": submission_pk = request.data.get("submission") + submission = get_submission_model(submission_pk) + # Update submission_input_file for is_static_dataset_code_upload submission evaluation + if ( + request.FILES.get("submission_input_file") + and submission.challenge_phase.challenge.is_static_dataset_code_upload + ): + serializer = SubmissionSerializer( + submission, + data=request.data, + context={ + "request": request, + }, + partial=True, + ) + if serializer.is_valid(): + serializer.save() + message = { + "challenge_pk": challenge_pk, + "phase_pk": submission.challenge_phase.pk, + "submission_pk": submission_pk, + "is_static_dataset_code_upload_submission": False, + } + # publish message in the queue + publish_submission_message(message) + response_data = serializer.data + return Response(response_data, status=status.HTTP_200_OK) + else: + return Response( + serializer.errors, status=status.HTTP_400_BAD_REQUEST + ) submission_status = request.data.get("submission_status", "").lower() job_name = request.data.get("job_name", "").lower() - submission = get_submission_model(submission_pk) jobs = submission.job_name if job_name: jobs.append(job_name) diff --git a/scripts/workers/code_upload_submission_worker.py b/scripts/workers/code_upload_submission_worker.py index 4ae33a77ab..908d6ca4fc 100755 --- a/scripts/workers/code_upload_submission_worker.py +++ b/scripts/workers/code_upload_submission_worker.py @@ -31,6 +31,7 @@ def exit_gracefully(self, signum, frame): "EVALAI_API_SERVER", "http://localhost:8000" ) QUEUE_NAME = os.environ.get("QUEUE_NAME", "evalai_submission_queue") +script_config_map_name = "evalai-scripts-cm" def get_volume_mount_object(mount_path, name, read_only=False): @@ -92,6 +93,18 @@ def create_config_map_object(config_map_name, file_paths): return config_map +def create_configmap(core_v1_api_instance, config_map): + try: + core_v1_api_instance.create_namespaced_config_map( + namespace="default", + body=config_map, + ) + except Exception as e: + logger.exception( + "Exception while creating configmap with error {}".format(e) + ) + + def create_script_config_map(config_map_name): submission_script_file_path = ( "/code/scripts/workers/code_upload_worker_utils/make_submission.sh" @@ -252,9 +265,6 @@ def create_static_code_upload_submission_job_object(message): # Get dataset volume and volume mounts volume_mount_list = get_volume_mount_list("/dataset", True) volume_list = get_volume_list() - # Create and Mount Script Volume - script_config_map_name = "evalai-scripts-cm" - create_script_config_map(script_config_map_name) script_volume_name = "evalai-scripts" script_volume = get_config_map_volume_object( script_config_map_name, script_volume_name @@ -553,35 +563,40 @@ def main(): cluster_details = evalai.get_aws_eks_cluster_details(challenge.get("id")) cluster_name = cluster_details.get("name") cluster_endpoint = cluster_details.get("cluster_endpoint") - api_instance = get_api_client( + api_instance_client = get_api_client( + cluster_name, cluster_endpoint, challenge, evalai + ) + install_gpu_drivers(api_instance_client) + api_instance = get_api_object( cluster_name, cluster_endpoint, challenge, evalai ) - install_gpu_drivers(api_instance) + core_v1_api_instance = get_core_v1_api_object( + cluster_name, cluster_endpoint, challenge, evalai + ) + if challenge.get("is_static_dataset_code_upload"): + # Create and Mount Script Volume + script_config_map = create_script_config_map(script_config_map_name) + create_configmap(core_v1_api_instance, script_config_map) submission_meta = {} - submission_meta["submission_time_limit"] = challenge.submission_time_limit + submission_meta["submission_time_limit"] = challenge.get( + "submission_time_limit" + ) while True: message = evalai.get_message_from_sqs_queue() message_body = message.get("body") - message_body["submission_meta"] = submission_meta if message_body: - if ( - challenge.is_static_dataset_code_upload - and not message_body.get( - "is_static_dataset_code_upload_submission" - ) + if challenge.get( + "is_static_dataset_code_upload" + ) and not message_body.get( + "is_static_dataset_code_upload_submission" ): continue + message_body["submission_meta"] = submission_meta submission_pk = message_body.get("submission_pk") challenge_pk = message_body.get("challenge_pk") phase_pk = message_body.get("phase_pk") submission = evalai.get_submission_by_pk(submission_pk) if submission: - api_instance = get_api_object( - cluster_name, cluster_endpoint, challenge, evalai - ) - core_v1_api_instance = get_core_v1_api_object( - cluster_name, cluster_endpoint, challenge, evalai - ) if ( submission.get("status") == "finished" or submission.get("status") == "failed" diff --git a/scripts/workers/code_upload_worker_utils/make_submission.sh b/scripts/workers/code_upload_worker_utils/make_submission.sh index 58f5b63bf5..a043fd5212 100644 --- a/scripts/workers/code_upload_worker_utils/make_submission.sh +++ b/scripts/workers/code_upload_worker_utils/make_submission.sh @@ -1,6 +1,6 @@ #!/bin/sh -submission_url="$EVALAI_API_SERVER/api/jobs/challenge/$CHALLENGE_PK/challenge_phase/$PHASE_PK/submission/$SUBMISSION_PK" -submission_curl_base_request="curl --location --request PATCH '$submission_url' --header 'Authorization: Bearer $AUTH_TOKEN'" +submission_url="$EVALAI_API_SERVER/api/jobs/challenge/$CHALLENGE_PK/update_submission/" +submission_curl_base_request="curl --location --request PATCH '$submission_url' --header 'Authorization: Bearer $AUTH_TOKEN' -F 'submission=$SUBMISSION_PK'" if [ -f "$SUBMISSION_PATH/submission.csv" ] then file_path="submission.csv" From 96c2b5e02bfbcceba6d1c672d20529a059fd6af8 Mon Sep 17 00:00:00 2001 From: Ram Ramrakhya Date: Tue, 29 Jun 2021 23:45:50 +0530 Subject: [PATCH 03/31] Revert "Deployment: Update travis auto deployment script for prometheus and grafana(#3447)" (#3508) This reverts commit afd863ece9766d5b8cefe4cec31e14af5b1f2044. --- docker-compose-staging.yml | 26 -------------------- docker/prod/django/uwsgi.ini | 1 - docker/prod/docker_staging.env | 5 ---- monitoring/prometheus/prometheus_staging.yml | 14 ----------- scripts/deployment/deploy.sh | 14 ----------- 5 files changed, 60 deletions(-) delete mode 100644 monitoring/prometheus/prometheus_staging.yml diff --git a/docker-compose-staging.yml b/docker-compose-staging.yml index b97e615eca..27583d886f 100644 --- a/docker-compose-staging.yml +++ b/docker-compose-staging.yml @@ -94,29 +94,3 @@ services: build: context: ./ dockerfile: docker/prod/code-upload-worker/Dockerfile - - prometheus: - image: prom/prometheus:latest - user: "1000" - volumes: - - ./monitoring/prometheus/prometheus_staging.yml:/etc/prometheus/prometheus.yml - - ./monitoring/prometheus/prometheus_db:/var/lib/prometheus - - ./monitoring/prometheus/prometheus_db:/prometheus - - ./monitoring/prometheus/prometheus_db:/etc/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - restart: unless-stopped - ports: - - '9090:9090' - - grafana: - image: grafana/grafana:latest - user: "1000" - env_file: - - docker/prod/docker_staging.env - volumes: - - ./monitoring/grafana/grafana_db:/var/lib/grafana - depends_on: - - prometheus - ports: - - '3000:3000' diff --git a/docker/prod/django/uwsgi.ini b/docker/prod/django/uwsgi.ini index f113e1d5e4..bf5d31768c 100644 --- a/docker/prod/django/uwsgi.ini +++ b/docker/prod/django/uwsgi.ini @@ -8,4 +8,3 @@ vaccum = true python-autoreload = 1 buffer-size=32768 chmod-socket = 777 -lazy-apps = true diff --git a/docker/prod/docker_staging.env b/docker/prod/docker_staging.env index fa459d7471..1a5273bbff 100644 --- a/docker/prod/docker_staging.env +++ b/docker/prod/docker_staging.env @@ -55,8 +55,3 @@ EKS_NODEGROUP_ROLE_ARN=x SLACK_WEB_HOOK_URL=x ENVIRONMENT=staging SERVICE_DNS=x - -GF_SECURITY_ADMIN_USER=x -GF_SECURITY_ADMIN_PASSWORD=x - -prometheus_multiproc_dir=/code/monitoring/prometheus/django diff --git a/monitoring/prometheus/prometheus_staging.yml b/monitoring/prometheus/prometheus_staging.yml deleted file mode 100644 index 66870e94b6..0000000000 --- a/monitoring/prometheus/prometheus_staging.yml +++ /dev/null @@ -1,14 +0,0 @@ -global: - scrape_interval: 30s - external_labels: - monitor: 'evalai-monitor' - -scrape_configs: - - job_name: 'prometheus' - static_configs: - - targets: ['localhost:9090'] - - - job_name: 'django-prometheus' - metrics_path: '/api/prometheus/metrics' - static_configs: - - targets: ['https://staging.eval.ai'] diff --git a/scripts/deployment/deploy.sh b/scripts/deployment/deploy.sh index 205f0a4274..08f348135f 100755 --- a/scripts/deployment/deploy.sh +++ b/scripts/deployment/deploy.sh @@ -135,16 +135,6 @@ case $opt in echo "Deployed worker docker container for queue: " $queue done ;; - deploy-prometheus) - echo "Deploying prometheus docker container..." - docker-compose -f docker-compose-${env}.yml up -d prometheus - echo "Completed deploy operation." - ;; - deploy-grafana) - echo "Deploying grafana docker container..." - docker-compose -f docker-compose-${env}.yml up -d grafana - echo "Completed deploy operation." - ;; scale) service=${3} instances=${4} @@ -182,10 +172,6 @@ case $opt in echo " Eg. ./scripts/deployment/deploy.sh deploy-remote-worker production " echo " deploy-workers : Deploy worker containers in the respective environment." echo " Eg. ./scripts/deployment/deploy.sh deploy production " - echo " deploy-prometheus : Deploy prometheus container in the respective environment." - echo " Eg. ./scripts/deployment/deploy.sh deploy-prometheus production" - echo " deploy-grafana : Deploy grafana container in the respective environment." - echo " Eg. ./scripts/deployment/deploy.sh deploy-grafana production" echo " scale : Scale particular docker service in an environment." echo " Eg. ./scripts/deployment/deploy.sh scale production django 5" echo " clean : Remove all docker containers and images." From ae5eceeed100b5df881108f947a57fe75b6aef6c Mon Sep 17 00:00:00 2001 From: Aviral Jain Date: Wed, 30 Jun 2021 23:17:02 +0530 Subject: [PATCH 04/31] Revert "Backend: Integrate prometheus-django library and add django as scrape target for prometheus(#3446)" (#3510) This reverts commit 4ac30efc123fa8286df6f4ef800596883cb45d2c. --- .gitignore | 2 -- docker/dev/django/uwsgi.ini | 1 - docker/dev/docker.env | 2 -- evalai/urls.py | 1 - monitoring/prometheus/django/.gitkeep | 0 monitoring/prometheus/prometheus.yml | 6 ----- requirements/common.txt | 1 - settings/common.py | 2 -- .../postgres_psycopg2/base.py | 26 ------------------- settings/dev.py | 3 +-- settings/prod.py | 8 ++---- 11 files changed, 3 insertions(+), 49 deletions(-) delete mode 100644 monitoring/prometheus/django/.gitkeep delete mode 100644 settings/custom_databases/postgres_psycopg2/base.py diff --git a/.gitignore b/.gitignore index a1b1f0d3a7..8b796207f5 100644 --- a/.gitignore +++ b/.gitignore @@ -71,5 +71,3 @@ out/ monitoring/prometheus/prometheus_db/* !monitoring/prometheus/prometheus_db/.gitkeep monitoring/grafana/grafana_db/grafana.db -monitoring/prometheus/django/* -!monitoring/prometheus/django/.gitkeep diff --git a/docker/dev/django/uwsgi.ini b/docker/dev/django/uwsgi.ini index 6b38f6805c..3e79d7accc 100644 --- a/docker/dev/django/uwsgi.ini +++ b/docker/dev/django/uwsgi.ini @@ -8,4 +8,3 @@ vaccum = true python-autoreload = 1 buffer-size=32768 chmod-socket = 777 -lazy-apps = true diff --git a/docker/dev/docker.env b/docker/dev/docker.env index 51b5a31a3d..bc0357f963 100644 --- a/docker/dev/docker.env +++ b/docker/dev/docker.env @@ -35,5 +35,3 @@ SERVICE_DNS=localhost GF_SECURITY_ADMIN_USER=admin GF_SECURITY_ADMIN_PASSWORD=password - -prometheus_multiproc_dir=/code/monitoring/prometheus/django diff --git a/evalai/urls.py b/evalai/urls.py index 8f94b949a2..9e09a5811d 100644 --- a/evalai/urls.py +++ b/evalai/urls.py @@ -92,7 +92,6 @@ schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc", ), - url(r"^api/prometheus/", include("django_prometheus.urls")), ] # DJANGO-SPAGHETTI-AND-MEATBALLS URLs available during development only. diff --git a/monitoring/prometheus/django/.gitkeep b/monitoring/prometheus/django/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index 0e30b7ebe1..d578b68534 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -2,13 +2,7 @@ global: scrape_interval: 30s external_labels: monitor: 'evalai-monitor' - scrape_configs: - job_name: 'prometheus' static_configs: - targets: ['localhost:9090'] - - - job_name: 'django-prometheus' - metrics_path: '/api/prometheus/metrics' - static_configs: - - targets: ['django:8000'] diff --git a/requirements/common.txt b/requirements/common.txt index 9dfddfd99c..5e4c87be10 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -8,7 +8,6 @@ django==2.2.20 django-allauth==0.43.0 django-filter==2.4.0 django-import-export==2.4.0 -django-prometheus==2.1.0 djangorestframework==3.10.3 djangorestframework-expiring-authtoken==0.1.4 djangorestframework-simplejwt==4.6.0 diff --git a/settings/common.py b/settings/common.py index d06b73bc9c..b2461a408b 100755 --- a/settings/common.py +++ b/settings/common.py @@ -82,7 +82,6 @@ INSTALLED_APPS = DEFAULT_APPS + OUR_APPS + THIRD_PARTY_APPS MIDDLEWARE = [ - "django_prometheus.middleware.PrometheusBeforeMiddleware", "corsheaders.middleware.CorsMiddleware", "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", @@ -91,7 +90,6 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - "django_prometheus.middleware.PrometheusAfterMiddleware", ] ROOT_URLCONF = "evalai.urls" diff --git a/settings/custom_databases/postgres_psycopg2/base.py b/settings/custom_databases/postgres_psycopg2/base.py deleted file mode 100644 index a6a813788c..0000000000 --- a/settings/custom_databases/postgres_psycopg2/base.py +++ /dev/null @@ -1,26 +0,0 @@ -import psycopg2.extensions -from django.db.backends.postgresql_psycopg2 import base -from django_prometheus.db.common import ( - DatabaseWrapperMixin, - ExportingCursorWrapper, -) - - -class DatabaseFeatures(base.DatabaseFeatures): - """Our database has the exact same features as the base one.""" - - pass - - -class DatabaseWrapper(DatabaseWrapperMixin, base.DatabaseWrapper): - def get_connection_params(self): - conn_params = super().get_connection_params() - conn_params["cursor_factory"] = ExportingCursorWrapper( - psycopg2.extensions.cursor, self.alias, self.vendor - ) - return conn_params - - def create_cursor(self, name=None): - # cursor_factory is a kwarg to connect() so restore create_cursor()'s - # default behavior - return base.DatabaseWrapper.create_cursor(self, name=name) diff --git a/settings/dev.py b/settings/dev.py index e0c67d4a43..0591ada401 100644 --- a/settings/dev.py +++ b/settings/dev.py @@ -11,7 +11,7 @@ DATABASES = { "default": { - "ENGINE": "settings.custom_databases.postgres_psycopg2", + "ENGINE": "django.db.backends.postgresql_psycopg2", "NAME": os.environ.get("POSTGRES_NAME", "evalai"), # noqa: ignore=F405 "USER": os.environ.get( # noqa: ignore=F405 "POSTGRES_USER", "postgres" @@ -35,7 +35,6 @@ # DJANGO-SPAGHETTI-AND-MEATBALLS SETTINGS INSTALLED_APPS += [ # noqa: ignore=F405 - "django_prometheus", "django_spaghetti", "autofixture", "debug_toolbar", diff --git a/settings/prod.py b/settings/prod.py index a9e72b270b..36a708ecd1 100755 --- a/settings/prod.py +++ b/settings/prod.py @@ -20,7 +20,7 @@ DATABASES = { "default": { - "ENGINE": "settings.custom_databases.postgres_psycopg2", + "ENGINE": "django.db.backends.postgresql_psycopg2", "NAME": os.environ.get("RDS_DB_NAME"), "USER": os.environ.get("RDS_USERNAME"), "PASSWORD": os.environ.get("RDS_PASSWORD"), @@ -35,11 +35,7 @@ MIDDLEWARE += ["middleware.metrics.DatadogMiddleware"] # noqa -INSTALLED_APPS += ( # noqa - "storages", - "raven.contrib.django.raven_compat", - "django_prometheus", -) +INSTALLED_APPS += ("storages", "raven.contrib.django.raven_compat") # noqa AWS_STORAGE_BUCKET_NAME = os.environ.get("AWS_STORAGE_BUCKET_NAME") AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID") From c8f965c0df9c30aa81a5388997b9bbe9c2bad5e3 Mon Sep 17 00:00:00 2001 From: Ram Ramrakhya Date: Thu, 1 Jul 2021 01:31:10 +0530 Subject: [PATCH 05/31] Frontend_V2: Migrate frontend v2 UI to the staging server(#3513) --- docker-compose-staging.yml | 2 +- docker/prod/nodejs_v2/nginx_staging.conf | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose-staging.yml b/docker-compose-staging.yml index 27583d886f..3f53000f9f 100644 --- a/docker-compose-staging.yml +++ b/docker-compose-staging.yml @@ -49,7 +49,6 @@ services: NODE_ENV: staging ports: - '80:80' - - '443:443' volumes: - /code/node_modules - /code/bower_components @@ -69,6 +68,7 @@ services: NODE_ENV: staging ports: - "9999:80" + - "443:443" volumes: - /code/node_modules logging: diff --git a/docker/prod/nodejs_v2/nginx_staging.conf b/docker/prod/nodejs_v2/nginx_staging.conf index 633655c2dc..f74ba4f35f 100644 --- a/docker/prod/nodejs_v2/nginx_staging.conf +++ b/docker/prod/nodejs_v2/nginx_staging.conf @@ -3,13 +3,13 @@ upstream django_app { } server { - server_name beta-staging.eval.ai; + server_name staging.eval.ai; listen 80; - return 301 https://beta-staging.eval.ai$request_uri; + return 301 https://staging.eval.ai$request_uri; } server { - server_name beta-staging.eval.ai; + server_name staging.eval.ai; listen 443 ssl; location / { root /usr/share/nginx/html; @@ -21,7 +21,7 @@ server { ssl_certificate /etc/ssl/eval_ai.crt; ssl_certificate_key /etc/ssl/eval_ai.key; ssl_prefer_server_ciphers on; - #enables all versions of TLS, but not SSLv2 or 3 which are weak and now deprecated. + # enables all versions of TLS, but not SSLv2 or 3 which are weak and now deprecated. ssl_protocols TLSv1 TLSv1.1 TLSv1.2; access_log /var/log/nginx/access.log; From bcdd695ce9013f6d415e41c3e4cce3adea54a576 Mon Sep 17 00:00:00 2001 From: Gautam Jajoo Date: Thu, 1 Jul 2021 12:34:53 +0530 Subject: [PATCH 06/31] Frontend_V2: Fix console error on challenge list page(#3514) Co-authored-by: Rishabh Jain --- .../publiclists/challengelist/challengelist.component.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend_v2/src/app/components/publiclists/challengelist/challengelist.component.ts b/frontend_v2/src/app/components/publiclists/challengelist/challengelist.component.ts index 4584fc91d2..329c033ae8 100644 --- a/frontend_v2/src/app/components/publiclists/challengelist/challengelist.component.ts +++ b/frontend_v2/src/app/components/publiclists/challengelist/challengelist.component.ts @@ -231,8 +231,10 @@ export class ChallengelistComponent implements OnInit { */ @HostListener('window:scroll', []) onWindowScroll(): void { - const RECT = this.document.getElementById('ongoing-challenges').getBoundingClientRect(); - this.isScrollbtnVisible = RECT.top < 0; + if(this.document.getElementById('ongoing-challenges')) { + const RECT = this.document.getElementById('ongoing-challenges').getBoundingClientRect(); + this.isScrollbtnVisible = RECT.top < 0; + } } /** From 02fb8fe3ae93833ab16918f6d8e6a7d112c127df Mon Sep 17 00:00:00 2001 From: Ram Ramrakhya Date: Thu, 1 Jul 2021 22:17:42 +0530 Subject: [PATCH 07/31] Frontend_V2: Change Django backend to staging server(#3515) Co-authored-by: Rishabh Jain --- frontend_v2/src/environments/environment.staging.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend_v2/src/environments/environment.staging.ts b/frontend_v2/src/environments/environment.staging.ts index b71eef3dbf..68413cea57 100644 --- a/frontend_v2/src/environments/environment.staging.ts +++ b/frontend_v2/src/environments/environment.staging.ts @@ -8,5 +8,5 @@ */ export const environment = { production: true, - api_endpoint: 'https://beta-staging.eval.ai/api/', + api_endpoint: 'https://staging.eval.ai/api/', }; From 2e5912f1b113c3120528de3db68f54d2528c235b Mon Sep 17 00:00:00 2001 From: Aviral Jain Date: Sat, 3 Jul 2021 19:58:26 +0530 Subject: [PATCH 08/31] Feat: Integrate statsd exporter to fetch django metrics (#3511) --- docker-compose.yml | 11 ++++++ docker/dev/docker.env | 3 ++ middleware/statsd/__init__.py | 3 ++ middleware/statsd/statsd_middleware.py | 50 ++++++++++++++++++++++++++ monitoring/prometheus/prometheus.yml | 5 +++ requirements/common.txt | 1 + requirements/prod.txt | 1 - settings/common.py | 1 + 8 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 middleware/statsd/__init__.py create mode 100644 middleware/statsd/statsd_middleware.py diff --git a/docker-compose.yml b/docker-compose.yml index df65f130f1..138bbc9232 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,7 @@ services: depends_on: - db - sqs + - statsd-exporter volumes: - .:/code @@ -90,3 +91,13 @@ services: - prometheus ports: - '3000:3000' + + statsd-exporter: + hostname: statsd + image: prom/statsd-exporter:latest + command: + - '--log.level=info' + - '--web.telemetry-path=/statsd/metrics' + ports: + - '9125:9125' + - '9102:9102' diff --git a/docker/dev/docker.env b/docker/dev/docker.env index bc0357f963..9352952bcf 100644 --- a/docker/dev/docker.env +++ b/docker/dev/docker.env @@ -35,3 +35,6 @@ SERVICE_DNS=localhost GF_SECURITY_ADMIN_USER=admin GF_SECURITY_ADMIN_PASSWORD=password + +STATSD_ENDPOINT=statsd +STATSD_PORT=9125 diff --git a/middleware/statsd/__init__.py b/middleware/statsd/__init__.py new file mode 100644 index 0000000000..830e0128c3 --- /dev/null +++ b/middleware/statsd/__init__.py @@ -0,0 +1,3 @@ +from .statsd_middleware import StatsdMetricsMiddleware + +__all__ = [StatsdMetricsMiddleware] diff --git a/middleware/statsd/statsd_middleware.py b/middleware/statsd/statsd_middleware.py new file mode 100644 index 0000000000..9f6a2e1f26 --- /dev/null +++ b/middleware/statsd/statsd_middleware.py @@ -0,0 +1,50 @@ +import os +import time + +from datadog import DogStatsd +from django.utils.deprecation import MiddlewareMixin + + +statsd_host = os.environ.get("STATSD_ENDPOINT") +statsd_port = int(os.environ.get("STATSD_PORT")) +statsd = DogStatsd(host=statsd_host, port=statsd_port) + +REQUEST_LATENCY_METRIC_NAME = "django_request_latency_seconds" +REQUEST_COUNT_METRIC_NAME = "django_request_count" + + +class StatsdMetricsMiddleware(MiddlewareMixin): + def process_request(self, request): + request.start_time = time.time() + + def get_view_name(self, request): + view_name = "" + if hasattr(request, "resolver_match"): + if request.resolver_match is not None: + if request.resolver_match.view_name is not None: + view_name = request.resolver_match.view_name + return view_name + + def process_response(self, request, response): + statsd.increment( + REQUEST_COUNT_METRIC_NAME, + tags=[ + "service:django_worker", + "method:%s" % request.method, + "view:%s" % self.get_view_name(request), + "status:%s" % str(response.status_code), + ], + ) + + resp_time = (time.time() - request.start_time) * 1000 + + statsd.histogram( + REQUEST_LATENCY_METRIC_NAME, + resp_time, + tags=[ + "service:django_worker", + "view:%s" % self.get_view_name(request), + ], + ) + + return response diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index d578b68534..6aa39327d9 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -6,3 +6,8 @@ scrape_configs: - job_name: 'prometheus' static_configs: - targets: ['localhost:9090'] + + - job_name: 'statsd' + metrics_path: '/statsd/metrics' + static_configs: + - targets: ['statsd:9102'] diff --git a/requirements/common.txt b/requirements/common.txt index 5e4c87be10..8f483f62fd 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -28,3 +28,4 @@ PyYaml==5.1 proc==1.0 rstr==2.2.6 sendgrid==6.4.8 +datadog==0.39.0 diff --git a/requirements/prod.txt b/requirements/prod.txt index 25ce1ae299..d371e54c26 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -5,4 +5,3 @@ django-storages==1.7.1 python-memcached==1.59 raven==6.10.0 uWSGI==2.0.19.1 -datadog==0.39.0 diff --git a/settings/common.py b/settings/common.py index b2461a408b..513a2504e3 100755 --- a/settings/common.py +++ b/settings/common.py @@ -82,6 +82,7 @@ INSTALLED_APPS = DEFAULT_APPS + OUR_APPS + THIRD_PARTY_APPS MIDDLEWARE = [ + "middleware.statsd.StatsdMetricsMiddleware", "corsheaders.middleware.CorsMiddleware", "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", From 3fa2d95e23aceff0999098e5907af5c2dcf02695 Mon Sep 17 00:00:00 2001 From: Aviral Jain Date: Sun, 4 Jul 2021 14:11:57 +0530 Subject: [PATCH 09/31] Deployment: Update travis auto deploy scripts & add configs for staging environment (#3512) * add statsd exporter to fetch django metrics * cleanup unncessary files with gitignore and fix import style * switch to views instead of endpoint for metrics * fix import order * add auto deployment scripts for travis and add configs for staging environ * add grafana env variables for staging * rebase on top of statsd * fix uwsgi file for dev * update auto deployment command name --- docker-compose-staging.yml | 36 ++++++++++++++++++++ docker/prod/docker_staging.env | 6 ++++ docker/prod/nodejs/nginx_staging.conf | 11 ++++++ monitoring/prometheus/prometheus_staging.yml | 14 ++++++++ scripts/deployment/deploy.sh | 21 ++++++++++++ 5 files changed, 88 insertions(+) create mode 100644 monitoring/prometheus/prometheus_staging.yml diff --git a/docker-compose-staging.yml b/docker-compose-staging.yml index 3f53000f9f..05781fc689 100644 --- a/docker-compose-staging.yml +++ b/docker-compose-staging.yml @@ -94,3 +94,39 @@ services: build: context: ./ dockerfile: docker/prod/code-upload-worker/Dockerfile + + prometheus: + image: prom/prometheus:latest + user: "1000" + volumes: + - ./monitoring/prometheus/prometheus_staging.yml:/etc/prometheus/prometheus.yml + - ./monitoring/prometheus/prometheus_db:/var/lib/prometheus + - ./monitoring/prometheus/prometheus_db:/prometheus + - ./monitoring/prometheus/prometheus_db:/etc/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + restart: unless-stopped + ports: + - '9090:9090' + + grafana: + image: grafana/grafana:latest + user: "1000" + env_file: + - docker/prod/docker_staging.env + volumes: + - ./monitoring/grafana/grafana_db:/var/lib/grafana + depends_on: + - prometheus + ports: + - '3000:3000' + + statsd-exporter: + hostname: statsd + image: prom/statsd-exporter:latest + command: + - '--log.level=info' + - '--web.telemetry-path=/statsd/metrics' + ports: + - '9125:9125' + - '9102:9102' diff --git a/docker/prod/docker_staging.env b/docker/prod/docker_staging.env index 1a5273bbff..cd62caf1f1 100644 --- a/docker/prod/docker_staging.env +++ b/docker/prod/docker_staging.env @@ -55,3 +55,9 @@ EKS_NODEGROUP_ROLE_ARN=x SLACK_WEB_HOOK_URL=x ENVIRONMENT=staging SERVICE_DNS=x + +GF_SECURITY_ADMIN_USER=x +GF_SECURITY_ADMIN_PASSWORD=x + +STATSD_ENDPOINT=statsd +STATSD_PORT=9125 diff --git a/docker/prod/nodejs/nginx_staging.conf b/docker/prod/nodejs/nginx_staging.conf index b9af1864d4..a40d40a9c9 100644 --- a/docker/prod/nodejs/nginx_staging.conf +++ b/docker/prod/nodejs/nginx_staging.conf @@ -2,6 +2,10 @@ upstream django_app { server django:8000 fail_timeout=0; } +upstream statsd_exporter { + server statsd:9102 fail_timeout=0; +} + server { server_name staging-evalai.cloudcv.org evalai-staging.cloudcv.org; listen 80; @@ -53,6 +57,13 @@ server { proxy_pass http://django_app; } + location /statsd { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://statsd_exporter; + } + ssl on; ssl_certificate /etc/ssl/eval_ai.crt; ssl_certificate_key /etc/ssl/eval_ai.key; diff --git a/monitoring/prometheus/prometheus_staging.yml b/monitoring/prometheus/prometheus_staging.yml new file mode 100644 index 0000000000..afe071df50 --- /dev/null +++ b/monitoring/prometheus/prometheus_staging.yml @@ -0,0 +1,14 @@ +global: + scrape_interval: 30s + external_labels: + monitor: 'evalai-monitor' + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'statsd' + metrics_path: '/statsd/metrics' + static_configs: + - targets: ['https://staging.eval.ai'] diff --git a/scripts/deployment/deploy.sh b/scripts/deployment/deploy.sh index 08f348135f..03c64d77cb 100755 --- a/scripts/deployment/deploy.sh +++ b/scripts/deployment/deploy.sh @@ -135,6 +135,21 @@ case $opt in echo "Deployed worker docker container for queue: " $queue done ;; + deploy-prometheus) + echo "Deploying prometheus docker container..." + docker-compose -f docker-compose-${env}.yml up -d prometheus + echo "Completed deploy operation." + ;; + deploy-grafana) + echo "Deploying grafana docker container..." + docker-compose -f docker-compose-${env}.yml up -d grafana + echo "Completed deploy operation." + ;; + deploy-statsd) + echo "Deploying statsd docker container..." + docker-compose -f docker-compose-${env}.yml up -d statsd-exporter + echo "Completed deploy operation." + ;; scale) service=${3} instances=${4} @@ -172,6 +187,12 @@ case $opt in echo " Eg. ./scripts/deployment/deploy.sh deploy-remote-worker production " echo " deploy-workers : Deploy worker containers in the respective environment." echo " Eg. ./scripts/deployment/deploy.sh deploy production " + echo " deploy-prometheus : Deploy prometheus container in the respective environment." + echo " Eg. ./scripts/deployment/deploy.sh deploy-prometheus production" + echo " deploy-grafana : Deploy grafana container in the respective environment." + echo " Eg. ./scripts/deployment/deploy.sh deploy-grafana production" + echo " deploy-statsd : Deploy statsd container in the respective environment." + echo " Eg. ./scripts/deployment/deploy.sh deploy-statsd production" echo " scale : Scale particular docker service in an environment." echo " Eg. ./scripts/deployment/deploy.sh scale production django 5" echo " clean : Remove all docker containers and images." From b67145f35483e230b271474f56eafe835fc1bdd5 Mon Sep 17 00:00:00 2001 From: Aviral Jain Date: Sun, 4 Jul 2021 21:05:27 +0530 Subject: [PATCH 10/31] Deployment: Add statsd nginx config to frontend-v2 (#3519) --- docker/prod/nodejs_v2/nginx_staging.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker/prod/nodejs_v2/nginx_staging.conf b/docker/prod/nodejs_v2/nginx_staging.conf index f74ba4f35f..f2c7eb15df 100644 --- a/docker/prod/nodejs_v2/nginx_staging.conf +++ b/docker/prod/nodejs_v2/nginx_staging.conf @@ -2,6 +2,10 @@ upstream django_app { server django:8000 fail_timeout=0; } +upstream statsd_exporter { + server statsd:9102 fail_timeout=0; +} + server { server_name staging.eval.ai; listen 80; @@ -33,4 +37,11 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://django_app; } + + location /statsd { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://statsd_exporter; + } } From 04a16b5420a75d5e0ab4bff4e91afd60625a34e7 Mon Sep 17 00:00:00 2001 From: savish28 <32800267+savish28@users.noreply.github.com> Date: Wed, 7 Jul 2021 21:05:53 +0530 Subject: [PATCH 11/31] [Worker] Evaluate static dataset code upload challenge submissions (#3480) * Refactor submission worker to evaluate static dataset code upload challenge submissions * Update submission_input_file name error. * Bug submission message body json error --- scripts/workers/submission_worker.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py index faba1a6210..ff63dca0ba 100644 --- a/scripts/workers/submission_worker.py +++ b/scripts/workers/submission_worker.py @@ -347,7 +347,11 @@ def extract_submission_data(submission_id): # does not exist return None - submission_input_file = submission.input_file.url + if submission.challenge_phase.challenge.is_static_dataset_code_upload: + input_file = submission.submission_input_file + else: + input_file = submission.input_file + submission_input_file = input_file.url submission_input_file = return_file_url_per_environment( submission_input_file ) @@ -355,7 +359,7 @@ def extract_submission_data(submission_id): submission_data_directory = SUBMISSION_DATA_DIR.format( submission_id=submission.id ) - submission_input_file_name = os.path.basename(submission.input_file.name) + submission_input_file_name = os.path.basename(input_file.name) submission_input_file_path = SUBMISSION_INPUT_FILE_PATH.format( submission_id=submission.id, input_file=submission_input_file_name ) @@ -637,9 +641,15 @@ def process_submission_message(message): ) raise + if ( + submission_instance.challenge_phase.challenge.is_static_dataset_code_upload + ): + input_file_name = submission_instance.submission_input_file.name + else: + input_file_name = submission_instance.input_file.name user_annotation_file_path = join( SUBMISSION_DATA_DIR.format(submission_id=submission_id), - os.path.basename(submission_instance.input_file.name), + os.path.basename(input_file_name), ) run_submission( challenge_id, @@ -785,6 +795,10 @@ def main(): queue = get_or_create_sqs_queue(queue_name) while True: for message in queue.receive_messages(): + if json.loads(message.body).get( + "is_static_dataset_code_upload_submission" + ): + continue if settings.DEBUG or settings.TEST: if eval(LIMIT_CONCURRENT_SUBMISSION_PROCESSING): current_running_submissions_count = ( From 9846067455e017522763f5807496c2286ee4c41b Mon Sep 17 00:00:00 2001 From: Ram Ramrakhya Date: Wed, 7 Jul 2021 22:38:27 +0530 Subject: [PATCH 12/31] Deployment: Add statsd-exporter auto deployment(#3525) --- scripts/deployment/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deployment/deploy.sh b/scripts/deployment/deploy.sh index 03c64d77cb..3446ce30c3 100755 --- a/scripts/deployment/deploy.sh +++ b/scripts/deployment/deploy.sh @@ -47,7 +47,7 @@ case $opt in aws s3 cp s3://cloudcv-secrets/evalai/${env}/docker_${env}.env ./docker/prod/docker_${env}.env docker-compose -f docker-compose-${env}.yml rm -s -v -f docker-compose -f docker-compose-${env}.yml pull - docker-compose -f docker-compose-${env}.yml up -d --force-recreate --remove-orphans django nodejs nodejs_v2 celery + docker-compose -f docker-compose-${env}.yml up -d --force-recreate --remove-orphans statsd-exporter django nodejs nodejs_v2 celery ENDSSH2 ENDSSH ;; From cae7f871adbdbb0ec8c871e7d2d4e9de98e4bd9d Mon Sep 17 00:00:00 2001 From: Gautam Jajoo Date: Wed, 7 Jul 2021 22:39:44 +0530 Subject: [PATCH 13/31] Frontend_V2: Fix organization logos on the challenge list page(#3521) Co-authored-by: Rishabh Jain --- .../challengelist/challengecard/challengecard.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend_v2/src/app/components/publiclists/challengelist/challengecard/challengecard.component.scss b/frontend_v2/src/app/components/publiclists/challengelist/challengecard/challengecard.component.scss index a427c2f91c..dee9eec866 100644 --- a/frontend_v2/src/app/components/publiclists/challengelist/challengecard/challengecard.component.scss +++ b/frontend_v2/src/app/components/publiclists/challengelist/challengecard/challengecard.component.scss @@ -5,7 +5,7 @@ } .bg-img { - width: 100%; + object-fit: cover; height: 100%; margin: auto; } From d3c2baef57e255847696ed09ca8b72039b0ea47f Mon Sep 17 00:00:00 2001 From: Gautam Jajoo Date: Wed, 7 Jul 2021 22:40:56 +0530 Subject: [PATCH 14/31] Frontend_V2: Fix notification message(#3517) Co-authored-by: Rishabh Jain --- frontend_v2/src/app/app.component.html | 10 +++++----- .../app/components/utility/toast/toast.component.scss | 1 + frontend_v2/src/styles/base.scss | 9 +++++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/frontend_v2/src/app/app.component.html b/frontend_v2/src/app/app.component.html index 6be740b4ca..dd80668e1f 100644 --- a/frontend_v2/src/app/app.component.html +++ b/frontend_v2/src/app/app.component.html @@ -1,11 +1,11 @@
+ (currentRoutePath == '/' || currentRoutePath == '/auth/login' || currentRoutePath == '/auth/signup') + ? '' : (currentRoutePath == '/about' || currentRoutePath == '/contact' || currentRoutePath == '/our-team' || + currentRoutePath == '/get-involved' || currentRoutePath == '/privacy-policy') + ? 'page-wrap' : 'page-wrap-challenge' + " >
diff --git a/frontend_v2/src/app/components/utility/toast/toast.component.scss b/frontend_v2/src/app/components/utility/toast/toast.component.scss index c72f49e24d..3daf1c35bd 100644 --- a/frontend_v2/src/app/components/utility/toast/toast.component.scss +++ b/frontend_v2/src/app/components/utility/toast/toast.component.scss @@ -4,6 +4,7 @@ .toast-container { visibility: hidden; word-wrap: break-word; + z-index: 1; display: inline-block; color: #fff; text-align: center; diff --git a/frontend_v2/src/styles/base.scss b/frontend_v2/src/styles/base.scss index aba40b2aaf..2ec8cecaa5 100644 --- a/frontend_v2/src/styles/base.scss +++ b/frontend_v2/src/styles/base.scss @@ -2152,6 +2152,15 @@ tr { } .page-wrap { + max-width: 100% !important; + overflow: hidden; + margin: auto !important; + position: relative !important; + background: white; + zoom: 1 !important; + } + + .page-wrap-challenge { max-width: 1440px !important; overflow: hidden; margin: auto !important; From 8efffaa2dac747c7454410088a13991126198a09 Mon Sep 17 00:00:00 2001 From: DXGatech Date: Wed, 7 Jul 2021 13:12:03 -0400 Subject: [PATCH 15/31] FAQs: Add question for docker image issue(#2746) * update FAQ with docker image issue * modify FAQ message * Update faq(developers).md Co-authored-by: Rishabh Jain --- docs/source/faq(developers).md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/faq(developers).md b/docs/source/faq(developers).md index e3d006a4ce..b3c597d205 100644 --- a/docs/source/faq(developers).md +++ b/docs/source/faq(developers).md @@ -258,3 +258,12 @@ rm -rf bower_components npm install bower install ``` + +#### Q. While trying to build EvalAI from the master branch and run the command docker-compose up: + +``` +ERROR: Service 'celery' failed to build: pull access denied for evalai_django, repository does not exist or may require 'docker login': denied: requested access to the resource is denied +``` + +Please make sure to clone EvalAI in its default directory with name evalai. This happens because the parent directory changes the name of docker images. +For instance, the image evalai_django gets renamed to evalai_dev_django if your directory is renamed to EvalAI_dev. From 394ec36909e80b4fe2a5358f73401a510274bb12 Mon Sep 17 00:00:00 2001 From: Aviral Jain Date: Wed, 7 Jul 2021 23:12:39 +0530 Subject: [PATCH 16/31] Backend: Fix restart policy and staging endpoint for prometheus (#3520) Co-authored-by: Rishabh Jain --- docker-compose-staging.yml | 1 - docker-compose.yml | 1 - monitoring/prometheus/prometheus_staging.yml | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose-staging.yml b/docker-compose-staging.yml index 05781fc689..948fb296f0 100644 --- a/docker-compose-staging.yml +++ b/docker-compose-staging.yml @@ -105,7 +105,6 @@ services: - ./monitoring/prometheus/prometheus_db:/etc/prometheus command: - '--config.file=/etc/prometheus/prometheus.yml' - restart: unless-stopped ports: - '9090:9090' diff --git a/docker-compose.yml b/docker-compose.yml index 138bbc9232..16d99d04bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,7 +76,6 @@ services: - ./monitoring/prometheus/prometheus_db:/etc/prometheus command: - '--config.file=/etc/prometheus/prometheus.yml' - restart: unless-stopped ports: - '9090:9090' diff --git a/monitoring/prometheus/prometheus_staging.yml b/monitoring/prometheus/prometheus_staging.yml index afe071df50..7935d04fe6 100644 --- a/monitoring/prometheus/prometheus_staging.yml +++ b/monitoring/prometheus/prometheus_staging.yml @@ -11,4 +11,4 @@ scrape_configs: - job_name: 'statsd' metrics_path: '/statsd/metrics' static_configs: - - targets: ['https://staging.eval.ai'] + - targets: ['staging.eval.ai'] From 6a7d8d90e48c55e11c27c8a807e2696cfb9d8ea4 Mon Sep 17 00:00:00 2001 From: Aviral Jain Date: Thu, 8 Jul 2021 23:15:20 +0530 Subject: [PATCH 17/31] Backend: Add CPU metrics using node-exporter and setup push gateway for prometheus(#3481) * add view name to statsd metrics * cleanup unncessary files with gitignore and fix import style * switch to views instead of endpoint for metrics * fix import order * rebase on top of statsd * install node exporter and add as a scrape target to prometheus * add pushgateway for django-prometheus and export sample push custom metrics * add configs to deploy gateway and nodeexporter to staging setup * fix failing tests for pushgateway metrics * add monitoring for submission queue using prometheus counters * fix staging configs and add labels to queue counters * fix staging configs and add labels to queue counters * fix tests for submission worker * push metrics to gateway using prometheus client * rebase on top of master * cleanup unncessary files * switch to pushgateway for metrics for submission loaded into queue * fix configs for staging server * fix metrics path for node exporter * fix push metrics key for submission worker * fix flake issues * fix import order * cleanup unncessary files with gitignore * change counters name for queue monitoring * add nginx config to frontend-v2 * fix staging endpoints and prometheus restart policy * fix hostnames and pathnames for node_exporter and push_gateway * fix job names for node-exporter and pushgateway * fix command names for autodeployment * update auto-deploy command --- apps/jobs/metrics.py | 18 +++++++++++++ apps/jobs/sender.py | 7 +++++ docker-compose-staging.yml | 14 ++++++++++ docker-compose.yml | 15 +++++++++++ docker/dev/docker.env | 1 + docker/prod/docker_staging.env | 1 + docker/prod/nodejs/nginx_staging.conf | 22 +++++++++++++++ docker/prod/nodejs_v2/nginx_staging.conf | 22 +++++++++++++++ monitoring/prometheus/prometheus.yml | 10 +++++++ monitoring/prometheus/prometheus_staging.yml | 10 +++++++ requirements/common.txt | 1 + sample.txt | 0 scripts/deployment/deploy.sh | 16 ++++++++++- scripts/workers/submission_worker.py | 28 ++++++++++++++++++++ 14 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 apps/jobs/metrics.py create mode 100644 sample.txt diff --git a/apps/jobs/metrics.py b/apps/jobs/metrics.py new file mode 100644 index 0000000000..632ee6039e --- /dev/null +++ b/apps/jobs/metrics.py @@ -0,0 +1,18 @@ +import os + +from prometheus_client import Counter, pushadd_to_gateway, CollectorRegistry + + +pushgateway_registry = CollectorRegistry() + +num_submissions_in_queue = Counter( + "num_submissions_in_queue", + "Counter for number of submissions pushed into queue", + ["submission_pk", "queue_name"], + registry=pushgateway_registry, +) + + +def push_metrics_to_pushgateway(job_id): + pushgateway_endpoint = os.environ.get("PUSHGATEWAY_ENDPOINT") + pushadd_to_gateway(pushgateway_endpoint, job=job_id, registry=pushgateway_registry) diff --git a/apps/jobs/sender.py b/apps/jobs/sender.py index 0927a9685e..d018328c68 100644 --- a/apps/jobs/sender.py +++ b/apps/jobs/sender.py @@ -11,6 +11,7 @@ from base.utils import send_slack_notification from challenges.models import Challenge from .utils import get_submission_model +from .metrics import push_metrics_to_pushgateway, num_submissions_in_queue logger = logging.getLogger(__name__) @@ -83,6 +84,12 @@ def publish_submission_message(message): queue_name = challenge.queue slack_url = challenge.slack_webhook_url queue = get_or_create_sqs_queue(queue_name) + # increase counter for submission pushed into queue + submission_pk = message["submission_pk"] + num_submissions_in_queue.labels(submission_pk, queue_name).inc() + # push metrics to pushgateway as a submission is pushed into queue + job_id = "submission_{}".format(submission_pk) + push_metrics_to_pushgateway(job_id) response = queue.send_message(MessageBody=json.dumps(message)) # send slack notification if slack_url: diff --git a/docker-compose-staging.yml b/docker-compose-staging.yml index 948fb296f0..55de63db05 100644 --- a/docker-compose-staging.yml +++ b/docker-compose-staging.yml @@ -129,3 +129,17 @@ services: ports: - '9125:9125' - '9102:9102' + + node_exporter: + hostname: node_exporter + image: prom/node-exporter + ports: + - '9100:9100' + + push_gateway: + hostname: push_gateway + image: prom/pushgateway + command: + - '--web.route-prefix=/push_gateway' + ports: + - '9091:9091' diff --git a/docker-compose.yml b/docker-compose.yml index 16d99d04bb..3e8003b3e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,7 @@ services: - db - sqs - statsd-exporter + - push_gateway volumes: - .:/code @@ -100,3 +101,17 @@ services: ports: - '9125:9125' - '9102:9102' + + node_exporter: + hostname: node_exporter + image: prom/node-exporter + ports: + - '9100:9100' + + push_gateway: + hostname: push_gateway + image: prom/pushgateway + command: + - '--web.route-prefix=/push_gateway' + ports: + - '9091:9091' diff --git a/docker/dev/docker.env b/docker/dev/docker.env index 9352952bcf..bad5755a20 100644 --- a/docker/dev/docker.env +++ b/docker/dev/docker.env @@ -38,3 +38,4 @@ GF_SECURITY_ADMIN_PASSWORD=password STATSD_ENDPOINT=statsd STATSD_PORT=9125 +PUSHGATEWAY_ENDPOINT=http://push_gateway:9091/push_gateway diff --git a/docker/prod/docker_staging.env b/docker/prod/docker_staging.env index cd62caf1f1..36539acd64 100644 --- a/docker/prod/docker_staging.env +++ b/docker/prod/docker_staging.env @@ -61,3 +61,4 @@ GF_SECURITY_ADMIN_PASSWORD=x STATSD_ENDPOINT=statsd STATSD_PORT=9125 +PUSHGATEWAY_ENDPOINT=x diff --git a/docker/prod/nodejs/nginx_staging.conf b/docker/prod/nodejs/nginx_staging.conf index a40d40a9c9..6d0f3db692 100644 --- a/docker/prod/nodejs/nginx_staging.conf +++ b/docker/prod/nodejs/nginx_staging.conf @@ -6,6 +6,14 @@ upstream statsd_exporter { server statsd:9102 fail_timeout=0; } +upstream node_exporter { + server node_exporter:9100 fail_timeout=0; +} + +upstream push_gateway { + server push_gateway:9091 fail_timeout=0; +} + server { server_name staging-evalai.cloudcv.org evalai-staging.cloudcv.org; listen 80; @@ -64,6 +72,20 @@ server { proxy_pass http://statsd_exporter; } + location /node_exporter { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://node_exporter/metrics; + } + + location /push_gateway { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://push_gateway; + } + ssl on; ssl_certificate /etc/ssl/eval_ai.crt; ssl_certificate_key /etc/ssl/eval_ai.key; diff --git a/docker/prod/nodejs_v2/nginx_staging.conf b/docker/prod/nodejs_v2/nginx_staging.conf index f2c7eb15df..3458673164 100644 --- a/docker/prod/nodejs_v2/nginx_staging.conf +++ b/docker/prod/nodejs_v2/nginx_staging.conf @@ -6,6 +6,14 @@ upstream statsd_exporter { server statsd:9102 fail_timeout=0; } +upstream node_exporter { + server node_exporter:9100 fail_timeout=0; +} + +upstream push_gateway { + server push_gateway:9091 fail_timeout=0; +} + server { server_name staging.eval.ai; listen 80; @@ -44,4 +52,18 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://statsd_exporter; } + + location /node_exporter { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://node_exporter/metrics; + } + + location /push_gateway { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://push_gateway; + } } diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index 6aa39327d9..a3a1e710c0 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -11,3 +11,13 @@ scrape_configs: metrics_path: '/statsd/metrics' static_configs: - targets: ['statsd:9102'] + + - job_name: 'node_exporter' + metrics_path: '/metrics' + static_configs: + - targets: ['node_exporter:9100'] + + - job_name: 'push_gateway' + metrics_path: '/push_gateway/metrics' + static_configs: + - targets: ['push_gateway:9091'] diff --git a/monitoring/prometheus/prometheus_staging.yml b/monitoring/prometheus/prometheus_staging.yml index 7935d04fe6..53783a7e20 100644 --- a/monitoring/prometheus/prometheus_staging.yml +++ b/monitoring/prometheus/prometheus_staging.yml @@ -12,3 +12,13 @@ scrape_configs: metrics_path: '/statsd/metrics' static_configs: - targets: ['staging.eval.ai'] + + - job_name: 'node_exporter' + metrics_path: '/node_exporter' + static_configs: + - targets: ['staging.eval.ai'] + + - job_name: 'push_gateway' + metrics_path: '/push_gateway/metrics' + static_configs: + - targets: ['staging.eval.ai'] diff --git a/requirements/common.txt b/requirements/common.txt index 8f483f62fd..d39796754e 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -22,6 +22,7 @@ moto==1.3.14 pika==1.1.0 pickleshare==0.7.5 Pillow==7.1.0 +prometheus-client==0.11.0 psycopg2==2.8.4 pycurl==7.43.0.6 PyYaml==5.1 diff --git a/sample.txt b/sample.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/deployment/deploy.sh b/scripts/deployment/deploy.sh index 3446ce30c3..10b60551fe 100755 --- a/scripts/deployment/deploy.sh +++ b/scripts/deployment/deploy.sh @@ -47,7 +47,7 @@ case $opt in aws s3 cp s3://cloudcv-secrets/evalai/${env}/docker_${env}.env ./docker/prod/docker_${env}.env docker-compose -f docker-compose-${env}.yml rm -s -v -f docker-compose -f docker-compose-${env}.yml pull - docker-compose -f docker-compose-${env}.yml up -d --force-recreate --remove-orphans statsd-exporter django nodejs nodejs_v2 celery + docker-compose -f docker-compose-${env}.yml up -d --force-recreate --remove-orphans statsd-exporter django nodejs nodejs_v2 celery node_exporter push_gateway ENDSSH2 ENDSSH ;; @@ -150,6 +150,16 @@ case $opt in docker-compose -f docker-compose-${env}.yml up -d statsd-exporter echo "Completed deploy operation." ;; + deploy-node-exporter) + echo "Deploying node_exporter docker container..." + docker-compose -f docker-compose-${env}.yml up -d node_exporter + echo "Completed deploy operation." + ;; + deploy-push-gateway) + echo "Deploying push_gateway docker container..." + docker-compose -f docker-compose-${env}.yml up -d push_gateway + echo "Completed deploy operation." + ;; scale) service=${3} instances=${4} @@ -193,6 +203,10 @@ case $opt in echo " Eg. ./scripts/deployment/deploy.sh deploy-grafana production" echo " deploy-statsd : Deploy statsd container in the respective environment." echo " Eg. ./scripts/deployment/deploy.sh deploy-statsd production" + echo " deploy-node-exporter : Deploy node_exporter container in the respective environment." + echo " Eg. ./scripts/deployment/deploy.sh deploy-node-exporter production" + echo " deploy-push-gateway : Deploy push_gateway container in the respective environment." + echo " Eg. ./scripts/deployment/deploy.sh deploy-push-gateway production" echo " scale : Scale particular docker service in an environment." echo " Eg. ./scripts/deployment/deploy.sh scale production django 5" echo " clean : Remove all docker containers and images." diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py index ff63dca0ba..ca1f47d7e3 100644 --- a/scripts/workers/submission_worker.py +++ b/scripts/workers/submission_worker.py @@ -25,6 +25,7 @@ from django.core.files.base import ContentFile from django.utils import timezone +from prometheus_client import pushadd_to_gateway, CollectorRegistry, Counter # all challenge and submission will be stored in temp directory BASE_TEMP_DIR = tempfile.mkdtemp() @@ -40,6 +41,15 @@ logger = logging.getLogger(__name__) logger.addHandler(handler) logger.setLevel(logging.INFO) + +pushgateway_registry = CollectorRegistry() +num_processed_submissions = Counter( + "num_processed_submissions", + "Counter for number of submissions processed from the queue", + ["submission_pk", "queue_name"], + registry=pushgateway_registry, +) + django.setup() # Load django app settings @@ -749,6 +759,21 @@ def load_challenge_and_return_max_submissions(q_params): return maximum_concurrent_submissions, challenge +def increment_and_push_metrics_to_pushgateway(body, queue_name): + try: + submission_pk = json.loads(body)["submission_pk"] + num_processed_submissions.labels(submission_pk, queue_name).inc() + pushgateway_endpoint = os.environ.get("PUSHGATEWAY_ENDPOINT") + job_id = "submission_worker_{}".format(submission_pk) + pushadd_to_gateway(pushgateway_endpoint, job=job_id, registry=pushgateway_registry) + except Exception as e: + logger.exception( + "{} Exception when pushing metrics to push gateway: {}".format( + SUBMISSION_LOGS_PREFIX, e + ) + ) + + def main(): killer = GracefulKiller() logger.info( @@ -821,6 +846,7 @@ def main(): process_submission_callback(message.body) # Let the queue know that the message is processed message.delete() + increment_and_push_metrics_to_pushgateway(message.body, queue_name) else: logger.info( "{} Processing message body: {}".format( @@ -830,6 +856,7 @@ def main(): process_submission_callback(message.body) # Let the queue know that the message is processed message.delete() + increment_and_push_metrics_to_pushgateway(message.body, queue_name) else: current_running_submissions_count = Submission.objects.filter( challenge_phase__challenge=challenge.id, status="running" @@ -848,6 +875,7 @@ def main(): process_submission_callback(message.body) # Let the queue know that the message is processed message.delete() + increment_and_push_metrics_to_pushgateway(message.body, queue_name) if killer.kill_now: break time.sleep(0.1) From 9d0e043b5822f526d8938314cd2ea016dddf8dbb Mon Sep 17 00:00:00 2001 From: savish28 <32800267+savish28@users.noreply.github.com> Date: Thu, 8 Jul 2021 23:32:17 +0530 Subject: [PATCH 18/31] Code Upload Worker: Refactor code upload challenge worker for stuck submissions(#3492) * Refactor code upload challenge submission stuck * Delete message and job when exception in reading job * Add: cleanup_submission in code upload worker * Add: docstrings cleanup_submission Co-authored-by: Rishabh Jain --- .../workers/code_upload_submission_worker.py | 139 +++++++++++++----- 1 file changed, 99 insertions(+), 40 deletions(-) diff --git a/scripts/workers/code_upload_submission_worker.py b/scripts/workers/code_upload_submission_worker.py index 908d6ca4fc..f0df85f439 100755 --- a/scripts/workers/code_upload_submission_worker.py +++ b/scripts/workers/code_upload_submission_worker.py @@ -471,6 +471,53 @@ def read_job(api_instance, job_name): return api_response +def cleanup_submission( + api_instance, + evalai, + job_name, + submission_pk, + challenge_pk, + phase_pk, + stderr, + message, +): + """Function to update status of submission to EvalAi, Delete corrosponding job from cluster and messaage from SQS. + Arguments: + api_instance {[AWS EKS API object]} -- API object for deleting job + evalai {[EvalAI class object]} -- EvalAI class object imported from worker_utils + job_name {[string]} -- Name of the job to be terminated + submission_pk {[int]} -- Submission id + challenge_pk {[int]} -- Challenge id + phase_pk {[int]} -- Challenge Phase id + stderr {[string]} -- Reason of failure for submission/job + message {[dict]} -- Submission message from AWS SQS queue + """ + try: + submission_data = { + "challenge_phase": phase_pk, + "submission": submission_pk, + "stdout": "", + "stderr": stderr, + "submission_status": "FAILED", + "result": "[]", + "metadata": "", + } + evalai.update_submission_data(submission_data, challenge_pk, phase_pk) + try: + delete_job(api_instance, job_name) + except Exception as e: + logger.exception("Failed to delete submission job: {}".format(e)) + message_receipt_handle = message.get("receipt_handle") + if message_receipt_handle: + evalai.delete_message_from_sqs_queue(message_receipt_handle) + except Exception as e: + logger.exception( + "Exception while cleanup Submission {}: {}".format( + submission_pk, e + ) + ) + + def update_failed_jobs_and_send_logs( api_instance, core_v1_api_instance, @@ -479,48 +526,59 @@ def update_failed_jobs_and_send_logs( submission_pk, challenge_pk, phase_pk, + message, ): - job_def = read_job(api_instance, job_name) - controller_uid = job_def.metadata.labels["controller-uid"] - pod_label_selector = "controller-uid=" + controller_uid - pods_list = core_v1_api_instance.list_namespaced_pod( - namespace="default", - label_selector=pod_label_selector, - timeout_seconds=10, - ) - for container in pods_list.items[0].status.container_statuses: - if container.name in ["agent", "submission"]: - if container.state.terminated is not None: - if container.state.terminated.reason == "Error": - pod_name = pods_list.items[0].metadata.name - try: - pod_log_response = ( - core_v1_api_instance.read_namespaced_pod_log( - name=pod_name, - namespace="default", - _return_http_data_only=True, - _preload_content=False, - container=container.name, + try: + job_def = read_job(api_instance, job_name) + controller_uid = job_def.metadata.labels["controller-uid"] + pod_label_selector = "controller-uid=" + controller_uid + pods_list = core_v1_api_instance.list_namespaced_pod( + namespace="default", + label_selector=pod_label_selector, + timeout_seconds=10, + ) + for container in pods_list.items[0].status.container_statuses: + if container.name in ["agent", "submission"]: + if container.state.terminated is not None: + if container.state.terminated.reason == "Error": + pod_name = pods_list.items[0].metadata.name + try: + pod_log_response = ( + core_v1_api_instance.read_namespaced_pod_log( + name=pod_name, + namespace="default", + _return_http_data_only=True, + _preload_content=False, + container=container.name, + ) ) - ) - pod_log = pod_log_response.data.decode("utf-8") - submission_data = { - "challenge_phase": phase_pk, - "submission": submission_pk, - "stdout": "", - "stderr": pod_log, - "submission_status": "FAILED", - "result": "[]", - "metadata": "", - } - response = evalai.update_submission_data( - submission_data, challenge_pk, phase_pk - ) - print(response) - except client.rest.ApiException as e: - logger.exception( - "Exception while reading Job logs {}".format(e) - ) + pod_log = pod_log_response.data.decode("utf-8") + cleanup_submission( + api_instance, + evalai, + job_name, + submission_pk, + challenge_pk, + phase_pk, + pod_log, + message, + ) + except client.rest.ApiException as e: + logger.exception( + "Exception while reading Job logs {}".format(e) + ) + except Exception as e: + logger.exception("Exception while reading Job {}".format(e)) + cleanup_submission( + api_instance, + evalai, + job_name, + submission_pk, + challenge_pk, + phase_pk, + pod_log, + message, + ) def install_gpu_drivers(api_instance): @@ -628,6 +686,7 @@ def main(): submission_pk, challenge_pk, phase_pk, + message, ) else: logger.info( From 2bc8762313934a92b585ec950221786807e8ff26 Mon Sep 17 00:00:00 2001 From: Gautam Jajoo Date: Sun, 11 Jul 2021 15:07:20 +0530 Subject: [PATCH 19/31] [FrontendV2] Add challenge phase detail edit feature (#3491) --- apps/challenges/serializers.py | 1 + .../editphasemodal.component.html | 38 +++ .../editphasemodal.component.ts | 20 +- .../challengesettings.component.html | 43 +++- .../challengesettings.component.scss | 8 + .../challengesettings.component.ts | 230 +++++++++++++++--- .../challengesubmissions.component.html | 2 +- .../challengesubmit.component.html | 3 +- .../challengesubmit.component.ts | 6 + .../selectphase/selectphase.component.ts | 15 +- .../src/app/services/challenge.service.ts | 10 + tests/unit/challenges/test_views.py | 9 + 12 files changed, 338 insertions(+), 47 deletions(-) diff --git a/apps/challenges/serializers.py b/apps/challenges/serializers.py index 6e28744f12..4fa6a24bee 100644 --- a/apps/challenges/serializers.py +++ b/apps/challenges/serializers.py @@ -103,6 +103,7 @@ class Meta: "allowed_submission_file_types", "default_submission_meta_attributes", "allowed_email_ids", + "is_submission_public", ) diff --git a/frontend_v2/src/app/components/challenge/challengephases/editphasemodal/editphasemodal.component.html b/frontend_v2/src/app/components/challenge/challengephases/editphasemodal/editphasemodal.component.html index 6af6dac21a..924da573c7 100644 --- a/frontend_v2/src/app/components/challenge/challengephases/editphasemodal/editphasemodal.component.html +++ b/frontend_v2/src/app/components/challenge/challengephases/editphasemodal/editphasemodal.component.html @@ -76,6 +76,7 @@ +
Submissions/day @@ -134,6 +135,43 @@ >
+
+ +
+
+
+ Max Concurrent Submissions Allowed + +
+
+ +
+
+
Allowed Submission File Types
+ +
+
+
diff --git a/frontend_v2/src/app/components/challenge/challengephases/editphasemodal/editphasemodal.component.ts b/frontend_v2/src/app/components/challenge/challengephases/editphasemodal/editphasemodal.component.ts index 043db1c87c..b74a9fab77 100644 --- a/frontend_v2/src/app/components/challenge/challengephases/editphasemodal/editphasemodal.component.ts +++ b/frontend_v2/src/app/components/challenge/challengephases/editphasemodal/editphasemodal.component.ts @@ -27,6 +27,11 @@ export class EditphasemodalComponent implements OnInit { * Challenge phase name */ name = ''; + + /** + * Challenge phase allowed submission file types + */ + allowedSubmissionFileTypes = ''; /** * Challenge phase description @@ -58,6 +63,11 @@ export class EditphasemodalComponent implements OnInit { */ maxSubmissions: number; + /** + * Challenge phase max concurrent submissions allowed + */ + maxConcurrentSubmissionsAllowed: number; + /** * If editor error message */ @@ -141,7 +151,9 @@ export class EditphasemodalComponent implements OnInit { * Constructor. * @param globalService GlobalService Injection. */ - constructor(private globalService: GlobalService) {} + constructor( + private globalService: GlobalService, + ) {} ngOnInit() { if (this.params) { @@ -172,6 +184,12 @@ export class EditphasemodalComponent implements OnInit { if (this.params['maxSubmissions']) { this.maxSubmissions = this.params['maxSubmissions']; } + if (this.params['maxConcurrentSubmissionsAllowed']) { + this.maxConcurrentSubmissionsAllowed = this.params['maxConcurrentSubmissionsAllowed']; + } + if (this.params['allowedSubmissionFileTypes']) { + this.allowedSubmissionFileTypes = this.params['allowedSubmissionFileTypes']; + } if (this.params['confirm']) { this.confirm = this.params['confirm']; } diff --git a/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.html b/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.html index d69aa6a310..796a9092f8 100644 --- a/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.html +++ b/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.html @@ -106,16 +106,41 @@
Challenge Settings

- -
-
+
+ + +
+ +
+ + Is Public + + + +      + + Submission Visibility + + + + +
+
+
diff --git a/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.scss b/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.scss index 0de5a6dbb1..543e84f175 100644 --- a/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.scss +++ b/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.scss @@ -13,6 +13,14 @@ } } +.phase-card { + width:50vw; +} + +.phase-button { + margin-left: 25px; +} + @include screen-medium { .settings-section { padding: 10px 20px !important; diff --git a/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.ts b/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.ts index 924a462226..96265fcc37 100644 --- a/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.ts +++ b/frontend_v2/src/app/components/challenge/challengesettings/challengesettings.component.ts @@ -89,13 +89,44 @@ export class ChallengesettingsComponent implements OnInit, OnDestroy { * Email error message */ message: string; + + /** + * If the submission is public + */ + isSubmissionPublic : boolean = false; + + /** + * If the phase is public + */ + isPhasePublic : boolean = false; + + /** + * If leaderboard is public + */ + isLeaderboardPublic : boolean = false; + + /** + * phase visibility state and it's icon + */ + phaseVisibility = { + state: 'Private', + icon: 'fa fa-toggle-off', + }; + + /** + * submission visibility state and it's icon + */ + submissionVisibility = { + state: 'Private', + icon: 'fa fa-toggle-off', + }; /** * publish challenge state and it's icon */ publishChallenge = { state: 'Not Published', - icon: 'fa fa-eye-slash red-text', + icon: 'fa fa-toggle-off', }; /** @@ -188,49 +219,180 @@ export class ChallengesettingsComponent implements OnInit, OnDestroy { const SELF = this; return (phase) => { SELF.selectedPhase = phase; + SELF.isPhasePublic = SELF.selectedPhase['is_public']; + SELF.isSubmissionPublic = SELF.selectedPhase['is_submission_public']; + SELF.isLeaderboardPublic = SELF.selectedPhase['leaderboard_public']; + if (SELF.isPhasePublic) { + SELF.phaseVisibility.state = 'Public'; + SELF.phaseVisibility.icon = 'fa fa-toggle-on green-text'; + } + else { + SELF.phaseVisibility.state = 'Private'; + SELF.phaseVisibility.icon = 'fa fa-toggle-off grey-text text-darken-1'; + } + if (SELF.isSubmissionPublic) { + SELF.submissionVisibility.state = 'Public'; + SELF.submissionVisibility.icon = 'fa fa-toggle-on green-text'; + } + else { + SELF.submissionVisibility.state = 'Private'; + SELF.submissionVisibility.icon = 'fa fa-toggle-off grey-text text-darken-1'; + } + }; + } - SELF.apiCall = (params) => { - const FORM_DATA: FormData = new FormData(); - for (const key in params) { - if (params[key]) { - FORM_DATA.append(key, params[key]); - } + editPhaseDetails() { + const SELF = this; + SELF.apiCall = (params) => { + const FORM_DATA: FormData = new FormData(); + for (const key in params) { + if (params[key]) { + FORM_DATA.append(key, params[key]); } + } + SELF.apiService + .patchFileUrl( + SELF.endpointsService.updateChallengePhaseDetailsURL(SELF.challenge.id, SELF.selectedPhase['id']), + FORM_DATA + ) + .subscribe( + (data) => { + SELF.selectedPhase = data; + SELF.challengeService.fetchPhases(SELF.challenge['id']); + SELF.challengeService.changePhaseSelected(true); + SELF.selectedPhase = false; + SELF.globalService.showToast('success', 'The challenge phase details are successfully updated!'); + }, + (err) => { + SELF.globalService.showToast('error', err); + }, + () => {this.logger.info('PHASE-UPDATE-FINISHED')} + ); + }; + + const PARAMS = { + title: 'Edit Challenge Phase Details', + name: SELF.selectedPhase['name'], + label: 'description', + description: SELF.selectedPhase['description'], + startDate: SELF.selectedPhase['start_date'], + endDate: SELF.selectedPhase['end_date'], + maxSubmissionsPerDay: SELF.selectedPhase['max_submissions_per_day'], + maxSubmissionsPerMonth: SELF.selectedPhase['max_submissions_per_month'], + maxSubmissions: SELF.selectedPhase['max_submissions'], + maxConcurrentSubmissionsAllowed: SELF.selectedPhase['max_concurrent_submissions_allowed'], + allowedSubmissionFileTypes: SELF.selectedPhase['allowed_submission_file_types'], + confirm: 'Submit', + deny: 'Cancel', + confirmCallback: SELF.apiCall, + }; + SELF.globalService.showEditPhaseModal(PARAMS); +} + + /** + * Phase Visibility toggle function + */ + togglePhaseVisibility() { + const SELF = this; + let togglePhaseVisibilityState, isPublic; + if (SELF.phaseVisibility.state === 'Public') { + togglePhaseVisibilityState = 'private'; + isPublic = false; + SELF.phaseVisibility.state = 'Private'; + SELF.phaseVisibility.icon = 'fa fa-toggle-off'; + } else { + togglePhaseVisibilityState = 'public'; + isPublic = true; + SELF.phaseVisibility.state = 'Public'; + SELF.phaseVisibility.icon = 'fa fa-toggle-on green-text'; + } + const BODY: FormData = new FormData(); + BODY.append("is_public", isPublic); + SELF.apiService + .patchFileUrl( + SELF.endpointsService.updateChallengePhaseDetailsURL(SELF.selectedPhase['challenge'], SELF.selectedPhase['id']), + BODY + ) + .subscribe( + (data) => { + SELF.challengeService.fetchPhases(SELF.selectedPhase['challenge']); + SELF.challengeService.changePhaseSelected(true); + SELF.selectedPhase = false; + SELF.globalService.showToast( + 'success', + 'The phase was successfully made ' + togglePhaseVisibilityState, + 5 + ); + }, + (err) => { + SELF.globalService.handleApiError(err, true); + SELF.globalService.showToast('error', err); + if (isPublic) { + SELF.phaseVisibility.state = 'Private'; + SELF.phaseVisibility.icon = 'fa fa-toggle-off'; + } else { + SELF.phaseVisibility.state = 'Public'; + SELF.phaseVisibility.icon = 'fa fa-toggle-on green-text'; + } + }, + () => this.logger.info('PHASE-VISIBILITY-UPDATE-FINISHED') + ); + } + + /** + * Submission Visibility toggle function + */ + toggleSubmissionVisibility() { + const SELF = this; + if(SELF.isLeaderboardPublic == true) { + let toggleSubmissionVisibilityState, isSubmissionPublic; + if (SELF.submissionVisibility.state === 'Public') { + toggleSubmissionVisibilityState = 'private'; + isSubmissionPublic = false; + SELF.submissionVisibility.state = 'Private'; + SELF.submissionVisibility.icon = 'fa fa-toggle-off'; + } else { + toggleSubmissionVisibilityState = 'public'; + isSubmissionPublic = true; + SELF.submissionVisibility.state = 'Public'; + SELF.submissionVisibility.icon = 'fa fa-toggle-on green-text'; + } + const BODY: FormData = new FormData(); + BODY.append("is_submission_public", isSubmissionPublic); SELF.apiService - .patchFileUrl( - SELF.endpointsService.updateChallengePhaseDetailsURL(SELF.challenge.id, SELF.selectedPhase['id']), - FORM_DATA - ) + .patchFileUrl( + SELF.endpointsService.updateChallengePhaseDetailsURL(SELF.selectedPhase['challenge'], SELF.selectedPhase['id']), + BODY + ) .subscribe( (data) => { - SELF.selectedPhase = data; - SELF.challengeService.fetchPhases(SELF.challenge['id']); - SELF.globalService.showToast('success', 'The challenge phase details are successfully updated!'); + SELF.challengeService.fetchPhases(SELF.selectedPhase['challenge']); + SELF.challengeService.changePhaseSelected(true); + SELF.selectedPhase = false; + SELF.globalService.showToast( + 'success', + 'The submissions were successfully made ' + toggleSubmissionVisibilityState, + 5 + ); }, (err) => { + SELF.globalService.handleApiError(err, true); SELF.globalService.showToast('error', err); + if (isSubmissionPublic) { + SELF.submissionVisibility.state = 'Private'; + SELF.submissionVisibility.icon = 'fa fa-toggle-off'; + } else { + SELF.submissionVisibility.state = 'Public'; + SELF.submissionVisibility.icon = 'fa fa-toggle-on green-text'; + } }, - () => {this.logger.info('PHASE-UPDATE-FINISHED')} + () => this.logger.info('SUBMISSION-VISIBILITY-UPDATE-FINISHED') ); - }; - - const PARAMS = { - title: 'Edit Challenge Phase Details', - name: SELF.selectedPhase['name'], - label: 'description', - description: SELF.selectedPhase['description'], - startDate: SELF.selectedPhase['start_date'], - endDate: SELF.selectedPhase['end_date'], - maxSubmissionsPerDay: SELF.selectedPhase['max_submissions_per_day'], - maxSubmissionsPerMonth: SELF.selectedPhase['max_submissions_per_month'], - maxSubmissions: SELF.selectedPhase['max_submissions'], - confirm: 'Submit', - deny: 'Cancel', - confirmCallback: SELF.apiCall, - }; - SELF.globalService.showEditPhaseModal(PARAMS); - }; - } + } + else { + SELF.globalService.showToast('error', "Leaderboard is private, please make the leaderbaord public"); + } + } /** * Remove banned email chip diff --git a/frontend_v2/src/app/components/challenge/challengesubmissions/challengesubmissions.component.html b/frontend_v2/src/app/components/challenge/challengesubmissions/challengesubmissions.component.html index 5b61795a72..e633b57a7b 100644 --- a/frontend_v2/src/app/components/challenge/challengesubmissions/challengesubmissions.component.html +++ b/frontend_v2/src/app/components/challenge/challengesubmissions/challengesubmissions.component.html @@ -139,7 +139,7 @@
My Submissions
Show on leaderboard diff --git a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html index 458ae03599..c18a55e034 100644 --- a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html +++ b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html @@ -286,13 +286,14 @@
Make Submission
diff --git a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.ts b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.ts index 6a5746b031..b19619d9dd 100644 --- a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.ts +++ b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.ts @@ -53,6 +53,11 @@ export class ChallengesubmitComponent implements OnInit { */ isPublicSubmission:boolean = true; + /** + * Is submittion allowed by host + */ + isSubmissionPublic:boolean = false; + /** * Challenge object */ @@ -467,6 +472,7 @@ export class ChallengesubmitComponent implements OnInit { const SELF = this; return (phase) => { SELF.selectedPhase = phase; + SELF.isSubmissionPublic = phase['is_submission_public']; if (SELF.challenge['id'] && phase['id']) { SELF.getMetaDataDetails(SELF.challenge['id'], phase['id']); SELF.fetchRemainingSubmissions(SELF.challenge['id'], phase['id']); diff --git a/frontend_v2/src/app/components/utility/selectphase/selectphase.component.ts b/frontend_v2/src/app/components/utility/selectphase/selectphase.component.ts index e24b401af4..6abdfe0c55 100644 --- a/frontend_v2/src/app/components/utility/selectphase/selectphase.component.ts +++ b/frontend_v2/src/app/components/utility/selectphase/selectphase.component.ts @@ -46,6 +46,11 @@ export class SelectphaseComponent implements OnInit, OnChanges { */ phaseVisibility = false; + /** + * If phase selected + */ + isPhaseSelected : boolean = false; + /** * Currently selected phase */ @@ -86,7 +91,15 @@ export class SelectphaseComponent implements OnInit, OnChanges { * Component on changes detected in Input. * @param change changes detected */ - ngOnChanges(change) {} + ngOnChanges(change) { + this.challengeService.isPhaseSelected.subscribe((isPhaseSelected) => { + this.isPhaseSelected = isPhaseSelected; + }); + if(this.isPhaseSelected == true) { + this.challengeService.changePhaseSelected(false); + this.phaseName = ''; + } + } /** * Select a particular phase. diff --git a/frontend_v2/src/app/services/challenge.service.ts b/frontend_v2/src/app/services/challenge.service.ts index 3e78be9cf6..942b3317b8 100644 --- a/frontend_v2/src/app/services/challenge.service.ts +++ b/frontend_v2/src/app/services/challenge.service.ts @@ -33,6 +33,8 @@ export class ChallengeService { currentHostTeam = this.hostTeamSource.asObservable(); private challengeHostSource = new BehaviorSubject(false); isChallengeHost = this.challengeHostSource.asObservable(); + private phaseSelected = new BehaviorSubject(false); + isPhaseSelected = this.phaseSelected.asObservable(); private challengePublishSource = new BehaviorSubject(this.defaultPublishChallenge); currentChallengePublishState = this.challengePublishSource.asObservable(); @@ -66,6 +68,14 @@ export class ChallengeService { this.challengeHostSource.next(isChallengeHost); } + /** + * Update the status for selectPhase component after details are updated + * @param selectedPhase new updated phase details status + */ + changePhaseSelected(selectedPhase: boolean) { + this.phaseSelected.next(selectedPhase); + } + /** * Update challenge publish state and icon for current challenge. * @param publishChallenge new challenge publish status and icon. diff --git a/tests/unit/challenges/test_views.py b/tests/unit/challenges/test_views.py index 6a3f95923b..e2d7a4d846 100644 --- a/tests/unit/challenges/test_views.py +++ b/tests/unit/challenges/test_views.py @@ -2057,6 +2057,7 @@ def test_get_challenge_phase(self): "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types, "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes, "allowed_email_ids": self.challenge_phase.allowed_email_ids, + "is_submission_public": self.challenge_phase.is_submission_public, }, { "id": self.private_challenge_phase.id, @@ -2084,6 +2085,7 @@ def test_get_challenge_phase(self): "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types, "default_submission_meta_attributes": self.private_challenge_phase.default_submission_meta_attributes, "allowed_email_ids": self.challenge_phase.allowed_email_ids, + "is_submission_public": self.challenge_phase.is_submission_public, }, ] @@ -2119,6 +2121,7 @@ def test_get_challenge_phase_when_user_is_not_authenticated(self): "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types, "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes, "allowed_email_ids": self.challenge_phase.allowed_email_ids, + "is_submission_public": self.challenge_phase.is_submission_public, } ] self.client.force_authenticate(user=None) @@ -2164,6 +2167,7 @@ def test_get_challenge_phase_when_user_is_host(self): "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types, "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes, "allowed_email_ids": self.challenge_phase.allowed_email_ids, + "is_submission_public": self.challenge_phase.is_submission_public, }, { "id": self.private_challenge_phase.id, @@ -2191,6 +2195,7 @@ def test_get_challenge_phase_when_user_is_host(self): "is_partial_submission_evaluation_enabled": self.private_challenge_phase.is_partial_submission_evaluation_enabled, "default_submission_meta_attributes": self.private_challenge_phase.default_submission_meta_attributes, "allowed_email_ids": self.private_challenge_phase.allowed_email_ids, + "is_submission_public": self.private_challenge_phase.is_submission_public, }, ] @@ -2540,6 +2545,7 @@ def test_get_particular_challenge_phase_if_user_is_participant(self): "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types, "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes, "allowed_email_ids": self.challenge_phase.allowed_email_ids, + "is_submission_public": self.challenge_phase.is_submission_public, } self.client.force_authenticate(user=self.participant_user) response = self.client.get(self.url, {}) @@ -2640,6 +2646,7 @@ def test_update_challenge_phase_when_user_is_its_creator(self): "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types, "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes, "allowed_email_ids": self.challenge_phase.allowed_email_ids, + "is_submission_public": self.challenge_phase.is_submission_public, } response = self.client.put( self.url, {"name": new_name, "description": new_description} @@ -2740,6 +2747,7 @@ def test_particular_challenge_phase_partial_update(self): "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types, "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes, "allowed_email_ids": self.challenge_phase.allowed_email_ids, + "is_submission_public": self.challenge_phase.is_submission_public, } response = self.client.patch(self.url, self.partial_update_data) self.assertEqual(response.data, expected) @@ -4257,6 +4265,7 @@ def test_get_challenge_phase_by_pk(self): "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types, "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes, "allowed_email_ids": self.challenge_phase.allowed_email_ids, + "is_submission_public": self.challenge_phase.is_submission_public, } response = self.client.get(self.url, {}) self.assertEqual(response.data, expected) From 9c8c510ec9072d0c39dd405edec58ee31eaf2598 Mon Sep 17 00:00:00 2001 From: savish28 <32800267+savish28@users.noreply.github.com> Date: Wed, 14 Jul 2021 20:51:22 +0530 Subject: [PATCH 20/31] Code Upload Worker: Cleanup submission file before TLE(#3523) * Refactor code upload challenge submission stuck * Delete message and job when exception in reading job * Add: cleanup_submission in code upload worker * Add: docstrings cleanup_submission * Cleanup submission code upload: file not found * Submission container check execution function abstraction * Refactor code structuring Co-authored-by: Rishabh Jain --- .../workers/code_upload_submission_worker.py | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/scripts/workers/code_upload_submission_worker.py b/scripts/workers/code_upload_submission_worker.py index f0df85f439..52230bf36d 100755 --- a/scripts/workers/code_upload_submission_worker.py +++ b/scripts/workers/code_upload_submission_worker.py @@ -518,6 +518,21 @@ def cleanup_submission( ) +def is_submission_evaluation_done(container_state, container_state_map): + """Function to check if submission container is successfully terminated and sidecar-container is running + Arguments: + container_state {[AWS EKS API Container state object]} -- State of the container + container_state_map {[dict]} -- Map of container_name to container_state + """ + if ( + container_state.terminated.reason == "Completed" + and container_state_map.get("sidecar-container") + and container_state_map.get("sidecar-container").terminated is None + ): + return True + return False + + def update_failed_jobs_and_send_logs( api_instance, core_v1_api_instance, @@ -528,6 +543,7 @@ def update_failed_jobs_and_send_logs( phase_pk, message, ): + clean_submission = False try: job_def = read_job(api_instance, job_name) controller_uid = job_def.metadata.labels["controller-uid"] @@ -537,10 +553,13 @@ def update_failed_jobs_and_send_logs( label_selector=pod_label_selector, timeout_seconds=10, ) + container_state_map = {} for container in pods_list.items[0].status.container_statuses: - if container.name in ["agent", "submission"]: - if container.state.terminated is not None: - if container.state.terminated.reason == "Error": + container_state_map[container.name] = container.state + for container_name, container_state in container_state_map.items(): + if container_name in ["agent", "submission"]: + if container_state.terminated is not None: + if container_state.terminated.reason == "Error": pod_name = pods_list.items[0].metadata.name try: pod_log_response = ( @@ -549,26 +568,31 @@ def update_failed_jobs_and_send_logs( namespace="default", _return_http_data_only=True, _preload_content=False, - container=container.name, + container=container_name, ) ) pod_log = pod_log_response.data.decode("utf-8") - cleanup_submission( - api_instance, - evalai, - job_name, - submission_pk, - challenge_pk, - phase_pk, - pod_log, - message, - ) + clean_submission = True + submission_error = pod_log except client.rest.ApiException as e: logger.exception( "Exception while reading Job logs {}".format(e) ) + elif ( + container_name == "submission" + and is_submission_evaluation_done( + container_state, container_state_map + ) + ): + clean_submission = True + submission_error = ( + "submission.json/submission.csv file not found." + ) except Exception as e: logger.exception("Exception while reading Job {}".format(e)) + clean_submission = True + submission_error = "Submission Job Failed." + if clean_submission: cleanup_submission( api_instance, evalai, @@ -576,7 +600,7 @@ def update_failed_jobs_and_send_logs( submission_pk, challenge_pk, phase_pk, - pod_log, + submission_error, message, ) From fb749a4871eabf36951a4ce89b69811b05d8e182 Mon Sep 17 00:00:00 2001 From: savish28 <32800267+savish28@users.noreply.github.com> Date: Wed, 14 Jul 2021 21:46:58 +0530 Subject: [PATCH 21/31] Code Upload Worker: Add check before creating config maps in code upload worker (#3522) * Add check before creating config maps code upload worker * Add time stop for submission worker to read * Add comment and revert time delta Co-authored-by: Rishabh Jain --- scripts/workers/code_upload_submission_worker.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/workers/code_upload_submission_worker.py b/scripts/workers/code_upload_submission_worker.py index 52230bf36d..492699b5e3 100755 --- a/scripts/workers/code_upload_submission_worker.py +++ b/scripts/workers/code_upload_submission_worker.py @@ -3,6 +3,7 @@ import os import signal import yaml +import time from worker_utils import EvalAI_Interface @@ -95,6 +96,14 @@ def create_config_map_object(config_map_name, file_paths): def create_configmap(core_v1_api_instance, config_map): try: + config_maps = core_v1_api_instance.list_namespaced_config_map( + namespace="default" + ) + if ( + len(config_maps.items) + and config_maps.items[0].metadata.name == script_config_map_name + ): + return core_v1_api_instance.create_namespaced_config_map( namespace="default", body=config_map, @@ -664,6 +673,9 @@ def main(): "submission_time_limit" ) while True: + # Equal distribution of queue messages among submission worker and code upload worker + if challenge.get("is_static_dataset_code_upload"): + time.sleep(2.1) message = evalai.get_message_from_sqs_queue() message_body = message.get("body") if message_body: From ab69de409e5d1b1a0f60c6d019ecbc75599ba4d7 Mon Sep 17 00:00:00 2001 From: savish28 <32800267+savish28@users.noreply.github.com> Date: Mon, 19 Jul 2021 00:02:40 +0530 Subject: [PATCH 22/31] Backend: Restart static code upload worker on modifying evaluation script(#3535) --- apps/challenges/aws_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/challenges/aws_utils.py b/apps/challenges/aws_utils.py index acc1e8181f..d9fae7905c 100644 --- a/apps/challenges/aws_utils.py +++ b/apps/challenges/aws_utils.py @@ -1146,7 +1146,10 @@ def restart_workers(queryset): count = 0 failures = [] for challenge in queryset: - if challenge.is_docker_based: + if ( + challenge.is_docker_based + and not challenge.is_static_dataset_code_upload + ): response = "Sorry. This feature is not available for code upload/docker based challenges." failures.append( {"message": response, "challenge_pk": challenge.pk} From 4c763ffac82b4139f624bc5e57570cbd61c03dc4 Mon Sep 17 00:00:00 2001 From: savish28 <32800267+savish28@users.noreply.github.com> Date: Mon, 19 Jul 2021 00:03:11 +0530 Subject: [PATCH 23/31] Code Upload Worker: Update api_instacne for kubernetes token refresh (#3533) --- scripts/workers/code_upload_submission_worker.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/workers/code_upload_submission_worker.py b/scripts/workers/code_upload_submission_worker.py index 492699b5e3..a90b66acba 100755 --- a/scripts/workers/code_upload_submission_worker.py +++ b/scripts/workers/code_upload_submission_worker.py @@ -685,6 +685,12 @@ def main(): "is_static_dataset_code_upload_submission" ): continue + api_instance = get_api_object( + cluster_name, cluster_endpoint, challenge, evalai + ) + core_v1_api_instance = get_core_v1_api_object( + cluster_name, cluster_endpoint, challenge, evalai + ) message_body["submission_meta"] = submission_meta submission_pk = message_body.get("submission_pk") challenge_pk = message_body.get("challenge_pk") From b2b8234bfc3be41af4ba07db230d289d86f46988 Mon Sep 17 00:00:00 2001 From: Gautam Jajoo Date: Mon, 19 Jul 2021 00:03:46 +0530 Subject: [PATCH 24/31] Frontend_V2: Fix challenge template page component (#3532) * fix challenge template page and add footer * fix challenge title * add pointer to calendar icon Co-authored-by: Rishabh Jain --- .../template-challenge-create.component.html | 9 ++++----- .../template-challenge-create.component.scss | 10 +++++++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/frontend_v2/src/app/components/template-challenge-create/template-challenge-create.component.html b/frontend_v2/src/app/components/template-challenge-create/template-challenge-create.component.html index 0e7bb212cf..860294ec62 100644 --- a/frontend_v2/src/app/components/template-challenge-create/template-challenge-create.component.html +++ b/frontend_v2/src/app/components/template-challenge-create/template-challenge-create.component.html @@ -5,9 +5,7 @@
- - Create Challenge - +

Create Challenge


@@ -43,9 +41,9 @@
- Challenge Phases + Challenge Phases
-
+
Name for Phase {{phase['id']}}
+
diff --git a/frontend_v2/src/app/components/template-challenge-create/template-challenge-create.component.scss b/frontend_v2/src/app/components/template-challenge-create/template-challenge-create.component.scss index 8c41306b58..5bc6db6909 100644 --- a/frontend_v2/src/app/components/template-challenge-create/template-challenge-create.component.scss +++ b/frontend_v2/src/app/components/template-challenge-create/template-challenge-create.component.scss @@ -13,6 +13,14 @@ } } +.challenge-phase { + margin-top: 2rem; +} + +::-webkit-calendar-picker-indicator:hover{ + cursor:pointer; +} + @media only screen and (max-width: $med-screen) { .web-container { width: 100%; @@ -43,7 +51,7 @@ p { } .web-container { - width: calc(100vw - 223px); + width: calc(100% - 223px); float: right; padding-top: 70px; overflow-x: hidden; From 8f0cf0e76ac87b0b5dae7f8a07bbab12a1423526 Mon Sep 17 00:00:00 2001 From: Gautam Jajoo Date: Mon, 19 Jul 2021 00:04:26 +0530 Subject: [PATCH 25/31] Frontend_V2: Highlight tab of side-bar when moving to challenge components (#3531) * add service * add the functionality for tab highlight * fix final issues * remove console statement * fix error Co-authored-by: Rishabh Jain --- .../header-static.component.html | 3 +- .../header-static/header-static.component.ts | 34 ++++++++++- .../utility/side-bar/side-bar.component.html | 12 ++-- .../utility/side-bar/side-bar.component.scss | 1 + .../utility/side-bar/side-bar.component.ts | 58 ++++++++++++++++++- .../src/app/services/global.service.ts | 10 ++++ 6 files changed, 110 insertions(+), 8 deletions(-) diff --git a/frontend_v2/src/app/components/nav/header-static/header-static.component.html b/frontend_v2/src/app/components/nav/header-static/header-static.component.html index c48a1dc74f..8c619e3ce2 100644 --- a/frontend_v2/src/app/components/nav/header-static/header-static.component.html +++ b/frontend_v2/src/app/components/nav/header-static/header-static.component.html @@ -9,7 +9,8 @@ />
  • - All Challenges + All Challenges
  • diff --git a/frontend_v2/src/app/components/nav/header-static/header-static.component.ts b/frontend_v2/src/app/components/nav/header-static/header-static.component.ts index e3f45c0dfd..5d53f335b0 100644 --- a/frontend_v2/src/app/components/nav/header-static/header-static.component.ts +++ b/frontend_v2/src/app/components/nav/header-static/header-static.component.ts @@ -11,6 +11,7 @@ import { } from '@angular/core'; import { GlobalService } from '../../../services/global.service'; import { AuthService } from '../../../services/auth.service'; +import { filter } from "rxjs/internal/operators"; import { RouterModule, Router, ActivatedRoute, NavigationEnd } from '@angular/router'; import { DOCUMENT } from '@angular/common'; import { ApiService } from '../../../services/api.service'; @@ -62,6 +63,16 @@ export class HeaderStaticComponent implements OnInit, OnDestroy { */ isLoggedIn: any = false; + /** + * Current name of tab which needs to be active + */ + tabHighlight: string = "allChallenges"; + + /** + * Returns true if the string is not a number + */ + isChallengeComponent : boolean = false; + /** * Inner width */ @@ -88,7 +99,7 @@ export class HeaderStaticComponent implements OnInit, OnDestroy { private apiService: ApiService, @Inject(DOCUMENT) private document: Document ) { - this.authState = authService.authState; + this.authState = authService.authState; } /** @@ -107,6 +118,27 @@ export class HeaderStaticComponent implements OnInit, OnDestroy { ngOnInit() { this.updateElements(); this.checkInnerWidth(); + + this.router.events + .pipe(filter(event => event instanceof NavigationEnd)) + .subscribe((event) => { + if(event) { + if(this.router.url.split('/')[length] == "all") { + this.tabHighlight = "allChallenges"; + this.globalService.changeTabActiveStatus("allChallenges"); + } + else if(this.router.url.split('/')[1] == "profile") { + this.tabHighlight = "profile"; + this.globalService.changeTabActiveStatus("profile"); + } + } + }); + this.isChallengeComponent = isNaN(parseInt(this.router.url.split('/')[length])); + + this.globalService.nameTabHighlight.subscribe((tabHighlight) => { + this.tabHighlight = tabHighlight; + }); + this.authServiceSubscription = this.authService.change.subscribe((authState) => { this.authState = authState; if (this.authService.isLoggedIn()) { diff --git a/frontend_v2/src/app/components/utility/side-bar/side-bar.component.html b/frontend_v2/src/app/components/utility/side-bar/side-bar.component.html index 577e5af12b..e00acdb49f 100644 --- a/frontend_v2/src/app/components/utility/side-bar/side-bar.component.html +++ b/frontend_v2/src/app/components/utility/side-bar/side-bar.component.html @@ -3,25 +3,29 @@
    • - +
      Participated challenges
    • - +
      Hosted challenges
    • - +
      Create challenge
    • - +
      My participant teams
       diff --git a/frontend_v2/src/app/components/utility/side-bar/side-bar.component.scss b/frontend_v2/src/app/components/utility/side-bar/side-bar.component.scss index c5952ab61f..8d6b256471 100644 --- a/frontend_v2/src/app/components/utility/side-bar/side-bar.component.scss +++ b/frontend_v2/src/app/components/utility/side-bar/side-bar.component.scss @@ -36,6 +36,7 @@ border-right: 1px solid transparent; &:hover { color: $highlight; + border-right: 1px solid $highlight; } &.side-active-link { color: $med-black; diff --git a/frontend_v2/src/app/components/utility/side-bar/side-bar.component.ts b/frontend_v2/src/app/components/utility/side-bar/side-bar.component.ts index 8c531c3c72..943212da8a 100644 --- a/frontend_v2/src/app/components/utility/side-bar/side-bar.component.ts +++ b/frontend_v2/src/app/components/utility/side-bar/side-bar.component.ts @@ -1,4 +1,7 @@ import { Component, OnInit } from '@angular/core'; +import { Router, NavigationEnd } from '@angular/router'; +import { filter } from "rxjs/internal/operators"; +import { GlobalService } from '../../../services/global.service'; @Component({ selector: 'app-side-bar', @@ -6,7 +9,58 @@ import { Component, OnInit } from '@angular/core'; styleUrls: ['./side-bar.component.scss'], }) export class SideBarComponent implements OnInit { - constructor() {} - ngOnInit() {} + /** + * Current name of tab which needs to be active + */ + tabHighlight: string = "allChallenges"; + + /** + * Returns true if the string is not a number + */ + isChallengeComponent : boolean = false; + + /** + * Constructor + * @param globalService GlobalService Injection. + * @param router Router + */ + constructor( + private globalService: GlobalService, + private router: Router + ) { } + + ngOnInit() { + this.globalService.nameTabHighlight.subscribe((tabHighlight) => { + this.tabHighlight = tabHighlight; + }); + + this.router.events + .pipe(filter(event => event instanceof NavigationEnd)) + .subscribe((event) => { + if(event) { + if(this.router.url.split('/')[2] == "me") { + this.tabHighlight = "hostedChallenges"; + this.globalService.changeTabActiveStatus("hostedChallenges"); + } + else if(this.router.url.split('/')[2] == "hosts") { + this.tabHighlight = "createChallenge"; + this.globalService.changeTabActiveStatus("createChallenge"); + } + else if(this.router.url.split('/')[2] == "participants") { + this.tabHighlight = "myParticipantTeams"; + this.globalService.changeTabActiveStatus("myParticipantTeams"); + } + else if(this.router.url.split('/')[2] == "participated") { + this.tabHighlight = "participatedChallenges"; + this.globalService.changeTabActiveStatus("participatedChallenges"); + } + else if(this.router.url.split('/')[2] == "all") { + this.tabHighlight = "allChallenges"; + this.globalService.changeTabActiveStatus("allChallenges"); + } + this.isChallengeComponent = isNaN(parseInt(this.router.url.split('/')[length])); + } + }); + } } diff --git a/frontend_v2/src/app/services/global.service.ts b/frontend_v2/src/app/services/global.service.ts index 099ba2ca9f..f5c52f768c 100644 --- a/frontend_v2/src/app/services/global.service.ts +++ b/frontend_v2/src/app/services/global.service.ts @@ -75,6 +75,8 @@ export class GlobalService { editPhaseModalParams = this.editPhasemodalSource.asObservable(); private termsAndConditionsSource = new BehaviorSubject(this.termsAndConditionsModalDefault); termsAndConditionsModalParams = this.termsAndConditionsSource.asObservable(); + private tabHighlight = new BehaviorSubject(" "); + nameTabHighlight = this.tabHighlight.asObservable(); @Output() toast: EventEmitter = new EventEmitter(); @Output() loading: EventEmitter = new EventEmitter(); @@ -256,6 +258,14 @@ export class GlobalService { } } + /** + * Update the status for highlighting the active tab + * @param activeTab new updated name of tab + */ + changeTabActiveStatus(activeTab: string) { + this.tabHighlight.next(activeTab); + } + /** * This triggers the logout function in auth service (to avoid a cyclic dependency). */ From a5d2aabad698637337ce39aa24063cf1bbb462ae Mon Sep 17 00:00:00 2001 From: Aviral Jain Date: Mon, 19 Jul 2021 00:06:15 +0530 Subject: [PATCH 26/31] Backend: Add configs for migrating monitoring-setup containers to a new instance(#3529) * migrate pushgateway configs for separate ec2 instance * add nginx container for monitoring setup * update image name for monitoring nginx * add auto deploy commands to setup-pushgateway * update auto deploy commands for monitoring instance * change nginx service name to nginx-ingress * fix indentation of auto-deploy script * fix indentation of auto-deployment script * change dns name to monitoring-staging Co-authored-by: Rishabh Jain --- docker-compose-staging.yml | 18 +++++++ docker/prod/docker_staging.env | 2 + docker/prod/nginx-ingress/Dockerfile | 4 ++ docker/prod/nginx-ingress/nginx_staging.conf | 57 ++++++++++++++++++++ docker/prod/nodejs/nginx_staging.conf | 11 ---- docker/prod/nodejs_v2/nginx_staging.conf | 11 ---- monitoring/prometheus/prometheus_staging.yml | 3 +- scripts/deployment/deploy.sh | 14 ++++- settings/staging.py | 4 +- 9 files changed, 98 insertions(+), 26 deletions(-) create mode 100644 docker/prod/nginx-ingress/Dockerfile create mode 100644 docker/prod/nginx-ingress/nginx_staging.conf diff --git a/docker-compose-staging.yml b/docker-compose-staging.yml index 55de63db05..8aabe115a0 100644 --- a/docker-compose-staging.yml +++ b/docker-compose-staging.yml @@ -96,6 +96,7 @@ services: dockerfile: docker/prod/code-upload-worker/Dockerfile prometheus: + hostname: prometheus image: prom/prometheus:latest user: "1000" volumes: @@ -105,10 +106,12 @@ services: - ./monitoring/prometheus/prometheus_db:/etc/prometheus command: - '--config.file=/etc/prometheus/prometheus.yml' + - '--web.external-url=http://localhost:9090/prometheus' ports: - '9090:9090' grafana: + hostname: grafana image: grafana/grafana:latest user: "1000" env_file: @@ -143,3 +146,18 @@ services: - '--web.route-prefix=/push_gateway' ports: - '9091:9091' + + nginx-ingress: + image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-staging-nginx-ingress:${COMMIT_ID} + build: + context: ./ + dockerfile: docker/prod/nginx-ingress/Dockerfile + args: + MONITORING_ENV: staging + depends_on: + - prometheus + - grafana + - push_gateway + ports: + - '80:80' + - '443:443' diff --git a/docker/prod/docker_staging.env b/docker/prod/docker_staging.env index 36539acd64..8a8bab8d33 100644 --- a/docker/prod/docker_staging.env +++ b/docker/prod/docker_staging.env @@ -58,6 +58,8 @@ SERVICE_DNS=x GF_SECURITY_ADMIN_USER=x GF_SECURITY_ADMIN_PASSWORD=x +GF_SERVER_ROOT_URL=http://localhost:3000/grafana +GF_SERVER_SERVE_FROM_SUB_PATH=true STATSD_ENDPOINT=statsd STATSD_PORT=9125 diff --git a/docker/prod/nginx-ingress/Dockerfile b/docker/prod/nginx-ingress/Dockerfile new file mode 100644 index 0000000000..0137e342f1 --- /dev/null +++ b/docker/prod/nginx-ingress/Dockerfile @@ -0,0 +1,4 @@ +FROM nginx:1.13-alpine +ARG MONITORING_ENV +COPY docker/prod/nginx-ingress/nginx_${MONITORING_ENV}.conf /etc/nginx/conf.d/default.conf +COPY /ssl /etc/ssl diff --git a/docker/prod/nginx-ingress/nginx_staging.conf b/docker/prod/nginx-ingress/nginx_staging.conf new file mode 100644 index 0000000000..79e169dfea --- /dev/null +++ b/docker/prod/nginx-ingress/nginx_staging.conf @@ -0,0 +1,57 @@ +upstream prometheus { + server prometheus:9090 fail_timeout=0; +} + +upstream grafana { + server grafana:3000 fail_timeout=0; +} + +upstream push_gateway { + server push_gateway:9091 fail_timeout=0; +} + +server { + server_name monitoring-staging.eval.ai; + listen 80; + return 301 https://monitoring-staging.eval.ai$request_uri; +} + +server { + server_name monitoring-staging.eval.ai; + listen 443 ssl; + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + ssl on; + ssl_certificate /etc/ssl/eval_ai.crt; + ssl_certificate_key /etc/ssl/eval_ai.key; + ssl_prefer_server_ciphers on; + # enables all versions of TLS, but not SSLv2 or 3 which are weak and now deprecated. + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + location /prometheus { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass https://prometheus; + } + + location /grafana { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass https://grafana; + } + + location /push_gateway { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass https://push_gateway; + } +} diff --git a/docker/prod/nodejs/nginx_staging.conf b/docker/prod/nodejs/nginx_staging.conf index 6d0f3db692..8e7cdd02ce 100644 --- a/docker/prod/nodejs/nginx_staging.conf +++ b/docker/prod/nodejs/nginx_staging.conf @@ -10,10 +10,6 @@ upstream node_exporter { server node_exporter:9100 fail_timeout=0; } -upstream push_gateway { - server push_gateway:9091 fail_timeout=0; -} - server { server_name staging-evalai.cloudcv.org evalai-staging.cloudcv.org; listen 80; @@ -79,13 +75,6 @@ server { proxy_pass http://node_exporter/metrics; } - location /push_gateway { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://push_gateway; - } - ssl on; ssl_certificate /etc/ssl/eval_ai.crt; ssl_certificate_key /etc/ssl/eval_ai.key; diff --git a/docker/prod/nodejs_v2/nginx_staging.conf b/docker/prod/nodejs_v2/nginx_staging.conf index 3458673164..9044522769 100644 --- a/docker/prod/nodejs_v2/nginx_staging.conf +++ b/docker/prod/nodejs_v2/nginx_staging.conf @@ -10,10 +10,6 @@ upstream node_exporter { server node_exporter:9100 fail_timeout=0; } -upstream push_gateway { - server push_gateway:9091 fail_timeout=0; -} - server { server_name staging.eval.ai; listen 80; @@ -59,11 +55,4 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://node_exporter/metrics; } - - location /push_gateway { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://push_gateway; - } } diff --git a/monitoring/prometheus/prometheus_staging.yml b/monitoring/prometheus/prometheus_staging.yml index 53783a7e20..b67c3177f6 100644 --- a/monitoring/prometheus/prometheus_staging.yml +++ b/monitoring/prometheus/prometheus_staging.yml @@ -5,6 +5,7 @@ global: scrape_configs: - job_name: 'prometheus' + metrics_path: '/prometheus/metrics' static_configs: - targets: ['localhost:9090'] @@ -21,4 +22,4 @@ scrape_configs: - job_name: 'push_gateway' metrics_path: '/push_gateway/metrics' static_configs: - - targets: ['staging.eval.ai'] + - targets: ['monitoring-staging.eval.ai'] diff --git a/scripts/deployment/deploy.sh b/scripts/deployment/deploy.sh index 10b60551fe..afd01fd297 100755 --- a/scripts/deployment/deploy.sh +++ b/scripts/deployment/deploy.sh @@ -26,8 +26,10 @@ JUMPBOX=${JUMPBOX_INSTANCE} if [[ ${env} == "production" ]]; then INSTANCE=${PRODUCTION_INSTANCE} + MONITORING_INSTANCE=${PRODUCTION_MONITORING_INSTANCE} elif [[ ${env} == "staging" ]]; then INSTANCE=${STAGING_INSTANCE} + MONITORING_INSTANCE=${STAGING_MONITORING_INSTANCE} else echo "Skipping deployment since commit not on staging or production branch." exit 0 @@ -47,7 +49,17 @@ case $opt in aws s3 cp s3://cloudcv-secrets/evalai/${env}/docker_${env}.env ./docker/prod/docker_${env}.env docker-compose -f docker-compose-${env}.yml rm -s -v -f docker-compose -f docker-compose-${env}.yml pull - docker-compose -f docker-compose-${env}.yml up -d --force-recreate --remove-orphans statsd-exporter django nodejs nodejs_v2 celery node_exporter push_gateway + docker-compose -f docker-compose-${env}.yml up -d --force-recreate --remove-orphans statsd-exporter django nodejs nodejs_v2 celery node_exporter + ENDSSH2 + ssh ubuntu@${MONITORING_INSTANCE} -o StrictHostKeyChecking=no AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID} COMMIT_ID=${COMMIT_ID} env=${env} 'bash -s' <<-'ENDSSH2' + cd ~/Projects/EvalAI + export AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID} + export COMMIT_ID=${COMMIT_ID} + eval $(aws ecr get-login --no-include-email) + aws s3 cp s3://cloudcv-secrets/evalai/${env}/docker_${env}.env ./docker/prod/docker_${env}.env + docker-compose -f docker-compose-${env}.yml rm -s -v -f + docker-compose -f docker-compose-${env}.yml pull + docker-compose -f docker-compose-${env}.yml up -d --force-recreate --remove-orphans nginx-ingress prometheus grafana push_gateway ENDSSH2 ENDSSH ;; diff --git a/settings/staging.py b/settings/staging.py index 7335fc67ee..dd6ab3b647 100644 --- a/settings/staging.py +++ b/settings/staging.py @@ -1,6 +1,6 @@ from .prod import * # noqa: ignore=F405 -ALLOWED_HOSTS = ["staging.eval.ai", "beta-staging.eval.ai"] +ALLOWED_HOSTS = ["staging.eval.ai", "monitoring-staging.eval.ai"] CORS_ORIGIN_ALLOW_ALL = False @@ -8,5 +8,5 @@ "https://evalai.s3.amazonaws.com", "https://staging-evalai.s3.amazonaws.com", "https://staging.eval.ai", - "https://beta-staging.eval.ai", + "https://monitoring-staging.eval.ai", ) From d1c32369f20e1b9206c54ac834e4b771446f36e9 Mon Sep 17 00:00:00 2001 From: Ram Ramrakhya Date: Mon, 19 Jul 2021 22:09:07 +0530 Subject: [PATCH 27/31] Frontend_V2: Increase nginx upload limit to 200 MB(#3538) --- docker/prod/nodejs_v2/nginx_staging.conf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker/prod/nodejs_v2/nginx_staging.conf b/docker/prod/nodejs_v2/nginx_staging.conf index 9044522769..74214c17a1 100644 --- a/docker/prod/nodejs_v2/nginx_staging.conf +++ b/docker/prod/nodejs_v2/nginx_staging.conf @@ -25,6 +25,16 @@ server { try_files $uri $uri/ /index.html =404; } + client_max_body_size 200M; + gzip on; + gzip_http_version 1.1; + gzip_disable "MSIE [1-6]\."; + gzip_min_length 256; + gzip_vary on; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_comp_level 9; + ssl on; ssl_certificate /etc/ssl/eval_ai.crt; ssl_certificate_key /etc/ssl/eval_ai.key; From c0860d872bda48f4d92ce9e68a00dc1eb0353acc Mon Sep 17 00:00:00 2001 From: Aviral Jain Date: Tue, 20 Jul 2021 00:16:13 +0530 Subject: [PATCH 28/31] Revert pushgateway from monitoring setup (#3540) --- apps/jobs/metrics.py | 18 ------------- apps/jobs/sender.py | 7 ----- docker-compose-staging.yml | 9 ------- docker-compose.yml | 9 ------- docker/dev/docker.env | 1 - docker/prod/docker_staging.env | 1 - docker/prod/nginx-ingress/nginx_staging.conf | 10 -------- monitoring/prometheus/prometheus.yml | 5 ---- monitoring/prometheus/prometheus_staging.yml | 5 ---- requirements/common.txt | 1 - scripts/deployment/deploy.sh | 9 +------ scripts/workers/submission_worker.py | 27 -------------------- 12 files changed, 1 insertion(+), 101 deletions(-) delete mode 100644 apps/jobs/metrics.py diff --git a/apps/jobs/metrics.py b/apps/jobs/metrics.py deleted file mode 100644 index 632ee6039e..0000000000 --- a/apps/jobs/metrics.py +++ /dev/null @@ -1,18 +0,0 @@ -import os - -from prometheus_client import Counter, pushadd_to_gateway, CollectorRegistry - - -pushgateway_registry = CollectorRegistry() - -num_submissions_in_queue = Counter( - "num_submissions_in_queue", - "Counter for number of submissions pushed into queue", - ["submission_pk", "queue_name"], - registry=pushgateway_registry, -) - - -def push_metrics_to_pushgateway(job_id): - pushgateway_endpoint = os.environ.get("PUSHGATEWAY_ENDPOINT") - pushadd_to_gateway(pushgateway_endpoint, job=job_id, registry=pushgateway_registry) diff --git a/apps/jobs/sender.py b/apps/jobs/sender.py index d018328c68..0927a9685e 100644 --- a/apps/jobs/sender.py +++ b/apps/jobs/sender.py @@ -11,7 +11,6 @@ from base.utils import send_slack_notification from challenges.models import Challenge from .utils import get_submission_model -from .metrics import push_metrics_to_pushgateway, num_submissions_in_queue logger = logging.getLogger(__name__) @@ -84,12 +83,6 @@ def publish_submission_message(message): queue_name = challenge.queue slack_url = challenge.slack_webhook_url queue = get_or_create_sqs_queue(queue_name) - # increase counter for submission pushed into queue - submission_pk = message["submission_pk"] - num_submissions_in_queue.labels(submission_pk, queue_name).inc() - # push metrics to pushgateway as a submission is pushed into queue - job_id = "submission_{}".format(submission_pk) - push_metrics_to_pushgateway(job_id) response = queue.send_message(MessageBody=json.dumps(message)) # send slack notification if slack_url: diff --git a/docker-compose-staging.yml b/docker-compose-staging.yml index 8aabe115a0..4789017ebd 100644 --- a/docker-compose-staging.yml +++ b/docker-compose-staging.yml @@ -138,14 +138,6 @@ services: image: prom/node-exporter ports: - '9100:9100' - - push_gateway: - hostname: push_gateway - image: prom/pushgateway - command: - - '--web.route-prefix=/push_gateway' - ports: - - '9091:9091' nginx-ingress: image: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/evalai-staging-nginx-ingress:${COMMIT_ID} @@ -157,7 +149,6 @@ services: depends_on: - prometheus - grafana - - push_gateway ports: - '80:80' - '443:443' diff --git a/docker-compose.yml b/docker-compose.yml index 3e8003b3e7..b3d7cab852 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,6 @@ services: - db - sqs - statsd-exporter - - push_gateway volumes: - .:/code @@ -107,11 +106,3 @@ services: image: prom/node-exporter ports: - '9100:9100' - - push_gateway: - hostname: push_gateway - image: prom/pushgateway - command: - - '--web.route-prefix=/push_gateway' - ports: - - '9091:9091' diff --git a/docker/dev/docker.env b/docker/dev/docker.env index bad5755a20..9352952bcf 100644 --- a/docker/dev/docker.env +++ b/docker/dev/docker.env @@ -38,4 +38,3 @@ GF_SECURITY_ADMIN_PASSWORD=password STATSD_ENDPOINT=statsd STATSD_PORT=9125 -PUSHGATEWAY_ENDPOINT=http://push_gateway:9091/push_gateway diff --git a/docker/prod/docker_staging.env b/docker/prod/docker_staging.env index 8a8bab8d33..e561b8a2be 100644 --- a/docker/prod/docker_staging.env +++ b/docker/prod/docker_staging.env @@ -63,4 +63,3 @@ GF_SERVER_SERVE_FROM_SUB_PATH=true STATSD_ENDPOINT=statsd STATSD_PORT=9125 -PUSHGATEWAY_ENDPOINT=x diff --git a/docker/prod/nginx-ingress/nginx_staging.conf b/docker/prod/nginx-ingress/nginx_staging.conf index 79e169dfea..b8e31a57ec 100644 --- a/docker/prod/nginx-ingress/nginx_staging.conf +++ b/docker/prod/nginx-ingress/nginx_staging.conf @@ -6,10 +6,6 @@ upstream grafana { server grafana:3000 fail_timeout=0; } -upstream push_gateway { - server push_gateway:9091 fail_timeout=0; -} - server { server_name monitoring-staging.eval.ai; listen 80; @@ -48,10 +44,4 @@ server { proxy_pass https://grafana; } - location /push_gateway { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass https://push_gateway; - } } diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index a3a1e710c0..23e7b7b5f6 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -16,8 +16,3 @@ scrape_configs: metrics_path: '/metrics' static_configs: - targets: ['node_exporter:9100'] - - - job_name: 'push_gateway' - metrics_path: '/push_gateway/metrics' - static_configs: - - targets: ['push_gateway:9091'] diff --git a/monitoring/prometheus/prometheus_staging.yml b/monitoring/prometheus/prometheus_staging.yml index b67c3177f6..fd7a2c7307 100644 --- a/monitoring/prometheus/prometheus_staging.yml +++ b/monitoring/prometheus/prometheus_staging.yml @@ -18,8 +18,3 @@ scrape_configs: metrics_path: '/node_exporter' static_configs: - targets: ['staging.eval.ai'] - - - job_name: 'push_gateway' - metrics_path: '/push_gateway/metrics' - static_configs: - - targets: ['monitoring-staging.eval.ai'] diff --git a/requirements/common.txt b/requirements/common.txt index d39796754e..8f483f62fd 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -22,7 +22,6 @@ moto==1.3.14 pika==1.1.0 pickleshare==0.7.5 Pillow==7.1.0 -prometheus-client==0.11.0 psycopg2==2.8.4 pycurl==7.43.0.6 PyYaml==5.1 diff --git a/scripts/deployment/deploy.sh b/scripts/deployment/deploy.sh index afd01fd297..10130d8be6 100755 --- a/scripts/deployment/deploy.sh +++ b/scripts/deployment/deploy.sh @@ -59,7 +59,7 @@ case $opt in aws s3 cp s3://cloudcv-secrets/evalai/${env}/docker_${env}.env ./docker/prod/docker_${env}.env docker-compose -f docker-compose-${env}.yml rm -s -v -f docker-compose -f docker-compose-${env}.yml pull - docker-compose -f docker-compose-${env}.yml up -d --force-recreate --remove-orphans nginx-ingress prometheus grafana push_gateway + docker-compose -f docker-compose-${env}.yml up -d --force-recreate --remove-orphans nginx-ingress prometheus grafana ENDSSH2 ENDSSH ;; @@ -167,11 +167,6 @@ case $opt in docker-compose -f docker-compose-${env}.yml up -d node_exporter echo "Completed deploy operation." ;; - deploy-push-gateway) - echo "Deploying push_gateway docker container..." - docker-compose -f docker-compose-${env}.yml up -d push_gateway - echo "Completed deploy operation." - ;; scale) service=${3} instances=${4} @@ -217,8 +212,6 @@ case $opt in echo " Eg. ./scripts/deployment/deploy.sh deploy-statsd production" echo " deploy-node-exporter : Deploy node_exporter container in the respective environment." echo " Eg. ./scripts/deployment/deploy.sh deploy-node-exporter production" - echo " deploy-push-gateway : Deploy push_gateway container in the respective environment." - echo " Eg. ./scripts/deployment/deploy.sh deploy-push-gateway production" echo " scale : Scale particular docker service in an environment." echo " Eg. ./scripts/deployment/deploy.sh scale production django 5" echo " clean : Remove all docker containers and images." diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py index ca1f47d7e3..f432de5378 100644 --- a/scripts/workers/submission_worker.py +++ b/scripts/workers/submission_worker.py @@ -25,7 +25,6 @@ from django.core.files.base import ContentFile from django.utils import timezone -from prometheus_client import pushadd_to_gateway, CollectorRegistry, Counter # all challenge and submission will be stored in temp directory BASE_TEMP_DIR = tempfile.mkdtemp() @@ -42,14 +41,6 @@ logger.addHandler(handler) logger.setLevel(logging.INFO) -pushgateway_registry = CollectorRegistry() -num_processed_submissions = Counter( - "num_processed_submissions", - "Counter for number of submissions processed from the queue", - ["submission_pk", "queue_name"], - registry=pushgateway_registry, -) - django.setup() # Load django app settings @@ -759,21 +750,6 @@ def load_challenge_and_return_max_submissions(q_params): return maximum_concurrent_submissions, challenge -def increment_and_push_metrics_to_pushgateway(body, queue_name): - try: - submission_pk = json.loads(body)["submission_pk"] - num_processed_submissions.labels(submission_pk, queue_name).inc() - pushgateway_endpoint = os.environ.get("PUSHGATEWAY_ENDPOINT") - job_id = "submission_worker_{}".format(submission_pk) - pushadd_to_gateway(pushgateway_endpoint, job=job_id, registry=pushgateway_registry) - except Exception as e: - logger.exception( - "{} Exception when pushing metrics to push gateway: {}".format( - SUBMISSION_LOGS_PREFIX, e - ) - ) - - def main(): killer = GracefulKiller() logger.info( @@ -846,7 +822,6 @@ def main(): process_submission_callback(message.body) # Let the queue know that the message is processed message.delete() - increment_and_push_metrics_to_pushgateway(message.body, queue_name) else: logger.info( "{} Processing message body: {}".format( @@ -856,7 +831,6 @@ def main(): process_submission_callback(message.body) # Let the queue know that the message is processed message.delete() - increment_and_push_metrics_to_pushgateway(message.body, queue_name) else: current_running_submissions_count = Submission.objects.filter( challenge_phase__challenge=challenge.id, status="running" @@ -875,7 +849,6 @@ def main(): process_submission_callback(message.body) # Let the queue know that the message is processed message.delete() - increment_and_push_metrics_to_pushgateway(message.body, queue_name) if killer.kill_now: break time.sleep(0.1) From 733904883d4dd71849e8000cd65e304f6060ee1a Mon Sep 17 00:00:00 2001 From: Gautam Jajoo Date: Thu, 22 Jul 2021 23:40:58 +0530 Subject: [PATCH 29/31] Frontend_V2: Restructure submissions page(#3534) * add code element hover * change classes of elements --- .../challengesubmit.component.html | 522 +++++++++--------- .../challengesubmit.component.scss | 13 +- 2 files changed, 276 insertions(+), 259 deletions(-) diff --git a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html index c18a55e034..9520c20526 100644 --- a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html +++ b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html @@ -83,6 +83,7 @@
      Submission Guidelines
      +
      Submission limits

      @@ -154,13 +155,14 @@
      Submission Guidelines
      -
      + +
      Make Submission
      -
      +
        @@ -210,266 +212,10 @@
        Make Submission
      -
      -

      Submission instructions

      -
        -
      1. Install evalai-cli
        - $ pip install evalai{{ cliVersion }} - -
        2. -
      2. Add your EvalAI account token to evalai-cli
        - $ evalai set_token {{authToken}} - -
        3. -
      3. Make Submission
        - $ evalai challenge {{challengeId}} phase {{phaseId}} submit --file <submission_file_path> --large - -
        4. -
      4. -
        Use   --private  or  --public   flag in the submission - command to make the submission private or public respectively.
        -
        -
        5. -
      5. For more commands, please refer to evalai-cli documentation. -
        6. -
      -
      -
      -
      -
      -
      - -
      {{ inputErrorMessage }}
      -
      -
      - -
      -
      -
      -
      -
      -

      Select submission visibility:

      - - -
      - - -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      - {{ attribute.name }} - {{ attribute.description }}
      - - -
      -
      -
      -
      -
      -
      - {{ attribute.name }} - {{ attribute.description }}
      -
      - - - - - -
      -
      -
      -
      -
      -
      - {{ attribute.name }} - {{ attribute.description }}
      -
      - - - -
      -
      -
      -
      -
      -
      - {{ attribute.name }} - {{ attribute.description }}
      -
      - - -
      - -       -
      -
      -
      -
      -
      -
      -
      - {{ submissionError }}

      -
      - -
      -
      -
      -
      - - - {{ selectedPhaseSubmissions.maxExceededMessage }} - - -
      -
      -
      @@ -502,6 +248,266 @@
      Make Submission
      + +
      +
      +

      Submission instructions

      +
        +
      1. Install evalai-cli
        + $ pip install evalai{{ cliVersion }} + +
        2. +
      2. Add your EvalAI account token to evalai-cli
        + $ evalai set_token {{authToken}} + +
        3. +
      3. Make Submission
        + $ evalai challenge {{challengeId}} phase {{phaseId}} submit --file <submission_file_path> --large + +
        4. +
      4. +
        Use   --private  or  --public   flag in the submission + command to make the submission private or public respectively.
        +
        +
        5. +
      5. For more commands, please refer to evalai-cli documentation. +
        6. +
      +
      +
      + +
      +
      +
      +
      + +
      {{ inputErrorMessage }}
      +
      +
      + +
      +
      +
      +
      +
      +

      Select submission visibility:

      + + +
      + + +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      + {{ attribute.name }} - {{ attribute.description }}
      + + +
      +
      +
      +
      +
      +
      + {{ attribute.name }} - {{ attribute.description }}
      +
      + + + + + +
      +
      +
      +
      +
      +
      + {{ attribute.name }} - {{ attribute.description }}
      +
      + + + +
      +
      +
      +
      +
      +
      + {{ attribute.name }} - {{ attribute.description }}
      +
      + + +
      + +       +
      +
      +
      +
      +
      +
      +
      + {{ submissionError }}

      +
      + +
      + +
      +
      +
      + + + {{ selectedPhaseSubmissions.maxExceededMessage }} + + +
      +
      +
      diff --git a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.scss b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.scss index 57e9e0d982..25f420ea7d 100644 --- a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.scss +++ b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.scss @@ -5,6 +5,12 @@ padding: 10px 30px; } +.ev-challenge-view { + margin-top: 0px; + padding-top: 30px; + margin-bottom: 20px; +} + .row .row-lr-margin { margin-bottom: 20px; } @@ -56,7 +62,7 @@ th { padding-right: 36px; } ul { - padding-left: 70px; + padding-left: 60px; } } @@ -134,3 +140,8 @@ th { flex-direction: row; align-items: center; } + +.cli-card-container { + margin: 14.5px 0px; + padding: 0px 0px 0px 70px; +} From 625d6b5f318397393769ee81048057f8e36baad9 Mon Sep 17 00:00:00 2001 From: Gautam Jajoo Date: Fri, 23 Jul 2021 00:00:22 +0530 Subject: [PATCH 30/31] Frontend_V2: Show/hide private/public button on submit page(#3541) * show/hide public button on submit page * Update submit component --- .../challengesubmit/challengesubmit.component.html | 6 ++++-- .../challengesubmit/challengesubmit.component.ts | 10 +++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html index 9520c20526..5d3e6b3e0b 100644 --- a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html +++ b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.html @@ -327,14 +327,16 @@
      Make Submission

      Select submission visibility:

      diff --git a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.ts b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.ts index b19619d9dd..4acf2e5af9 100644 --- a/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.ts +++ b/frontend_v2/src/app/components/challenge/challengesubmit/challengesubmit.component.ts @@ -44,19 +44,19 @@ export class ChallengesubmitComponent implements OnInit { isLoggedIn = false; /** - * Is submittion submitted + * Is submission submitted */ isSubmitted = false; /** - * Is submittion submitted + * Is submission submitted */ isPublicSubmission:boolean = true; /** - * Is submittion allowed by host + * Is submission allowed by host */ - isSubmissionPublic:boolean = false; + isLeaderboardPublic:boolean = false; /** * Challenge object @@ -472,7 +472,7 @@ export class ChallengesubmitComponent implements OnInit { const SELF = this; return (phase) => { SELF.selectedPhase = phase; - SELF.isSubmissionPublic = phase['is_submission_public']; + SELF.isLeaderboardPublic = phase['leaderboard_public']; if (SELF.challenge['id'] && phase['id']) { SELF.getMetaDataDetails(SELF.challenge['id'], phase['id']); SELF.fetchRemainingSubmissions(SELF.challenge['id'], phase['id']); From 5353625707fb92278de1560c188abf7668d3da6c Mon Sep 17 00:00:00 2001 From: Gautam Jajoo Date: Fri, 23 Jul 2021 00:12:15 +0530 Subject: [PATCH 31/31] Frontend_V2: Increase decimal to 20 and display value received from backend(#3543) Co-authored-by: Rishabh Jain --- .../challengeleaderboard.component.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/frontend_v2/src/app/components/challenge/challengeleaderboard/challengeleaderboard.component.ts b/frontend_v2/src/app/components/challenge/challengeleaderboard/challengeleaderboard.component.ts index 7bc11aea02..8b8d1edbc2 100644 --- a/frontend_v2/src/app/components/challenge/challengeleaderboard/challengeleaderboard.component.ts +++ b/frontend_v2/src/app/components/challenge/challengeleaderboard/challengeleaderboard.component.ts @@ -343,6 +343,19 @@ export class ChallengeleaderboardComponent implements OnInit, AfterViewInit, OnD phaseSplitSelected(phaseSplit) { const SELF = this; SELF.selectedPhaseSplit = phaseSplit; + + const API_PATH = SELF.endpointsService.particularChallengePhaseSplitUrl(this.selectedPhaseSplit['id']); + SELF.apiService.getUrl(API_PATH).subscribe( + (data) => { + SELF.leaderboardPrecisionValue = data.leaderboard_decimal_precision; + SELF.setLeaderboardPrecisionValue = `1.${ SELF.leaderboardPrecisionValue }-${ SELF.leaderboardPrecisionValue }`; + }, + (err) => { + SELF.globalService.handleApiError(err); + }, + () => {} + ); + if (SELF.selectedPhaseSplit && SELF.router.url.endsWith('leaderboard/' + phaseSplit['id'])) { SELF.fetchLeaderboard(SELF.selectedPhaseSplit['id']); SELF.showLeaderboardByLatest = SELF.selectedPhaseSplit.show_leaderboard_by_latest_submission; @@ -620,7 +633,7 @@ export class ChallengeleaderboardComponent implements OnInit, AfterViewInit, OnD SELF.apiService.patchUrl(API_PATH, BODY).subscribe( (data) => { this.minusDisabled = SELF.leaderboardPrecisionValue === 0 ? true : false; - this.plusDisabled = SELF.leaderboardPrecisionValue === 5 ? true : false; + this.plusDisabled = SELF.leaderboardPrecisionValue === 20 ? true : false; }, (err) => { SELF.globalService.handleApiError(err, true);