Merge pull request #398 from isb-cgc/prod-sp

Sprint 38

.circleci/config.yml

@@ -121,6 +121,7 @@ jobs:
             - ./Dockerfile
             - ./app.yaml
             - ./openapi-appengine.yaml
+            - ./gunicorn.conf.py
             - ./settings.py
             - ./txt
             - ./json

Dockerfile

@@ -71,11 +71,6 @@ ADD . /app
 RUN pip3 install -r /app/requirements.txt -t /app/lib/ --upgrade
 RUN pip3 install gunicorn==19.9.0
 
-# Install Cloud SDK
-RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && apt-get update -y && apt-get install google-cloud-sdk -y
-# Install the Python Libraries
-RUN apt-get -y install google-cloud-sdk-app-engine-python
-
 ENV PYTHONPATH=/app:/app/apiv4:/app/lib:/app/ISB-CGC-Common:${PYTHONPATH}
 
-CMD gunicorn -b :$PORT apiv4:app -w 3 -t 130
+CMD gunicorn -c gunicorn.conf.py -b :$PORT apiv4:app -w 3 -t 130

apiv4/__init__.py

@@ -25,6 +25,7 @@
 from flask_talisman import Talisman
 
 app = Flask(__name__, static_folder='api_static')
+
 Talisman(app, strict_transport_security_max_age=300, content_security_policy={
     'default-src': [
         '\'self\'',
@@ -46,6 +47,7 @@
 from sample_case_routes import *
 from file_routes import *
 from user_routes import *
+from deprecated.user_routes import *
 
 logger = logging.getLogger(settings.LOGGER_NAME)
 
@@ -56,7 +58,6 @@ def load_spec():
         json_spec = ""
         try:
             yaml = ruamel.yaml.YAML(typ='safe')
-            logger.debug(os.path.split(os.path.abspath(dirname(__file__)))[0] + '/openapi-appengine.yaml')
             with open(os.path.split(os.path.abspath(dirname(__file__)))[0] + '/openapi-appengine.yaml') as fpi:
                 data = yaml.load(fpi)
                 del data['paths']['/swagger']

apiv4/cohorts_routes.py

@@ -42,10 +42,7 @@ def cohort(cohort_id):
         response_obj = None
 
         if not user:
-            response_obj = {
-                'message': 'Encountered an error while attempting to identify this user.'
-            }
-            code = 500
+            raise Exception('Encountered an error while attempting to identify this user.')
         else:
             if cohort_id <= 0:
                 logger.warn("[WARNING] Invalid cohort ID {}".format(str(cohort_id)))
@@ -62,12 +59,8 @@ def cohort(cohort_id):
                     cohort_info = edit_cohort(cohort_id, user, delete=(request.method == 'DELETE'))
 
                 if cohort_info:
-                    response_obj['data'] = cohort_info
-
-                    if 'message' in cohort_info:
-                        code = 400
-                    else:
-                        code = 200
+                    response_obj = {'data': cohort_info}
+                    code = 400 if 'message' in cohort_info else 200
                 else:
                     response_obj = {
                         'message': "Cohort ID {} was not found.".format(str(cohort_id))
@@ -111,36 +104,23 @@ def cohorts():
         user = validate_user(user_info['email'])
 
         if not user:
-            response_obj = {
-                'code': 500,
-                'message': 'Encountered an error while attempting to identify this user.'
-            }
-            code = 500
+            raise Exception('Encountered an error while attempting to identify this user.')
         else:
             st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, request.full_path))
-            if request.method == 'GET':
-                info = get_cohorts(user_info['email'])
-            else:
-                info = create_cohort(user)
+            info = get_cohorts(user_info['email']) if request.method == 'GET' else create_cohort(user)
 
             if info:
-                response_obj = {}
-
-                if 'message' in info:
-                    code = 400
-                else:
-                    code = 200
+                response_obj = {
+                    'data': info
+                }
 
-                response_obj['data'] = info
+                code = 400 if 'message' in info else 200
 
             # Lack of a valid object means something went wrong on the server
             else:
-                response_obj = {
-                    'message': "Error while attempting to {}.".format(
-                        'retrieve the cohort list' if request.method == 'GET' else 'create this cohort'
-                    )
-                }
-                code = 500
+                raise Exception("Invalid response while attempting to {}.".format(
+                    'retrieve the cohort list' if request.method == 'GET' else 'create this cohort'
+                ))
 
     except UserValidationException as e:
         response_obj = {
@@ -161,7 +141,7 @@ def cohorts():
     response_obj['code'] = code
     response = jsonify(response_obj)
     response.status_code = code
-
+        
     return response
 
 
@@ -180,10 +160,7 @@ def cohort_file_manifest(cohort_id):
         user = validate_user(user_info['email'], cohort_id)
 
         if not user:
-            response_obj = {
-                'message': 'Encountered an error while attempting to identify this user.'
-            }
-            code = 500
+            raise Exception('Encountered an error while attempting to identify this user.')
         else:
             if cohort_id <= 0:
                 logger.warn("[WARNING] Invalid cohort ID {}".format(str(cohort_id)))
@@ -205,10 +182,7 @@ def cohort_file_manifest(cohort_id):
                             'data': file_manifest
                         }
                 else:
-                    response_obj = {
-                        'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id))
-                    }
-                    code = 500
+                    raise Exception("Invalid response while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id)))
 
     except UserValidationException as e:
         response_obj = {
@@ -256,11 +230,8 @@ def cohort_preview():
 
         # Lack of a valid object means something went wrong on the server
         else:
-            response_obj = {
-                'message': "Error while attempting to retrieve case and sample counts for these filters."
-            }
-            code = 500
-
+            raise Exception("Invalid response while attempting to retrieve case and sample counts for these filters.")
+
     except Exception as e:
         logger.exception(e)
         response_obj = {

apiv4/deprecated/user_routes.py

@@ -0,0 +1,48 @@
+# 
+# Copyright 2019, Institute for Systems Biology
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import json
+from apiv4 import app
+from flask import jsonify, request
+
+HTTP_405_METHOD_NOT_ALLOWED = 405
+
+
+def make_405_response():
+    response = jsonify({
+        'code': HTTP_405_METHOD_NOT_ALLOWED,
+        'message': "The 'gcp' path has been deprecated in version 4.1 in favor of /cloud_projects and subroutes."
+    })
+
+    response.status_code = HTTP_405_METHOD_NOT_ALLOWED
+
+    return response
+
+
+@app.route('/v4/users/gcp/validate/<gcp_id>/', methods=['GET'], strict_slashes=False)
+def validate_gcp_old(gcp_id):
+    return make_405_response()
+
+
+@app.route('/v4/users/gcp/<gcp_id>/', methods=['DELETE', 'PATCH', 'GET'], strict_slashes=False)
+def user_gcp_old(gcp_id):
+    return make_405_response()
+
+
+@app.route('/v4/users/gcp/', methods=['POST', 'GET'], strict_slashes=False)
+def user_gcps_old():
+    return make_405_response()

apiv4/main_routes.py

@@ -16,7 +16,7 @@
 
 import logging
 import json
-from flask import jsonify, request, render_template
+from flask import jsonify, request, render_template, redirect, url_for
 from django.conf import settings
 from apiv4 import app
 from api_logging import *
@@ -31,12 +31,12 @@ def apiv4():
     """Base response"""
 
     st_logger.write_text_log_entry(log_name, activity_message.format(request.method, request.full_path))
-
+    
     response = jsonify({
         'code': 200,
         'message': 'Welcome to the ISB-CGC API, Version 4.',
         'documentation': 'SwaggerUI interface available at <{}/swagger/>.'.format(settings.BASE_API_URL) +
-             'Documentation available at <https://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html>'
+             'Documentation available at <https://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/progAPI-v4/Programmatic-Demo.html>'
     })
     response.status_code = 200
     return response

apiv4/program_routes.py

@@ -20,44 +20,67 @@
 from apiv4 import app
 from django.conf import settings
 from django.db import close_old_connections
-from program_views import get_programs
+from program_views import get_cohort_programs, get_dataset_for_reg
 from api_logging import *
 
 logger = logging.getLogger(settings.LOGGER_NAME)
 
 
 @app.route('/v4/programs/', methods=['GET'], strict_slashes=False)
 def programs():
-    """Retrieve the list of programs and builds currently available for cohort creation."""
+    response = jsonify({
+        'code': 405,
+        'message': "The 'programs' path has been deprecated in version 4.1 in favor of /data/availabile and subroutes."
+    })
+
+    response.status_code=405
+
+    return response
+
+@app.route('/v4/data/available/', methods=['GET'], strict_slashes=False)
+def data(routes=None):
+    """Retrieve the list of all data available via ISB-CGC"""
     response = None
+    response_obj = {}
+    response_code = None
 
     st_logger.write_text_log_entry(log_name, activity_message.format(request.method, request.full_path))
-
-    try:
 
-        program_info = get_programs()
-
-        if program_info:   
-            response = jsonify({
-                'code': 200,
-                'data': program_info
-            })
-            response.status_code = 200
-        else:
-            response = jsonify({
-                'code': 500,
-                'message': 'Encountered an error while retrieving the program list.'
-            })
-            response.status_code = 500
+    try:
+        if not routes or 'cohorts' in routes:
+            program_info = get_cohort_programs()
+            response_obj['programs_for_cohorts'] = program_info if program_info and len(program_info) > 0 else 'None found'
+
+        if not routes or 'registration' in routes:
+            reg_info = get_dataset_for_reg()
+            response_obj['datasets_for_registration'] = reg_info if reg_info and len(reg_info) > 0 else 'None found'
+
+        response_code = 200
     except Exception as e:
-        logger.error("[ERROR] While retrieving program information:")
+        logger.error("[ERROR] While retrieving data availability:")
         logger.exception(e)
-        response = jsonify({
-            'code': 500,
-            'message': 'Encountered an error while retrieving the program list.'
-        })
-        response.status_code = 500
+        response_obj = {
+            'message': 'Encountered an error while retrieving data availability.'
+        }
+        response_code = 500
     finally:
         close_old_connections()
-
+
+    response_obj['code'] = response_code
+    response = jsonify(response_obj)
+    response.status_code = response_code
+
     return response
+
+
+@app.route('/v4/data/available/registration/', methods=['GET'], strict_slashes=False)
+def data_for_reg():
+    """Retrieve the list of all data available for GCP project and service account registration via ISB-CGC"""
+    return data(['registration'])
+
+
+@app.route('/v4/data/available/cohorts/', methods=['GET'], strict_slashes=False)
+def data_for_cohorts():
+    """Retrieve the list of all data available for cohort creation via ISB-CGC"""
+    return data(['cohorts'])
+

apiv4/program_views.py

@@ -23,23 +23,56 @@
 from django.conf import settings
 
 from projects.models import Program, Project
+from accounts.models import AuthorizedDataset
 
 logger = logging.getLogger(settings.LOGGER_NAME)
 
 
-def get_programs():
+def get_cohort_programs():
     django.setup()
     program_info = None
     try:
+        name = request.args.get('name', default='%', type=str) if 'name' in request.args else None
+        desc = request.args.get('desc', default='%', type=str) if 'desc' in request.args else None
+
+        results = Program.get_public_programs(name=name, desc=desc)
+
         program_info = [
             {
                 'name': x.name,
                 'description': x.description,
-                'projects': [{'name': y.name, 'description': y.description} for y in Project.objects.filter(program=x,active=1)]
+                'program_privacy': "Public" if x.is_public else "User",
+                'projects': [{'name': y.name, 'description': y.description} for y in x.get_all_projects()]
             }
-            for x in Program.get_public_programs()
+            for x in results
         ]
     except Exception as e:
         logger.exception(e)
 
     return program_info
+
+
+def get_dataset_for_reg():
+    django.setup()
+    datasets = None
+    try:
+        name = request.args.get('name', default='%', type=str) if 'name' in request.args else None
+        id = request.args.get('id', default='%', type=str) if 'id' in request.args else None
+        access = request.args.get('access', default='controlled', type=str) if 'access' in request.args else None
+
+        public = True if access.lower()=='open' else False if access.lower()=='controlled' else None
+
+        results = AuthorizedDataset.get_datasets(name=name, whitelist_id=id, public=public)
+
+        datasets = [
+            {
+                'name': x.name,
+                'dataset_id': x.whitelist_id,
+                'dataset_access': "Open" if x.public else "Controlled"
+            }
+            for x in results
+        ]
+    except Exception as e:
+        logger.exception(e)
+
+    return datasets