diff --git a/.circleci/config.yml b/.circleci/config.yml index 002e1d16..b3832f36 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,6 +121,7 @@ jobs: - ./Dockerfile - ./app.yaml - ./openapi-appengine.yaml + - ./gunicorn.conf.py - ./settings.py - ./txt - ./json diff --git a/Dockerfile b/Dockerfile index e2332ce5..beb048a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,11 +71,6 @@ ADD . /app RUN pip3 install -r /app/requirements.txt -t /app/lib/ --upgrade RUN pip3 install gunicorn==19.9.0 -# Install Cloud SDK -RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && apt-get update -y && apt-get install google-cloud-sdk -y -# Install the Python Libraries -RUN apt-get -y install google-cloud-sdk-app-engine-python - ENV PYTHONPATH=/app:/app/apiv4:/app/lib:/app/ISB-CGC-Common:${PYTHONPATH} -CMD gunicorn -b :$PORT apiv4:app -w 3 -t 130 +CMD gunicorn -c gunicorn.conf.py -b :$PORT apiv4:app -w 3 -t 130 diff --git a/apiv4/__init__.py b/apiv4/__init__.py index d8b2cf00..eb2632ce 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -25,6 +25,7 @@ from flask_talisman import Talisman app = Flask(__name__, static_folder='api_static') + Talisman(app, strict_transport_security_max_age=300, content_security_policy={ 'default-src': [ '\'self\'', @@ -46,6 +47,7 @@ from sample_case_routes import * from file_routes import * from user_routes import * +from deprecated.user_routes import * logger = logging.getLogger(settings.LOGGER_NAME) @@ -56,7 +58,6 @@ def load_spec(): json_spec = "" try: yaml = ruamel.yaml.YAML(typ='safe') - logger.debug(os.path.split(os.path.abspath(dirname(__file__)))[0] + '/openapi-appengine.yaml') with open(os.path.split(os.path.abspath(dirname(__file__)))[0] + '/openapi-appengine.yaml') as fpi: data = yaml.load(fpi) del data['paths']['/swagger'] diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 899e4048..3009308b 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -42,10 +42,7 @@ def cohort(cohort_id): response_obj = None if not user: - response_obj = { - 'message': 'Encountered an error while attempting to identify this user.' - } - code = 500 + raise Exception('Encountered an error while attempting to identify this user.') else: if cohort_id <= 0: logger.warn("[WARNING] Invalid cohort ID {}".format(str(cohort_id))) @@ -62,12 +59,8 @@ def cohort(cohort_id): cohort_info = edit_cohort(cohort_id, user, delete=(request.method == 'DELETE')) if cohort_info: - response_obj['data'] = cohort_info - - if 'message' in cohort_info: - code = 400 - else: - code = 200 + response_obj = {'data': cohort_info} + code = 400 if 'message' in cohort_info else 200 else: response_obj = { 'message': "Cohort ID {} was not found.".format(str(cohort_id)) @@ -111,36 +104,23 @@ def cohorts(): user = validate_user(user_info['email']) if not user: - response_obj = { - 'code': 500, - 'message': 'Encountered an error while attempting to identify this user.' - } - code = 500 + raise Exception('Encountered an error while attempting to identify this user.') else: st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, request.full_path)) - if request.method == 'GET': - info = get_cohorts(user_info['email']) - else: - info = create_cohort(user) + info = get_cohorts(user_info['email']) if request.method == 'GET' else create_cohort(user) if info: - response_obj = {} - - if 'message' in info: - code = 400 - else: - code = 200 + response_obj = { + 'data': info + } - response_obj['data'] = info + code = 400 if 'message' in info else 200 # Lack of a valid object means something went wrong on the server else: - response_obj = { - 'message': "Error while attempting to {}.".format( - 'retrieve the cohort list' if request.method == 'GET' else 'create this cohort' - ) - } - code = 500 + raise Exception("Invalid response while attempting to {}.".format( + 'retrieve the cohort list' if request.method == 'GET' else 'create this cohort' + )) except UserValidationException as e: response_obj = { @@ -161,7 +141,7 @@ def cohorts(): response_obj['code'] = code response = jsonify(response_obj) response.status_code = code - + return response @@ -180,10 +160,7 @@ def cohort_file_manifest(cohort_id): user = validate_user(user_info['email'], cohort_id) if not user: - response_obj = { - 'message': 'Encountered an error while attempting to identify this user.' - } - code = 500 + raise Exception('Encountered an error while attempting to identify this user.') else: if cohort_id <= 0: logger.warn("[WARNING] Invalid cohort ID {}".format(str(cohort_id))) @@ -205,10 +182,7 @@ def cohort_file_manifest(cohort_id): 'data': file_manifest } else: - response_obj = { - 'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id)) - } - code = 500 + raise Exception("Invalid response while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id))) except UserValidationException as e: response_obj = { @@ -256,11 +230,8 @@ def cohort_preview(): # Lack of a valid object means something went wrong on the server else: - response_obj = { - 'message': "Error while attempting to retrieve case and sample counts for these filters." - } - code = 500 - + raise Exception("Invalid response while attempting to retrieve case and sample counts for these filters.") + except Exception as e: logger.exception(e) response_obj = { diff --git a/apiv4/deprecated/__init__.py b/apiv4/deprecated/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apiv4/deprecated/user_routes.py b/apiv4/deprecated/user_routes.py new file mode 100644 index 00000000..0e5083b0 --- /dev/null +++ b/apiv4/deprecated/user_routes.py @@ -0,0 +1,48 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +from apiv4 import app +from flask import jsonify, request + +HTTP_405_METHOD_NOT_ALLOWED = 405 + + +def make_405_response(): + response = jsonify({ + 'code': HTTP_405_METHOD_NOT_ALLOWED, + 'message': "The 'gcp' path has been deprecated in version 4.1 in favor of /cloud_projects and subroutes." + }) + + response.status_code = HTTP_405_METHOD_NOT_ALLOWED + + return response + + +@app.route('/v4/users/gcp/validate//', methods=['GET'], strict_slashes=False) +def validate_gcp_old(gcp_id): + return make_405_response() + + +@app.route('/v4/users/gcp//', methods=['DELETE', 'PATCH', 'GET'], strict_slashes=False) +def user_gcp_old(gcp_id): + return make_405_response() + + +@app.route('/v4/users/gcp/', methods=['POST', 'GET'], strict_slashes=False) +def user_gcps_old(): + return make_405_response() diff --git a/apiv4/main_routes.py b/apiv4/main_routes.py index 20b90d3f..7dfbb7bd 100644 --- a/apiv4/main_routes.py +++ b/apiv4/main_routes.py @@ -16,7 +16,7 @@ import logging import json -from flask import jsonify, request, render_template +from flask import jsonify, request, render_template, redirect, url_for from django.conf import settings from apiv4 import app from api_logging import * @@ -31,12 +31,12 @@ def apiv4(): """Base response""" st_logger.write_text_log_entry(log_name, activity_message.format(request.method, request.full_path)) - + response = jsonify({ 'code': 200, 'message': 'Welcome to the ISB-CGC API, Version 4.', 'documentation': 'SwaggerUI interface available at <{}/swagger/>.'.format(settings.BASE_API_URL) + - 'Documentation available at ' + 'Documentation available at ' }) response.status_code = 200 return response diff --git a/apiv4/program_routes.py b/apiv4/program_routes.py index 39df6f32..b2c97f6c 100644 --- a/apiv4/program_routes.py +++ b/apiv4/program_routes.py @@ -20,7 +20,7 @@ from apiv4 import app from django.conf import settings from django.db import close_old_connections -from program_views import get_programs +from program_views import get_cohort_programs, get_dataset_for_reg from api_logging import * logger = logging.getLogger(settings.LOGGER_NAME) @@ -28,36 +28,59 @@ @app.route('/v4/programs/', methods=['GET'], strict_slashes=False) def programs(): - """Retrieve the list of programs and builds currently available for cohort creation.""" + response = jsonify({ + 'code': 405, + 'message': "The 'programs' path has been deprecated in version 4.1 in favor of /data/availabile and subroutes." + }) + + response.status_code=405 + + return response + +@app.route('/v4/data/available/', methods=['GET'], strict_slashes=False) +def data(routes=None): + """Retrieve the list of all data available via ISB-CGC""" response = None + response_obj = {} + response_code = None st_logger.write_text_log_entry(log_name, activity_message.format(request.method, request.full_path)) - - try: - program_info = get_programs() - - if program_info: - response = jsonify({ - 'code': 200, - 'data': program_info - }) - response.status_code = 200 - else: - response = jsonify({ - 'code': 500, - 'message': 'Encountered an error while retrieving the program list.' - }) - response.status_code = 500 + try: + if not routes or 'cohorts' in routes: + program_info = get_cohort_programs() + response_obj['programs_for_cohorts'] = program_info if program_info and len(program_info) > 0 else 'None found' + + if not routes or 'registration' in routes: + reg_info = get_dataset_for_reg() + response_obj['datasets_for_registration'] = reg_info if reg_info and len(reg_info) > 0 else 'None found' + + response_code = 200 except Exception as e: - logger.error("[ERROR] While retrieving program information:") + logger.error("[ERROR] While retrieving data availability:") logger.exception(e) - response = jsonify({ - 'code': 500, - 'message': 'Encountered an error while retrieving the program list.' - }) - response.status_code = 500 + response_obj = { + 'message': 'Encountered an error while retrieving data availability.' + } + response_code = 500 finally: close_old_connections() - + + response_obj['code'] = response_code + response = jsonify(response_obj) + response.status_code = response_code + return response + + +@app.route('/v4/data/available/registration/', methods=['GET'], strict_slashes=False) +def data_for_reg(): + """Retrieve the list of all data available for GCP project and service account registration via ISB-CGC""" + return data(['registration']) + + +@app.route('/v4/data/available/cohorts/', methods=['GET'], strict_slashes=False) +def data_for_cohorts(): + """Retrieve the list of all data available for cohort creation via ISB-CGC""" + return data(['cohorts']) + diff --git a/apiv4/program_views.py b/apiv4/program_views.py index dabb1d0d..4b95730a 100644 --- a/apiv4/program_views.py +++ b/apiv4/program_views.py @@ -23,23 +23,56 @@ from django.conf import settings from projects.models import Program, Project +from accounts.models import AuthorizedDataset logger = logging.getLogger(settings.LOGGER_NAME) -def get_programs(): +def get_cohort_programs(): django.setup() program_info = None try: + name = request.args.get('name', default='%', type=str) if 'name' in request.args else None + desc = request.args.get('desc', default='%', type=str) if 'desc' in request.args else None + + results = Program.get_public_programs(name=name, desc=desc) + program_info = [ { 'name': x.name, 'description': x.description, - 'projects': [{'name': y.name, 'description': y.description} for y in Project.objects.filter(program=x,active=1)] + 'program_privacy': "Public" if x.is_public else "User", + 'projects': [{'name': y.name, 'description': y.description} for y in x.get_all_projects()] } - for x in Program.get_public_programs() + for x in results ] except Exception as e: logger.exception(e) return program_info + + +def get_dataset_for_reg(): + django.setup() + datasets = None + try: + name = request.args.get('name', default='%', type=str) if 'name' in request.args else None + id = request.args.get('id', default='%', type=str) if 'id' in request.args else None + access = request.args.get('access', default='controlled', type=str) if 'access' in request.args else None + + public = True if access.lower()=='open' else False if access.lower()=='controlled' else None + + results = AuthorizedDataset.get_datasets(name=name, whitelist_id=id, public=public) + + datasets = [ + { + 'name': x.name, + 'dataset_id': x.whitelist_id, + 'dataset_access': "Open" if x.public else "Controlled" + } + for x in results + ] + except Exception as e: + logger.exception(e) + + return datasets diff --git a/apiv4/user_routes.py b/apiv4/user_routes.py index be93ca0e..b15461a8 100644 --- a/apiv4/user_routes.py +++ b/apiv4/user_routes.py @@ -16,7 +16,7 @@ import logging import json -from flask import jsonify, request +from flask import jsonify, request, redirect, url_for from apiv4 import app from auth import auth_info, UserValidationException, validate_user, get_user from user_views import get_user_acls, get_account_details, gcp_validation, gcp_registration, gcp_unregistration, gcp_info @@ -24,6 +24,8 @@ from django.db import close_old_connections from api_logging import * +HTTP_301_MOVED_PERMANENTLY = 301 + logger = logging.getLogger(settings.LOGGER_NAME) @@ -40,11 +42,7 @@ def account_details(): response = None if not user: - response = jsonify({ - 'code': 500, - 'message': 'Encountered an error while attempting to identify this user.' - }) - response.status_code = 500 + raise Exception("Encountered an error while attempting to identify this user.") else: st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, request.full_path)) @@ -87,79 +85,58 @@ def account_details(): return response -@app.route('/v4/users/gcp/validate//', methods=['GET'], strict_slashes=False) +@app.route('/v4/users/cloud_projects/validate//', methods=['GET'], strict_slashes=False) def validate_gcp(gcp_id): """ GET: Validate a Google Cloud Project for registration and return the results to the user """ + response_obj = None + try: user_info = auth_info() user = validate_user(user_info['email']) - response = None - if not user: - response = jsonify({ - 'code': 500, - 'message': 'Encountered an error while attempting to identify this user.' - }) - response.status_code = 500 + raise Exception("Encountered an error while attempting to identify this user.") else: st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, request.full_path)) validation = gcp_validation(user, gcp_id) if validation: - response_obj = {} - code = None - - if 'message' in validation: - response_obj['message'] = validation['message'] - if 'notes' in validation: - response_obj['notes'] = validation['notes'] - - if 'roles' not in validation: - code = 400 - else: - code = 200 - response_obj['gcp_project_id'] = validation['gcp_id'] + response_obj = { + 'data': validation + } - response_obj['code'] = code - response = jsonify(response_obj) - response.status_code = code + code = 400 if 'roles' not in validation else 200 # Lack of a valid object means something went wrong on the server else: - response = jsonify({ - 'code': 500, - 'message': "Encountered an error while attempting to validate Google Cloud Platform project ID {}.".format(gcp_id) - }) - response.status_code = 500 + raise Exception("Encountered an error while attempting to validate Google Cloud Platform project ID {}.".format(gcp_id)) except UserValidationException as e: - response = jsonify({ - 'code': 403, - 'message': str(e) - }) - response.status_code = 403 + response_obj = {'message': str(e)} + code = 403 except Exception as e: logger.exception(e) - response = jsonify({ - 'code': 500, + response_obj = { 'message': 'Encountered an error while attempting to validate Google Cloud Platform project ID {}.'.format(gcp_id) - }) - response.status_code = 500 + } + code = 500 finally: close_old_connections() - + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + return response -@app.route('/v4/users/gcp//', methods=['POST', 'DELETE', 'PATCH', 'GET'], strict_slashes=False) +@app.route('/v4/users/cloud_projects//', methods=['DELETE', 'PATCH', 'GET'], strict_slashes=False) def user_gcp(gcp_id): """ - POST: Register a Google Cloud Project with ISB-CGC PATCH: Update the Google Cloud Project's user list with ISB-CGC DELETE: Unregister the Google Cloud Project with ISB-CGC GET: Fetch details about the Google Cloud Project @@ -173,10 +150,7 @@ def user_gcp(gcp_id): user = validate_user(user_info['email']) if not user: - response_obj = { - 'message': 'Encountered an error while attempting to identify this user.' - } - code = 500 + raise Exception('Encountered an error while attempting to identify this user.') else: st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, request.full_path)) @@ -184,7 +158,7 @@ def user_gcp(gcp_id): result = None success = False - if request.method == 'POST' or request.method == 'PATCH': + if request.method == 'PATCH': action, success = gcp_registration(user, gcp_id, (request.method == 'PATCH')) elif request.method == 'GET': result, success = gcp_info(user, gcp_id) @@ -237,7 +211,7 @@ def user_gcp(gcp_id): } except Exception as e: - logger.error("[ERROR] For route /v4/users/gcp/ method {}:".format(request.method)) + logger.error("[ERROR] For route /v4/users/cloud_projects/ method {}:".format(request.method)) logger.exception(e) code = 500 response_obj = { @@ -253,12 +227,11 @@ def user_gcp(gcp_id): return response -@app.route('/v4/users/gcp/', methods=['POST', 'GET'], strict_slashes=False) + +@app.route('/v4/users/cloud_projects/', methods=['POST', 'GET'], strict_slashes=False) def user_gcps(): """ POST: Register a Google Cloud Project with ISB-CGC - PATCH: Update the Google Cloud Project's user list with ISB-CGC - DELETE: Unregister the Google Cloud Project with ISB-CGC GET: Fetch details about the Google Cloud Project """ @@ -271,10 +244,7 @@ def user_gcps(): user = validate_user(user_info['email']) if not user: - response_obj = { - 'message': 'Encountered an error while attempting to identify this user.' - } - code = 500 + raise Exception('Encountered an error while attempting to identify this user.') else: st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, request.full_path)) @@ -295,8 +265,10 @@ def user_gcps(): if result is not None: code = 404 response_obj['message'] = 'No Google Cloud Platform projects found for user {}'.format( - gcp_id, user.email + user.email ) + elif action is not None: + code = 400 else: code = 200 @@ -332,7 +304,7 @@ def user_gcps(): } except Exception as e: - logger.error("[ERROR] For route /v4/users/gcp/{gcp_id} method {}:".format(request.method)) + logger.error("[ERROR] For route /v4/users/cloud_projects/{gcp_id} method {}:".format(request.method)) logger.exception(e) code = 500 response_obj = { @@ -349,6 +321,279 @@ def user_gcps(): return response +@app.route('/v4/users/cloud_projects//service_accounts', methods=['POST', 'GET'], strict_slashes=False) +def user_sas(gcp_id): + """ + POST: Register a Service Account with ISB-CGC + GET: Fetch a list of all Service Accounts from a Google Cloud Platform project. + """ + + response_obj = {} + code = None + + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + if not user: + raise Exception('Encountered an error while attempting to identify this user.') + else: + st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, + request.full_path)) + action = None + result = None + success = False + + if request.method == 'POST': + action, success = sa_registration(user, gcp_id, None, (request.method == 'PATCH')) + elif request.method == 'GET': + result, success = sa_info(user, gcp_id, None) + else: + raise Exception("Method not recognized: {}".format(request.method)) + + code = 200 + + if action is not None: + response_obj['gcp_project_id'] = gcp_id + response_obj['service_account_id'] = sa_id + if 'message' in action: + response_obj['message'] = action['message'] + if 'notes' in action: + response_obj['notes'] = action['notes'] + if not success: + code = 400 + elif result is not None: + # The case of an empty result set is handled above + if success: + response_obj['data'] = result + else: + code = 404 + response_obj['message'] = 'A Google Cloud Platform service account with ID {} in project ID {} was not found for user {}'.format( + sa_id, gcp_id, user.email + ) + + # Lack of a valid object means something went wrong on the server + else: + code = 500 + response_obj = { + 'message': "Encountered an error while attempting to list service accounts for Google Cloud Platform Project {}.".format( + gcp_id + ) + } + + except UserValidationException as e: + code = 403 + response_obj = { + 'message': str(e) + } + + except Exception as e: + logger.error("[ERROR] For route /v4/users/cloud_projects//service_accounts/ method {}:".format(request.method)) + logger.exception(e) + code = 500 + response_obj = { + 'message': 'Encountered an error while attempting to register service account ID {} from Google Cloud Platform project {}.'.format( + sa_id, gcp_id) + } + finally: + close_old_connections() + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + return response + + +@app.route('/v4/users/cloud_projects//service_accounts/validate/', methods=['GET'], strict_slashes=False) +def validate_sa(gcp_id, sa_id): + """ + POST: Register a Service Account with ISB-CGC + PATCH: Refresh a Service Account's expiration time with ISB-CGC + DELETE: Unregister a Service Account with ISB-CGC + GET: Fetch details about a Service Account + """ + + response_obj = {} + code = None + + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + if not user: + raise Exception('Encountered an error while attempting to identify this user.') + else: + st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, + request.full_path)) + action = None + result = None + success = False + + if request.method == 'POST' or request.method == 'PATCH': + action, success = sa_registration(user, gcp_id, sa_id, (request.method == 'PATCH')) + elif request.method == 'GET': + result, success = sa_info(user, gcp_id, sa_id) + elif request.method == 'DELETE': + action, success = sa_unregistration(user, gcp_id, sa_id) + else: + raise Exception("Method not recognized: {}".format(request.method)) + + code = 200 + + if action is not None: + response_obj['gcp_project_id'] = gcp_id + response_obj['service_account_id'] = sa_id + if 'message' in action: + response_obj['message'] = action['message'] + if 'notes' in action: + response_obj['notes'] = action['notes'] + if not success: + code = 400 + elif result is not None: + # The case of an empty result set is handled above + if success: + response_obj['data'] = result + else: + code = 404 + response_obj['message'] = 'A Google Cloud Platform service account with ID {} in project ID {} was not found for user {}'.format( + sa_id, gcp_id, user.email + ) + + # Lack of a valid object means something went wrong on the server + else: + code = 500 + act = "fetch" + if request.method == 'POST': + act = "register" + if request.method == 'DELETE': + act = "unregister" + if request.method == "PATCH": + act = "refresh" + response_obj = { + 'message': "Encountered an error while attempting to {} Service Account ID {} from Google Cloud Platform Project {}.".format( + act, + sa_id, + gcp_id + ) + } + + except UserValidationException as e: + code = 403 + response_obj = { + 'message': str(e) + } + + except Exception as e: + logger.error("[ERROR] For route /v4/users/cloud_projects//service_account/ method {}:".format(request.method)) + logger.exception(e) + code = 500 + response_obj = { + 'message': 'Encountered an error while attempting to register service account ID {} from Google Cloud Platform project {}.'.format( + sa_id, gcp_id) + } + finally: + close_old_connections() + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + return response + + +@app.route('/v4/users/cloud_projects//service_accounts/', methods=['DELETE', 'PATCH', 'GET'], strict_slashes=False) +def user_sa(gcp_id, sa_id): + """ + POST: Register a Service Account with ISB-CGC + PATCH: Refresh a Service Account's expiration time with ISB-CGC + DELETE: Unregister a Service Account with ISB-CGC + GET: Fetch details about a Service Account + """ + response_obj = {} + code = None + try: + user_info = auth_info() + user = validate_user(user_info['email']) + if not user: + raise Exception('Encountered an error while attempting to identify this user.') + else: + st_logger.write_text_log_entry(log_name, user_activity_message.format(user_info['email'], request.method, + request.full_path)) + action = None + result = None + success = False + + if request.method == 'POST' or request.method == 'PATCH': + action, success = sa_registration(user, gcp_id, sa_id, (request.method == 'PATCH')) + elif request.method == 'GET': + result, success = sa_info(user, gcp_id, sa_id) + elif request.method == 'DELETE': + action, success = sa_unregistration(user, gcp_id, sa_id) + else: + raise Exception("Method not recognized: {}".format(request.method)) + + code = 200 + + if action is not None: + response_obj['gcp_project_id'] = gcp_id + response_obj['service_account_id'] = sa_id + if 'message' in action: + response_obj['message'] = action['message'] + if 'notes' in action: + response_obj['notes'] = action['notes'] + if not success: + code = 400 + elif result is not None: + # The case of an empty result set is handled above + if success: + response_obj['data'] = result + else: + code = 404 + response_obj['message'] = 'A Google Cloud Platform service account with ID {} in project ID {} was not found for user {}'.format( + sa_id, gcp_id, user.email + ) + + # Lack of a valid object means something went wrong on the server + else: + code = 500 + act = "fetch" + if request.method == 'POST': + act = "register" + if request.method == 'DELETE': + act = "unregister" + if request.method == "PATCH": + act = "refresh" + response_obj = { + 'message': "Encountered an error while attempting to {} Service Account ID {} from Google Cloud Platform Project {}.".format( + act, + sa_id, + gcp_id + ) + } + + except UserValidationException as e: + code = 403 + response_obj = { + 'message': str(e) + } + + except Exception as e: + logger.error("[ERROR] For route /v4/users/cloud_projects//service_account/ method {}:".format(request.method)) + logger.exception(e) + code = 500 + response_obj = { + 'message': 'Encountered an error while attempting to register service account ID {} from Google Cloud Platform project {}.'.format( + sa_id, gcp_id) + } + finally: + close_old_connections() + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + return response diff --git a/apiv4/user_views.py b/apiv4/user_views.py index 32a18190..65a52e36 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -27,6 +27,7 @@ from django.conf import settings from accounts.sa_utils import auth_dataset_whitelists_for_user +from accounts.dcf_support import verify_sa_at_dcf, register_sa_at_dcf from accounts.utils import register_or_refresh_gcp, verify_gcp_for_reg, api_gcp_delete, get_user_gcps from accounts.models import AuthorizedDataset from projects.models import Program @@ -79,34 +80,34 @@ def gcp_info(user, gcp_id=None): def gcp_validation(user, gcp_id, refresh=False): validation = None + result = {} try: validation, status = verify_gcp_for_reg(user, gcp_id, refresh) - logger.info("Validation result: {}".format(str(validation))) - if validation: - if 'roles' in validation: - unregs = [x for x in validation['roles'] if not validation['roles'][x]['registered_user']] + if 'roles' in validation: + result['registered_users'] = [{'email': x, 'project_roles': validation['roles'][x]['roles']} for x in validation['roles'] if validation['roles'][x]['registered_user']] + unregs = [{'email': x, 'project_roles': validation['roles'][x]['roles']} for x in validation['roles'] if not validation['roles'][x]['registered_user']] if len(unregs): - validation['notes'] = "The following users are not registered in our system. Please note that if GCP {} ".format(gcp_id) + \ - "is intended for use with controlled access data, all users must log in to the ISB-CGC " + \ + result['unregistered_users'] = unregs + result['notes'] = "Users listed under 'unregistered users' are not registered in the ISB-CGC WebApp. Please note that if GCP Project {} ".format(gcp_id) + \ + "is intended for use with controlled access data, all users on the project must log in to the ISB-CGC " + \ "web application at and link their Google Account to their eRA " + \ - "Commons ID. The link to do so is found in Account Settings. Unregistered users: " + \ - "{}".format("; ".join(unregs)) - - if 'message' not in validation: - validation['message'] = "Google Cloud Platform project ID {} was successfully validated for registration.".format(gcp_id) + "Commons ID. The link to do so is found in Account Settings." + result['message'] = "Google Cloud Platform project ID {} was successfully validated for registration.".format(gcp_id) \ + if 'message' not in validation else validation['message'] + result['gcp_project_id'] = validation['gcp_id'] else: - logger.warn("[WARNING] Validation of {} by user {} was unsuccessful!".format(gcp_id, user.email)) + logger.warn("[WARNING] Validation of GCP ID {} by user {} was unsuccessful!".format(gcp_id, user.email)) except Exception as e: logger.error("[ERROR] While attempting to validate a project for registration:") logger.exception(e) - return validation + return result def gcp_registration(user, gcp_id, refresh): @@ -117,18 +118,20 @@ def gcp_registration(user, gcp_id, refresh): validation = gcp_validation(user, gcp_id, refresh) if validation: - if 'roles' in validation: + if 'users' in validation: - registered_users = [x for x, y in validation['roles'].items() if y['registered_user']] + registered_users = [x for x, y in validation['users'].items() if y['registered_user']] registration, status = register_or_refresh_gcp(user, gcp_id, registered_users, refresh) - logger.info("Registration: {}".format(str(registration))) if status == 200: success = True + registration['registered_users'] = validation['registered_users'] if 'notes' in validation: registration['notes'] = validation['notes'] if 'message' not in registration: registration['message'] = "Google Cloud Platform project ID {} was successfully {}.".format(gcp_id, 'refreshed' if refresh else 'registered') + if 'unregistered_users' in validation: + registration['unregistered_users'] = validation['unregistered_users'] else: registration = validation logger.warn("[WARNING] Validation of {} by user {} was unsuccessful! This project was not {}".format(gcp_id, user.email, 'refreshed' if refresh else 'registered')) @@ -137,7 +140,7 @@ def gcp_registration(user, gcp_id, refresh): logger.warn("[WARNING] Validation of {} by user {} was unsuccessful!".format(gcp_id, user.email)) except Exception as e: - logger.error("[ERROR] While registering a GCP:") + logger.error("[ERROR] While registering GCP ID {}:".format(gcp_id)) logger.exception(e) return registration, success @@ -157,8 +160,51 @@ def gcp_unregistration(user, gcp_id): else: logger.warn("[WARNING] Unregistration of {} by user {} was unsuccessful!".format(gcp_id, user.email)) + unreg['gcp_project_id'] = gcp_id + except Exception as e: logger.error("[ERROR] While unregistering a GCP:") logger.exception(e) return unreg, success + + +def sa_info(user, gcp_id=None, sa_id=None): + return None + + +def sa_registration(user, gcp_id=None, sa_id=None, action=None): + + result = {} + + try: + request_data = request.get_json() + sa_id = request_data['sa_id'] if 'sa_id' in request_data and not sa_id else None + datasets = request.args.get('datasets', default=None, type=str).split(',') if 'datasets' in request.args else None + + if not sa_id: + raise Exception("Service Account ID not provided!") + + if not len(datasets): + raise Exception("Dataset list not provided!") + + result = verify_sa_at_dcf(user, gcp_id, sa_id, datasets, {}) + + except RefreshTokenExpired as e: + logger.error("[ERROR] RefreshTokenExpired for user {} registering SA ID {}".format(user.email,sa_id)) + result['message'] = "Your DCF login has expired. Please go to our web application at https://isb-cgc.appspot.com and refresh your DCF login, then try to register your Service Account again." + except TokenFailure as e: + logger.error("[ERROR] TokenFailure for user {} registering SA ID {}".format(user.email,sa_id)) + result['message'] = "Your DCF login has expired or been disconnected. Please go to our web application at https://isb-cgc.appspot.com and renew your DCF login, then try to register your Service Account again." + except Exception as e: + logger.error("[ERROR] While registering service account {}:".format(sa_id)) + logger.exception(e) + result['message'] = "Encountered a server error while attempting to register service account {}. Please contact the administrator.".format(sa_id) + + return result + + +def sa_unregistration(user, gcp_id=None, sa_id=None): + return None + + diff --git a/gunicorn.conf.py b/gunicorn.conf.py new file mode 100644 index 00000000..08757196 --- /dev/null +++ b/gunicorn.conf.py @@ -0,0 +1,3 @@ + +# Enforce HTTPS +secure_scheme_headers = {'X-FORWARDED-PROTO': 'https'} diff --git a/requirements.txt b/requirements.txt index 062d26a3..4ee88c54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ django==1.11.23 mysqlclient==1.3.13 requests==2.20.0 -ecdsa==0.13 +ecdsa==0.13.3 google-api-python-client==1.6.1 httplib2==0.9.2 oauth2client==3.0.0 @@ -17,7 +17,7 @@ GoogleAppEngineCloudStorageClient==1.9.22.1 django-finalware==1.0.0 django-allauth==0.35.0 jsonschema==2.6.0 -Flask==1.0.2 +Flask==1.1.1 flask-cors==3.0.7 flask-talisman==0.6.0 pyjwt==1.6.1 diff --git a/settings.py b/settings.py index a6e2442a..672d712e 100644 --- a/settings.py +++ b/settings.py @@ -21,11 +21,18 @@ from dotenv import load_dotenv from socket import gethostname, gethostbyname +SECURE_LOCAL_PATH = os.environ.get('SECURE_LOCAL_PATH', '') + env_path = '' -if os.environ.get('SECURE_LOCAL_PATH', None): +env_file = '.env' + +if SECURE_LOCAL_PATH: env_path += os.environ.get('SECURE_LOCAL_PATH') -load_dotenv(dotenv_path=join(dirname(__file__), env_path+'.env')) +if os.environ.get('ENV_FILE', None): + env_file = os.environ.get('ENV_FILE') + +load_dotenv(dotenv_path=join(dirname(__file__), env_path+env_file)) APP_ENGINE_FLEX = 'aef-' APP_ENGINE = 'Google App Engine/' @@ -40,36 +47,10 @@ 'ISB-CGC-Common' ] -# The Google AppEngine library and the Google Cloud APIs don't play nice. Teach them to get along. -# This unfortunately requires either hardcoding the path to the SDK, or sorting out a way to -# provide an environment variable indicating where it is. -# From https://github.com/GoogleCloudPlatform/python-repo-tools/blob/master/gcp_devrel/testing/appengine.py#L26 -def setup_sdk_imports(): - """Sets up appengine SDK third-party imports.""" - sdk_path = os.environ.get('GAE_SDK_PATH', '/usr/lib/google-cloud-sdk') - - # Trigger loading of the Cloud APIs so they're in sys.modules - import google.cloud - - # The libraries are specifically under platform/google_appengine - if os.path.exists(os.path.join(sdk_path, 'platform/google_appengine')): - sdk_path = os.path.join(sdk_path, 'platform/google_appengine') - - # This sets up libraries packaged with the SDK, but puts them last in - # sys.path to prevent clobbering newer versions - if 'google' in sys.modules: - sys.modules['google'].__path__.append( - os.path.join(sdk_path, 'google')) - - sys.path.append(sdk_path) - - # Add the shared Django application subdirectory to the Python module search path for directory_name in SHARED_SOURCE_DIRECTORIES: sys.path.append(os.path.join(BASE_DIR, directory_name)) -setup_sdk_imports() - ALLOWED_HOSTS = list(set(os.environ.get('ALLOWED_HOST', 'localhost').split(',') + ['localhost', '127.0.0.1', '[::1]', gethostname(), gethostbyname(gethostname()),])) # Testing health checks problem # ALLOWED_HOSTS = ['*'] @@ -247,28 +228,6 @@ def GET_BQ_COHORT_SETTINGS(): # Make this unique, and don't share it with anybody. SECRET_KEY = os.environ.get('DJANGO_SECRET_KEY', '') -SECURE_HSTS_INCLUDE_SUBDOMAINS = (os.environ.get('SECURE_HSTS_INCLUDE_SUBDOMAINS','True') == 'True') -SECURE_HSTS_PRELOAD = (os.environ.get('SECURE_HSTS_PRELOAD','True') == 'True') -SECURE_HSTS_SECONDS = int(os.environ.get('SECURE_HSTS_SECONDS','3600')) - -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'GenespotRE.checkreqsize_middleware.CheckReqSize', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'adminrestrict.middleware.AdminPagesRestrictMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - # Uncomment the next line for simple clickjacking protection: - 'django.middleware.clickjacking.XFrameOptionsMiddleware' -] - -ROOT_URLCONF = 'GenespotRE.urls' - -# Python dotted path to the WSGI application used by Django's runserver. -WSGI_APPLICATION = 'GenespotRE.wsgi.application' - INSTALLED_APPS = ( 'django.contrib.auth', 'django.contrib.contenttypes', @@ -390,8 +349,7 @@ def GET_BQ_COHORT_SETTINGS(): 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', 'django.template.context_processors.tz', - 'finalware.context_processors.contextify', - 'GenespotRE.context_processor.additional_context', + 'finalware.context_processors.contextify' ), # add any loaders here; if using the defaults, we can comment it out # 'loaders': ( @@ -430,7 +388,7 @@ def GET_BQ_COHORT_SETTINGS(): ########################## # Path to application runtime JSON key -GOOGLE_APPLICATION_CREDENTIALS = os.path.join(os.path.dirname(__file__), os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')) if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') else '' +GOOGLE_APPLICATION_CREDENTIALS = join(dirname(__file__), '{}{}'.format(SECURE_LOCAL_PATH,os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''))) # OAuth2 client ID for the API API_CLIENT_ID = os.environ.get('API_CLIENT_ID', '') # Client ID for the API @@ -439,7 +397,7 @@ def GET_BQ_COHORT_SETTINGS(): MONITORING_SA_CLIENT_EMAIL = os.environ.get('MONITORING_SA_CLIENT_EMAIL', '') # GCP monitoring Service Account key -MONITORING_SA_ACCESS_CREDENTIALS = os.environ.get('MONITORING_SA_ACCESS_CREDENTIALS', '') +MONITORING_SA_ACCESS_CREDENTIALS = join(dirname(__file__), '{}{}'.format(SECURE_LOCAL_PATH,os.environ.get('MONITORING_SA_ACCESS_CREDENTIALS', ''))) ################################# # For NIH/eRA Commons login # @@ -523,6 +481,12 @@ def GET_BQ_COHORT_SETTINGS(): # Rough max file size to allow for eg. barcode list upload, to revent triggering RequestDataTooBig FILE_SIZE_UPLOAD_MAX = 1950000 +# Apache Solr settings +SOLR_URI = os.environ.get('SOLR_URI', '') +SOLR_LOGIN = os.environ.get('SOLR_LOGIN', '') +SOLR_PASSWORD = os.environ.get('SOLR_PASSWORD', '') +SOLR_CERT = os.environ.get('SOLR_CERT', '') + ############################################################## # MailGun Email Settings ############################################################## diff --git a/shell/vagrant-set-env.sh b/shell/vagrant-set-env.sh index a88b6c18..c2eea8ff 100644 --- a/shell/vagrant-set-env.sh +++ b/shell/vagrant-set-env.sh @@ -1,2 +1,2 @@ -echo 'export PYTHONPATH=/home/vagrant/API:/home/vagrant/google_appengine:/home/vagrant/google_appengine/lib/protorpc-1.0:/home/vagrant/API/lib' | tee -a /home/vagrant/.bash_profile +echo 'export PYTHONPATH=/home/vagrant/API:/home/vagrant/API/lib' | tee -a /home/vagrant/.bash_profile chmod +x /home/vagrant/API/shell/python-su.sh \ No newline at end of file