Skip to content

Commit

Permalink
Merge pull request #818 from CIMAC-CIDC/master
Browse files Browse the repository at this point in the history 
data freeze
  • Loading branch information
@crouchcd
crouchcd authored Aug 16, 2023
2 parents d3271a8 + ac084da commit 40b2c82
Show file tree
Hide file tree
Showing 18 changed files with 393 additions and 203 deletions.
46 changes: 24 additions & 22 deletions cidc_api/resources/permissions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,35 +78,37 @@ def get_permission(permission: Permissions) -> Permissions:
@permissions_bp.route("/", methods=["POST"])
@requires_auth("permissions_item", allowed_roles=[CIDCRole.ADMIN.value])
@unmarshal_request(permission_schema, "permission")
@marshal_response(permission_schema, 201)
# @marshal_response(permission_schema, 201)
def create_permission(permission: Permissions) -> Permissions:
"""Create a new permission record."""
if permission.granted_by_user is None:
granter = get_current_user()
permission.granted_by_user = granter.id
try:
permission.insert()
except IntegrityError as e:
raise BadRequest(str(e.orig))
except IAMException as e:
# We return info on this internal error, since this is an admin-only endpoint
raise InternalServerError(str(e))
return "Data Freeze", 503
# if permission.granted_by_user is None:
# granter = get_current_user()
# permission.granted_by_user = granter.id
# try:
# permission.insert()
# except IntegrityError as e:
# raise BadRequest(str(e.orig))
# except IAMException as e:
# # We return info on this internal error, since this is an admin-only endpoint
# raise InternalServerError(str(e))

return permission
# return permission


@permissions_bp.route("/<int:permission>", methods=["DELETE"])
@requires_auth("permissions_item", allowed_roles=[CIDCRole.ADMIN.value])
@with_lookup(Permissions, "permission", check_etag=True)
def delete_permission(permission: Permissions):
"""Delete a permission record."""
try:
deleter = get_current_user()
permission.delete(deleted_by=deleter)
except NoResultFound as e:
raise NotFound(str(e.orig))
except IAMException as e:
# We return info on this internal error, since this is an admin-only endpoint
raise InternalServerError(str(e))

return delete_response()
return "Data Freeze", 503
# try:
# deleter = get_current_user()
# permission.delete(deleted_by=deleter)
# except NoResultFound as e:
# raise NotFound(str(e.orig))
# except IAMException as e:
# # We return info on this internal error, since this is an admin-only endpoint
# raise InternalServerError(str(e))

# return delete_response()
61 changes: 32 additions & 29 deletions cidc_api/resources/trial_metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,17 @@ def list_trial_metadata(args, pagination_args):
@trial_metadata_bp.route("/", methods=["POST"])
@requires_auth("trial_metadata_item", trial_modifier_roles)
@unmarshal_request(trial_metadata_schema, "trial")
@marshal_response(trial_metadata_schema, 201)
# @marshal_response(trial_metadata_schema, 201)
def create_trial_metadata(trial):
"""Create a new trial metadata record."""
try:
# metadata was already validated by unmarshal_request
trial.insert(validate_metadata=False)
except IntegrityError as e:
raise BadRequest(str(e.orig))
return "Data Freeze", 503
# try:
# # metadata was already validated by unmarshal_request
# trial.insert(validate_metadata=False)
# except IntegrityError as e:
# raise BadRequest(str(e.orig))

return trial
# return trial


@trial_metadata_bp.route("/summaries", methods=["GET"])
Expand All @@ -92,35 +93,37 @@ def get_trial_metadata_by_trial_id(trial):
TrialMetadata, "trial", check_etag=True, find_func=TrialMetadata.find_by_trial_id
)
@unmarshal_request(partial_trial_metadata_schema, "trial_updates", load_sqla=False)
@marshal_response(trial_metadata_schema, 200)
# @marshal_response(trial_metadata_schema, 200)
def update_trial_metadata_by_trial_id(trial, trial_updates):
"""Update an existing trial metadata record by trial_id."""
# Block updates to protected metadata JSON fields
metadata_updates = trial_updates.get("metadata_json")
if trial.metadata_json or metadata_updates:
for field in TrialMetadata.PROTECTED_FIELDS:
if trial.metadata_json.get(field) != metadata_updates.get(field):
raise BadRequest(
f"updating metadata_json['{field}'] via the API is prohibited"
)
return "Data Freeze", 503
# metadata_updates = trial_updates.get("metadata_json")
# if trial.metadata_json or metadata_updates:
# for field in TrialMetadata.PROTECTED_FIELDS:
# if trial.metadata_json.get(field) != metadata_updates.get(field):
# raise BadRequest(
# f"updating metadata_json['{field}'] via the API is prohibited"
# )

trial.update(changes=trial_updates)
# trial.update(changes=trial_updates)

return trial
# return trial


@trial_metadata_bp.route("/new_manifest", methods=["POST"])
@requires_auth("new_manifest", [CIDCRole.ADMIN.value])
def add_new_manifest_from_json():
try:
# schemas JSON blob hook
insert_manifest_into_blob(request.json, uploader_email=get_current_user().email)

except Exception as e:
res = jsonify(error=str(e))
res.status_code = 500
else:
res = jsonify(status="success")
res.status_code = 200
finally:
return res
return "Data Freeze", 503
# try:
# # schemas JSON blob hook
# insert_manifest_into_blob(request.json, uploader_email=get_current_user().email)

# except Exception as e:
# res = jsonify(error=str(e))
# res.status_code = 500
# else:
# res = jsonify(status="success")
# res.status_code = 200
# finally:
# return res
191 changes: 101 additions & 90 deletions cidc_api/resources/upload_jobs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,34 +155,35 @@ def wrapped(args, *pos_args, **kwargs):
"upload_job_updates",
load_sqla=False,
)
@marshal_response(upload_job_schema, 200)
# @marshal_response(upload_job_schema, 200)
def update_upload_job(upload_job: UploadJobs, upload_job_updates: dict):
"""Update an upload_job."""
try:
if "gcs_file_map" in upload_job_updates and upload_job.gcs_file_map is not None:
upload_job_updates["metadata_patch"] = upload_job.metadata_patch.copy()
for uri, uuid in upload_job.gcs_file_map.items():
if uri not in upload_job_updates["gcs_file_map"]:
upload_job_updates[
"metadata_patch"
] = _remove_optional_uuid_recursive(
upload_job_updates["metadata_patch"], uuid
)

upload_job.update(changes=upload_job_updates)
except ValueError as e:
raise BadRequest(str(e))

# If this is a successful upload job, publish this info to Pub/Sub
if upload_job.status == UploadJobStatus.UPLOAD_COMPLETED.value:
gcloud_client.publish_upload_success(upload_job.id)

# Revoke the uploading user's bucket access, since their querying
# this endpoint indicates a completed / failed upload attempt.
gcloud_client.revoke_upload_access(upload_job.uploader_email)

# this is not user-input due to @with_lookup, so safe to return
return upload_job
return "Data Freeze", 503
# try:
# if "gcs_file_map" in upload_job_updates and upload_job.gcs_file_map is not None:
# upload_job_updates["metadata_patch"] = upload_job.metadata_patch.copy()
# for uri, uuid in upload_job.gcs_file_map.items():
# if uri not in upload_job_updates["gcs_file_map"]:
# upload_job_updates[
# "metadata_patch"
# ] = _remove_optional_uuid_recursive(
# upload_job_updates["metadata_patch"], uuid
# )

# upload_job.update(changes=upload_job_updates)
# except ValueError as e:
# raise BadRequest(str(e))

# # If this is a successful upload job, publish this info to Pub/Sub
# if upload_job.status == UploadJobStatus.UPLOAD_COMPLETED.value:
# gcloud_client.publish_upload_success(upload_job.id)

# # Revoke the uploading user's bucket access, since their querying
# # this endpoint indicates a completed / failed upload attempt.
# gcloud_client.revoke_upload_access(upload_job.uploader_email)

# # this is not user-input due to @with_lookup, so safe to return
# return upload_job


def _remove_optional_uuid_recursive(target: dict, uuid: str):
Expand Down Expand Up @@ -471,31 +472,32 @@ def upload_manifest(
Response:
201 if the upload succeeds. Otherwise, some error status code and message.
"""
try:
trial = TrialMetadata.patch_manifest(trial.trial_id, md_patch, commit=False)
except ValidationError as e:
raise BadRequest(json_validation.format_validation_error(e))
except ValidationMultiError as e:
raise BadRequest({"errors": e.args[0]})

# TODO maybe rely on default session
session = Session.object_session(trial)

manifest_upload = UploadJobs.create(
upload_type=template_type,
uploader_email=user.email,
metadata=md_patch,
gcs_xlsx_uri="", # not saving xlsx so we won't have phi-ish stuff in it
gcs_file_map=None,
session=session,
send_email=True,
status=UploadJobStatus.MERGE_COMPLETED.value,
)

# Publish that a manifest upload has been received
gcloud_client.publish_patient_sample_update(manifest_upload.id)

return jsonify({"metadata_json_patch": md_patch})
return "Data Freeze", 503
# try:
# trial = TrialMetadata.patch_manifest(trial.trial_id, md_patch, commit=False)
# except ValidationError as e:
# raise BadRequest(json_validation.format_validation_error(e))
# except ValidationMultiError as e:
# raise BadRequest({"errors": e.args[0]})

# # TODO maybe rely on default session
# session = Session.object_session(trial)

# manifest_upload = UploadJobs.create(
# upload_type=template_type,
# uploader_email=user.email,
# metadata=md_patch,
# gcs_xlsx_uri="", # not saving xlsx so we won't have phi-ish stuff in it
# gcs_file_map=None,
# session=session,
# send_email=True,
# status=UploadJobStatus.MERGE_COMPLETED.value,
# )

# # Publish that a manifest upload has been received
# gcloud_client.publish_patient_sample_update(manifest_upload.id)

# return jsonify({"metadata_json_patch": md_patch})


@ingestion_bp.route("/upload_assay", methods=["POST"])
Expand All @@ -505,7 +507,8 @@ def upload_manifest(
@upload_handler(prism.SUPPORTED_ASSAYS)
def upload_assay(*args, **kwargs):
"""Handle assay metadata / file uploads."""
return upload_data_files(*args, **kwargs)
# return upload_data_files(*args, **kwargs)
return "Data Freeze", 503


@ingestion_bp.route("/upload_analysis", methods=["POST"])
Expand All @@ -515,7 +518,8 @@ def upload_assay(*args, **kwargs):
@upload_handler(prism.SUPPORTED_ANALYSES)
def upload_analysis(*args, **kwargs):
"""Handle analysis metadata / file uploads."""
return upload_data_files(*args, **kwargs)
# return upload_data_files(*args, **kwargs)
return "Data Freeze", 503


def upload_data_files(
Expand Down Expand Up @@ -643,38 +647,38 @@ def extra_assay_metadata():
[artifact_uuid_2]: [open extra metadata file 2]
}
"""
return "Data Freeze", 503
# if not request.form:
# raise BadRequest(
# "Expected form content in request body, or failed to parse form content"
# )

if not request.form:
raise BadRequest(
"Expected form content in request body, or failed to parse form content"
)

if "job_id" not in request.form:
raise BadRequest("Expected job_id in form")
# if "job_id" not in request.form:
# raise BadRequest("Expected job_id in form")

if not request.files:
raise BadRequest(
"Expected files in request (mapping from artifact uuids to open files)"
)
# if not request.files:
# raise BadRequest(
# "Expected files in request (mapping from artifact uuids to open files)"
# )

job_id = request.form["job_id"]
# job_id = request.form["job_id"]

files = request.files.to_dict()
# files = request.files.to_dict()

try:
UploadJobs.merge_extra_metadata(job_id, files)
except ValueError as e:
# thrown by parser itself if file cannot be parsed, e.g. wrong file uploaded
# wrapped by merger to include uuid / assay_hint information, just use that message
# thrown by UploadJobs.merge_extra_metadata if job_id doesn't exist or is already merged
# thrown by getting artifact if uuid doesn't exist in the trial
raise BadRequest(f"{e!s}")
# try:
# UploadJobs.merge_extra_metadata(job_id, files)
# except ValueError as e:
# # thrown by parser itself if file cannot be parsed, e.g. wrong file uploaded
# # wrapped by merger to include uuid / assay_hint information, just use that message
# # thrown by UploadJobs.merge_extra_metadata if job_id doesn't exist or is already merged
# # thrown by getting artifact if uuid doesn't exist in the trial
# raise BadRequest(f"{e!s}")

# Uncaught i.e. internal errors
# TypeError thrown by parser itself if file is not the right type
# # Uncaught i.e. internal errors
# # TypeError thrown by parser itself if file is not the right type

# TODO: return something here?
return jsonify({})
# # TODO: return something here?
# return jsonify({})


INTAKE_ROLES = [
Expand All @@ -697,11 +701,17 @@ def create_intake_bucket(args):
"""
user = get_current_user()
intake_bucket = gcloud_client.create_intake_bucket(user.email)
bucket_subdir = f'{intake_bucket.name}/{args["trial_id"]}/{args["upload_type"]}'
gs_url = f"gs://{bucket_subdir}"
console_url = f"https://console.cloud.google.com/storage/browser/{bucket_subdir}"
# For Data Freeze, only return bucket if it exists
if intake_bucket:
bucket_subdir = f'{intake_bucket.name}/{args["trial_id"]}/{args["upload_type"]}'
gs_url = f"gs://{bucket_subdir}"
console_url = (
f"https://console.cloud.google.com/storage/browser/{bucket_subdir}"
)

return jsonify({"gs_url": gs_url, "console_url": console_url})
return jsonify({"gs_url": gs_url, "console_url": console_url})
else:
return "Data Freeze", 503


@ingestion_bp.route("/intake_metadata", methods=["POST"])
Expand Down Expand Up @@ -730,11 +740,12 @@ def send_intake_metadata(form_args, file_args):
"""
Send an email to the CIDC Admin mailing list with the provided metadata attached.
"""
user = get_current_user()
xlsx_gcp_url = gcloud_client.upload_xlsx_to_intake_bucket(
user.email, form_args["trial_id"], form_args["assay_type"], file_args["xlsx"]
)
emails.intake_metadata(
user, **form_args, xlsx_gcp_url=xlsx_gcp_url, send_email=True
)
return jsonify("ok")
return "Data Freeze", 503
# user = get_current_user()
# xlsx_gcp_url = gcloud_client.upload_xlsx_to_intake_bucket(
# user.email, form_args["trial_id"], form_args["assay_type"], file_args["xlsx"]
# )
# emails.intake_metadata(
# user, **form_args, xlsx_gcp_url=xlsx_gcp_url, send_email=True
# )
# return jsonify("ok")
Loading

0 comments on commit 40b2c82

Please sign in to comment.