From 4debcc3d934156bce99600942d72901e839d5719 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Sat, 4 Feb 2023 09:20:28 -0500 Subject: [PATCH] Only mount reference disks if requested [WX-925] --- ...ference_disk_mounted_only_if_requested.wdl | 49 +++++++++++++++++++ ..._disk_mounted_only_if_requested_false.test | 14 ++++++ ...e_disk_mounted_only_if_requested_true.test | 14 ++++++ ...mounted_only_if_requested_unspecified.test | 14 ++++++ ...inesApiAsyncBackendJobExecutionActor.scala | 4 +- 5 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_mounted_only_if_requested.wdl create mode 100644 centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_false.test create mode 100644 centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_true.test create mode 100644 centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_unspecified.test diff --git a/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_mounted_only_if_requested.wdl b/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_mounted_only_if_requested.wdl new file mode 100644 index 00000000000..33153c1157c --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_mounted_only_if_requested.wdl @@ -0,0 +1,49 @@ +version 1.0 + + +workflow ReferenceDiskMountedOnlyIfRequested { + call MentionsNirvanaReference {} + output { + Boolean disk_mounted = MentionsNirvanaReference.disk_mounted + } +} + + +task MentionsNirvanaReference { + input { + # Tiny 116 byte reference file, certainly not worth attaching a 55 GiB Nirvana reference disk for this. + File mention = + "gs://broad-public-datasets/gvs/vat-annotations/Nirvana/3.18.1/SupplementaryAnnotation/GRCh38/MITOMAP_20200819.nsa.idx" + } + command <<< + PS4='\D{+%F %T} \w $ ' + set -o nounset -o xtrace + + # Debug output + lsblk + + CANDIDATE_MOUNT_POINT=$(lsblk | sed -E -n 's!.*(/mnt/[a-f0-9]+).*!\1!p') + + if [[ ! -z ${CANDIDATE_MOUNT_POINT} ]]; then + echo "Found unexpected mounted disk, investigating further." + find ${CANDIDATE_MOUNT_POINT} -print | tee find.out + + if grep -i nirvana find.out; then + echo "Found what appears to be a Nirvana reference disk." + else + echo "Found unknown volume mounted, see 'find.out' for manifest." + fi + echo true > disk_mounted.out + else + echo false > disk_mounted.out + fi + >>> + runtime { + docker: "ubuntu:latest" + backend: "Papiv2-Reference-Disk-Localization" + } + output { + Boolean disk_mounted = read_boolean("disk_mounted.out") + File? find_out = "find.out" + } +} diff --git a/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_false.test b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_false.test new file mode 100644 index 00000000000..32c6d83e5c3 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_false.test @@ -0,0 +1,14 @@ +name: reference_disk_mounted_only_if_requested_false +testFormat: workflowsuccess +backends: [Papiv2-Reference-Disk-Localization] + +files { + workflow: reference_disk/reference_disk_mounted_only_if_requested.wdl + options: reference_disk/reference_disk_test_false.options.json +} + +metadata { + workflowName: ReferenceDiskMountedOnlyIfRequested + status: Succeeded + "outputs.ReferenceDiskMountedOnlyIfRequested.disk_mounted": false +} diff --git a/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_true.test b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_true.test new file mode 100644 index 00000000000..165e1c6d1f4 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_true.test @@ -0,0 +1,14 @@ +name: reference_disk_mounted_only_if_requested_true +testFormat: workflowsuccess +backends: [Papiv2-Reference-Disk-Localization] + +files { + workflow: reference_disk/reference_disk_mounted_only_if_requested.wdl + options: reference_disk/reference_disk_test_true.options.json +} + +metadata { + workflowName: ReferenceDiskMountedOnlyIfRequested + status: Succeeded + "outputs.ReferenceDiskMountedOnlyIfRequested.disk_mounted": true +} diff --git a/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_unspecified.test b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_unspecified.test new file mode 100644 index 00000000000..860159b594e --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_unspecified.test @@ -0,0 +1,14 @@ +name: reference_disk_mounted_only_if_requested_unspecified +testFormat: workflowsuccess +backends: [Papiv2-Reference-Disk-Localization] + +files { + workflow: reference_disk/reference_disk_mounted_only_if_requested.wdl + options: reference_disk/reference_disk_test_unspecified.options.json +} + +metadata { + workflowName: ReferenceDiskMountedOnlyIfRequested + status: Succeeded + "outputs.ReferenceDiskMountedOnlyIfRequested.disk_mounted": false +} diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala index c1c68b7dd39..745e11bee35 100644 --- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala @@ -452,7 +452,9 @@ class PipelinesApiAsyncBackendJobExecutionActor(override val standardParams: Sta val inputFilePaths = inputOutputParameters.jobInputParameters.map(_.cloudPath.pathAsString).toSet val referenceDisksToMount = - jesAttributes.referenceFileToDiskImageMappingOpt.map(getReferenceDisksToMount(_, inputFilePaths)) + if (useReferenceDisks) + jesAttributes.referenceFileToDiskImageMappingOpt.map(getReferenceDisksToMount(_, inputFilePaths)) + else None val workflowOptions = workflowDescriptor.workflowOptions