diff --git a/backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala b/backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala index 40f8f1f7b07..beac15b1610 100644 --- a/backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala +++ b/backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala @@ -3,7 +3,7 @@ package cromwell.backend import java.util.UUID import cromwell.backend.io.JobPaths -import cromwell.backend.validation.{CpuValidation, MemoryValidation} +import cromwell.backend.validation.MemoryValidation import cromwell.core.path.Path import eu.timepit.refined.api.Refined import eu.timepit.refined.numeric.Positive @@ -27,7 +27,8 @@ object RuntimeEnvironmentBuilder { callRoot.resolve(s"tmp.$hash").pathAsString } - val cores: Int Refined Positive = CpuValidation.instanceMin.validate(runtimeAttributes).getOrElse(minimums.cores) + // This class is going away in https://github.com/broadinstitute/cromwell/pull/7369 + val cores: Int Refined Positive = minimums.cores val memoryInMB: Double = MemoryValidation diff --git a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala index 0cc06a7dfa4..0038290010c 100644 --- a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala @@ -24,10 +24,6 @@ import wom.values.{WomInteger, WomValue} object CpuValidation { lazy val instance: RuntimeAttributesValidation[Int Refined Positive] = new CpuValidation(CpuKey) lazy val optional: OptionalRuntimeAttributesValidation[Int Refined Positive] = instance.optional - lazy val instanceMin: RuntimeAttributesValidation[Int Refined Positive] = new CpuValidation(CpuMinKey) - lazy val optionalMin: OptionalRuntimeAttributesValidation[Int Refined Positive] = instanceMin.optional - lazy val instanceMax: RuntimeAttributesValidation[Int Refined Positive] = new CpuValidation(CpuMaxKey) - lazy val optionalMax: OptionalRuntimeAttributesValidation[Int Refined Positive] = instanceMax.optional lazy val defaultMin: WomValue = WomInteger(1) def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config) diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index 6d3bcb6bafa..8f692d6486a 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -410,20 +410,10 @@ class RuntimeAttributesValidationSpec ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") val memoryVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, Some(backendConfig)) - val memoryMinVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryMinKey, Some(backendConfig)) - val memoryMaxVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryMaxKey, Some(backendConfig)) MemoryValidation .withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get) .runtimeAttributeDefinition .factoryDefault shouldBe Some(WomLong(2147483648L)) - MemoryValidation - .withDefaultMemory(RuntimeAttributesKeys.MemoryMinKey, memoryMinVal.get) - .runtimeAttributeDefinition - .factoryDefault shouldBe Some(WomLong(322122547L)) - MemoryValidation - .withDefaultMemory(RuntimeAttributesKeys.MemoryMaxKey, memoryMaxVal.get) - .runtimeAttributeDefinition - .factoryDefault shouldBe Some(WomLong(429496729L)) } "shouldn't throw up if the value for a default-runtime-attribute key cannot be coerced into an expected WomType" in { diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala index 87f04cc44dc..447f48ae2b9 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributes.scala @@ -97,10 +97,6 @@ object AwsBatchRuntimeAttributes { CpuValidation.instance .withDefault(CpuValidation.configDefaultWomValue(runtimeConfig) getOrElse CpuValidation.defaultMin) - private def cpuMinValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int Refined Positive] = - CpuValidation.instanceMin - .withDefault(CpuValidation.configDefaultWomValue(runtimeConfig) getOrElse CpuValidation.defaultMin) - private def failOnStderrValidation(runtimeConfig: Option[Config]) = FailOnStderrValidation.default(runtimeConfig) private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = @@ -120,13 +116,6 @@ object AwsBatchRuntimeAttributes { MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, runtimeConfig) getOrElse MemoryDefaultValue ) - private def memoryMinValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[MemorySize] = - MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryMinKey, - MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryMinKey, - runtimeConfig - ) getOrElse MemoryDefaultValue - ) - private def noAddressValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Boolean] = noAddressValidationInstance .withDefault(noAddressValidationInstance.configDefaultWomValue(runtimeConfig) getOrElse NoAddressDefaultValue) @@ -152,11 +141,9 @@ object AwsBatchRuntimeAttributes { .default(runtimeConfig) .withValidation( cpuValidation(runtimeConfig), - cpuMinValidation(runtimeConfig), disksValidation(runtimeConfig), zonesValidation(runtimeConfig), memoryValidation(runtimeConfig), - memoryMinValidation(runtimeConfig), noAddressValidation(runtimeConfig), dockerValidation, queueArnValidation(runtimeConfig), @@ -166,11 +153,9 @@ object AwsBatchRuntimeAttributes { .default(runtimeConfig) .withValidation( cpuValidation(runtimeConfig), - cpuMinValidation(runtimeConfig), disksValidation(runtimeConfig), zonesValidation(runtimeConfig), memoryValidation(runtimeConfig), - memoryMinValidation(runtimeConfig), noAddressValidation(runtimeConfig), dockerValidation, queueArnValidation(runtimeConfig) diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActor.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActor.scala index eba54bdc4c3..dbd63740ad0 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActor.scala @@ -48,13 +48,11 @@ import shapeless.Coproduct import org.apache.commons.codec.digest.DigestUtils import org.apache.commons.csv.{CSVFormat, CSVPrinter} import org.apache.commons.io.output.ByteArrayOutputStream -import wdl4s.parser.MemoryUnit import wom.callable.Callable.OutputDefinition import wom.callable.MetaValueElement.{MetaValueElementBoolean, MetaValueElementObject} import wom.callable.{AdHocValue, RuntimeEnvironment} import wom.core.FullyQualifiedName import wom.expression.{FileEvaluation, NoIoFunctionSet} -import wom.format.MemorySize import wom.values._ import java.io.OutputStreamWriter @@ -590,30 +588,6 @@ class GcpBatchAsyncBackendJobExecutionActor(override val standardParams: Standar token <- data.privateDockerEncryptedToken } yield CreateBatchDockerKeyAndToken(key, token) - /* - * Right now this doesn't cost anything, because sizeOption returns the size if it was previously already fetched - * for some reason (expression evaluation for instance), but otherwise does not retrieve it and returns None. - * In CWL-land we tend to be aggressive in pre-fetching the size in order to be able to evaluate JS expressions, - * but less in WDL as we can get it last minute and on demand because size is a WDL function, whereas in CWL - * we don't inspect the JS to know if size is called and therefore always pre-fetch it. - * - * We could decide to call withSize before in which case we would retrieve the size for all files and have - * a guaranteed more accurate total size, but there might be performance impacts ? - */ - val inputFileSize = Option(callInputFiles.values.flatMap(_.flatMap(_.sizeOption)).sum) - - // Attempt to adjust the disk size by taking into account the size of input files - val adjustedSizeDisks = - inputFileSize.map(size => MemorySize.apply(size.toDouble, MemoryUnit.Bytes).to(MemoryUnit.GB)) map { - inputFileSizeInformation => - runtimeAttributes.disks.adjustWorkingDiskWithNewMin( - inputFileSizeInformation, - jobLogger.info( - s"Adjusted working disk size to ${inputFileSizeInformation.amount} GB to account for input files" - ) - ) - } getOrElse runtimeAttributes.disks - val inputFilePaths = inputOutputParameters.jobInputParameters.map(_.cloudPath.pathAsString).toSet val referenceDisksToMount = @@ -667,7 +641,7 @@ class GcpBatchAsyncBackendJobExecutionActor(override val standardParams: Standar jobShell = batchConfiguration.jobShell, privateDockerKeyAndEncryptedToken = dockerKeyAndToken, womOutputRuntimeExtractor = jobDescriptor.workflowDescriptor.outputRuntimeExtractor, - adjustedSizeDisks = adjustedSizeDisks, + disks = runtimeAttributes.disks, virtualPrivateCloudConfiguration = batchAttributes.virtualPrivateCloudConfiguration, retryWithMoreMemoryKeys = retryWithMoreMemoryKeys, fuseEnabled = fuseEnabled(jobDescriptor.workflowDescriptor), diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/api/GcpBatchRequestFactory.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/api/GcpBatchRequestFactory.scala index 3c69c3c8fb4..19421acef8b 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/api/GcpBatchRequestFactory.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/api/GcpBatchRequestFactory.scala @@ -80,7 +80,7 @@ object GcpBatchRequestFactory { jobShell: String, privateDockerKeyAndEncryptedToken: Option[CreateBatchDockerKeyAndToken], womOutputRuntimeExtractor: Option[WomOutputRuntimeExtractor], - adjustedSizeDisks: Seq[GcpBatchAttachedDisk], + disks: Seq[GcpBatchAttachedDisk], virtualPrivateCloudConfiguration: VirtualPrivateCloudConfiguration, retryWithMoreMemoryKeys: Option[List[String]], fuseEnabled: Boolean, diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/api/GcpBatchRequestFactoryImpl.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/api/GcpBatchRequestFactoryImpl.scala index a8f6519b7a5..f1f81a6f03b 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/api/GcpBatchRequestFactoryImpl.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/api/GcpBatchRequestFactoryImpl.scala @@ -161,7 +161,7 @@ class GcpBatchRequestFactoryImpl()(implicit gcsTransferConfiguration: GcsTransfe val createParameters = data.createParameters val retryCount = data.gcpBatchParameters.runtimeAttributes.preemptible val allDisksToBeMounted: Seq[GcpBatchAttachedDisk] = - createParameters.adjustedSizeDisks ++ createParameters.referenceDisksForLocalizationOpt.getOrElse(List.empty) + createParameters.disks ++ createParameters.referenceDisksForLocalizationOpt.getOrElse(List.empty) val gcpBootDiskSizeMb = convertGbToMib(runtimeAttributes) // set parent for metadata storage of job information diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/io/GcpBatchAttachedDisk.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/io/GcpBatchAttachedDisk.scala index 50a0704dd4e..d0fc00134d5 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/io/GcpBatchAttachedDisk.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/io/GcpBatchAttachedDisk.scala @@ -7,8 +7,6 @@ import common.exception.MessageAggregation import common.validation.ErrorOr._ import cromwell.backend.DiskPatterns._ import cromwell.core.path.{DefaultPathBuilder, Path} -import wdl4s.parser.MemoryUnit -import wom.format.MemorySize import wom.values._ import scala.util.Try @@ -58,15 +56,6 @@ object GcpBatchAttachedDisk { case _: IllegalArgumentException => s"$value not convertible to a Long".invalidNel } - implicit class EnhancedDisks(val disks: Seq[GcpBatchAttachedDisk]) extends AnyVal { - def adjustWorkingDiskWithNewMin(minimum: MemorySize, onAdjustment: => Unit): Seq[GcpBatchAttachedDisk] = disks map { - case disk: GcpBatchWorkingDisk - if disk == GcpBatchWorkingDisk.Default && disk.sizeGb < minimum.to(MemoryUnit.GB).amount.toInt => - onAdjustment - disk.copy(sizeGb = minimum.to(MemoryUnit.GB).amount.toInt) - case other => other - } - } } trait GcpBatchAttachedDisk { diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala index cdf3826f7af..ef9e2389d4b 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributes.scala @@ -11,7 +11,6 @@ import cromwell.backend.validation._ import eu.timepit.refined._ import eu.timepit.refined.api.Refined import eu.timepit.refined.numeric.Positive -import wdl4s.parser.MemoryUnit import wom.RuntimeAttributesKeys import wom.format.MemorySize import wom.types.{WomArrayType, WomStringType, WomType} @@ -79,10 +78,6 @@ object GcpBatchRuntimeAttributes { val CpuPlatformIntelCascadeLakeValue = "Intel Cascade Lake" val CpuPlatformAMDRomeValue = "AMD Rome" - private def cpuMinValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int Refined Positive] = - CpuValidation.instanceMin - .withDefault(CpuValidation.configDefaultWomValue(runtimeConfig) getOrElse CpuValidation.defaultMin) - val UseDockerImageCacheKey = "useDockerImageCache" private val useDockerImageCacheValidationInstance = new BooleanRuntimeAttributesValidation( UseDockerImageCacheKey @@ -111,9 +106,6 @@ object GcpBatchRuntimeAttributes { private def gpuCountValidation( runtimeConfig: Option[Config] ): OptionalRuntimeAttributesValidation[Int Refined Positive] = GpuValidation.optional - private def gpuMinValidation( - runtimeConfig: Option[Config] - ): OptionalRuntimeAttributesValidation[Int Refined Positive] = GpuValidation.optionalMin private val dockerValidation: RuntimeAttributesValidation[String] = DockerValidation.instance @@ -143,13 +135,6 @@ object GcpBatchRuntimeAttributes { MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, runtimeConfig) getOrElse MemoryDefaultValue ) - private def memoryMinValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[MemorySize] = - MemoryValidation.withDefaultMemory( - RuntimeAttributesKeys.MemoryMinKey, - MemoryValidation - .configDefaultString(RuntimeAttributesKeys.MemoryMinKey, runtimeConfig) getOrElse MemoryDefaultValue - ) - private def bootDiskSizeValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = bootDiskValidationInstance .withDefault(bootDiskValidationInstance.configDefaultWomValue(runtimeConfig) getOrElse BootDiskDefaultValue) @@ -166,15 +151,6 @@ object GcpBatchRuntimeAttributes { ): OptionalRuntimeAttributesValidation[Boolean] = useDockerImageCacheValidationInstance - private val outDirMinValidation: OptionalRuntimeAttributesValidation[MemorySize] = - InformationValidation.optional(RuntimeAttributesKeys.OutDirMinKey, MemoryUnit.MB, allowZero = true) - - private val tmpDirMinValidation: OptionalRuntimeAttributesValidation[MemorySize] = - InformationValidation.optional(RuntimeAttributesKeys.TmpDirMinKey, MemoryUnit.MB, allowZero = true) - - private val inputDirMinValidation: OptionalRuntimeAttributesValidation[MemorySize] = - InformationValidation.optional(RuntimeAttributesKeys.DnaNexusInputDirMinKey, MemoryUnit.MB, allowZero = true) - def runtimeAttributesBuilder(batchConfiguration: GcpBatchConfiguration): StandardValidatedRuntimeAttributesBuilder = { val runtimeConfig = batchConfiguration.runtimeConfig StandardValidatedRuntimeAttributesBuilder @@ -185,21 +161,15 @@ object GcpBatchRuntimeAttributes { gpuDriverValidation(runtimeConfig), cpuValidation(runtimeConfig), cpuPlatformValidation(runtimeConfig), - cpuMinValidation(runtimeConfig), - gpuMinValidation(runtimeConfig), disksValidation(runtimeConfig), noAddressValidation(runtimeConfig), zonesValidation(runtimeConfig), preemptibleValidation(runtimeConfig), memoryValidation(runtimeConfig), - memoryMinValidation(runtimeConfig), bootDiskSizeValidation(runtimeConfig), useDockerImageCacheValidation(runtimeConfig), checkpointFileValidationInstance, - dockerValidation, - outDirMinValidation, - tmpDirMinValidation, - inputDirMinValidation + dockerValidation ) } @@ -257,21 +227,6 @@ object GcpBatchRuntimeAttributes { validatedRuntimeAttributes ) - val outDirMin: Option[MemorySize] = RuntimeAttributesValidation - .extractOption(outDirMinValidation.key, validatedRuntimeAttributes) - val tmpDirMin: Option[MemorySize] = RuntimeAttributesValidation - .extractOption(tmpDirMinValidation.key, validatedRuntimeAttributes) - val inputDirMin: Option[MemorySize] = RuntimeAttributesValidation - .extractOption(inputDirMinValidation.key, validatedRuntimeAttributes) - - val totalExecutionDiskSizeBytes = List(inputDirMin.map(_.bytes), - outDirMin.map(_.bytes), - tmpDirMin.map(_.bytes) - ).flatten.fold(MemorySize(0, MemoryUnit.Bytes).bytes)(_ + _) - val totalExecutionDiskSize = MemorySize(totalExecutionDiskSizeBytes, MemoryUnit.Bytes) - - val adjustedDisks = disks.adjustWorkingDiskWithNewMin(totalExecutionDiskSize, ()) - new GcpBatchRuntimeAttributes( cpu, cpuPlatform, @@ -280,7 +235,7 @@ object GcpBatchRuntimeAttributes { preemptible, bootDiskSize, memory, - adjustedDisks, + disks, docker, failOnStderr, continueOnReturnCode, diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/util/GpuValidation.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/util/GpuValidation.scala index 51c19813229..001bf0efb84 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/util/GpuValidation.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/util/GpuValidation.scala @@ -13,17 +13,13 @@ import cromwell.backend.validation.{ import eu.timepit.refined.api.Refined import eu.timepit.refined.numeric.Positive import eu.timepit.refined.refineV -import wom.RuntimeAttributesKeys.{GpuKey, GpuMaxKey, GpuMinKey} +import wom.RuntimeAttributesKeys.GpuKey import wom.types.WomIntegerType import wom.values.{WomInteger, WomValue} object GpuValidation { lazy val instance: RuntimeAttributesValidation[Int Refined Positive] = new GpuValidation(GpuKey) lazy val optional: OptionalRuntimeAttributesValidation[Int Refined Positive] = instance.optional - lazy val instanceMin: RuntimeAttributesValidation[Int Refined Positive] = new GpuValidation(GpuMinKey) - lazy val optionalMin: OptionalRuntimeAttributesValidation[Int Refined Positive] = instanceMin.optional - lazy val instanceMax: RuntimeAttributesValidation[Int Refined Positive] = new GpuValidation(GpuMaxKey) - lazy val optionalMax: OptionalRuntimeAttributesValidation[Int Refined Positive] = instanceMax.optional lazy val defaultMin: WomValue = WomInteger(0) def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config) diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala index 4253b1877be..f8d6e071b31 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala @@ -936,12 +936,10 @@ class GcpBatchAsyncBackendJobExecutionActorSpec "runtimeAttributes:bootDiskSizeGb" -> "10", "runtimeAttributes:continueOnReturnCode" -> "0", "runtimeAttributes:cpu" -> "1", - "runtimeAttributes:cpuMin" -> "1", "runtimeAttributes:disks" -> "local-disk 200 SSD", "runtimeAttributes:docker" -> "ubuntu:latest", "runtimeAttributes:failOnStderr" -> "false", "runtimeAttributes:memory" -> "2 GB", - "runtimeAttributes:memoryMin" -> "2 GB", "runtimeAttributes:noAddress" -> "false", "runtimeAttributes:preemptible" -> "0", "runtimeAttributes:zones" -> "us-central1-b,us-central1-a", diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/GpuValidation.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/GpuValidation.scala index 916410df293..ca21d11b200 100644 --- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/GpuValidation.scala +++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/GpuValidation.scala @@ -13,17 +13,13 @@ import cromwell.backend.validation.{ import eu.timepit.refined.api.Refined import eu.timepit.refined.numeric.Positive import eu.timepit.refined.refineV -import wom.RuntimeAttributesKeys.{GpuKey, GpuMaxKey, GpuMinKey} +import wom.RuntimeAttributesKeys.GpuKey import wom.types.WomIntegerType import wom.values.{WomInteger, WomValue} object GpuValidation { lazy val instance: RuntimeAttributesValidation[Int Refined Positive] = new GpuValidation(GpuKey) lazy val optional: OptionalRuntimeAttributesValidation[Int Refined Positive] = instance.optional - lazy val instanceMin: RuntimeAttributesValidation[Int Refined Positive] = new GpuValidation(GpuMinKey) - lazy val optionalMin: OptionalRuntimeAttributesValidation[Int Refined Positive] = instanceMin.optional - lazy val instanceMax: RuntimeAttributesValidation[Int Refined Positive] = new GpuValidation(GpuMaxKey) - lazy val optionalMax: OptionalRuntimeAttributesValidation[Int Refined Positive] = instanceMax.optional lazy val defaultMin: WomValue = WomInteger(0) def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config) diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala index fa079ea86b2..965dc4e1244 100644 --- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala @@ -57,13 +57,11 @@ import cromwell.services.keyvalue.KeyValueServiceActor._ import cromwell.services.metadata.CallMetadataKeys import mouse.all._ import shapeless.Coproduct -import wdl4s.parser.MemoryUnit import wom.callable.Callable.OutputDefinition import wom.callable.MetaValueElement.{MetaValueElementBoolean, MetaValueElementObject} import wom.callable.{AdHocValue, RuntimeEnvironment} import wom.core.FullyQualifiedName import wom.expression.{FileEvaluation, NoIoFunctionSet} -import wom.format.MemorySize import wom.types.{WomArrayType, WomSingleFileType} import wom.values._ @@ -490,30 +488,6 @@ class PipelinesApiAsyncBackendJobExecutionActor(override val standardParams: Sta token <- data.privateDockerEncryptedToken } yield CreatePipelineDockerKeyAndToken(key, token) - /* - * Right now this doesn't cost anything, because sizeOption returns the size if it was previously already fetched - * for some reason (expression evaluation for instance), but otherwise does not retrieve it and returns None. - * In CWL-land we tend to be aggressive in pre-fetching the size in order to be able to evaluate JS expressions, - * but less in WDL as we can get it last minute and on demand because size is a WDL function, whereas in CWL - * we don't inspect the JS to know if size is called and therefore always pre-fetch it. - * - * We could decide to call withSize before in which case we would retrieve the size for all files and have - * a guaranteed more accurate total size, but there might be performance impacts ? - */ - val inputFileSize = Option(callInputFiles.values.flatMap(_.flatMap(_.sizeOption)).sum) - - // Attempt to adjust the disk size by taking into account the size of input files - val adjustedSizeDisks = - inputFileSize.map(size => MemorySize.apply(size.toDouble, MemoryUnit.Bytes).to(MemoryUnit.GB)) map { - inputFileSizeInformation => - runtimeAttributes.disks.adjustWorkingDiskWithNewMin( - inputFileSizeInformation, - jobLogger.info( - s"Adjusted working disk size to ${inputFileSizeInformation.amount} GB to account for input files" - ) - ) - } getOrElse runtimeAttributes.disks - val inputFilePaths = inputOutputParameters.jobInputParameters.map(_.cloudPath.pathAsString).toSet val referenceDisksToMount = if (useReferenceDisks) @@ -571,7 +545,7 @@ class PipelinesApiAsyncBackendJobExecutionActor(override val standardParams: Sta jobShell = pipelinesConfiguration.jobShell, privateDockerKeyAndEncryptedToken = dockerKeyAndToken, womOutputRuntimeExtractor = jobDescriptor.workflowDescriptor.outputRuntimeExtractor, - adjustedSizeDisks = adjustedSizeDisks, + disks = runtimeAttributes.disks, virtualPrivateCloudConfiguration = jesAttributes.virtualPrivateCloudConfiguration, retryWithMoreMemoryKeys = retryWithMoreMemoryKeys, fuseEnabled = fuseEnabled(jobDescriptor.workflowDescriptor), diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala index 3e2ee2446e0..a472007b611 100644 --- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala +++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiRuntimeAttributes.scala @@ -13,7 +13,6 @@ import cromwell.backend.validation.{BooleanRuntimeAttributesValidation, _} import eu.timepit.refined._ import eu.timepit.refined.api.Refined import eu.timepit.refined.numeric.Positive -import wdl4s.parser.MemoryUnit import wom.RuntimeAttributesKeys import wom.format.MemorySize import wom.types._ @@ -101,10 +100,6 @@ object PipelinesApiRuntimeAttributes { private def cpuPlatformValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[String] = cpuPlatformValidationInstance - private def cpuMinValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int Refined Positive] = - CpuValidation.instanceMin - .withDefault(CpuValidation.configDefaultWomValue(runtimeConfig) getOrElse CpuValidation.defaultMin) - private def gpuTypeValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[GpuType] = GpuTypeValidation.optional @@ -116,10 +111,6 @@ object PipelinesApiRuntimeAttributes { runtimeConfig: Option[Config] ): OptionalRuntimeAttributesValidation[Int Refined Positive] = GpuValidation.optional - private def gpuMinValidation( - runtimeConfig: Option[Config] - ): OptionalRuntimeAttributesValidation[Int Refined Positive] = GpuValidation.optionalMin - private def failOnStderrValidation(runtimeConfig: Option[Config]) = FailOnStderrValidation.default(runtimeConfig) private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = @@ -144,13 +135,6 @@ object PipelinesApiRuntimeAttributes { MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, runtimeConfig) getOrElse MemoryDefaultValue ) - private def memoryMinValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[MemorySize] = - MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryMinKey, - MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryMinKey, - runtimeConfig - ) getOrElse MemoryDefaultValue - ) - private def bootDiskSizeValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = bootDiskValidationInstance .withDefault(bootDiskValidationInstance.configDefaultWomValue(runtimeConfig) getOrElse BootDiskDefaultValue) @@ -166,15 +150,6 @@ object PipelinesApiRuntimeAttributes { private val dockerValidation: RuntimeAttributesValidation[String] = DockerValidation.instance - private val outDirMinValidation: OptionalRuntimeAttributesValidation[MemorySize] = - InformationValidation.optional(RuntimeAttributesKeys.OutDirMinKey, MemoryUnit.MB, allowZero = true) - - private val tmpDirMinValidation: OptionalRuntimeAttributesValidation[MemorySize] = - InformationValidation.optional(RuntimeAttributesKeys.TmpDirMinKey, MemoryUnit.MB, allowZero = true) - - private val inputDirMinValidation: OptionalRuntimeAttributesValidation[MemorySize] = - InformationValidation.optional(RuntimeAttributesKeys.DnaNexusInputDirMinKey, MemoryUnit.MB, allowZero = true) - def runtimeAttributesBuilder( jesConfiguration: PipelinesApiConfiguration ): StandardValidatedRuntimeAttributesBuilder = { @@ -186,22 +161,16 @@ object PipelinesApiRuntimeAttributes { gpuTypeValidation(runtimeConfig), gpuDriverValidation(runtimeConfig), cpuValidation(runtimeConfig), - cpuMinValidation(runtimeConfig), - gpuMinValidation(runtimeConfig), disksValidation(runtimeConfig), zonesValidation(runtimeConfig), preemptibleValidation(runtimeConfig), memoryValidation(runtimeConfig), - memoryMinValidation(runtimeConfig), bootDiskSizeValidation(runtimeConfig), noAddressValidation(runtimeConfig), cpuPlatformValidation(runtimeConfig), useDockerImageCacheValidation(runtimeConfig), checkpointFileValidationInstance, - dockerValidation, - outDirMinValidation, - tmpDirMinValidation, - inputDirMinValidation + dockerValidation ) } @@ -261,21 +230,6 @@ object PipelinesApiRuntimeAttributes { validatedRuntimeAttributes ) - val outDirMin: Option[MemorySize] = - RuntimeAttributesValidation.extractOption(outDirMinValidation.key, validatedRuntimeAttributes) - val tmpDirMin: Option[MemorySize] = - RuntimeAttributesValidation.extractOption(tmpDirMinValidation.key, validatedRuntimeAttributes) - val inputDirMin: Option[MemorySize] = - RuntimeAttributesValidation.extractOption(inputDirMinValidation.key, validatedRuntimeAttributes) - - val totalExecutionDiskSizeBytes = List(inputDirMin.map(_.bytes), - outDirMin.map(_.bytes), - tmpDirMin.map(_.bytes) - ).flatten.fold(MemorySize(0, MemoryUnit.Bytes).bytes)(_ + _) - val totalExecutionDiskSize = MemorySize(totalExecutionDiskSizeBytes, MemoryUnit.Bytes) - - val adjustedDisks = disks.adjustWorkingDiskWithNewMin(totalExecutionDiskSize, ()) - new PipelinesApiRuntimeAttributes( cpu, cpuPlatform, @@ -284,7 +238,7 @@ object PipelinesApiRuntimeAttributes { preemptible, bootDiskSize, memory, - adjustedDisks, + disks, docker, failOnStderr, continueOnReturnCode, diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestFactory.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestFactory.scala index 71bc4203b62..188d5a1e9e9 100644 --- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestFactory.scala +++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiRequestFactory.scala @@ -83,7 +83,7 @@ object PipelinesApiRequestFactory { jobShell: String, privateDockerKeyAndEncryptedToken: Option[CreatePipelineDockerKeyAndToken], womOutputRuntimeExtractor: Option[WomOutputRuntimeExtractor], - adjustedSizeDisks: Seq[PipelinesApiAttachedDisk], + disks: Seq[PipelinesApiAttachedDisk], virtualPrivateCloudConfiguration: VirtualPrivateCloudConfiguration, retryWithMoreMemoryKeys: Option[List[String]], fuseEnabled: Boolean, diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/io/PipelinesApiAttachedDisk.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/io/PipelinesApiAttachedDisk.scala index fec1ad22c52..6126384cb27 100644 --- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/io/PipelinesApiAttachedDisk.scala +++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/io/PipelinesApiAttachedDisk.scala @@ -7,8 +7,6 @@ import common.exception.MessageAggregation import common.validation.ErrorOr._ import cromwell.backend.DiskPatterns._ import cromwell.core.path.{DefaultPathBuilder, Path} -import wdl4s.parser.MemoryUnit -import wom.format.MemorySize import wom.values._ import scala.util.Try @@ -55,16 +53,6 @@ object PipelinesApiAttachedDisk { case _: IllegalArgumentException => s"$value not convertible to a Long".invalidNel } - implicit class EnhancedDisks(val disks: Seq[PipelinesApiAttachedDisk]) extends AnyVal { - def adjustWorkingDiskWithNewMin(minimum: MemorySize, onAdjustment: => Unit): Seq[PipelinesApiAttachedDisk] = - disks map { - case disk: PipelinesApiWorkingDisk - if disk == PipelinesApiWorkingDisk.Default && disk.sizeGb < minimum.to(MemoryUnit.GB).amount.toInt => - onAdjustment - disk.copy(sizeGb = minimum.to(MemoryUnit.GB).amount.toInt) - case other => other - } - } } trait PipelinesApiAttachedDisk { diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala index 37221ff8534..41e826a4dc2 100644 --- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActorSpec.scala @@ -1723,12 +1723,10 @@ class PipelinesApiAsyncBackendJobExecutionActorSpec "runtimeAttributes:bootDiskSizeGb" -> "10", "runtimeAttributes:continueOnReturnCode" -> "0", "runtimeAttributes:cpu" -> "1", - "runtimeAttributes:cpuMin" -> "1", "runtimeAttributes:disks" -> "local-disk 200 SSD", "runtimeAttributes:docker" -> "ubuntu:latest", "runtimeAttributes:failOnStderr" -> "false", "runtimeAttributes:memory" -> "2 GB", - "runtimeAttributes:memoryMin" -> "2 GB", "runtimeAttributes:noAddress" -> "false", "runtimeAttributes:preemptible" -> "0", "runtimeAttributes:zones" -> "us-central1-b,us-central1-a", diff --git a/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala b/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala index 3b38f578d6a..2db30f868e6 100644 --- a/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala +++ b/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala @@ -69,7 +69,7 @@ case class GenomicsFactory(applicationName: String, authMode: GoogleAuthMode, en case _ => new Network().setUsePrivateAddress(createPipelineParameters.runtimeAttributes.noAddress) } - val allDisksToBeMounted = createPipelineParameters.adjustedSizeDisks ++ + val allDisksToBeMounted = createPipelineParameters.disks ++ createPipelineParameters.referenceDisksForLocalizationOpt.getOrElse(List.empty) // Disks defined in the runtime attributes and reference-files-localization disks diff --git a/supportedBackends/google/pipelines/v2beta/src/main/scala/cromwell/backend/google/pipelines/v2beta/LifeSciencesFactory.scala b/supportedBackends/google/pipelines/v2beta/src/main/scala/cromwell/backend/google/pipelines/v2beta/LifeSciencesFactory.scala index 2c601007e2e..0d4996f63cc 100644 --- a/supportedBackends/google/pipelines/v2beta/src/main/scala/cromwell/backend/google/pipelines/v2beta/LifeSciencesFactory.scala +++ b/supportedBackends/google/pipelines/v2beta/src/main/scala/cromwell/backend/google/pipelines/v2beta/LifeSciencesFactory.scala @@ -75,7 +75,7 @@ case class LifeSciencesFactory(applicationName: String, authMode: GoogleAuthMode case _ => new Network().setUsePrivateAddress(createPipelineParameters.runtimeAttributes.noAddress) } - val allDisksToBeMounted = createPipelineParameters.adjustedSizeDisks ++ + val allDisksToBeMounted = createPipelineParameters.disks ++ createPipelineParameters.referenceDisksForLocalizationOpt.getOrElse(List.empty) // Disks defined in the runtime attributes and reference-files-localization disks diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala index 1896011fdec..4a697758819 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala @@ -25,11 +25,7 @@ object ConfigConstants { /* Runtime attributes that may be specified within the RuntimeAttributesConfig. */ - val DockerRuntimeAttribute = RuntimeAttributesKeys.DockerKey - val CpuRuntimeAttribute = RuntimeAttributesKeys.CpuKey val MemoryRuntimeAttribute = RuntimeAttributesKeys.MemoryKey - val MemoryMinRuntimeAttribute = RuntimeAttributesKeys.MemoryMinKey - val MemoryMaxRuntimeAttribute = RuntimeAttributesKeys.MemoryMaxKey // See: MemoryDeclarationValidation val MemoryRuntimeAttributePrefix = "memory_" val MemoryMinRuntimeAttributePrefix = "memoryMin_" diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala index 7b5bcd4e7fc..87fb72bd240 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala @@ -33,8 +33,6 @@ object DeclarationValidation { case name if name == DockerValidation.instance.key => new DeclarationValidation(declaration, DockerValidation.instance, usedInCallCachingOverride = None) case RuntimeAttributesKeys.CpuKey => new CpuDeclarationValidation(declaration, CpuValidation.instance) - case RuntimeAttributesKeys.CpuMinKey => new CpuDeclarationValidation(declaration, CpuValidation.instanceMin) - case RuntimeAttributesKeys.CpuMaxKey => new CpuDeclarationValidation(declaration, CpuValidation.instanceMax) // See MemoryDeclarationValidation for more info case name if MemoryDeclarationValidation.isMemoryDeclaration(name, @@ -42,18 +40,6 @@ object DeclarationValidation { MemoryRuntimeAttributePrefix ) => new MemoryDeclarationValidation(declaration, MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix) - case name - if MemoryDeclarationValidation.isMemoryDeclaration(name, - MemoryMinRuntimeAttribute, - MemoryRuntimeAttributePrefix - ) => - new MemoryDeclarationValidation(declaration, MemoryMinRuntimeAttribute, MemoryMinRuntimeAttributePrefix) - case name - if MemoryDeclarationValidation.isMemoryDeclaration(name, - MemoryMaxRuntimeAttribute, - MemoryRuntimeAttributePrefix - ) => - new MemoryDeclarationValidation(declaration, MemoryMaxRuntimeAttribute, MemoryMaxRuntimeAttributePrefix) case name if MemoryDeclarationValidation.isMemoryDeclaration(name, DiskRuntimeAttribute, DiskRuntimeAttributePrefix) => new MemoryDeclarationValidation(declaration, DiskRuntimeAttribute, DiskRuntimeAttributePrefix) diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala index 48722a0dc66..ccc175cbbdb 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala @@ -39,48 +39,11 @@ class MemoryDeclarationValidationSpec ("Float? memory_gb = 3.0", None, Option(3), None), ("Float? memory_gb = 3.0", Option(2), Option(3), Option(WomFloat(2))), ("Float? memory_gb = 3", None, Option(3), None), - ("Float? memory_gb = 3", Option(2), Option(3), Option(WomFloat(2))), - ("Int memoryMin", Option(2), None, Option(WomLong(2L * 1024L * 1024L * 1024L))), - ("Int memoryMin_gb", Option(2), None, Option(WomLong(2))), - ("Int memoryMin_gb = 3", None, Option(3), None), - ("Int memoryMin_gb = 3", Option(2), Option(3), Option(WomLong(2))), - ("Int? memoryMin_gb", None, None, None), - ("Int? memoryMin_gb", Option(2), None, Option(WomLong(2))), - ("Int? memoryMin_gb = 3", None, Option(3), None), - ("Int? memoryMin_gb = 3", Option(2), Option(3), Option(WomLong(2))), - ("Float memoryMin", Option(2), None, Option(WomFloat(2f * 1024f * 1024f * 1024f))), - ("Float memoryMin_gb", Option(2), None, Option(WomFloat(2))), - ("Float memoryMin_gb = 3.0", None, Option(3), None), - ("Float memoryMin_gb = 3.0", Option(2), Option(3), Option(WomFloat(2))), - ("Float? memoryMin_gb", None, None, None), - ("Float? memoryMin_gb", Option(2), None, Option(WomFloat(2))), - ("Float? memoryMin_gb = 3.0", None, Option(3), None), - ("Float? memoryMin_gb = 3.0", Option(2), Option(3), Option(WomFloat(2))), - ("Int memoryMax", Option(2), None, Option(WomLong(2L * 1024L * 1024L * 1024L))), - ("Int memoryMax_gb", Option(2), None, Option(WomLong(2))), - ("Int memoryMax_gb = 3", None, Option(3), None), - ("Int memoryMax_gb = 3", Option(2), Option(3), Option(WomLong(2))), - ("Int? memoryMax_gb", None, None, None), - ("Int? memoryMax_gb", Option(2), None, Option(WomLong(2))), - ("Int? memoryMax_gb = 3", None, Option(3), None), - ("Int? memoryMax_gb = 3", Option(2), Option(3), Option(WomLong(2))), - ("Float memoryMax", Option(2), None, Option(WomFloat(2f * 1024f * 1024f * 1024f))), - ("Float memoryMax_gb", Option(2), None, Option(WomFloat(2))), - ("Float memoryMax_gb = 3.0", None, Option(3), None), - ("Float memoryMax_gb = 3.0", Option(2), Option(3), Option(WomFloat(2))), - ("Float? memoryMax_gb", None, None, None), - ("Float? memoryMax_gb", Option(2), None, Option(WomFloat(2))), - ("Float? memoryMax_gb = 3.0", None, Option(3), None), - ("Float? memoryMax_gb = 3.0", Option(2), Option(3), Option(WomFloat(2))) + ("Float? memory_gb = 3", Option(2), Option(3), Option(WomFloat(2))) ) forAll(validDeclaredAmounts) { (declaration, runtimeAmount, expectedDefaultAmount, expectedExtracted) => it should s"extract memory from declared $declaration with memory set to ${runtimeAmount.getOrElse("none")}" in { - val memoryKey = - if (declaration.contains("Min")) MemoryMinRuntimeAttribute - else if (declaration.contains("Max")) MemoryMaxRuntimeAttribute - else MemoryRuntimeAttribute - val memoryPrefix = if (declaration.contains("Min")) MemoryMinRuntimeAttributePrefix else if (declaration.contains("Max")) MemoryMaxRuntimeAttributePrefix @@ -92,9 +55,10 @@ class MemoryDeclarationValidationSpec val configWdlNamespace = new ConfigWdlNamespace(config) val runtimeDeclaration = configWdlNamespace.runtimeDeclarations.head - val memoryDeclarationValidation = new MemoryDeclarationValidation(runtimeDeclaration, memoryKey, memoryPrefix) + val memoryDeclarationValidation = + new MemoryDeclarationValidation(runtimeDeclaration, MemoryRuntimeAttribute, memoryPrefix) val attributes = runtimeAmount - .map(amount => memoryKey -> MemorySize(amount.toDouble, MemoryUnit.GB)) + .map(amount => MemoryRuntimeAttribute -> MemorySize(amount.toDouble, MemoryUnit.GB)) .toMap val validatedRuntimeAttributes = ValidatedRuntimeAttributes(attributes) @@ -105,7 +69,7 @@ class MemoryDeclarationValidationSpec .map(amount => WomLong(MemorySize(amount.toDouble, MemoryUnit.GB).bytes.toLong)) MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName, - memoryKey, + MemoryRuntimeAttribute, memoryPrefix ) should be(true) default should be(expectedDefault) diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/config/DeclarationValidationSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/config/DeclarationValidationSpec.scala index 681e4a91baa..265821ad3f8 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/config/DeclarationValidationSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/config/DeclarationValidationSpec.scala @@ -32,9 +32,7 @@ class DeclarationValidationSpec it should "validate cpu attributes" in { val keys = Table( "key", - "cpu", - "cpuMin", - "cpuMax" + "cpu" ) forAll(keys)(validateCpu) diff --git a/wom/src/main/scala/wom/RuntimeAttributes.scala b/wom/src/main/scala/wom/RuntimeAttributes.scala index 486d69d5e97..2c66b6859d3 100644 --- a/wom/src/main/scala/wom/RuntimeAttributes.scala +++ b/wom/src/main/scala/wom/RuntimeAttributes.scala @@ -6,33 +6,13 @@ object RuntimeAttributesKeys { val DockerKey = "docker" val MaxRetriesKey = "maxRetries" - /** - * Equivalent to CPUMinKey - */ val CpuKey = "cpu" val CpuPlatformKey = "cpuPlatform" - val CpuMinKey = "cpuMin" - val CpuMaxKey = "cpuMax" - /** - * Equivalent to GPUMinKey - */ val GpuKey = "gpuCount" - val GpuMinKey = "gpuCountMin" - val GpuMaxKey = "gpuCountMax" val GpuTypeKey = "gpuType" - val DnaNexusInputDirMinKey = "dnaNexusInputDirMin" - /** - * Equivalent to MemoryMinKey - */ val MemoryKey = "memory" - val MemoryMinKey = "memoryMin" - val MemoryMaxKey = "memoryMax" - val TmpDirMinKey = "tmpDirMin" - val TmpDirMaxKey = "tmpDirMax" - val OutDirMinKey = "outDirMin" - val OutDirMaxKey = "outDirMax" val FailOnStderrKey = "failOnStderr" val ContinueOnReturnCodeKey = "continueOnReturnCode" }