Skip to content

Commit

Permalink
Revert "Format code"
Browse files Browse the repository at this point in the history 
This reverts commit 06e5932.
  • Loading branch information
@aednichols
aednichols committed Jan 18, 2024
1 parent 06e5932 commit 32b6d91
Show file tree
Hide file tree
Showing 3 changed files with 142 additions and 131 deletions.
48 changes: 24 additions & 24 deletions common/src/main/scala/common/util/UriUtil.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,30 @@ object UriUtil {
.getOrElse(uri)
}

private def maskSensitiveQuery(query: String): String = {
val parsedQuery: Array[Seq[String]] =
query
.split("&")
.map { param =>
param.split("=", 2).toSeq match {
case seq @ Seq(_, _) => seq
case _ => Seq(param)
}
}

if (!parsedQuery.exists(param => isSensitiveKey(param.head))) {
// Mask the entire query just in case
"masked"
} else {
parsedQuery
.map {
case Seq(name, _) if isSensitiveKey(name) => s"$name=masked"
case seq => seq.mkString("=")
}
.mkString("&")
}
}

/*
Parts of these examples have been redacted even if they will not be masked.
Expand Down Expand Up @@ -64,30 +88,6 @@ object UriUtil {
"sig"
)

private def maskSensitiveQuery(query: String): String = {
val parsedQuery: Array[Seq[String]] =
query
.split("&")
.map { param =>
param.split("=", 2).toSeq match {
case seq @ Seq(_, _) => seq
case _ => Seq(param)
}
}

if (!parsedQuery.exists(param => isSensitiveKey(param.head))) {
// Mask the entire query just in case
"masked"
} else {
parsedQuery
.map {
case Seq(name, _) if isSensitiveKey(name) => s"$name=masked"
case seq => seq.mkString("=")
}
.mkString("&")
}
}

private def isSensitiveKey(name: String): Boolean = {
val lower = name.toLowerCase
SensitiveKeyParts.exists(lower.contains(_))
Expand Down
219 changes: 116 additions & 103 deletions supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,88 +34,9 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
outputMode: OutputMode
) {

lazy val inputs: Seq[Input] = {
val result =
TesTask.buildTaskInputs(callInputFiles ++ writeFunctionFiles, workflowName, mapCommandLineWomFile) ++ Seq(
commandScript
)
jobLogger.info(
s"Calculated TES inputs (found ${result.size}): " + result.mkString(System.lineSeparator(),
System.lineSeparator(),
System.lineSeparator()
)
)
result
}
// TODO add TES logs to standard outputs
private lazy val standardOutputs = Seq("rc", "stdout", "stderr").map { f =>
Output(
name = Option(f),
description = Option(fullyQualifiedTaskName + "." + f),
url = Option(tesPaths.storageOutput(f)),
path = tesPaths.containerOutput(containerWorkDir, f),
`type` = Option("FILE")
)
}
private lazy val cwdOutput = Output(
name = Option("execution.dir.output"),
description = Option(fullyQualifiedTaskName + "." + "execution.dir.output"),
url = Option(tesPaths.callExecutionRoot.pathAsString),
path = containerWorkDir.pathAsString,
`type` = Option("DIRECTORY")
)
val name: String = fullyQualifiedTaskName
val description: String = jobDescriptor.toString
// TODO validate "project" field of workflowOptions
val project =
workflowDescriptor.workflowOptions.getOrElse("project", "")
val outputs: Seq[Output] = {
val result = outputMode match {
case OutputMode.GRANULAR => standardOutputs ++ Seq(commandScriptOut) ++ womOutputs ++ additionalGlobOutput
case OutputMode.ROOT => List(cwdOutput) ++ additionalGlobOutput
}

jobLogger.info(
s"Calculated TES outputs (found ${result.size}): " + result.mkString(System.lineSeparator(),
System.lineSeparator(),
System.lineSeparator()
)
)

result
}
val preferedWorkflowExecutionIdentity = TesTask.getPreferredWorkflowExecutionIdentity(
workflowExecutionIdentityConfig,
workflowExecutionIdentityOption
)
val executors = Seq(
Executor(
image = dockerImageUsed,
command = Seq(jobShell, commandScript.path),
workdir = runtimeAttributes.dockerWorkingDir,
stdout = Option(tesPaths.containerOutput(containerWorkDir, "stdout")),
stderr = Option(tesPaths.containerOutput(containerWorkDir, "stderr")),
stdin = None,
env = None
)
)
val resources: Resources = TesTask.makeResources(
runtimeAttributes,
preferedWorkflowExecutionIdentity,
Option(tesPaths.tesTaskRoot)
)
val tags: Map[String, Option[String]] = TesTask.makeTags(jobDescriptor.workflowDescriptor)
private val workflowDescriptor = jobDescriptor.workflowDescriptor
private val workflowName = workflowDescriptor.callable.name
private val fullyQualifiedTaskName = jobDescriptor.taskCall.fullyQualifiedName

// TODO extract output file variable names and match with Files below
// The problem is that we only care about the files CREATED, so stdout and input redirects are ignored and
// thus we can't directly match the names returned here to the files returned below. Also we have to consider Arrays
//
// private val outputFileNames = jobDescriptor.call.task.outputs
// .filter(o => o.womType.toWdlString == "Array[File]" || o.womType.toWdlString == "File")
// .map(_.unqualifiedName)
private val workflowExecutionIdentityConfig: Option[WorkflowExecutionIdentityConfig] =
configurationDescriptor.backendConfig
.getAs[String]("workflow-execution-identity")
Expand All @@ -125,6 +46,13 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
.get(TesWorkflowOptionKeys.WorkflowExecutionIdentity)
.toOption
.map(WorkflowExecutionIdentityOption)
val name: String = fullyQualifiedTaskName
val description: String = jobDescriptor.toString

// TODO validate "project" field of workflowOptions
val project =
workflowDescriptor.workflowOptions.getOrElse("project", "")

// contains the script to be executed
private val commandScript = Input(
name = Option("commandScript"),
Expand All @@ -134,17 +62,55 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
`type` = Option("FILE"),
content = None
)

private val commandScriptOut = Output(
name = Option("commandScript"),
description = Option(fullyQualifiedTaskName + ".commandScript"),
url = Option(tesPaths.script.toString),
path = tesPaths.callExecutionDockerRoot.resolve("script").toString,
`type` = Option("FILE")
)
private def writeFunctionFiles: Map[FullyQualifiedName, Seq[WomFile]] =
instantiatedCommand.createdFiles map { f => f.file.value.md5SumShort -> List(f.file) } toMap

private val callInputFiles: Map[FullyQualifiedName, Seq[WomFile]] = jobDescriptor.fullyQualifiedInputs
.safeMapValues {
_.collectAsSeq { case w: WomFile => w }
}

lazy val inputs: Seq[Input] = {
val result =
TesTask.buildTaskInputs(callInputFiles ++ writeFunctionFiles, workflowName, mapCommandLineWomFile) ++ Seq(
commandScript
)
jobLogger.info(
s"Calculated TES inputs (found ${result.size}): " + result.mkString(System.lineSeparator(),
System.lineSeparator(),
System.lineSeparator()
)
)
result
}

// TODO add TES logs to standard outputs
private lazy val standardOutputs = Seq("rc", "stdout", "stderr").map { f =>
Output(
name = Option(f),
description = Option(fullyQualifiedTaskName + "." + f),
url = Option(tesPaths.storageOutput(f)),
path = tesPaths.containerOutput(containerWorkDir, f),
`type` = Option("FILE")
)
}

// TODO extract output file variable names and match with Files below
// The problem is that we only care about the files CREATED, so stdout and input redirects are ignored and
// thus we can't directly match the names returned here to the files returned below. Also we have to consider Arrays
//
// private val outputFileNames = jobDescriptor.call.task.outputs
// .filter(o => o.womType.toWdlString == "Array[File]" || o.womType.toWdlString == "File")
// .map(_.unqualifiedName)

// extract output files
// if output paths are absolute we will ignore them here and assume they are redirects
private val outputWomFiles: Seq[WomFile] = {
Expand All @@ -164,6 +130,31 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
.flatMap(evaluateFiles)
.filter(o => !DefaultPathBuilder.get(o.valueString).isAbsolute)
}

def handleGlobFile(g: WomGlobFile, index: Int) = {
val globName = GlobFunctions.globName(g.value)
val globDirName = "globDir." + index
val globDirectory = globName + "/"
val globListName = "globList." + index
val globListFile = globName + ".list"
Seq(
Output(
name = Option(globDirName),
description = Option(fullyQualifiedTaskName + "." + globDirName),
url = Option(tesPaths.storageOutput(globDirectory)),
path = tesPaths.containerOutput(containerWorkDir, globDirectory),
`type` = Option("DIRECTORY")
),
Output(
name = Option(globListName),
description = Option(fullyQualifiedTaskName + "." + globListName),
url = Option(tesPaths.storageOutput(globListFile)),
path = tesPaths.containerOutput(containerWorkDir, globListFile),
`type` = Option("FILE")
)
)
}

private val womOutputs = outputWomFiles
.flatMap(_.flattenFiles)
.zipWithIndex
Expand Down Expand Up @@ -202,35 +193,58 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
)
)
}

private val additionalGlobOutput =
jobDescriptor.taskCall.callable.additionalGlob.toList.flatMap(handleGlobFile(_, womOutputs.size))

def handleGlobFile(g: WomGlobFile, index: Int) = {
val globName = GlobFunctions.globName(g.value)
val globDirName = "globDir." + index
val globDirectory = globName + "/"
val globListName = "globList." + index
val globListFile = globName + ".list"
Seq(
Output(
name = Option(globDirName),
description = Option(fullyQualifiedTaskName + "." + globDirName),
url = Option(tesPaths.storageOutput(globDirectory)),
path = tesPaths.containerOutput(containerWorkDir, globDirectory),
`type` = Option("DIRECTORY")
),
Output(
name = Option(globListName),
description = Option(fullyQualifiedTaskName + "." + globListName),
url = Option(tesPaths.storageOutput(globListFile)),
path = tesPaths.containerOutput(containerWorkDir, globListFile),
`type` = Option("FILE")
private lazy val cwdOutput = Output(
name = Option("execution.dir.output"),
description = Option(fullyQualifiedTaskName + "." + "execution.dir.output"),
url = Option(tesPaths.callExecutionRoot.pathAsString),
path = containerWorkDir.pathAsString,
`type` = Option("DIRECTORY")
)

val outputs: Seq[Output] = {
val result = outputMode match {
case OutputMode.GRANULAR => standardOutputs ++ Seq(commandScriptOut) ++ womOutputs ++ additionalGlobOutput
case OutputMode.ROOT => List(cwdOutput) ++ additionalGlobOutput
}

jobLogger.info(
s"Calculated TES outputs (found ${result.size}): " + result.mkString(System.lineSeparator(),
System.lineSeparator(),
System.lineSeparator()
)
)

result
}

private def writeFunctionFiles: Map[FullyQualifiedName, Seq[WomFile]] =
instantiatedCommand.createdFiles map { f => f.file.value.md5SumShort -> List(f.file) } toMap
val preferedWorkflowExecutionIdentity = TesTask.getPreferredWorkflowExecutionIdentity(
workflowExecutionIdentityConfig,
workflowExecutionIdentityOption
)

val executors = Seq(
Executor(
image = dockerImageUsed,
command = Seq(jobShell, commandScript.path),
workdir = runtimeAttributes.dockerWorkingDir,
stdout = Option(tesPaths.containerOutput(containerWorkDir, "stdout")),
stderr = Option(tesPaths.containerOutput(containerWorkDir, "stderr")),
stdin = None,
env = None
)
)

val resources: Resources = TesTask.makeResources(
runtimeAttributes,
preferedWorkflowExecutionIdentity,
Option(tesPaths.tesTaskRoot)
)

val tags: Map[String, Option[String]] = TesTask.makeTags(jobDescriptor.workflowDescriptor)
}

object TesTask {
Expand Down Expand Up @@ -358,8 +372,7 @@ final case class Input(name: Option[String],
import common.util.StringUtil.EnhancedString

// Mask SAS token signature in query
this.getClass.getName + Seq(name, description, url.map(_.maskSensitiveUri), path, `type`, content)
.mkString("(", ",", ")")
this.getClass.getName + Seq(name, description, url.map(_.maskSensitiveUri), path, `type`, content).mkString("(",",",")")

Check warning on line 375 in supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala

View check run for this annotation

Codecov / codecov/patch

supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala#L375 

Added line #L375 was not covered by tests
}
}

Expand Down
6 changes: 2 additions & 4 deletions supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTaskSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,10 +222,8 @@ class TesTaskSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wit
val input = Input(
Option("asdf"),
Option("asdf"),
url = Option(
"https://lz304a1e79fd7359e5327eda.blob.core.windows.net/sc-705b830a-d699-478e-9da6-49661b326e77" +
"?sv=2021-12-02&spr=https&st=2023-12-13T20%3A27%3A55Z&se=2023-12-14T04%3A42%3A55Z&sr=c&sp=racwdlt&sig=SECRET&rscd=foo"
),
url = Option("https://lz304a1e79fd7359e5327eda.blob.core.windows.net/sc-705b830a-d699-478e-9da6-49661b326e77" +
"?sv=2021-12-02&spr=https&st=2023-12-13T20%3A27%3A55Z&se=2023-12-14T04%3A42%3A55Z&sr=c&sp=racwdlt&sig=SECRET&rscd=foo"),
"asdf",
Option("asdf"),
Option("asdf")
Expand Down

0 comments on commit 32b6d91

Please sign in to comment.