Features : Optional_Files, jobTimeout and LogStream to metadata

backend/src/main/scala/cromwell/backend/backend.scala

@@ -146,6 +146,7 @@ object CommonBackendConfigurationAttributes {
     "default-runtime-attributes.maxRetries",
     "default-runtime-attributes.awsBatchEvaluateOnExit",
     "default-runtime-attributes.sharedMemorySize",
+    "default-runtime-attributes.jobTimeout",
     "default-runtime-attributes.ulimits",
     "default-runtime-attributes.efsDelocalize",
     "default-runtime-attributes.efsMakeMD5",

core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala

@@ -57,6 +57,10 @@ object DefaultIoCommand {
   case class DefaultIoExistsOrThrowCommand(override val file: Path) extends IoExistsOrThrowCommand(file) {
     override def commandDescription: String = s"DefaultIoExistsOrThrowCommand file '$file'"
   }
+
+  case class DefaultIoNoopCommand(override val file: Path) extends IoNoopCommand(file) {
+    override def commandDescription: String = s"DefaultIoNoopCommand file '$file'"
+  }
 
   case class DefaultIoReadLinesCommand(override val file: Path) extends IoReadLinesCommand(file) {
     override def commandDescription: String = s"DefaultIoReadLinesCommand file '$file'"

core/src/main/scala/cromwell/core/io/IoCommand.scala

@@ -190,6 +190,14 @@ abstract class IoExistsOrThrowCommand(val file: Path) extends SingleFileIoComman
   override lazy val name = "exist"
 }
 
+/**
+  * No-op Command that does nothing
+  */
+abstract class IoNoopCommand(val file: Path) extends SingleFileIoCommand[Boolean] {
+  override def toString = s"No-op command on ${file.pathAsString} (does nothing)"
+  override lazy val name = "noop"
+}
+
 /**
   * Return the lines of a file in a collection
   */

core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala

@@ -22,6 +22,7 @@ abstract class PartialIoCommandBuilder {
   def touchCommand: PartialFunction[Path, Try[IoTouchCommand]] = PartialFunction.empty
   def existsCommand: PartialFunction[Path, Try[IoExistsCommand]] = PartialFunction.empty
   def existsOrThrowCommand: PartialFunction[Path, Try[IoExistsOrThrowCommand]] = PartialFunction.empty
+  def noopCommand: PartialFunction[Path, Try[IoNoopCommand]] = PartialFunction.empty
   def isDirectoryCommand: PartialFunction[Path, Try[IoIsDirectoryCommand]] = PartialFunction.empty
   def readLinesCommand: PartialFunction[Path, Try[IoReadLinesCommand]] = PartialFunction.empty
 }
@@ -98,6 +99,9 @@ class IoCommandBuilder(partialBuilders: List[PartialIoCommandBuilder] = List.emp
   def existsOrThrowCommand(file: Path): Try[IoExistsOrThrowCommand] =
     buildOrDefault(_.existsOrThrowCommand, file, DefaultIoExistsOrThrowCommand(file))
 
+  def noopCommand(file: Path): Try[IoNoopCommand] =
+    buildOrDefault(_.noopCommand, file, DefaultIoNoopCommand(file))
+
   def isDirectoryCommand(file: Path): Try[IoIsDirectoryCommand] =
     buildOrDefault(_.isDirectoryCommand, file, DefaultIoIsDirectoryCommand(file))
 

engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala

@@ -80,6 +80,7 @@ class NioFlow(parallelism: Int,
       case touchCommand: IoTouchCommand => touch(touchCommand) map touchCommand.success
       case existsCommand: IoExistsCommand => exists(existsCommand) map existsCommand.success
       case existsOrThrowCommand: IoExistsOrThrowCommand => existsOrThrow(existsOrThrowCommand) map existsOrThrowCommand.success
+      case noopCommand: IoNoopCommand => noop(noopCommand) map noopCommand.success
       case readLinesCommand: IoReadLinesCommand => readLines(readLinesCommand) map readLinesCommand.success
       case isDirectoryCommand: IoIsDirectoryCommand => isDirectory(isDirectoryCommand) map isDirectoryCommand.success
       case _ => IO.raiseError(new UnsupportedOperationException("Method not implemented"))
@@ -191,6 +192,9 @@ class NioFlow(parallelism: Int,
       case true => true
     }
   }
+  private def noop(noop: IoNoopCommand) = IO {
+    ()
+  }
 
   private def readLines(exists: IoReadLinesCommand) = IO {
     exists.file.withReader { reader =>

engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/job/EngineJobExecutionActor.scala

@@ -287,7 +287,6 @@ class EngineJobExecutionActor(replyTo: ActorRef,
         log.info(s"BT-322 {} cache hit copying nomatch: could not find a suitable cache hit.", jobTag)
         workflowLogger.debug(s"Could not copy a suitable cache hit for {$jobTag}. No copy attempts were made.")
       }
-
       runJob(data)
     case Event(hashes: CallCacheHashes, data: ResponsePendingData) =>
       addHashesAndStay(data, hashes)

filesystems/s3/src/main/scala/cromwell/filesystems/s3/batch/S3BatchCommandBuilder.scala

@@ -37,7 +37,7 @@ import cromwell.filesystems.s3.S3Path
 import org.slf4j.{Logger, LoggerFactory}
 import cromwell.core.io.DefaultIoCommand.DefaultIoHashCommand
 
-import scala.util.Try
+import scala.util.{Try}
 
 /**
   * Generates commands for IO operations on S3
@@ -84,16 +84,23 @@ private case object PartialS3BatchCommandBuilder extends PartialIoCommandBuilder
   }
 
   override def touchCommand: PartialFunction[Path, Try[S3BatchTouchCommand]] = { case path: S3Path =>
+    println(s"Touching $path")
     Try(S3BatchTouchCommand(path))
   }
 
   override def existsCommand: PartialFunction[Path, Try[S3BatchExistsCommand]] = { case path: S3Path =>
+    println(s"Checking existence of $path")
     Try(S3BatchExistsCommand(path))
   }
 
   override def existsOrThrowCommand: PartialFunction[Path, Try[S3BatchExistsOrThrowCommand]] = { case path: S3Path =>
     Try(S3BatchExistsOrThrowCommand(path))
   }
+
+  override def noopCommand: PartialFunction[Path, Try[S3BatchNoopCommand]] = { case path: S3Path =>
+    Try(S3BatchNoopCommand(path))
+  }
+
 }
 
 /**

filesystems/s3/src/main/scala/cromwell/filesystems/s3/batch/S3BatchIoCommand.scala

@@ -39,13 +39,15 @@ import cromwell.core.io.{
   IoDeleteCommand,
   IoExistsCommand,
   IoExistsOrThrowCommand,
+  IoNoopCommand,
   IoHashCommand,
   IoSizeCommand,
   IoTouchCommand
 }
 
 import cromwell.filesystems.s3.S3Path
 import java.nio.file.NoSuchFileException
+//import scala.util.{Try }
 
 /**
   * Io commands with S3 paths and some logic enabling batching of request.
@@ -162,3 +164,13 @@ case class S3BatchExistsOrThrowCommand(override val file: S3Path)
     }
   override def commandDescription: String = s"S3BatchExistsCommand file '$file'"
 }
+/**
+  * `IoCommand` that does nothing
+  * @param file the path to the object
+  */
+case class S3BatchNoopCommand(override val file: S3Path) extends IoNoopCommand(file) {
+  override def commandDescription: String = s"S3BatchNoopCommand file '$file'"
+}
+
+
+

project/Dependencies.scala

@@ -6,7 +6,7 @@ object Dependencies {
   private val akkaV = "2.5.32" // scala-steward:off (CROM-6637)
   private val ammoniteOpsV = "2.4.1"
   private val apacheHttpClientV = "4.5.13"
-  private val awsSdkV = "2.26.19"
+  private val awsSdkV = "2.29.20"
   // We would like to use the BOM to manage Azure SDK versions, but SBT doesn't support it.
   // https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/boms/azure-sdk-bom
   // https://github.com/sbt/sbt/issues/4531