Add cmd output type

docs/process.md

@@ -1057,43 +1057,41 @@ To sum up, the use of output files with static names over dynamic ones is prefer
 
 The `env` qualifier allows you to output a variable defined in the process execution environment:
 
-```groovy
-process myTask {
-    output:
-    env FOO
-
-    script:
-    '''
-    FOO=$(ls -la)
-    '''
-}
-
-workflow {
-    myTask | view { "directory contents: $it" }
-}
+```{literalinclude} snippets/process-out-env.nf
+:language: groovy
 ```
 
+:::{versionchanged} 23.12.0-edge
+Prior to this version, if the environment variable contained multiple lines of output, the output would be compressed to a single line by converting newlines to spaces.
+:::
+
 (process-stdout)=
 
 ### Output type `stdout`
 
 The `stdout` qualifier allows you to output the `stdout` of the executed process:
 
-```groovy
-process sayHello {
-    output:
-    stdout
+```{literalinclude} snippets/process-stdout.nf
+:language: groovy
+```
 
-    """
-    echo Hello world!
-    """
-}
+(process-out-cmd)=
 
-workflow {
-    sayHello | view { "I say... $it" }
-}
+### Output type `cmd`
+
+:::{versionadded} 23.12.0-edge
+:::
+
+The `cmd` qualifier allows you to capture the standard output of an arbitrary shell command:
+
+```{literalinclude} snippets/process-out-cmd.nf
+:language: groovy
 ```
 
+Only one-line Bash commands are supported. You can use a semi-colon `;` to specify multiple Bash commands on a single line, and many interpreters can execute arbitrary code on the command line, e.g. `python -c 'print("Hello world!")'`.
+
+If the command fails, the task will also fail. In Bash, you can append `|| true` to a command to suppress any command failure.
+
 (process-set)=
 
 ### Output type `set`

docs/snippets/process-out-cmd.nf

@@ -0,0 +1,12 @@
+process sayHello {
+    output:
+    cmd('bash --version')
+
+    """
+    echo Hello world!
+    """
+}
+
+workflow {
+    sayHello | view
+}

docs/snippets/process-out-cmd.out

@@ -0,0 +1,6 @@
+GNU bash, version 5.1.16(1)-release (x86_64-pc-linux-gnu)
+Copyright (C) 2020 Free Software Foundation, Inc.
+License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
+
+This is free software; you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.

docs/snippets/process-out-env.nf

@@ -0,0 +1,13 @@
+process myTask {
+    output:
+    env FOO
+
+    script:
+    '''
+    FOO=$(ls -a)
+    '''
+}
+
+workflow {
+    myTask | view
+}

docs/snippets/process-out-env.out

@@ -0,0 +1,8 @@
+.
+..
+.command.begin
+.command.err
+.command.log
+.command.out
+.command.run
+.command.sh

docs/snippets/process-stdout.nf

@@ -0,0 +1,12 @@
+process sayHello {
+    output:
+    stdout
+
+    """
+    echo Hello world!
+    """
+}
+
+workflow {
+    sayHello | view { "I say... $it" }
+}

docs/snippets/process-stdout.out

@@ -0,0 +1,2 @@
+I say... Hello world!
+

modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy

@@ -27,6 +27,7 @@ import nextflow.script.BaseScript
 import nextflow.script.BodyDef
 import nextflow.script.IncludeDef
 import nextflow.script.TaskClosure
+import nextflow.script.TokenCmdCall
 import nextflow.script.TokenEnvCall
 import nextflow.script.TokenFileCall
 import nextflow.script.TokenPathCall
@@ -955,7 +956,7 @@ class NextflowDSLImpl implements ASTTransformation {
             def nested = methodCall.objectExpression instanceof MethodCallExpression
             log.trace "convert > output method: $methodName"
 
-            if( methodName in ['val','env','file','set','stdout','path','tuple'] && !nested ) {
+            if( methodName in ['val','env','cmd','file','set','stdout','path','tuple'] && !nested ) {
                 // prefix the method name with the string '_out_'
                 methodCall.setMethod( new ConstantExpression('_out_' + methodName) )
                 fixMethodCall(methodCall)
@@ -1123,6 +1124,11 @@ class NextflowDSLImpl implements ASTTransformation {
                     return createX( TokenEnvCall, args )
                 }
 
+                if( methodCall.methodAsString == 'cmd' && withinTupleMethod ) {
+                    def args = (TupleExpression) varToStrX(methodCall.arguments)
+                    return createX( TokenCmdCall, args )
+                }
+
                 /*
                  * input:
                  *   tuple val(x), .. from q

modules/nextflow/src/main/groovy/nextflow/exception/ProcessCommandException.groovy

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.exception
+
+import groovy.transform.CompileStatic
+
+/**
+ * Exception thrown when a command output returns a non-zero exit status
+ * 
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+@CompileStatic
+class ProcessCommandException extends RuntimeException implements ShowOnlyExceptionMessage {
+
+    String command
+    String output
+    int status
+
+    ProcessCommandException(String message, String command, String output, int status) {
+        super(message)
+        this.command = command
+        this.output = output
+        this.status = status
+    }
+}

modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy

@@ -16,6 +16,8 @@
 
 package nextflow.executor
 
+import static java.nio.file.StandardOpenOption.*
+
 import java.nio.file.FileSystemException
 import java.nio.file.FileSystems
 import java.nio.file.Files
@@ -35,11 +37,7 @@ import nextflow.processor.TaskProcessor
 import nextflow.processor.TaskRun
 import nextflow.secret.SecretsLoader
 import nextflow.util.Escape
-
-import static java.nio.file.StandardOpenOption.*
-
 import nextflow.util.MemoryUnit
-
 /**
  * Builder to create the Bash script which is used to
  * wrap and launch the user task
@@ -177,18 +175,36 @@ class BashWrapperBuilder {
         }
     }
 
-    protected String getOutputEnvCaptureSnippet(List<String> names) {
-        def result = new StringBuilder()
-        result.append('\n')
-        result.append('# capture process environment\n')
-        result.append('set +u\n')
-        result.append('cd "$NXF_TASK_WORKDIR"\n')
-        for( int i=0; i<names.size(); i++) {
-            final key = names[i]
-            result.append "echo $key=\${$key[@]} "
-            result.append( i==0 ? '> ' : '>> ' )
-            result.append(TaskRun.CMD_ENV)
-            result.append('\n')
+    protected String getOutputEnvCaptureSnippet(List<String> outEnvs, Map<String,String> outCmds) {
+        // load the env template
+        def template = BashWrapperBuilder.class
+            .getResourceAsStream('command-env.txt')
+            .newReader()
+        def binding = Map.of('env_file', TaskRun.CMD_ENV)
+        def result = engine.render(template, binding)
+        // avoid nulls
+        if( outEnvs==null )
+            outEnvs = List.<String>of()
+        if( outCmds==null )
+            outCmds = Map.<String,String>of()
+        // out env
+        for( String key : outEnvs ) {
+            result += "#\n"
+            result += "echo $key=\"\${$key[@]}\" >> ${TaskRun.CMD_ENV}\n"
+            result += "echo /$key/ >> ${TaskRun.CMD_ENV}\n"
+        }
+        // out cmd
+        for( Map.Entry<String,String> cmd : outCmds ) {
+            result += "#\n"
+            result += "nxf_catch STDOUT STDERR ${cmd.value}\n"
+            result += 'status=$?\n'
+            result += 'if [ $status -eq 0 ]; then\n'
+            result += "  echo $cmd.key=\"\$STDOUT\" >> ${TaskRun.CMD_ENV}\n"
+            result += "  echo /$cmd.key/=exit:0 >> ${TaskRun.CMD_ENV}\n"
+            result += 'else\n'
+            result += "  echo $cmd.key=\"\$STDERR\" >> ${TaskRun.CMD_ENV}\n"
+            result += "  echo /$cmd.key/=exit:\$status >> ${TaskRun.CMD_ENV}\n"
+            result += 'fi\n'
         }
         result.toString()
     }
@@ -239,9 +255,12 @@ class BashWrapperBuilder {
          */
         final interpreter = TaskProcessor.fetchInterpreter(script)
 
-        if( outputEnvNames ) {
-            if( !isBash(interpreter) ) throw new IllegalArgumentException("Process output of type env is only allowed with Bash process command -- Current interpreter: $interpreter")
-            script += getOutputEnvCaptureSnippet(outputEnvNames)
+        if( outputEnvNames || outputCommands ) {
+            if( !isBash(interpreter) && outputEnvNames )
+                throw new IllegalArgumentException("Process output of type env is only allowed with Bash process scripts -- Current interpreter: $interpreter")
+            if( !isBash(interpreter) && outputCommands )
+                throw new IllegalArgumentException("Process output of type cmd is only allowed with Bash process scripts -- Current interpreter: $interpreter")
+            script += getOutputEnvCaptureSnippet(outputEnvNames, outputCommands)
         }
 
         final binding = new HashMap<String,String>(20)

modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy

@@ -73,6 +73,8 @@ class TaskBean implements Serializable, Cloneable {
 
     List<String> outputEnvNames
 
+    Map<String,String> outputCommands
+
     String beforeScript
 
     String afterScript
@@ -144,6 +146,7 @@ class TaskBean implements Serializable, Cloneable {
 
         // stats
         this.outputEnvNames = task.getOutputEnvNames()
+        this.outputCommands = task.getOutputCommands()
         this.statsEnabled = task.getProcessor().getSession().statsEnabled
 
         this.inputFiles = task.getInputFilesMap()