podman/singularity + nodejs

Fixes #803

cwltool/context.py

@@ -75,6 +75,8 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
         self.jobdefaults = None  # type: Optional[CommentedMap]
         self.doc_cache = True  # type: bool
         self.relax_path_checks = False  # type: bool
+        self.singularity = False  # type: bool
+        self.podman = False  # type: bool
 
         super().__init__(kwargs)
 

cwltool/factory.py

@@ -40,6 +40,9 @@ def __call__(self, **kwargs):
 class Factory:
     """Easy way to load a CWL document for execution."""
 
+    loading_context: LoadingContext
+    runtime_context: RuntimeContext
+
     def __init__(
         self,
         executor: Optional[JobExecutor] = None,
@@ -51,13 +54,16 @@ def __init__(
         if executor is None:
             executor = SingleJobExecutor()
         self.executor = executor
-        self.loading_context = loading_context
-        if loading_context is None:
-            self.loading_context = LoadingContext()
         if runtime_context is None:
             self.runtime_context = RuntimeContext()
         else:
             self.runtime_context = runtime_context
+        if loading_context is None:
+            self.loading_context = LoadingContext()
+            self.loading_context.singularity = self.runtime_context.singularity
+            self.loading_context.podman = self.runtime_context.podman
+        else:
+            self.loading_context = loading_context
 
     def make(self, cwl: Union[str, Dict[str, Any]]) -> Callable:
         """Instantiate a CWL object from a CWl document."""

cwltool/main.py

@@ -724,6 +724,8 @@ def setup_loadingContext(
 ) -> LoadingContext:
     if loadingContext is None:
         loadingContext = LoadingContext(vars(args))
+        loadingContext.singularity = runtimeContext.singularity
+        loadingContext.podman = runtimeContext.podman
     else:
         loadingContext = loadingContext.copy()
     loadingContext.loader = default_loader(

cwltool/process.py

@@ -718,10 +718,16 @@ def __init__(
                 avroname = classname
                 if self.doc_loader and classname in self.doc_loader.vocab:
                     avroname = avro_type_name(self.doc_loader.vocab[classname])
+                container_engine = "docker"
+                if loadingContext.podman:
+                    container_engine = "podman"
+                elif loadingContext.singularity:
+                    container_engine = "singularity"
                 validate_js_expressions(
                     toolpath_object,
                     self.doc_schema.names[avroname],
                     validate_js_options,
+                    container_engine,
                 )
 
         dockerReq, is_req = self.get_requirement("DockerRequirement")

cwltool/sandboxjs.py

@@ -14,6 +14,7 @@
 from schema_salad.utils import json_dumps
 
 from .loghandler import _logger
+from .singularity_utils import singularity_supports_userns
 from .utils import CWLOutputType, processes_to_kill
 
 
@@ -48,9 +49,10 @@ def check_js_threshold_version(working_alias: str) -> bool:
     return current_version >= minimum_node_version
 
 
-def new_js_proc(js_text: str, force_docker_pull: bool = False):
-    # type: (...) -> subprocess.Popen[str]
-
+def new_js_proc(
+    js_text: str, force_docker_pull: bool = False, container_engine: str = "docker"
+) -> 'subprocess.Popen[str]':
+    """Return a subprocess ready to submit javascript to."""
     required_node_version, docker = (False,) * 2
     nodejs = None  # type: Optional[subprocess.Popen[str]]
     trynodes = ("nodejs", "node")
@@ -79,36 +81,75 @@ def new_js_proc(js_text: str, force_docker_pull: bool = False):
 
     if nodejs is None or nodejs is not None and required_node_version is False:
         try:
-            nodeimg = "node:slim"
+            nodeimg = "docker.io/node:slim"
             global have_node_slim
 
             if not have_node_slim:
-                dockerimgs = subprocess.check_output(  # nosec
-                    ["docker", "images", "-q", nodeimg], universal_newlines=True
-                )
+                if container_engine in ("docker", "podman"):
+                    dockerimgs = subprocess.check_output(  # nosec
+                        [container_engine, "images", "-q", nodeimg],
+                        universal_newlines=True,
+                    )
+                elif container_engine == "singularity":
+                    nodeimg = f"docker://{nodeimg}"
+                else:
+                    raise Exception(f"Unknown container_engine: {container_engine}.")
                 # if output is an empty string
-                if (len(dockerimgs.split("\n")) <= 1) or force_docker_pull:
+                if (
+                    container_engine == "singularity"
+                    or len(dockerimgs.split("\n")) <= 1
+                    or force_docker_pull
+                ):
                     # pull node:slim docker container
+                    nodejs_pull_commands = [container_engine, "pull"]
+                    if container_engine == "singularity":
+                        nodejs_pull_commands.append("--force")
+                    nodejs_pull_commands.append(nodeimg)
                     nodejsimg = subprocess.check_output(  # nosec
-                        ["docker", "pull", nodeimg], universal_newlines=True
+                        nodejs_pull_commands, universal_newlines=True
+                    )
+                    _logger.info(
+                        "Pulled Docker image %s %s using %s",
+                        nodeimg,
+                        nodejsimg,
+                        container_engine,
                     )
-                    _logger.info("Pulled Docker image %s %s", nodeimg, nodejsimg)
                 have_node_slim = True
-            nodejs = subprocess.Popen(  # nosec
+            nodejs_commands = [
+                container_engine,
+            ]
+            if container_engine != "singularity":
+                nodejs_commands.extend(
+                    [
+                        "run",
+                        "--attach=STDIN",
+                        "--attach=STDOUT",
+                        "--attach=STDERR",
+                        "--sig-proxy=true",
+                        "--interactive",
+                        "--rm",
+                    ]
+                )
+            else:
+                nodejs_commands.extend(
+                    [
+                        "exec",
+                        "--contain",
+                        "--ipc",
+                        "--cleanenv",
+                        "--userns" if singularity_supports_userns() else "--pid",
+                    ]
+                )
+            nodejs_commands.extend(
                 [
-                    "docker",
-                    "run",
-                    "--attach=STDIN",
-                    "--attach=STDOUT",
-                    "--attach=STDERR",
-                    "--sig-proxy=true",
-                    "--interactive",
-                    "--rm",
                     nodeimg,
                     "node",
                     "--eval",
                     js_text,
                 ],
+            )
+            nodejs = subprocess.Popen(  # nosec
+                nodejs_commands,
                 universal_newlines=True,
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
@@ -129,7 +170,7 @@ def new_js_proc(js_text: str, force_docker_pull: bool = False):
         raise JavascriptException(
             "cwltool requires Node.js engine to evaluate and validate "
             "Javascript expressions, but couldn't find it.  Tried {}, "
-            "docker run node:slim".format(", ".join(trynodes))
+            f"{container_engine} run node:slim".format(", ".join(trynodes))
         )
 
     # docker failed, but nodejs is installed on system but the version is below the required version
@@ -153,6 +194,7 @@ def exec_js_process(
     js_console: bool = False,
     context: Optional[str] = None,
     force_docker_pull: bool = False,
+    container_engine: str = "docker",
 ) -> Tuple[int, str, str]:
 
     if not hasattr(localdata, "procs"):
@@ -184,7 +226,11 @@ def exec_js_process(
 
         created_new_process = True
 
-        new_proc = new_js_proc(js_engine_code, force_docker_pull=force_docker_pull)
+        new_proc = new_js_proc(
+            js_engine_code,
+            force_docker_pull=force_docker_pull,
+            container_engine=container_engine,
+        )
 
         if context is None:
             localdata.procs[js_engine] = new_proc

cwltool/singularity.py

@@ -24,33 +24,12 @@
 from .job import ContainerCommandLineJob
 from .loghandler import _logger
 from .pathmapper import MapperEnt, PathMapper
+from .singularity_utils import singularity_supports_userns
 from .utils import CWLObjectType, create_tmp_dir, ensure_non_writable, ensure_writable
 
-_USERNS = None  # type: Optional[bool]
 _SINGULARITY_VERSION = ""
 
 
-def _singularity_supports_userns() -> bool:
-    global _USERNS  # pylint: disable=global-statement
-    if _USERNS is None:
-        try:
-            hello_image = os.path.join(os.path.dirname(__file__), "hello.simg")
-            result = Popen(  # nosec
-                ["singularity", "exec", "--userns", hello_image, "true"],
-                stderr=PIPE,
-                stdout=DEVNULL,
-                universal_newlines=True,
-            ).communicate(timeout=60)[1]
-            _USERNS = (
-                "No valid /bin/sh" in result
-                or "/bin/sh doesn't exist in container" in result
-                or "executable file not found in" in result
-            )
-        except TimeoutExpired:
-            _USERNS = False
-    return _USERNS
-
-
 def get_version() -> str:
     global _SINGULARITY_VERSION  # pylint: disable=global-statement
     if _SINGULARITY_VERSION == "":
@@ -406,7 +385,7 @@ def create_runtime(
             "--ipc",
             "--cleanenv",
         ]
-        if _singularity_supports_userns():
+        if singularity_supports_userns():
             runtime.append("--userns")
         else:
             runtime.append("--pid")

cwltool/singularity_utils.py

@@ -0,0 +1,38 @@
+"""Support for executing Docker containers using the Singularity 2.x engine."""
+
+import os
+import os.path
+from subprocess import (  # nosec
+    DEVNULL,
+    PIPE,
+    Popen,
+    TimeoutExpired,
+    check_call,
+    check_output,
+)
+from typing import Optional
+
+
+_USERNS = None  # type: Optional[bool]
+
+
+def singularity_supports_userns() -> bool:
+    """Confirm if the version of Singularity install supports the --userns flag."""
+    global _USERNS  # pylint: disable=global-statement
+    if _USERNS is None:
+        try:
+            hello_image = os.path.join(os.path.dirname(__file__), "hello.simg")
+            result = Popen(  # nosec
+                ["singularity", "exec", "--userns", hello_image, "true"],
+                stderr=PIPE,
+                stdout=DEVNULL,
+                universal_newlines=True,
+            ).communicate(timeout=60)[1]
+            _USERNS = (
+                "No valid /bin/sh" in result
+                or "/bin/sh doesn't exist in container" in result
+                or "executable file not found in" in result
+            )
+        except TimeoutExpired:
+            _USERNS = False
+    return _USERNS

cwltool/validate_js.py

@@ -133,6 +133,7 @@ def jshint_js(
     js_text: str,
     globals: Optional[List[str]] = None,
     options: Optional[Dict[str, Union[List[str], str, int]]] = None,
+    container_engine: str = "docker",
 ) -> JSHintJSReturn:
     if globals is None:
         globals = []
@@ -165,6 +166,7 @@ def jshint_js(
         % json_dumps({"code": js_text, "options": options, "globals": globals}),
         timeout=30,
         context=jshint_functions_text,
+        container_engine=container_engine,
     )
 
     def dump_jshint_error():
@@ -213,6 +215,7 @@ def validate_js_expressions(
     tool: CommentedMap,
     schema: Schema,
     jshint_options: Optional[Dict[str, Union[List[str], str, int]]] = None,
+    container_engine: str = "docker",
 ) -> None:
 
     if tool.get("requirements") is None:
@@ -233,7 +236,7 @@ def validate_js_expressions(
 
     for i, expression_lib_line in enumerate(expression_lib):
         expression_lib_line_errors, expression_lib_line_globals = jshint_js(
-            expression_lib_line, js_globals, jshint_options
+            expression_lib_line, js_globals, jshint_options, container_engine
         )
         js_globals.extend(expression_lib_line_globals)
         print_js_hint_messages(
@@ -259,7 +262,7 @@ def validate_js_expressions(
                 code_fragment = unscanned_str[scan_slice[0] + 1 : scan_slice[1]]
                 code_fragment_js = code_fragment_to_js(code_fragment, "")
                 expression_errors, _ = jshint_js(
-                    code_fragment_js, js_globals, jshint_options
+                    code_fragment_js, js_globals, jshint_options, container_engine
                 )
                 print_js_hint_messages(expression_errors, source_line)