diff --git a/Makefile b/Makefile index dbe02adebc..5f94311cc6 100644 --- a/Makefile +++ b/Makefile @@ -148,11 +148,11 @@ diff-cover.html: coverage.xml ## test : run the ${MODULE} test suite test: check-python3 $(PYSOURCES) - python -m pytest ${PYTEST_EXTRA} + python -m pytest -rs ${PYTEST_EXTRA} ## testcov : run the ${MODULE} test suite and collect coverage testcov: check-python3 $(PYSOURCES) - python -m pytest --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA} + python -m pytest -rs --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA} sloccount.sc: $(PYSOURCES) Makefile sloccount --duplicates --wide --details $^ > $@ @@ -177,7 +177,7 @@ mypy: $(filter-out setup.py gittagger.py,$(PYSOURCES)) mypyc: $(PYSOURCES) MYPYPATH=typeshed CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \ - && pytest -vv ${PYTEST_EXTRA} + && pytest -rs -vv ${PYTEST_EXTRA} shellcheck: FORCE shellcheck build-cwltool-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ diff --git a/cwltool/builder.py b/cwltool/builder.py index 1136a8b334..a82274ce3e 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -179,6 +179,7 @@ def __init__( tmpdir: str, stagedir: str, cwlVersion: str, + container_engine: str, ) -> None: """Initialize this Builder.""" self.job = job @@ -215,6 +216,7 @@ def __init__( self.pathmapper = None # type: Optional[PathMapper] self.prov_obj = None # type: Optional[ProvenanceProfile] self.find_default_container = None # type: Optional[Callable[[], str]] + self.container_engine = container_engine def build_job_script(self, commands: List[str]) -> Optional[str]: if self.job_script_provider is not None: @@ -746,4 +748,5 @@ def do_eval( force_docker_pull=self.force_docker_pull, strip_whitespace=strip_whitespace, cwlVersion=self.cwlVersion, + container_engine=self.container_engine, ) diff --git a/cwltool/context.py b/cwltool/context.py index cefa515c81..272a3b2eb2 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -75,6 +75,8 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.jobdefaults = None # type: Optional[CommentedMap] self.doc_cache = True # type: bool self.relax_path_checks = False # type: bool + self.singularity = False # type: bool + self.podman = False # type: bool super().__init__(kwargs) diff --git a/cwltool/expression.py b/cwltool/expression.py index 8474dfe02d..bb8446086c 100644 --- a/cwltool/expression.py +++ b/cwltool/expression.py @@ -204,6 +204,7 @@ def evaluator( force_docker_pull: bool = False, debug: bool = False, js_console: bool = False, + container_engine: str = "docker", ) -> Optional[CWLOutputType]: match = param_re.match(ex) @@ -238,6 +239,7 @@ def evaluator( force_docker_pull=force_docker_pull, debug=debug, js_console=js_console, + container_engine=container_engine, ) else: if expression_parse_exception is not None: @@ -270,6 +272,7 @@ def interpolate( strip_whitespace: bool = True, escaping_behavior: int = 2, convert_to_expression: bool = False, + container_engine: str = "docker", ) -> Optional[CWLOutputType]: """ Interpolate and evaluate. @@ -303,6 +306,7 @@ def interpolate( force_docker_pull=force_docker_pull, debug=debug, js_console=js_console, + container_engine=container_engine, ) if w[0] == 0 and w[1] == len(scan) and len(parts) <= 1: return e @@ -367,6 +371,7 @@ def do_eval( js_console: bool = False, strip_whitespace: bool = True, cwlVersion: str = "", + container_engine: str = "docker", ) -> Optional[CWLOutputType]: runtime = cast(MutableMapping[str, Union[int, str, None]], copy.deepcopy(resources)) @@ -409,6 +414,7 @@ def do_eval( "v1.2.0-dev3", ) else 2, + container_engine=container_engine, ) except Exception as e: diff --git a/cwltool/factory.py b/cwltool/factory.py index 8931de76cb..4a93c3f052 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -40,6 +40,9 @@ def __call__(self, **kwargs): class Factory: """Easy way to load a CWL document for execution.""" + loading_context: LoadingContext + runtime_context: RuntimeContext + def __init__( self, executor: Optional[JobExecutor] = None, @@ -51,13 +54,16 @@ def __init__( if executor is None: executor = SingleJobExecutor() self.executor = executor - self.loading_context = loading_context - if loading_context is None: - self.loading_context = LoadingContext() if runtime_context is None: self.runtime_context = RuntimeContext() else: self.runtime_context = runtime_context + if loading_context is None: + self.loading_context = LoadingContext() + self.loading_context.singularity = self.runtime_context.singularity + self.loading_context.podman = self.runtime_context.podman + else: + self.loading_context = loading_context def make(self, cwl: Union[str, Dict[str, Any]]) -> Callable: """Instantiate a CWL object from a CWl document.""" diff --git a/cwltool/loghandler.py b/cwltool/loghandler.py index c720ddef89..8bc1811e72 100644 --- a/cwltool/loghandler.py +++ b/cwltool/loghandler.py @@ -1,7 +1,36 @@ """Shared logger for cwltool.""" import logging +import coloredlogs _logger = logging.getLogger("cwltool") # pylint: disable=invalid-name defaultStreamHandler = logging.StreamHandler() # pylint: disable=invalid-name _logger.addHandler(defaultStreamHandler) _logger.setLevel(logging.INFO) + + +def configure_logging( + stderr_handler: logging.Handler, + quiet: bool, + debug: bool, + enable_color: bool, + timestamps: bool, + base_logger: logging.Logger = _logger, +) -> None: + rdflib_logger = logging.getLogger("rdflib.term") + rdflib_logger.addHandler(stderr_handler) + rdflib_logger.setLevel(logging.ERROR) + if quiet: + # Silence STDERR, not an eventual provenance log file + stderr_handler.setLevel(logging.WARN) + if debug: + # Increase to debug for both stderr and provenance log file + base_logger.setLevel(logging.DEBUG) + stderr_handler.setLevel(logging.DEBUG) + rdflib_logger.setLevel(logging.DEBUG) + fmtclass = coloredlogs.ColoredFormatter if enable_color else logging.Formatter + formatter = fmtclass("%(levelname)s %(message)s") + if timestamps: + formatter = fmtclass( + "[%(asctime)s] %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S" + ) + stderr_handler.setFormatter(formatter) diff --git a/cwltool/main.py b/cwltool/main.py index 8284f34fe9..5389b55157 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -60,7 +60,7 @@ resolve_overrides, resolve_tool_uri, ) -from .loghandler import _logger, defaultStreamHandler +from .loghandler import _logger, defaultStreamHandler, configure_logging from .mpi import MpiConfig from .mutation import MutationManager from .pack import pack @@ -624,31 +624,6 @@ def supported_cwl_versions(enable_dev: bool) -> List[str]: return versions -def configure_logging( - args: argparse.Namespace, - stderr_handler: logging.Handler, - runtimeContext: RuntimeContext, -) -> None: - rdflib_logger = logging.getLogger("rdflib.term") - rdflib_logger.addHandler(stderr_handler) - rdflib_logger.setLevel(logging.ERROR) - if args.quiet: - # Silence STDERR, not an eventual provenance log file - stderr_handler.setLevel(logging.WARN) - if runtimeContext.debug: - # Increase to debug for both stderr and provenance log file - _logger.setLevel(logging.DEBUG) - stderr_handler.setLevel(logging.DEBUG) - rdflib_logger.setLevel(logging.DEBUG) - fmtclass = coloredlogs.ColoredFormatter if args.enable_color else logging.Formatter - formatter = fmtclass("%(levelname)s %(message)s") - if args.timestamps: - formatter = fmtclass( - "[%(asctime)s] %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S" - ) - stderr_handler.setFormatter(formatter) - - def setup_schema( args: argparse.Namespace, custom_schema_callback: Optional[Callable[[], None]] ) -> None: @@ -724,6 +699,8 @@ def setup_loadingContext( ) -> LoadingContext: if loadingContext is None: loadingContext = LoadingContext(vars(args)) + loadingContext.singularity = runtimeContext.singularity + loadingContext.podman = runtimeContext.podman else: loadingContext = loadingContext.copy() loadingContext.loader = default_loader( @@ -966,7 +943,13 @@ def main( if not hasattr(args, key): setattr(args, key, val) - configure_logging(args, stderr_handler, runtimeContext) + configure_logging( + stderr_handler, + args.quiet, + runtimeContext.debug, + args.enable_color, + args.timestamps, + ) if args.version: print(versionfunc()) diff --git a/cwltool/process.py b/cwltool/process.py index 58becf757d..ee39881f05 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -718,10 +718,16 @@ def __init__( avroname = classname if self.doc_loader and classname in self.doc_loader.vocab: avroname = avro_type_name(self.doc_loader.vocab[classname]) + container_engine = "docker" + if loadingContext.podman: + container_engine = "podman" + elif loadingContext.singularity: + container_engine = "singularity" validate_js_expressions( toolpath_object, self.doc_schema.names[avroname], validate_js_options, + container_engine, ) dockerReq, is_req = self.get_requirement("DockerRequirement") @@ -753,6 +759,11 @@ def __init__( var_spool_cwl_detector(self.tool) else: var_spool_cwl_detector(self.tool) + self.container_engine = "docker" + if loadingContext.podman: + self.container_engine = "podman" + elif loadingContext.singularity: + self.container_engine = "singularity" def _init_job( self, joborder: CWLObjectType, runtime_context: RuntimeContext @@ -903,6 +914,7 @@ def inc(d): # type: (List[int]) -> None tmpdir, stagedir, cwl_version, + self.container_engine, ) bindings.extend( diff --git a/cwltool/sandboxjs.py b/cwltool/sandboxjs.py index a677ba0dd9..b5247adeb9 100644 --- a/cwltool/sandboxjs.py +++ b/cwltool/sandboxjs.py @@ -14,6 +14,7 @@ from schema_salad.utils import json_dumps from .loghandler import _logger +from .singularity_utils import singularity_supports_userns from .utils import CWLOutputType, processes_to_kill @@ -48,9 +49,10 @@ def check_js_threshold_version(working_alias: str) -> bool: return current_version >= minimum_node_version -def new_js_proc(js_text: str, force_docker_pull: bool = False): - # type: (...) -> subprocess.Popen[str] - +def new_js_proc( + js_text: str, force_docker_pull: bool = False, container_engine: str = "docker" +) -> "subprocess.Popen[str]": + """Return a subprocess ready to submit javascript to.""" required_node_version, docker = (False,) * 2 nodejs = None # type: Optional[subprocess.Popen[str]] trynodes = ("nodejs", "node") @@ -79,36 +81,76 @@ def new_js_proc(js_text: str, force_docker_pull: bool = False): if nodejs is None or nodejs is not None and required_node_version is False: try: - nodeimg = "node:slim" + nodeimg = "docker.io/node:slim" global have_node_slim + if container_engine == "singularity": + nodeimg = f"docker://{nodeimg}" if not have_node_slim: - dockerimgs = subprocess.check_output( # nosec - ["docker", "images", "-q", nodeimg], universal_newlines=True - ) + if container_engine in ("docker", "podman"): + dockerimgs = subprocess.check_output( # nosec + [container_engine, "images", "-q", nodeimg], + universal_newlines=True, + ) + elif container_engine != "singularity": + raise Exception(f"Unknown container_engine: {container_engine}.") # if output is an empty string - if (len(dockerimgs.split("\n")) <= 1) or force_docker_pull: + if ( + container_engine == "singularity" + or len(dockerimgs.split("\n")) <= 1 + or force_docker_pull + ): # pull node:slim docker container + nodejs_pull_commands = [container_engine, "pull"] + if container_engine == "singularity": + nodejs_pull_commands.append("--force") + nodejs_pull_commands.append(nodeimg) nodejsimg = subprocess.check_output( # nosec - ["docker", "pull", nodeimg], universal_newlines=True + nodejs_pull_commands, universal_newlines=True + ) + _logger.debug( + "Pulled Docker image %s %s using %s", + nodeimg, + nodejsimg, + container_engine, ) - _logger.info("Pulled Docker image %s %s", nodeimg, nodejsimg) have_node_slim = True - nodejs = subprocess.Popen( # nosec + nodejs_commands = [ + container_engine, + ] + if container_engine != "singularity": + nodejs_commands.extend( + [ + "run", + "--attach=STDIN", + "--attach=STDOUT", + "--attach=STDERR", + "--sig-proxy=true", + "--interactive", + "--rm", + ] + ) + else: + nodejs_commands.extend( + [ + "exec", + "--contain", + "--ipc", + "--cleanenv", + "--userns" if singularity_supports_userns() else "--pid", + ] + ) + nodejs_commands.extend( [ - "docker", - "run", - "--attach=STDIN", - "--attach=STDOUT", - "--attach=STDERR", - "--sig-proxy=true", - "--interactive", - "--rm", nodeimg, "node", "--eval", js_text, ], + ) + _logger.debug("Running nodejs via %s", nodejs_commands[:-1]) + nodejs = subprocess.Popen( # nosec + nodejs_commands, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -129,7 +171,7 @@ def new_js_proc(js_text: str, force_docker_pull: bool = False): raise JavascriptException( "cwltool requires Node.js engine to evaluate and validate " "Javascript expressions, but couldn't find it. Tried {}, " - "docker run node:slim".format(", ".join(trynodes)) + f"{container_engine} run node:slim".format(", ".join(trynodes)) ) # docker failed, but nodejs is installed on system but the version is below the required version @@ -153,6 +195,7 @@ def exec_js_process( js_console: bool = False, context: Optional[str] = None, force_docker_pull: bool = False, + container_engine: str = "docker", ) -> Tuple[int, str, str]: if not hasattr(localdata, "procs"): @@ -184,7 +227,11 @@ def exec_js_process( created_new_process = True - new_proc = new_js_proc(js_engine_code, force_docker_pull=force_docker_pull) + new_proc = new_js_proc( + js_engine_code, + force_docker_pull=force_docker_pull, + container_engine=container_engine, + ) if context is None: localdata.procs[js_engine] = new_proc @@ -273,12 +320,17 @@ def execjs( force_docker_pull: bool = False, debug: bool = False, js_console: bool = False, + container_engine: str = "docker", ) -> CWLOutputType: fn = code_fragment_to_js(js, jslib) returncode, stdout, stderr = exec_js_process( - fn, timeout, js_console=js_console, force_docker_pull=force_docker_pull + fn, + timeout, + js_console=js_console, + force_docker_pull=force_docker_pull, + container_engine=container_engine, ) if js_console: diff --git a/cwltool/singularity.py b/cwltool/singularity.py index ea87acd144..a034a1bcd7 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -24,33 +24,12 @@ from .job import ContainerCommandLineJob from .loghandler import _logger from .pathmapper import MapperEnt, PathMapper +from .singularity_utils import singularity_supports_userns from .utils import CWLObjectType, create_tmp_dir, ensure_non_writable, ensure_writable -_USERNS = None # type: Optional[bool] _SINGULARITY_VERSION = "" -def _singularity_supports_userns() -> bool: - global _USERNS # pylint: disable=global-statement - if _USERNS is None: - try: - hello_image = os.path.join(os.path.dirname(__file__), "hello.simg") - result = Popen( # nosec - ["singularity", "exec", "--userns", hello_image, "true"], - stderr=PIPE, - stdout=DEVNULL, - universal_newlines=True, - ).communicate(timeout=60)[1] - _USERNS = ( - "No valid /bin/sh" in result - or "/bin/sh doesn't exist in container" in result - or "executable file not found in" in result - ) - except TimeoutExpired: - _USERNS = False - return _USERNS - - def get_version() -> str: global _SINGULARITY_VERSION # pylint: disable=global-statement if _SINGULARITY_VERSION == "": @@ -406,7 +385,7 @@ def create_runtime( "--ipc", "--cleanenv", ] - if _singularity_supports_userns(): + if singularity_supports_userns(): runtime.append("--userns") else: runtime.append("--pid") diff --git a/cwltool/singularity_utils.py b/cwltool/singularity_utils.py new file mode 100644 index 0000000000..164ffdc2fd --- /dev/null +++ b/cwltool/singularity_utils.py @@ -0,0 +1,38 @@ +"""Support for executing Docker containers using the Singularity 2.x engine.""" + +import os +import os.path +from subprocess import ( # nosec + DEVNULL, + PIPE, + Popen, + TimeoutExpired, + check_call, + check_output, +) +from typing import Optional + + +_USERNS = None # type: Optional[bool] + + +def singularity_supports_userns() -> bool: + """Confirm if the version of Singularity install supports the --userns flag.""" + global _USERNS # pylint: disable=global-statement + if _USERNS is None: + try: + hello_image = os.path.join(os.path.dirname(__file__), "hello.simg") + result = Popen( # nosec + ["singularity", "exec", "--userns", hello_image, "true"], + stderr=PIPE, + stdout=DEVNULL, + universal_newlines=True, + ).communicate(timeout=60)[1] + _USERNS = ( + "No valid /bin/sh" in result + or "/bin/sh doesn't exist in container" in result + or "executable file not found in" in result + ) + except TimeoutExpired: + _USERNS = False + return _USERNS diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index 64e0df8972..e994f1415d 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -133,6 +133,7 @@ def jshint_js( js_text: str, globals: Optional[List[str]] = None, options: Optional[Dict[str, Union[List[str], str, int]]] = None, + container_engine: str = "docker", ) -> JSHintJSReturn: if globals is None: globals = [] @@ -165,6 +166,7 @@ def jshint_js( % json_dumps({"code": js_text, "options": options, "globals": globals}), timeout=30, context=jshint_functions_text, + container_engine=container_engine, ) def dump_jshint_error(): @@ -213,6 +215,7 @@ def validate_js_expressions( tool: CommentedMap, schema: Schema, jshint_options: Optional[Dict[str, Union[List[str], str, int]]] = None, + container_engine: str = "docker", ) -> None: if tool.get("requirements") is None: @@ -233,7 +236,7 @@ def validate_js_expressions( for i, expression_lib_line in enumerate(expression_lib): expression_lib_line_errors, expression_lib_line_globals = jshint_js( - expression_lib_line, js_globals, jshint_options + expression_lib_line, js_globals, jshint_options, container_engine ) js_globals.extend(expression_lib_line_globals) print_js_hint_messages( @@ -259,7 +262,7 @@ def validate_js_expressions( code_fragment = unscanned_str[scan_slice[0] + 1 : scan_slice[1]] code_fragment_js = code_fragment_to_js(code_fragment, "") expression_errors, _ = jshint_js( - code_fragment_js, js_globals, jshint_options + code_fragment_js, js_globals, jshint_options, container_engine ) print_js_hint_messages(expression_errors, source_line) diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index 4efc302cd2..0eaa167971 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -1,11 +1,16 @@ -from typing import Any +"""Test sandboxjs.py and related code.""" +import shutil +import os +from pathlib import Path +from typing import Any, List import pytest from cwltool import sandboxjs from cwltool.factory import Factory +from cwltool.loghandler import _logger, configure_logging -from .util import get_data +from .util import get_data, needs_podman, needs_singularity node_versions = [ ("v0.8.26\n", False), @@ -15,6 +20,8 @@ ("v7.7.3\n", True), ] +configure_logging(_logger.handlers[-1], False, True, True, True) + @pytest.mark.parametrize("version,supported", node_versions) def test_node_version(version: str, supported: bool, mocker: Any) -> None: @@ -32,6 +39,61 @@ def test_value_from_two_concatenated_expressions() -> None: assert echo(file1=file) == {"out": "a string\n"} +def hide_nodejs(temp_dir: Path) -> str: + """Generate a new PATH that hides node{js,}.""" + paths: List[str] = os.environ.get("PATH", "").split(":") + names: List[str] = [] + for name in ("nodejs", "node"): + path = shutil.which(name) + if path: + names.append(path) + for name in names: + dirname = os.path.dirname(name) + if dirname in paths: + paths.remove(dirname) + new_dir = temp_dir / os.path.basename(dirname) + new_dir.mkdir() + for entry in os.listdir(dirname): + if entry not in ("nodejs", "node"): + os.symlink(os.path.join(dirname, entry), new_dir / entry) + paths.append(str(new_dir)) + return ":".join(paths) + + +@needs_podman +def test_value_from_two_concatenated_expressions_podman( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Javascript test using podman.""" + new_paths = hide_nodejs(tmp_path) + factory = Factory() + factory.loading_context.podman = True + factory.loading_context.debug = True + factory.runtime_context.debug = True + with monkeypatch.context() as m: + m.setenv("PATH", new_paths) + echo = factory.make(get_data("tests/wf/vf-concat.cwl")) + file = {"class": "File", "location": get_data("tests/wf/whale.txt")} + assert echo(file1=file) == {"out": "a string\n"} + + +@needs_singularity +def test_value_from_two_concatenated_expressions_singularity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Javascript test using Singularity.""" + new_paths = hide_nodejs(tmp_path) + factory = Factory() + factory.loading_context.singularity = True + factory.loading_context.debug = True + factory.runtime_context.debug = True + with monkeypatch.context() as m: + m.setenv("PATH", new_paths) + echo = factory.make(get_data("tests/wf/vf-concat.cwl")) + file = {"class": "File", "location": get_data("tests/wf/whale.txt")} + assert echo(file1=file) == {"out": "a string\n"} + + def test_caches_js_processes(mocker: Any) -> None: sandboxjs.exec_js_process("7", context="{}") diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 6cdc617ad1..b2c707b5e2 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -163,6 +163,7 @@ def test_docker_tmpdir_prefix(tmp_path: Path) -> None: runtime_context.get_tmpdir(), runtime_context.get_stagedir(), INTERNAL_VERSION, + "docker", ) job = DockerCommandLineJob(builder, {}, PathMapper, [], [], "") runtime: List[str] = [] diff --git a/tests/util.py b/tests/util.py index fbca2fb151..cc041f68a5 100644 --- a/tests/util.py +++ b/tests/util.py @@ -52,6 +52,11 @@ def get_data(filename: str) -> str: reason="Requires that version 3.x of singularity executable version is on the system path.", ) +needs_podman = pytest.mark.skipif( + not bool(shutil.which("podman")), + reason="Requires the podman executable on the system path.", +) + _env_accepts_null: Optional[bool] = None