diff --git a/shpc/client/sync.py b/shpc/client/sync.py index 6037a9a97..98cee4a8e 100644 --- a/shpc/client/sync.py +++ b/shpc/client/sync.py @@ -2,9 +2,10 @@ __copyright__ = "Copyright 2021-2022, Vanessa Sochat" __license__ = "MPL 2.0" +import os + import shpc.logger as logger import shpc.utils -import os def sync_registry(args, parser, extra, subparser): diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index 4d03f31b2..435a6a0eb 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -53,7 +53,6 @@ local containerPath = '{{ image }}' -- service environment variable to access docker URI setenv("PODMAN_CONTAINER", containerPath) -set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath) local shellCmd = "{{ command }} ${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} ${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath -- execCmd needs entrypoint to be the executor @@ -61,9 +60,6 @@ local execCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if local runCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath local inspectCmd = "{{ command }} ${PODMAN_OPTS} inspect ${PODMAN_COMMAND_OPTS} " .. containerPath --- set_shell_function takes bashStr and cshStr -set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) - -- conflict with modules with the same name conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) @@ -80,6 +76,11 @@ if (myShellName() == "bash") then {% endfor %} end{% endif %} +{% if wrapper_scripts %}{% else %}set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath) + +-- set_shell_function takes bashStr and cshStr +set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) + -- A customizable exec function set_shell_function("{|module_name|}-exec", execCmd .. " --entrypoint \"\" " .. containerPath .. " \"$@\"", execCmd .. " --entrypoint \"\" " .. containerPath) @@ -87,7 +88,7 @@ set_shell_function("{|module_name|}-exec", execCmd .. " --entrypoint \"\" " .. c set_shell_function("{|module_name|}-run", runCmd .. " \"$@\"", runCmd) -- Inspect runscript or deffile easily! -set_shell_function("{|module_name|}-inspect", inspectCmd, inspectCmd) +set_shell_function("{|module_name|}-inspect", inspectCmd, inspectCmd){% endif %} whatis("Name : " .. myModuleName()) whatis("Version : " .. myModuleVersion()) diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index 936cb6b61..ea2960d57 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -70,7 +70,6 @@ conflict {{ parsed_name.tool }} # service environment variable to access full SIF image path setenv PODMAN_CONTAINER "${containerPath}" -set-alias {|module_name|}-container "echo ${containerPath}" # interactive shell to any container, plus exec for aliases set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if settings.enable_tty %}t{% endif %} --entrypoint {{ shell }} {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" @@ -80,9 +79,6 @@ set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% set runCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}" -# set_shell_function takes bashStr and cshStr -set-alias {|module_name|}-shell "${shellCmd}" - # wrapper scripts? Add bin to path {% if wrapper_scripts %}prepend-path PATH "${moduleDir}/bin"{% endif %} @@ -101,6 +97,11 @@ set-alias {|module_name|}-shell "${shellCmd}" {% endfor %} }{% endif %} +{% if wrapper_scripts %}{% else %} +set-alias {|module_name|}-container "echo ${containerPath}" + +set-alias {|module_name|}-shell "${shellCmd}" + # A customizable exec function if { [ module-info shell bash ] } { set-alias {|module_name|}-exec "${execCmd} --entrypoint \"\" ${containerPath} \"\$@\"" @@ -116,7 +117,7 @@ if { [ module-info shell bash ] } { } # Inspect runscript or deffile easily! -set-alias {|module_name|}-inspect "${inspectCmd} ${containerPath}" +set-alias {|module_name|}-inspect "${inspectCmd} ${containerPath}"{% endif %} #===== # Module options diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index abef2e99a..6ad9c0cc3 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -56,7 +56,6 @@ if not os.getenv("SINGULARITY_COMMAND_OPTS") then setenv ("SINGULARITY_COMMAND_O local containerPath = '{{ container_sif }}' -- service environment variable to access full SIF image path setenv("SINGULARITY_CONTAINER", containerPath) -set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath) -- interactive shell to any container, plus exec for aliases local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath @@ -64,9 +63,6 @@ local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath local inspectCmd = "singularity ${SINGULARITY_OPTS} inspect ${SINGULARITY_COMMAND_OPTS} " --- set_shell_function takes bashStr and cshStr -set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) - -- conflict with modules with the same name conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) @@ -83,6 +79,12 @@ if (myShellName() == "bash") then {% endfor %} end{% endif %} +-- Only set shell functions if we don't use wrapper scripts +{% if wrapper_scripts %}{% else %}set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath) + +-- set_shell_function takes bashStr and cshStr +set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) + -- A customizable exec function set_shell_function("{|module_name|}-exec", execCmd .. containerPath .. " \"$@\"", execCmd .. containerPath) @@ -91,7 +93,8 @@ set_shell_function("{|module_name|}-run", runCmd .. " \"$@\"", runCmd) -- Inspect runscript or deffile easily! set_shell_function("{|module_name|}-inspect-runscript", inspectCmd .. " -r " .. containerPath, inspectCmd .. containerPath) -set_shell_function("{|module_name|}-inspect-deffile", inspectCmd .. " -d " .. containerPath, inspectCmd .. containerPath) +set_shell_function("{|module_name|}-inspect-deffile", inspectCmd .. " -d " .. containerPath, inspectCmd .. containerPath){% endif %} + whatis("Name : " .. myModuleName()) whatis("Version : " .. myModuleVersion()) diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl index aa86f6a14..452f3d758 100644 --- a/shpc/main/modules/templates/singularity.tcl +++ b/shpc/main/modules/templates/singularity.tcl @@ -78,7 +78,6 @@ setenv SINGULARITY_SHELL {{ settings.singularity_shell }} # service environment variable to access full SIF image path setenv SINGULARITY_CONTAINER "${containerPath}" -set-alias {|module_name|}-container "echo ${containerPath}" # interactive shell to any container, plus exec for aliases set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}" @@ -86,10 +85,6 @@ set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}" set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_OPTS} " -# set_shell_function takes bashStr and cshStr -set-alias {|module_name|}-shell "${shellCmd}" - - # if we have any wrapper scripts, add bin to path {% if wrapper_scripts %}prepend-path PATH "${moduleDir}/bin"{% endif %} @@ -108,7 +103,11 @@ set-alias {|module_name|}-shell "${shellCmd}" {% endfor %} }{% endif %} -# A customizable exec function +{% if wrapper_scripts %}{% else %} +set-alias {|module_name|}-shell "${shellCmd}" +set-alias {|module_name|}-container "echo ${containerPath}" + + if { [ module-info shell bash ] } { set-alias {|module_name|}-exec "${execCmd} ${containerPath} \"\$@\"" } else { @@ -124,8 +123,7 @@ if { [ module-info shell bash ] } { # Inspect runscript or deffile easily! set-alias {|module_name|}-inspect-runscript "${inspectCmd} -r ${containerPath}" -set-alias {|module_name|}-inspect-deffile "${inspectCmd} -d ${containerPath}" - +set-alias {|module_name|}-inspect-deffile "${inspectCmd} -d ${containerPath}"{% endif %} #===== # Module options diff --git a/shpc/main/registry/filesystem.py b/shpc/main/registry/filesystem.py index 5c7b6f6e3..a3c4e25bb 100644 --- a/shpc/main/registry/filesystem.py +++ b/shpc/main/registry/filesystem.py @@ -58,7 +58,7 @@ def load_wrapper_script(self, container_tech, script): """ wrapper_script = self.find_wrapper_script(container_tech, script) if wrapper_script: - return os.path.join(self.dirname, wrapper_script) + return shpc.utils.read_file(os.path.join(self.dirname, wrapper_script)) def override_exists(self, tag): """ diff --git a/shpc/main/wrappers/__init__.py b/shpc/main/wrappers/__init__.py index 36e3827e9..65f3e730d 100644 --- a/shpc/main/wrappers/__init__.py +++ b/shpc/main/wrappers/__init__.py @@ -2,21 +2,18 @@ __copyright__ = "Copyright 2022, Vanessa Sochat" __license__ = "MPL 2.0" -import os - -from shpc.logger import logger +from . import generators as gen from .base import WrapperScript -here = os.path.abspath(os.path.dirname(__file__)) - def generate(image, container, config, **kwargs): """ Generate one or more wrapper scripts for a container. Required arguments for all include container (class), image (path), settings - All kwargs go in optional. And this can be extended to include custom arguments. + All kwargs go in optional. The core set of constructor kwargs are provided + to each wrapper generator. This can be extended to include custom arguments. """ # Return list of generated templates generated = [] @@ -33,46 +30,30 @@ def generate(image, container, config, **kwargs): } # Default wrapper for container technology, used for aliases unless overridden + default_wrapper = load_default_wrapper(constructor_kwargs) + + # Generate wrappers for command aliases + generated += gen.alias_wrappers(aliases, default_wrapper, constructor_kwargs) + + # Generate wrappers for container interactions + generated += gen.container_wrappers(constructor_kwargs) + + # Container level wrapper scripts (allow eventually supporting custom podman) + generated += gen.custom_container_wrappers(constructor_kwargs) + return list(set(generated)) + + +def load_default_wrapper(constructor_kwargs): + """ + Given a container and user settings, load a default wrapper. + """ default_wrapper = None + container = constructor_kwargs["container"] + settings = constructor_kwargs["settings"] + default_template_name = settings.wrapper_scripts.get(container.command) if default_template_name: default_wrapper = WrapperScript(default_template_name, **constructor_kwargs) # include_container_dir not set -> only look in the global locations default_wrapper.load_template() - - # Command aliases - custom_wrapper_option_name = "%s_script" % container.templatefile - for alias in aliases: - # Allow overriding the template name in the script option - if custom_wrapper_option_name in alias: - wrapper = WrapperScript( - alias[custom_wrapper_option_name], **constructor_kwargs - ) - wrapper.load_template(include_container_dir=True) - elif default_wrapper: - wrapper = default_wrapper - else: - logger.exit( - "Can't generate a wrapper script for '%s' as there is no template defined for %s" - % (alias["name"], container.templatefile) - ) - - # NB: alias is a dictionary - generated += wrapper.generate(alias["name"], alias) - - # Container level wrapper scripts (allow eventually supporting custom podman) - scripts = { - "singularity": config.singularity_scripts, - "docker": config.docker_scripts, - "podman": config.docker_scripts, - } - # Additional commands defined in the custom container.yaml script section - listing = scripts.get(container.templatefile) or {} - for alias, template_name in listing.items(): - wrapper = WrapperScript(template_name, **constructor_kwargs) - # Template wrapper scripts may live alongside container.yaml - wrapper.load_template(include_container_dir=True) - # NB: alias is a string - generated += wrapper.generate(alias, alias) - - return list(set(generated)) + return default_wrapper diff --git a/shpc/main/wrappers/base.py b/shpc/main/wrappers/base.py index 395783242..f75cd1444 100644 --- a/shpc/main/wrappers/base.py +++ b/shpc/main/wrappers/base.py @@ -104,16 +104,21 @@ def load_template(self, include_container_dir=False): result = self.find_wrapper_script(template_paths, include_container_dir) loader = FileSystemLoader(template_paths) env = Environment(loader=loader) + + # Do we have a filesystem path to load directly? if "path" in result: self.template = env.get_template(self.wrapper_template) + + # Or string content to load? else: self.template = env.from_string(result["content"]) - def generate(self, wrapper_name, alias_definition): + def generate(self, wrapper_name, alias_definition=None): """ Template generation function. NB: alias_definition is a dictionary for command aliases, and a string - for additional arbitrary commands + for additional arbitrary commands. It is not required for container + interaction wrappers (e.g., exec, shell, etc.) """ # Write scripts into container directory diff --git a/shpc/main/wrappers/generators.py b/shpc/main/wrappers/generators.py new file mode 100644 index 000000000..aae32e16f --- /dev/null +++ b/shpc/main/wrappers/generators.py @@ -0,0 +1,116 @@ +__author__ = "Vanessa Sochat" +__copyright__ = "Copyright 2022, Vanessa Sochat" +__license__ = "MPL 2.0" + +import os + +from jinja2 import Template + +from shpc.logger import logger + +from .base import WrapperScript + +# These functions are under the generate namespace, so you can assume +# they generate the content being referenced + + +def alias_wrappers(aliases, default_wrapper, constructor_kwargs): + """ + Given a set of aliases and a default wrapper, generate alias-specific wrappers. + """ + container = constructor_kwargs["container"] + generated = [] + custom_wrapper_option_name = "%s_script" % container.templatefile + for alias in aliases: + # Allow overriding the template name in the script option + if custom_wrapper_option_name in alias: + wrapper = WrapperScript( + alias[custom_wrapper_option_name], **constructor_kwargs + ) + wrapper.load_template(include_container_dir=True) + elif default_wrapper: + wrapper = default_wrapper + else: + logger.exit( + "Can't generate a wrapper script for '%s' as there is no template defined for %s" + % (alias["name"], container.templatefile) + ) + + # NB: alias is a dictionary + generated += wrapper.generate(alias["name"], alias) + return generated + + +def custom_container_wrappers(constructor_kwargs): + """ + Generate wrappers for scripts defined in the container.yaml + """ + container = constructor_kwargs["container"] + config = constructor_kwargs["config"] + + generated = [] + scripts = { + "singularity": config.singularity_scripts, + "docker": config.docker_scripts, + "podman": config.docker_scripts, + } + # Additional commands defined in the custom container.yaml script section + listing = scripts.get(container.templatefile) or {} + for alias, template_name in listing.items(): + wrapper = WrapperScript(template_name, **constructor_kwargs) + # Template wrapper scripts may live alongside container.yaml + wrapper.load_template(include_container_dir=True) + # NB: alias is a string + generated += wrapper.generate(alias, alias) + return generated + + +def container_wrappers(constructor_kwargs): + """ + Generate wrappers for exec / shell / run etc. + """ + # We use the config preference against the settings to generate the prefix + # E.g., module "python" would generate python-shell -shell + config = constructor_kwargs["config"] + settings = constructor_kwargs["settings"] + container = constructor_kwargs["container"] + + template = Template(settings.module_name) + + # docker templates are also for podman + command = container.command + if command == "podman": + command = "docker" + + # Prepare template file-names with prefix (e.g., python) + prefix = template.render(parsed_name=config.name) + template_names = { + f"{prefix}-shell": os.path.join(command, "shell.sh"), + f"{prefix}-container": os.path.join(command, "container.sh"), + f"{prefix}-exec": os.path.join(command, "exec.sh"), + f"{prefix}-run": os.path.join(command, "run.sh"), + } + + # Only singularity has inspect-runscript / inspect-deffile + if command == "singularity": + template_names.update( + { + f"{prefix}-inspect-deffile": os.path.join( + container.command, "inspect-deffile.sh" + ), + f"{prefix}-inspect-runscript": os.path.join( + container.command, "inspect-runscript.sh" + ), + } + ) + else: + template_names.update( + {f"{prefix}-inspect": os.path.join(command, "inspect.sh")} + ) + + generated = [] + for script, template_name in template_names.items(): + wrapper = WrapperScript(template_name, **constructor_kwargs) + wrapper.load_template() + generated += wrapper.generate(script) + return generated diff --git a/shpc/main/wrappers/templates/docker.sh b/shpc/main/wrappers/templates/docker.sh index c9d8dfd6c..eda8b65f1 100755 --- a/shpc/main/wrappers/templates/docker.sh +++ b/shpc/main/wrappers/templates/docker.sh @@ -1,4 +1,4 @@ {% extends "bases/shell-script-base.sh" %} -{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file $moduleDir/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file $moduleDir/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} {% endblock %} diff --git a/shpc/main/wrappers/templates/docker/container.sh b/shpc/main/wrappers/templates/docker/container.sh new file mode 100644 index 000000000..5a9b455e5 --- /dev/null +++ b/shpc/main/wrappers/templates/docker/container.sh @@ -0,0 +1,3 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}echo "{{ image }}"{% endblock %} diff --git a/shpc/main/wrappers/templates/docker/exec.sh b/shpc/main/wrappers/templates/docker/exec.sh new file mode 100755 index 000000000..9de392494 --- /dev/null +++ b/shpc/main/wrappers/templates/docker/exec.sh @@ -0,0 +1,4 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file $moduleDir/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {{ image }} {{ alias.args }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% endblock %} diff --git a/shpc/main/wrappers/templates/docker/inspect.sh b/shpc/main/wrappers/templates/docker/inspect.sh new file mode 100644 index 000000000..3e63162ce --- /dev/null +++ b/shpc/main/wrappers/templates/docker/inspect.sh @@ -0,0 +1,3 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}singularity ${PODMAN_OPTS} inspect ${PODMAN_COMMAND_OPTS} {{ image }}{% endblock %} diff --git a/shpc/main/wrappers/templates/docker/run.sh b/shpc/main/wrappers/templates/docker/run.sh new file mode 100755 index 000000000..9de392494 --- /dev/null +++ b/shpc/main/wrappers/templates/docker/run.sh @@ -0,0 +1,4 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file $moduleDir/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {{ image }} {{ alias.args }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% endblock %} diff --git a/shpc/main/wrappers/templates/docker/shell.sh b/shpc/main/wrappers/templates/docker/shell.sh new file mode 100755 index 000000000..4081f6ff6 --- /dev/null +++ b/shpc/main/wrappers/templates/docker/shell.sh @@ -0,0 +1,4 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file $moduleDir/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} --entrypoint {{ settings.wrapper_shell }} {{ image }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% endblock %} diff --git a/shpc/main/wrappers/templates/singularity/container.sh b/shpc/main/wrappers/templates/singularity/container.sh new file mode 100644 index 000000000..5a9b455e5 --- /dev/null +++ b/shpc/main/wrappers/templates/singularity/container.sh @@ -0,0 +1,3 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}echo "{{ image }}"{% endblock %} diff --git a/shpc/main/wrappers/templates/singularity/exec.sh b/shpc/main/wrappers/templates/singularity/exec.sh new file mode 100644 index 000000000..d2b70eaf1 --- /dev/null +++ b/shpc/main/wrappers/templates/singularity/exec.sh @@ -0,0 +1,4 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B $moduleDir/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ image }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% endblock %} diff --git a/shpc/main/wrappers/templates/singularity/inspect-deffile.sh b/shpc/main/wrappers/templates/singularity/inspect-deffile.sh new file mode 100644 index 000000000..26fafcb7f --- /dev/null +++ b/shpc/main/wrappers/templates/singularity/inspect-deffile.sh @@ -0,0 +1,3 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}singularity ${SINGULARITY_OPTS} inspect ${SINGULARITY_COMMAND_OPTS} -d {{ image }}{% endblock %} diff --git a/shpc/main/wrappers/templates/singularity/inspect-runscript.sh b/shpc/main/wrappers/templates/singularity/inspect-runscript.sh new file mode 100644 index 000000000..7d8976e73 --- /dev/null +++ b/shpc/main/wrappers/templates/singularity/inspect-runscript.sh @@ -0,0 +1,3 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}singularity ${SINGULARITY_OPTS} inspect ${SINGULARITY_COMMAND_OPTS} -r {{ image }}{% endblock %} diff --git a/shpc/main/wrappers/templates/singularity/run.sh b/shpc/main/wrappers/templates/singularity/run.sh new file mode 100644 index 000000000..5047be356 --- /dev/null +++ b/shpc/main/wrappers/templates/singularity/run.sh @@ -0,0 +1,4 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B $moduleDir/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ image }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% endblock %} diff --git a/shpc/main/wrappers/templates/singularity/shell.sh b/shpc/main/wrappers/templates/singularity/shell.sh new file mode 100644 index 000000000..f1638686d --- /dev/null +++ b/shpc/main/wrappers/templates/singularity/shell.sh @@ -0,0 +1,4 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B $moduleDir/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} -s {{ settings.wrapper_shell }} {{ image }} +{% endblock %} diff --git a/shpc/tests/test_wrappers.py b/shpc/tests/test_wrappers.py index 69539e0f5..f066c1266 100644 --- a/shpc/tests/test_wrappers.py +++ b/shpc/tests/test_wrappers.py @@ -79,16 +79,81 @@ def test_get_registry_wrapper_script(tmp_path, remote): # This is an existing one assert result.find_wrapper_script("singularity", "singularity_fork.sh") script = result.load_wrapper_script("singularity", "singularity_fork.sh") - if not remote: - assert os.path.exists(script) - else: - assert '{% extends "bases/shell-script-base.sh" %}' in script + + # Regardless of source, we should get the correct content + assert '{% extends "bases/shell-script-base.sh" %}' in script # This is not assert not result.find_wrapper_script("singularity", "singularity.sh") assert not result.load_wrapper_script("singularity", "singularity.sh") +@pytest.mark.parametrize( + "module_sys,module_file,container_tech", + [ + ("lmod", "module.lua", "singularity"), + ("lmod", "module.lua", "podman"), + ("tcl", "module.tcl", "singularity"), + ("tcl", "module.tcl", "podman"), + ], +) +def test_wrapper_install(tmp_path, module_sys, module_file, container_tech): + """ + Test that install of wrapper scripts produces expected files in bin + """ + client = init_client(str(tmp_path), module_sys, container_tech) + + assert client.settings.get("wrapper_scripts:enabled") == True + + # Install known tag + client.install("python:3.9.2-alpine") + module_dir = os.path.join(client.settings.module_base, "python", "3.9.2-alpine") + assert os.path.exists(module_dir) + assert "bin" in os.listdir(module_dir) + module_bin = os.path.join(module_dir, "bin") + assert os.path.exists(module_bin) + + requireds = [ + "python-container", + "python-run", + "python-exec", + "python", + "python-shell", + ] + if container_tech == "singularity": + requireds += ["python-inspect-runscript", "python-inspect-deffile"] + else: + requireds += ["python-inspect"] + binaries = os.listdir(module_bin) + for required in requireds: + assert required in binaries + + +@pytest.mark.parametrize( + "module_sys,module_file,container_tech", + [ + ("lmod", "module.lua", "singularity"), + ("lmod", "module.lua", "podman"), + ("tcl", "module.tcl", "singularity"), + ("tcl", "module.tcl", "podman"), + ], +) +def test_disabled_wrapper_install(tmp_path, module_sys, module_file, container_tech): + """ + Test that install of wrapper scripts produces expected files in bin + """ + client = init_client(str(tmp_path), module_sys, container_tech) + + client.settings.set("wrapper_scripts:enabled", False) + assert client.settings.get("wrapper_scripts:enabled") == False + + # Install known tag + client.install("python:3.9.2-alpine") + module_dir = os.path.join(client.settings.module_base, "python", "3.9.2-alpine") + assert os.path.exists(module_dir) + assert "bin" not in os.listdir(module_dir) + + @pytest.mark.parametrize( "make_absolute,exists", [