Merge pull request #85 from databio/dev

Release 0.7.3

.github/workflows/pytest.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.9]
         os: [ubuntu-latest]
 
     steps:

README.md

@@ -1,7 +1,5 @@
 # Bulker
 
-[![PEP compatible](https://pepkit.github.io/img/PEP-compatible-green.svg)](http://pepkit.github.io) [![Build Status](https://travis-ci.org/databio/bulker.svg?branch=master)](https://travis-ci.org/databio/bulker)
-
 Bulker builds multi-container computing environments that are both **modular** and **interactive**. 
 
 For details, see the [bulker documentation](http://docs.bulker.io). You can find and share manifests at [bulker hub](http://hub.bulker.io).

bash_complete.sh

@@ -1,28 +1,28 @@
-_bulker_complete() {
-  local cur prev
-
-  COMPREPLY=()
-  cur=${COMP_WORDS[COMP_CWORD]}
-  prev=${COMP_WORDS[COMP_CWORD-1]}
-
-  if [ $COMP_CWORD -eq 1 ]; then
-    cmds=`bulker --commands`
-    COMPREPLY=( $(compgen -W "${cmds}" -- ${cur}) )
-  elif [ $COMP_CWORD -eq 2 ]; then
-    case "$prev" in
-      "run")
-        cmds=`bulker list --simple`
-        COMPREPLY=( $(compgen -W "${cmds}" -- ${cur}) )
-        ;;
-      "activate")
-        cmds=`bulker list --simple`
-        COMPREPLY=( $(compgen -W "${cmds}"  -- ${cur}) )
-        ;;
-      *)
-        COMPREPLY=()
-        ;;
+# Begin bulker bash autocomplete
+_bulker_autocomplete()
+{
+    local cur prev opts1 opts2
+    cur=${COMP_WORDS[COMP_CWORD]}
+    prev=${COMP_WORDS[COMP_CWORD-1]}
+    opts1=$(bulker --commands)
+    opts2=$(bulker list --simple)
+    case ${COMP_CWORD} in
+        1)
+            COMPREPLY=($(compgen -W "${opts1}" -- ${cur}))
+            ;;
+        2)
+            case ${prev} in
+                "activate"|"run")
+                    COMPREPLY=($(compgen -W "${opts2}" -- ${cur}))
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            ;;
+        *)
+            COMPREPLY=()
+            ;;
     esac
-  fi
-
-  return 0
-} && complete -F _bulker_complete bulker
+} && complete -o bashdefault -o default -F _bulker_autocomplete bulker
+# end bulker bash autocomplete

bulker/_version.py

@@ -1 +1 @@
-__version__ = "0.7.2"
+__version__ = "0.7.3"

bulker/autocomplete/autocomplete.bash

@@ -0,0 +1,44 @@
+# begin bulker bash completion
+_bulker_autocomplete()
+{
+   local cur prev opts
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    # opts=$(mm -l)
+    opts=$(bulker list --simple)
+    COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+    return 0
+}
+complete -o nospace -F _bulker_autocomplete bulker
+# end bulker bash completion
+
+
+_bulker_autocomplete_layer()
+{
+    local cur prev opts1 opts2
+    cur=${COMP_WORDS[COMP_CWORD]}
+    prev=${COMP_WORDS[COMP_CWORD-1]}
+    opts1=$(bulker --commands)
+    opts2=$(bulker list --simple)
+    case ${COMP_CWORD} in
+        1)
+            COMPREPLY=($(compgen -W "${opts1}" -- ${cur}))
+            ;;
+        2)
+            case ${prev} in
+                activate)
+                    COMPREPLY=($(compgen -W "${opts2}" -- ${cur}))
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            ;;
+        *)
+            COMPREPLY=()
+            ;;
+    esac
+}
+
+complete -o bashdefault -o default -F _bulker_autocomplete_layer bulker

bulker/bulker.py

@@ -339,7 +339,7 @@ def bulker_init(config_path, template_config_path, container_engine=None):
         # templates_subdir =  TEMPLATE_SUBDIR
         copy_tree(os.path.dirname(template_config_path), dest_templates_dir)
         new_template = os.path.join(dest_folder, os.path.basename(template_config_path))
-        bulker_config = yacman.YacAttMap(filepath=template_config_path, writable=True)
+        bulker_config = yacman.YacAttMap(filepath=template_config_path, writable=False, skip_read_lock=True)
         _LOGGER.debug("Engine used: {}".format(container_engine))
         bulker_config.bulker.container_engine = container_engine
         if bulker_config.bulker.container_engine == "docker":
@@ -1262,7 +1262,7 @@ def main():
 
     bulkercfg = select_bulker_config(args.config)
     bulker_config = yacman.YacAttMap(filepath=bulkercfg, writable=False)
-    _LOGGER.info("Bulker config: {}".format(bulkercfg))
+    # _LOGGER.info("Bulker config: {}".format(bulkercfg))
 
     if args.command == "envvars":
         if args.add:
@@ -1308,18 +1308,28 @@ def main():
 
         if args.simple:
             fmt = "{namespace}/{crate}:{tag}"
+            crateslist = []
+            if bulker_config.bulker.crates:
+                for namespace, crates in bulker_config.bulker.crates.items():
+                    for crate, tags in crates.items():
+                        for tag, path in tags.items():
+                            crateslist.append(fmt.format(namespace=namespace, crate=crate, 
+                                            tag=tag, path=path))
+
+            print(" ".join(crateslist))
+
         else:
             _LOGGER.info("Available crates:")
             fmt = "{namespace}/{crate}:{tag} -- {path}"
 
-        if bulker_config.bulker.crates:
-            for namespace, crates in bulker_config.bulker.crates.items():
-                for crate, tags in crates.items():
-                    for tag, path in tags.items():
-                        print(fmt.format(namespace=namespace, crate=crate, 
-                                         tag=tag, path=path))
-        else:
-            _LOGGER.info("No crates available. Use 'bulker load' to load a crate.")
+            if bulker_config.bulker.crates:
+                for namespace, crates in bulker_config.bulker.crates.items():
+                    for crate, tags in crates.items():
+                        for tag, path in tags.items():
+                            print(fmt.format(namespace=namespace, crate=crate, 
+                                            tag=tag, path=path))
+            else:
+                _LOGGER.info("No crates available. Use 'bulker load' to load a crate.")
         sys.exit(1)
 
     # For all remaining commands we need a crate identifier

docs/autocomplete.md

@@ -0,0 +1,6 @@
+# How to enable bash autocompletion
+
+Bulker provides bash autocompletion. The autocompletion script can be found in [bash_complete.sh](https://github.com/databio/bulker/blob/master/bash_complete.sh). Add the contents of this file to your `.bashrc` or `.profile` so that it runs whenever the shell starts.
+
+After reloading the shell, when you type `bulker` and hit `<tab><tab>`, it will populate the list of bulker commands. When you type `bulker run` or `bulker activate` and hit `<tab><tab>`, it will list your available crates.
+

docs/changelog.md

@@ -1,9 +1,13 @@
 # Changelog
 
+## [0.7.3] -- 2021-12-08
+- Fixed a bug that prevented use when installed in non-writable directory
+- Added ability to use bash autocompletion
+
 ## [0.7.2] -- 2021-06-24
 - Update to yacman 0.8.0, fixing references to internal config attributes.
 - Add documentation and clarity for the `shell_prompt` option
-- Fixed bug with relative crate root folders.
+- Fixed bug with relative crate root folders
 
 ## [0.7.1] -- 2021-03-03
 - Fixed bug in bulker reload.

docs/manifest.md

@@ -17,19 +17,30 @@ manifest:
 ```
 1. *command* is the executable name; this is what the user will type in to run the command (*e.g.* `cowsay`).
 
-2. *docker_args* is any additional arguments required by this tool. You should add `-i` for tools that need to read/write piped output to/from stdin and stdout, and add `-t` for commands like `python` or `R` that allocate a user interface.
+2. *docker_args* is any additional arguments required by this tool. You should add `-i` for tools that need to read/write piped output to/from stdin and stdout, and add `-t` for commands like `python` or `R` that allocate a user interface. For singularity, these additions aren't usually necessary, but you could adjust a singularity argument with `singularity_args` if you really needed to.
 
 3. *docker_image* is the location of the image.
 
 4. *docker_command* is the command that will be executed inside the container. This is often the same as the `command` itself for the user, but it doesn't have to be. If you leave this out, `bulker` will use the value of the `command` attribute by default.
 
 That's it. If you want your manifest to specify specific versions of images, make sure you include the tags in your `docker_image` strings.
 
+## Manifests and singularity
+
+The examples above are docker-centric, but if you're using singularity, you may wonder if you have to do anything different in your manifest. The answer is *not really* -- one of the goals of bulker is to be able to work with a single manifest, and have that manifest work with either docker *or* singularity. So the idea is to create a single manifest, and it should just work with both.
+
+That said, there is some potential for confusion. First, the bulker manifest points at docker images. When you're using singularity, bulker will use singularity to convert these into singularity images -- but they have to be docker images to begin with. This is how bulker manages to use a single manifest for either system. If I allowed you to use singularity images instead of docker images in the bulker manifest, then I'd need to convert those into docker images for the manifest to work with docker. It's just easier the other way around, because singularity has easy built-in methods to use docker containers, so that's what bulker uses.
+
+Second, the `docker_args` argument is only used with docker. In my use cases, I've never had a need to do something like this with singularity, because the nuances of the docker flags I use like `-i` and `-t` aren't relevant in a singularity command. Nevertheless, there is an analogous `singularity_args` argument if you find there are some singularity-specific flags you need to pass.
+
+Similarly, in an ideal world, you'd never need to use `docker_command` because the `command` would be the same. But for some containers, depending on how it uses a docker ENTRYPOINT, it might be required to tweak the `command` that is sent to docker. For singularity, bulker will try to use the same command specified under `docker_command`, but if you really have a situation where you need the command to differ between docker and singularity, the you can use `singularity_command` to specify the difference. I don't believe I've ever had to use this, though.
+
+In conclusion, all these example manifests should work in the exact same way with either docker or singularity. The reason the manifests appear docker-centric is that bulker considers the docker image the ground truth, and then uses singularity to build from that image, rather than the other way around -- but in no way does this mean you can't use singularity with bulker; as mentioned, that's one of the main strengths of bulker.
+
 ## Examples
 
 Check out these examples at [http://hub.bulker.io](http://hub.bulker.io):
 
-
 Demos:
 
 - [demo manifest](http://hub.bulker.io/bulker/demo.yaml) - Cowsay and fortune example

mkdocs.yml

@@ -26,6 +26,7 @@ nav:
     - Use a bulker registry: registry.md
     - Use images with entrypoints: entrypoint.md
     - Disable user or network map: advanced_templates.md
+    - Enable bash autocompletion: autocomplete.md
   - Reference:
     - Tips: tips.md
     - Support: https://github.com/databio/bulker/issues

requirements/requirements-all.txt

@@ -1,4 +1,4 @@
-yacman>=0.8.1
+yacman>=0.8.4
 pyyaml>=5.1
 logmuse>=0.2.0
 jinja2

tests/test_bulker.py

@@ -99,9 +99,9 @@ def test_nonconfig_load():
                                                      "demo",
                                                      None)
     manifest, cratevars = load_remote_registry_path(bulker_config, "demo", None)
-    exe_template = mkabs(bulker_config.bulker.executable_template, os.path.dirname(bulker_config._file_path))
+    exe_template = mkabs(bulker_config.bulker.executable_template, os.path.dirname(bulker_config.__internal.file_path))
     shell_template = mkabs(bulker_config.bulker.shell_template,
-                         os.path.dirname(bulker_config._file_path))        
+                         os.path.dirname(bulker_config.__internal.file_path))        
     import jinja2
     with open(exe_template, 'r') as f:
         contents = f.read()