From 5f7bb40259f644e74db6b3f662dc45425dc240e7 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Tue, 5 Oct 2021 20:45:53 +0000
Subject: [PATCH 01/26] implement deep_merge functionally

---
 scompose/client/__init__.py |  8 ++++-
 scompose/project/project.py | 63 ++++++++++++++++++++++++++++++-------
 2 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/scompose/client/__init__.py b/scompose/client/__init__.py
index 9393c08..ff62a3a 100644
--- a/scompose/client/__init__.py
+++ b/scompose/client/__init__.py
@@ -54,7 +54,8 @@ def get_parser():
         "-f",
         dest="file",
         help="specify compose file (default singularity-compose.yml)",
-        default="singularity-compose.yml",
+        action="append",
+        default=[],
     )
 
     parser.add_argument(
@@ -225,6 +226,11 @@ def show_help(return_code=0):
         print(scompose.__version__)
         sys.exit(0)
 
+    # argparse inherits a funny behaviour that appends default values to the dest value whether you've specified a value
+    # or not. The bug/behaviour is documented here: https://bugs.python.org/issue16399
+    if len(args.file) == 0:
+        args.file = ["singularity-compose.yml"]
+
     # Does the user want a shell?
     if args.command == "build":
         from scompose.client.build import main
diff --git a/scompose/project/project.py b/scompose/project/project.py
index ef80997..53e81a4 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -64,8 +64,8 @@ def set_filename(self, filename):
         ==========
         filename: the singularity-compose.yml file to use
         """
-        self.filename = filename or "singularity-compose.yml"
-        self.working_dir = os.path.dirname(os.path.abspath(self.filename))
+        self.filename = filename
+        self.working_dir = os.getcwd()
 
     def set_name(self, name):
         """set the filename to read the recipe from. If not provided, defaults
@@ -75,11 +75,9 @@ def set_name(self, name):
         ==========
         name: if a customize name is provided, use it
         """
-        pwd = os.path.basename(os.path.dirname(os.path.abspath(self.filename)))
-        self.name = (name or pwd).lower()
+        self.name = (name or self.working_dir).lower()
 
     # Listing
-
     def ps(self):
         """ps will print a table of instances, including pids and names."""
         instance_names = self.get_instance_names()
@@ -131,7 +129,6 @@ def get_instance(self, name):
         return instance
 
     # Loading Functions
-
     def get_already_running(self):
         """Since a user can bring select instances up and down, we need to
         derive a list of already running instances to include
@@ -148,15 +145,59 @@ def get_already_running(self):
     def load(self):
         """load a singularity-compose.yml recipe, and validate it."""
 
-        if not os.path.exists(self.filename):
-            bot.error("%s does not exist." % self.filename)
-            sys.exit(1)
-
         try:
-            self.config = read_yaml(self.filename, quiet=True)
+            yaml_files = []
+
+            for f in self.filename:
+                # ensure file exists
+                if not os.path.exists(f):
+                    bot.error("%s does not exist." % f)
+                    sys.exit(1)
+                # read yaml file
+                yaml_files.append(read_yaml(f, quiet=True))
+
+            # merge/override yaml properties where applicable
+            self.config = self.deep_merge(yaml_files)
         except:  # ParserError
             bot.exit("Cannot parse %s, invalid yaml." % self.filename)
 
+    def deep_merge(self, yaml_files):
+        """merge singularity-compose.yml files into a single dict"""
+        if len(yaml_files) == 1:
+            # nothing to merge as the user specified a single file
+            return yaml_files[0]
+
+        base_yaml = None
+        for idx, item in enumerate(yaml_files):
+            if idx == 0:
+                base_yaml = item
+            else:
+                base_yaml = self.merge(base_yaml, item)
+
+        return base_yaml
+
+    def merge(self, a, b):
+        """merge dict b into a"""
+        for key in b:
+            if key in a:
+                # merge dicts recursively
+                if isinstance(a[key], dict) and isinstance(b[key], dict):
+                    a[key] = self.merge(a[key], b[key])
+
+                # if types are equal, b takes precedence
+                elif isinstance(a[key], type(b[key])):
+                    a[key] = b[key]
+
+                # if nothing matches then this means a conflict of types which should exist in the first place
+                else:
+                    bot.error(
+                        "key %s has property type mismatch in different files." % key
+                    )
+                    sys.exit(1)
+            else:
+                a[key] = b[key]
+        return a
+
     def parse(self):
         """parse a loaded config"""
 

From 8c7a5921419863c277ead6bd32dc986ff38ebb71 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Tue, 5 Oct 2021 20:47:52 +0000
Subject: [PATCH 02/26] fix typo

---
 scompose/project/project.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scompose/project/project.py b/scompose/project/project.py
index 53e81a4..c6ba411 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -188,7 +188,7 @@ def merge(self, a, b):
                 elif isinstance(a[key], type(b[key])):
                     a[key] = b[key]
 
-                # if nothing matches then this means a conflict of types which should exist in the first place
+                # if nothing matches then this means a conflict of types which shouldn't exist in the first place
                 else:
                     bot.error(
                         "key %s has property type mismatch in different files." % key

From 5491961c049b0c4bf54ba6fb785063390f452af3 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Tue, 5 Oct 2021 22:47:07 +0000
Subject: [PATCH 03/26] implement some changes suggested during code review

---
 scompose/project/project.py | 66 +++++++------------------------------
 scompose/utils/__init__.py  | 49 +++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 55 deletions(-)

diff --git a/scompose/project/project.py b/scompose/project/project.py
index c6ba411..55cffff 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -10,7 +10,7 @@
 
 from scompose.templates import get_template
 from scompose.logger import bot
-from scompose.utils import read_yaml, read_file, write_file
+from scompose.utils import read_file, write_file, build_interpolated_config
 from spython.main import get_client
 from .instance import Instance
 from ipaddress import IPv4Network
@@ -64,7 +64,14 @@ def set_filename(self, filename):
         ==========
         filename: the singularity-compose.yml file to use
         """
-        self.filename = filename
+        default_value = ["singularity-compose.yml"]
+        if filename is None:
+            self.filenames = default_value
+        elif isinstance(filename, list):
+            self.filenames = filename or default_value
+        else:
+            self.filenames = [filename]
+
         self.working_dir = os.getcwd()
 
     def set_name(self, name):
@@ -144,59 +151,8 @@ def get_already_running(self):
 
     def load(self):
         """load a singularity-compose.yml recipe, and validate it."""
-
-        try:
-            yaml_files = []
-
-            for f in self.filename:
-                # ensure file exists
-                if not os.path.exists(f):
-                    bot.error("%s does not exist." % f)
-                    sys.exit(1)
-                # read yaml file
-                yaml_files.append(read_yaml(f, quiet=True))
-
-            # merge/override yaml properties where applicable
-            self.config = self.deep_merge(yaml_files)
-        except:  # ParserError
-            bot.exit("Cannot parse %s, invalid yaml." % self.filename)
-
-    def deep_merge(self, yaml_files):
-        """merge singularity-compose.yml files into a single dict"""
-        if len(yaml_files) == 1:
-            # nothing to merge as the user specified a single file
-            return yaml_files[0]
-
-        base_yaml = None
-        for idx, item in enumerate(yaml_files):
-            if idx == 0:
-                base_yaml = item
-            else:
-                base_yaml = self.merge(base_yaml, item)
-
-        return base_yaml
-
-    def merge(self, a, b):
-        """merge dict b into a"""
-        for key in b:
-            if key in a:
-                # merge dicts recursively
-                if isinstance(a[key], dict) and isinstance(b[key], dict):
-                    a[key] = self.merge(a[key], b[key])
-
-                # if types are equal, b takes precedence
-                elif isinstance(a[key], type(b[key])):
-                    a[key] = b[key]
-
-                # if nothing matches then this means a conflict of types which shouldn't exist in the first place
-                else:
-                    bot.error(
-                        "key %s has property type mismatch in different files." % key
-                    )
-                    sys.exit(1)
-            else:
-                a[key] = b[key]
-        return a
+        # merge/override yaml properties where applicable
+        self.config = build_interpolated_config(self.filenames)
 
     def parse(self):
         """parse a loaded config"""
diff --git a/scompose/utils/__init__.py b/scompose/utils/__init__.py
index 5e886dd..eadd926 100644
--- a/scompose/utils/__init__.py
+++ b/scompose/utils/__init__.py
@@ -150,6 +150,55 @@ def _read_yaml(section, quiet=False):
     return metadata
 
 
+def build_interpolated_config(file_list):
+    yaml_files = []
+    for f in file_list:
+        try:
+            # ensure file exists
+            if not os.path.exists(f):
+                print("%s does not exist." % f)
+                sys.exit(1)
+            # read yaml file
+            yaml_files.append(read_yaml(f, quiet=True))
+        except:  # ParserError
+            print("Cannot parse %s, invalid yaml." % f)
+            sys.exit(1)
+
+    # merge/override yaml properties where applicable
+    return _deep_merge(yaml_files)
+
+
+def _deep_merge(yaml_files):
+    """merge yaml files into a single dict"""
+    base_yaml = None
+    for idx, item in enumerate(yaml_files):
+        if idx == 0:
+            base_yaml = item
+        else:
+            base_yaml = _merge(base_yaml, item)
+
+    return base_yaml
+
+
+def _merge(self, a, b):
+    """merge dict b into a"""
+    for key in b:
+        if key in a:
+            # merge dicts recursively
+            if isinstance(a[key], dict) and isinstance(b[key], dict):
+                a[key] = self.merge(a[key], b[key])
+            # if types are equal, b takes precedence
+            elif isinstance(a[key], type(b[key])):
+                a[key] = b[key]
+            # if nothing matches then this means a conflict of types which shouldn't exist in the first place
+            else:
+                print("key '%s': type mismatch in different files." % key)
+                sys.exit(1)
+        else:
+            a[key] = b[key]
+    return a
+
+
 # Json
 
 

From 2285abac45d35d0beaca63b79da03f1c99bbe9d4 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Wed, 6 Oct 2021 03:54:33 +0000
Subject: [PATCH 04/26] add tests for the config override feature

---
 .../config_override/singularity-compose-1.yml |   8 ++
 .../config_override/singularity-compose-2.yml |  12 ++
 scompose/tests/test_utils.py                  | 106 ++++++++++++++++++
 scompose/utils/__init__.py                    |   4 +-
 4 files changed, 128 insertions(+), 2 deletions(-)
 create mode 100644 scompose/tests/configs/config_override/singularity-compose-1.yml
 create mode 100644 scompose/tests/configs/config_override/singularity-compose-2.yml

diff --git a/scompose/tests/configs/config_override/singularity-compose-1.yml b/scompose/tests/configs/config_override/singularity-compose-1.yml
new file mode 100644
index 0000000..f4b860c
--- /dev/null
+++ b/scompose/tests/configs/config_override/singularity-compose-1.yml
@@ -0,0 +1,8 @@
+version: "2.0"
+instances:
+  echo:
+    build:
+      context: .
+      recipe: Singularity
+    start:
+      args: "arg0 arg1 arg2"
diff --git a/scompose/tests/configs/config_override/singularity-compose-2.yml b/scompose/tests/configs/config_override/singularity-compose-2.yml
new file mode 100644
index 0000000..139e7b7
--- /dev/null
+++ b/scompose/tests/configs/config_override/singularity-compose-2.yml
@@ -0,0 +1,12 @@
+version: "2.0"
+instances:
+  echo:
+    start:
+      options:
+        - fakeroot
+      args: "arg0 arg1"
+
+  hello:
+    image: from_the_other_side.sif
+    start:
+      args: "how are you?"
diff --git a/scompose/tests/test_utils.py b/scompose/tests/test_utils.py
index e2f45d4..02637d9 100644
--- a/scompose/tests/test_utils.py
+++ b/scompose/tests/test_utils.py
@@ -9,6 +9,8 @@
 import os
 import pytest
 
+here = os.path.dirname(os.path.abspath(__file__))
+
 
 def test_write_read_files(tmp_path):
     """test_write_read_files will test the functions write_file and read_file"""
@@ -88,3 +90,107 @@ def test_print_json():
 
     result = print_json({1: 1})
     assert result == '{\n    "1": 1\n}'
+
+
+def test_merge():
+    print("Testing utils._merge")
+    from scompose.utils import _merge
+
+    # No override
+    a = {"a": 123}
+    b = {"b": 456}
+    assert _merge(a, b) == {"a": 123, "b": 456}
+
+    # Override + merge
+    a = {"a": 123}
+    b = {"b": 456, "a": 789}
+    assert _merge(a, b) == {"a": 789, "b": 456}
+
+    # Override only
+    a = {"a": 123}
+    b = {"a": 789}
+    assert _merge(a, b) == {"a": 789}
+
+    # Dict merge
+    a = {"a": 123, "b": {"c": "d"}}
+    b = {"b": {"e": "f"}}
+    assert _merge(a, b) == {"a": 123, "b": {"c": "d", "e": "f"}}
+
+    # Dict merge + key override
+    a = {"a": 123, "b": {"c": "d"}}
+    b = {"b": {"c": "f"}}
+    assert _merge(a, b) == {"a": 123, "b": {"c": "f"}}
+
+
+def test_deep_merge():
+    print("Testing utils._deep_merge")
+    from scompose.utils import _deep_merge, read_yaml
+    config_override = os.path.join(here, "configs", "config_override")
+
+    # single file
+    yaml_files = [
+        read_yaml(
+            os.path.join(config_override, "singularity-compose-1.yml"), quiet=True
+        )
+    ]
+    ret = _deep_merge(yaml_files)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1 arg2"},
+        }
+    }
+
+    # multiple files
+    yaml_files = [
+        read_yaml(
+            os.path.join(config_override, "singularity-compose-1.yml"), quiet=True
+        ),
+        read_yaml(
+            os.path.join(config_override, "singularity-compose-2.yml"), quiet=True
+        ),
+    ]
+    ret = _deep_merge(yaml_files)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1", "options": ["fakeroot"]},
+        },
+        "hello": {
+            "image": "from_the_other_side.sif",
+            "start": {"args": "how are you?"},
+        },
+    }
+
+
+def test_build_interpolated_config():
+    print("Testing utils.build_interpolated_config")
+    from scompose.utils import build_interpolated_config
+    config_override = os.path.join(here, "configs", "config_override")
+
+    # single file
+    file_list = [os.path.join(config_override, "singularity-compose-1.yml")]
+    ret = build_interpolated_config(file_list)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1 arg2"},
+        }
+    }
+
+    # multiple files
+    file_list = [
+            os.path.join(config_override, "singularity-compose-1.yml"),
+            os.path.join(config_override, "singularity-compose-2.yml"),
+    ]
+    ret = build_interpolated_config(file_list)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1", "options": ["fakeroot"]},
+        },
+        "hello": {
+            "image": "from_the_other_side.sif",
+            "start": {"args": "how are you?"},
+        },
+    }
diff --git a/scompose/utils/__init__.py b/scompose/utils/__init__.py
index eadd926..b52b940 100644
--- a/scompose/utils/__init__.py
+++ b/scompose/utils/__init__.py
@@ -180,13 +180,13 @@ def _deep_merge(yaml_files):
     return base_yaml
 
 
-def _merge(self, a, b):
+def _merge(a, b):
     """merge dict b into a"""
     for key in b:
         if key in a:
             # merge dicts recursively
             if isinstance(a[key], dict) and isinstance(b[key], dict):
-                a[key] = self.merge(a[key], b[key])
+                a[key] = _merge(a[key], b[key])
             # if types are equal, b takes precedence
             elif isinstance(a[key], type(b[key])):
                 a[key] = b[key]

From e2fbef0d688dd0b62329077f9171141e9859dd5e Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Wed, 6 Oct 2021 03:56:05 +0000
Subject: [PATCH 05/26] good old black formatting that I keep forgetting :)

---
 scompose/tests/test_utils.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scompose/tests/test_utils.py b/scompose/tests/test_utils.py
index 02637d9..5eb5b52 100644
--- a/scompose/tests/test_utils.py
+++ b/scompose/tests/test_utils.py
@@ -125,6 +125,7 @@ def test_merge():
 def test_deep_merge():
     print("Testing utils._deep_merge")
     from scompose.utils import _deep_merge, read_yaml
+
     config_override = os.path.join(here, "configs", "config_override")
 
     # single file
@@ -166,6 +167,7 @@ def test_deep_merge():
 def test_build_interpolated_config():
     print("Testing utils.build_interpolated_config")
     from scompose.utils import build_interpolated_config
+
     config_override = os.path.join(here, "configs", "config_override")
 
     # single file
@@ -180,8 +182,8 @@ def test_build_interpolated_config():
 
     # multiple files
     file_list = [
-            os.path.join(config_override, "singularity-compose-1.yml"),
-            os.path.join(config_override, "singularity-compose-2.yml"),
+        os.path.join(config_override, "singularity-compose-1.yml"),
+        os.path.join(config_override, "singularity-compose-2.yml"),
     ]
     ret = build_interpolated_config(file_list)
     assert ret["instances"] == {

From 7deee7ddcdb0cf5ba440b00e80cd2c94ba6c9ddf Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Wed, 6 Oct 2021 04:31:15 +0000
Subject: [PATCH 06/26] add docs

---
 docs/commands.md            | 125 +++++++++++++++++++++++++++++++++---
 scompose/client/__init__.py |   2 +-
 2 files changed, 117 insertions(+), 10 deletions(-)

diff --git a/docs/commands.md b/docs/commands.md
index 3e6b64c..f889486 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -3,7 +3,7 @@
 The following commands are currently supported. Remember, you must be in the 
 present working directory of the compose file to reference the correct instances.
 
-## Build
+## build
 
 Build will either build a container recipe, or pull a container to the
 instance folder. In both cases, it's named after the instance so we can
@@ -21,7 +21,7 @@ If the build requires sudo (if you've defined sections in the config that warran
 setting up networking with sudo) the build will instead give you an instruction
 to run with sudo.
 
-## Up
+## up
 
 If you want to both build and bring them up, you can use "up." Note that for
 builds that require sudo, this will still stop and ask you to build with sudo.
@@ -52,7 +52,7 @@ $ singularity-compose up --no-resolv
 Creating app
 ```
 
-## Create
+## create
 
 Given that you have built your containers with `singularity-compose build`,
 you can create your instances as follows:
@@ -93,7 +93,7 @@ INSTANCES  NAME PID     IMAGE
 3         nginx	6543	nginx.sif
 ```
 
-## Shell
+## shell
 
 It's sometimes helpful to peek inside a running instance, either to look at permissions,
 inspect binds, or manually test running something.
@@ -104,7 +104,7 @@ $ singularity-compose shell app
 Singularity app.sif:~/Documents/Dropbox/Code/singularity/singularity-compose-example> 
 ```
 
-## Exec
+## exec
 
 You can easily execute a command to a running instance:
 
@@ -134,7 +134,7 @@ usr
 var
 ```
 
-## Run
+## run
 
 If a container has a `%runscript` section (or a Docker entrypoint/cmd that 
 was converted to one), you can run that script easily:
@@ -147,7 +147,7 @@ If your container didn't have any kind of runscript, the startscript
 will be used instead.
 
 
-## Down
+## down
 
 You can bring one or more instances down (meaning, stopping them) by doing:
 
@@ -172,7 +172,7 @@ in order to kill instances after the specified number of seconds:
 singularity-compose down -t 100
 ```
 
-## Logs
+## logs
 
 You can of course view logs for all instances, or just specific named ones:
 
@@ -190,7 +190,7 @@ nginx: [emerg] host not found in upstream "uwsgi" in /etc/nginx/conf.d/default.c
 nginx: [emerg] host not found in upstream "uwsgi" in /etc/nginx/conf.d/default.conf:22
 ```
 
-## Config
+## config
 
 You can load and validate the configuration file (singularity-compose.yml) and
 print it to the screen as follows:
@@ -245,4 +245,111 @@ $ singularity-compose config
 }
 ```
 
+#Global arguments
+
+The following arguments are supported for all commands available.
+
+## debug
+
+Set logging verbosity to debug.
+
+```bash
+singularity-compose --debug version
+```
+
+This is equivalent to passing `--log-level=DEBUG` to the CLI.
+
+```bash
+singularity-compose --log-level='DEBUG' version
+```
+
+## log_level
+
+Change logging verbosity. Accepted values are: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`
+
+```bash
+singularity-compose --log-level='INFO' version
+```
+
+## file
+
+Specify the location of a Compose configuration file
+
+Default value: `singularity-compose.yml`
+
+Aliases `--file`, `-f`. 
+
+You can supply multiple `-f` configuration files. When you supply multiple files, `singularity-compose`
+ combines them into a single configuration. It builds the configuration in the order you supply the
+files. Subsequent files override and add to their predecessors.
+
+For example consider this command line:
+
+```bash
+singularity-compose -f singularity-compose.yml -f singularity-compose.dev.yml up
+```
+
+The `singularity-compose.yml` file might specify a `webapp` instance:
+
+```yaml
+instances:
+  webapp:
+    image: webapp.sif
+    start:
+      args: "start-daemon"
+    port:
+      - "80:80"
+    volumes:
+      - /mnt/shared_drive/folder:/webapp/data
+```
+
+if the `singularity-compose.dev.yml` also specifies this same service, any matching fields override 
+the previous files.
+
+```yaml
+instances:
+  webapp:
+    start:
+      args: "start-daemon -debug"
+    volumes:
+      - /home/user/folder:/webapp/data
+```
+
+The result of the examples above would be translated in runtime to:
+
+```yaml
+instances:
+  webapp:
+    image: webapp.sif
+    start:
+      args: "start-daemon -debug"
+    port:
+      - "80:80"
+    volumes:
+      - /home/user/folder:/webapp/data
+```
+
+## project-name
+
+Specify project name.
+
+Default value: `$PWD`
+
+Aliases `--project-name`, `-p`. 
+
+```bash
+singularity-compose --project-name 'my_cool_project' up
+```
+
+## project-directory
+
+Specify project working directory
+
+Default value: compose file location
+
+
+```bash
+singularity-compose --project-directory /home/user/myfolder up
+```
+
 [home](/README.md#singularity-compose)
diff --git a/scompose/client/__init__.py b/scompose/client/__init__.py
index ff62a3a..84a08b1 100644
--- a/scompose/client/__init__.py
+++ b/scompose/client/__init__.py
@@ -53,7 +53,7 @@ def get_parser():
         "--file",
         "-f",
         dest="file",
-        help="specify compose file (default singularity-compose.yml)",
+        help="specify compose file (default singularity-compose.yml). It can be used multiple times",
         action="append",
         default=[],
     )

From 746fb07e816805e57fcca8d7c399755d234cb0e9 Mon Sep 17 00:00:00 2001
From: vsoch <vsoch@users.noreply.github.com>
Date: Tue, 5 Oct 2021 22:42:42 -0600
Subject: [PATCH 07/26] adding support for singularity-compose check

this will add jsonschema as an optional dependency to check the validity of a
singularity-compose.yml file. We will eventually extend this to check (and preview)
combined files.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>
---
 CHANGELOG.md                 |   1 +
 docs/commands.md             |  12 ++++
 docs/spec/spec-2.0.md        |   3 +-
 scompose/client/__init__.py  |   8 +++
 scompose/client/check.py     |  24 +++++++
 scompose/project/__init__.py |   1 +
 scompose/project/config.py   | 121 +++++++++++++++++++++++++++++++++
 scompose/project/project.py  | 125 ++++++++++++++++++++++++-----------
 scompose/version.py          |   4 +-
 setup.py                     |   3 +-
 10 files changed, 259 insertions(+), 43 deletions(-)
 create mode 100644 scompose/client/check.py
 create mode 100644 scompose/project/config.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7f045c9..f2ebec8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pypi.
 
 ## [0.0.x](https://github.com/singularityhub/singularity-compose/tree/master) (0.0.x)
+ - adding jsonschema validation and check command (0.0.12)
  - add network->enable option on composer file (0.1.11)
  - add network->allocate_ip option on composer file (0.1.10)
  - version 2.0 of the spec with added fakeroot network, start, exec, and run options (0.1.0)
diff --git a/docs/commands.md b/docs/commands.md
index 3e6b64c..13ebc8b 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -3,6 +3,17 @@
 The following commands are currently supported. Remember, you must be in the 
 present working directory of the compose file to reference the correct instances.
 
+## Check
+
+To do a sanity check of your singularity-compose.yml, you can use `singularity-compose check`
+
+```bash
+$ singularity-compose check
+singularity-compose.yml is valid.
+```
+This will eventually be extended to allow checking for combined files, which
+is under development.
+
 ## Build
 
 Build will either build a container recipe, or pull a container to the
@@ -21,6 +32,7 @@ If the build requires sudo (if you've defined sections in the config that warran
 setting up networking with sudo) the build will instead give you an instruction
 to run with sudo.
 
+
 ## Up
 
 If you want to both build and bring them up, you can use "up." Note that for
diff --git a/docs/spec/spec-2.0.md b/docs/spec/spec-2.0.md
index 114cc40..72a7d41 100644
--- a/docs/spec/spec-2.0.md
+++ b/docs/spec/spec-2.0.md
@@ -293,8 +293,7 @@ The fields for instances are discussed below:
 
 |Name| Description |
 |----|-------------|
-|name|The name of the instance will be "nginx" unless the user provides a "name"
-field (not defined above).|
+|name|The name of the instance will be "nginx" unless the user provides a "name" field (not defined above).|
 |build| a section to define how and where to build the base container from.|
 |build.context| the folder with the Singularity file (and other relevant files). Must exist.
 |build.recipe| the Singularity recipe in the build context folder. It defaults to `Singularity`|
diff --git a/scompose/client/__init__.py b/scompose/client/__init__.py
index 9393c08..4b5357d 100644
--- a/scompose/client/__init__.py
+++ b/scompose/client/__init__.py
@@ -96,6 +96,12 @@ def get_parser():
 
     build = subparsers.add_parser("build", help="Build or rebuild containers")
 
+    # Check
+
+    check = subparsers.add_parser(
+        "check", help="check or validate singularity-compose.yml"
+    )
+
     # Config
 
     config = subparsers.add_parser("config", help="Validate and view the compose file")
@@ -228,6 +234,8 @@ def show_help(return_code=0):
     # Does the user want a shell?
     if args.command == "build":
         from scompose.client.build import main
+    elif args.command == "check":
+        from scompose.client.check import main
     elif args.command == "create":
         from scompose.client.create import main
     elif args.command == "config":
diff --git a/scompose/client/check.py b/scompose/client/check.py
new file mode 100644
index 0000000..ee9f508
--- /dev/null
+++ b/scompose/client/check.py
@@ -0,0 +1,24 @@
+"""
+
+Copyright (C) 2021 Vanessa Sochat.
+
+This Source Code Form is subject to the terms of the
+Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+
+from scompose.project.config import validate_config
+from scompose.logger import bot
+
+
+def main(args, parser, extra):
+    """validate a singularity-compose.yml for correctness.
+
+    Eventually this will also have a --preview flag to show combined configs.
+    """
+    result = validate_config(args.file)
+    if not result:
+        bot.info("%s is valid." % args.file)
+    else:
+        bot.exit("%s is not valid." % args.file)
diff --git a/scompose/project/__init__.py b/scompose/project/__init__.py
index f816466..6f1ff06 100644
--- a/scompose/project/__init__.py
+++ b/scompose/project/__init__.py
@@ -7,4 +7,5 @@
 with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """
+
 from .project import Project
diff --git a/scompose/project/config.py b/scompose/project/config.py
new file mode 100644
index 0000000..1ea2022
--- /dev/null
+++ b/scompose/project/config.py
@@ -0,0 +1,121 @@
+"""
+
+Copyright (C) 2021 Vanessa Sochat.
+
+This Source Code Form is subject to the terms of the
+Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+
+import os
+from scompose.utils import read_yaml
+
+# We don't require jsonschema, so catch import error and alert user
+try:
+    from jsonschema import validate
+except ImportError as e:
+    msg = "pip install jsonschema"
+    sys.exit("jsonschema is required for checking and validation: %s\n %s" % (e, msg))
+
+
+def validate_config(filepath):
+    """
+    Validate a singularity-compose.yaml file.
+    """
+    cfg = read_yaml(filepath, quiet=True)
+    return validate(cfg, compose_schema)
+
+
+## Singularity Compose Schema
+
+schema_url = "https://json-schema.org/draft-07/schema/#"
+
+# Common patterns of types
+string_list = {"type": "array", "items": {"type": "string"}}
+
+# Instance groups
+instance_build = {
+    "type": "object",
+    "properties": {
+        "recipe": {"type": "string"},
+        "context": {"type": "string"},
+        "options": string_list,
+    },
+}
+
+instance_network = {
+    "type": "object",
+    "properties": {
+        "allocate_ip": {"type": "boolean"},
+        "enable": {"type": "boolean"},
+    },
+}
+
+
+instance_start = {
+    "type": "object",
+    "properties": {
+        "args": {"type": ["string", "array"]},
+        "options": string_list,
+    },
+}
+
+instance_run = {
+    "type": "object",
+    "properties": {
+        "args": {"type": ["string", "array"]},
+        "options": string_list,
+    },
+}
+
+instance_post = {
+    "type": "object",
+    "properties": {
+        "commands": string_list,
+    },
+}
+
+instance_exec = {
+    "type": "object",
+    "properties": {"options": string_list, "command": {"type": "string"}},
+    "required": [
+        "command",
+    ],
+}
+
+# A single instance
+instance = {
+    "type": "object",
+    "properties": {
+        "image": {"type": "string"},
+        "build": instance_build,
+        "network": instance_network,
+        "ports": string_list,
+        "volumes": string_list,
+        "volumes_from": string_list,
+        "depends_on": string_list,
+        "start": instance_start,
+        "exec": instance_exec,
+        "run": {"oneOf": [instance_run, {"type": "array"}]},
+        "post": instance_post,
+    },
+}
+
+
+# instances define container services
+instances = {"type": "object", "patternProperties": {"\\w[\\w-]*": instance}}
+
+properties = {"version": {"type": "string"}, "instances": instances}
+
+compose_schema = {
+    "$schema": schema_url,
+    "title": "Singularity Compose Schema",
+    "type": "object",
+    "required": [
+        "version",
+        "instances",
+    ],
+    "properties": properties,
+    "additionalProperties": False,
+}
diff --git a/scompose/project/project.py b/scompose/project/project.py
index ef80997..be6feee 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -46,8 +46,10 @@ def __repr__(self):
         return self.__str__()
 
     def get_instance_names(self):
-        """return a list of names, if a config file is loaded, and instances
-        are defined.
+        """
+        Return a list of names.
+
+        Do this if a config file is loaded, and instances are defined.
         """
         names = []
         if self.instances is not None:
@@ -56,9 +58,10 @@ def get_instance_names(self):
         return names
 
     def set_filename(self, filename):
-        """set the filename to read the recipe from. If not provided, defaults
-        to singularity-compose.yml. The working directory is set to
-        be the directory name of the configuration file.
+        """Set the filename to read the recipe from.
+
+        If not provided, defaults to singularity-compose.yml. The working directory
+        is set to be the directory name of the configuration file.
 
         Parameters
         ==========
@@ -68,8 +71,10 @@ def set_filename(self, filename):
         self.working_dir = os.path.dirname(os.path.abspath(self.filename))
 
     def set_name(self, name):
-        """set the filename to read the recipe from. If not provided, defaults
-        to singularity-compose.yml
+        """
+        Set the filename to read the recipe from.
+
+        If not provided, defaults to singularity-compose.yml
 
         Parameters
         ==========
@@ -81,7 +86,9 @@ def set_name(self, name):
     # Listing
 
     def ps(self):
-        """ps will print a table of instances, including pids and names."""
+        """
+        Ps will print a table of instances, including pids and names.
+        """
         instance_names = self.get_instance_names()
         table = []
         for instance in self.client.instances(quiet=True, sudo=self.sudo):
@@ -100,8 +107,10 @@ def ps(self):
         bot.table(table)
 
     def iter_instances(self, names):
-        """yield instances one at a time. If an invalid name is given,
-        exit with error.
+        """
+        Yield instances one at a time.
+
+        If an invalid name is given, exit with error.
 
         Parameters
         ==========
@@ -116,9 +125,10 @@ def iter_instances(self, names):
             yield self.instances.get(name)
 
     def get_instance(self, name):
-        """get a specifically named instance. We first check that the
-        client has instances defined, and that the name we are looking
-        for is also included. If not found, we return None.
+        """Get a specifically named instance.
+
+        We first check that the client has instances defined, and that the name
+        we are looking for is also included. If not found, we return None.
 
         Parameters
         ==========
@@ -133,7 +143,10 @@ def get_instance(self, name):
     # Loading Functions
 
     def get_already_running(self):
-        """Since a user can bring select instances up and down, we need to
+        """
+        Get already running instances.
+
+        Since a user can bring select instances up and down, we need to
         derive a list of already running instances to include
         """
         # Get list of existing instances to skip addresses
@@ -146,7 +159,9 @@ def get_already_running(self):
         return {x["instance"]: x for x in instances}
 
     def load(self):
-        """load a singularity-compose.yml recipe, and validate it."""
+        """
+        Load a singularity-compose.yml recipe, and validate it.
+        """
 
         if not os.path.exists(self.filename):
             bot.error("%s does not exist." % self.filename)
@@ -158,7 +173,9 @@ def load(self):
             bot.exit("Cannot parse %s, invalid yaml." % self.filename)
 
     def parse(self):
-        """parse a loaded config"""
+        """
+        Parse a loaded config
+        """
 
         # If a port is defined, we need root.
         self.sudo = False
@@ -197,7 +214,9 @@ def parse(self):
                 instance.set_volumes_from(self.instances)
 
     def _sort_instances(self, instances):
-        """eventually reorder instances based on depends_on constraints"""
+        """
+        Eventually reorder instances based on depends_on constraints
+        """
         sorted_instances = []
         for instance in self.instances:
             depends_on = self.instances[instance].params.get("depends_on", [])
@@ -217,7 +236,10 @@ def _sort_instances(self, instances):
     # Networking
 
     def get_ip_lookup(self, names, bridge="10.22.0.0/16"):
-        """based on a bridge address that can serve other addresses (akin to
+        """
+        Generate a pre-determined address for each container.
+
+        Based on a bridge address that can serve other addresses (akin to
         a router, metaphorically, generate a pre-determined address for
         each container.
 
@@ -252,9 +274,11 @@ def get_ip_lookup(self, names, bridge="10.22.0.0/16"):
         return lookup
 
     def get_bridge_address(self, name="sbr0"):
-        """get the (named) bridge address on the host. It should be automatically
-        created by Singularity over 3.0. This function currently is not used,
-        but is left in case it's needed.
+        """
+        Get the (named) bridge address on the host.
+
+        It should be automatically created by Singularity over 3.0. This function
+        currently is not used, but is left in case it's needed.
 
         Parameters
         ==========
@@ -290,7 +314,9 @@ def create_hosts(self, lookup):
         return hosts_file
 
     def generate_resolv_conf(self):
-        """generate a resolv.conf file to bind to the containers.
+        """
+        Generate a resolv.conf file to bind to the containers.
+
         We use the template provided by scompose.
         """
         resolv_conf = os.path.join(self.working_dir, "resolv.conf")
@@ -302,7 +328,8 @@ def generate_resolv_conf(self):
     # Commands
 
     def shell(self, name):
-        """if an instance exists, shell into it.
+        """
+        If an instance exists, shell into it.
 
         Parameters
         ==========
@@ -316,7 +343,8 @@ def shell(self, name):
             self.client.shell(instance.instance.get_uri(), sudo=self.sudo)
 
     def run(self, name):
-        """if an instance exists, run it.
+        """
+        If an instance exists, run it.
 
         Parameters
         ==========
@@ -337,7 +365,8 @@ def run(self, name):
             print("".join([x for x in result["message"] if x]))
 
     def execute(self, name, commands):
-        """if an instance exists, execute a command to it.
+        """
+        If an instance exists, execute a command to it.
 
         Parameters
         ==========
@@ -363,7 +392,8 @@ def execute(self, name, commands):
     # Logs
 
     def clear_logs(self, names):
-        """clear_logs will remove all old error and output logs.
+        """
+        Clear_logs will remove all old error and output logs.
 
         Parameters
         ==========
@@ -374,7 +404,8 @@ def clear_logs(self, names):
             instance.clear_logs()
 
     def logs(self, names=None, tail=0):
-        """logs will print logs to the screen.
+        """
+        Logs will print logs to the screen.
 
         Parameters
         ==========
@@ -395,8 +426,9 @@ def view_config(self):
     # Down
 
     def down(self, names=None, timeout=None):
-        """stop one or more instances. If no names are provided, bring them
-        all down.
+        """
+        Stop one or more instances.
+        If no names are provided, bring them all down.
 
         Parameters
         ==========
@@ -417,18 +449,29 @@ def create(
         self, names=None, writable_tmpfs=True, bridge="10.22.0.0/16", no_resolv=False
     ):
 
-        """call the create function, which defaults to the command instance.create()"""
+        """
+        Call the create function, which defaults to the command instance.create()
+        """
         return self._create(names, writable_tmpfs=writable_tmpfs, no_resolv=no_resolv)
 
     def up(
-        self, names=None, writable_tmpfs=True, bridge="10.22.0.0/16", no_resolv=False,
+        self,
+        names=None,
+        writable_tmpfs=True,
+        bridge="10.22.0.0/16",
+        no_resolv=False,
     ):
 
-        """call the up function, instance.up(), which will build before if
-        a container binary does not exist.
+        """
+        Call the up function, instance.up().
+
+        This will build before if a container binary does not exist.
         """
         return self._create(
-            names, command="up", writable_tmpfs=writable_tmpfs, no_resolv=no_resolv,
+            names,
+            command="up",
+            writable_tmpfs=writable_tmpfs,
+            no_resolv=no_resolv,
         )
 
     def _create(
@@ -440,10 +483,12 @@ def _create(
         no_resolv=False,
     ):
 
-        """create one or more instances. "Command" determines the sub function
-        to call for the instance, which should be "create" or "up".
-        If the user provide a list of names, use them, otherwise default
-        to all instances.
+        """
+        Create one or more instances.
+
+        "Command" determines the sub function to call for the instance,
+        which should be "create" or "up". If the user provide a list of names,
+        use them, otherwise default to all instances.
 
         Parameters
         ==========
@@ -503,7 +548,9 @@ def _create(
     # Build
 
     def build(self, names=None):
-        """given a loaded project, build associated containers (or pull)."""
+        """
+        Given a loaded project, build associated containers (or pull).
+        """
         names = names or self.get_instance_names()
         for instance in self.iter_instances(names):
             instance.build(working_dir=self.working_dir)
diff --git a/scompose/version.py b/scompose/version.py
index c7cdb9f..e627bca 100644
--- a/scompose/version.py
+++ b/scompose/version.py
@@ -8,7 +8,7 @@
 
 """
 
-__version__ = "0.1.11"
+__version__ = "0.1.12"
 AUTHOR = "Vanessa Sochat"
 AUTHOR_EMAIL = "vsoch@users.noreply.github.com"
 NAME = "singularity-compose"
@@ -26,6 +26,8 @@
     ("pyaml", {"min_version": "5.1.1"}),
 )
 
+INSTALL_REQUIRES_CHECKS = INSTALL_REQUIRES + (("jsonschema", {"min_version": None}),)
+
 TESTS_REQUIRES = (("pytest", {"min_version": "4.6.2"}),)
 
 INSTALL_REQUIRES_ALL = INSTALL_REQUIRES
diff --git a/setup.py b/setup.py
index 42cbcac..f304bb0 100644
--- a/setup.py
+++ b/setup.py
@@ -68,6 +68,7 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"):
 
     INSTALL_REQUIRES = get_reqs(lookup)
     INSTALL_REQUIRES_ALL = get_reqs(lookup, "INSTALL_REQUIRES_ALL")
+    INSTALL_REQUIRES_CHECKS = get_reqs(lookup, "INSTALL_REQUIRES_CHECKS")
     TESTS_REQUIRES = get_reqs(lookup, "TESTS_REQUIRES")
 
     setup(
@@ -89,7 +90,7 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"):
         install_requires=INSTALL_REQUIRES,
         setup_requires=["pytest-runner"],
         tests_require=TESTS_REQUIRES,
-        extras_require={"all": [INSTALL_REQUIRES_ALL]},
+        extras_require={"all": [INSTALL_REQUIRES_ALL], "checks": [INSTALL_REQUIRES_CHECKS]},
         classifiers=[
             "Intended Audience :: Science/Research",
             "Intended Audience :: Developers",

From 41731fae138d81542c93aef5121d50eccdbe6dd7 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Tue, 5 Oct 2021 20:45:53 +0000
Subject: [PATCH 08/26] implement deep_merge functionally

---
 scompose/client/__init__.py |  8 ++++-
 scompose/project/project.py | 63 ++++++++++++++++++++++++++++++-------
 2 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/scompose/client/__init__.py b/scompose/client/__init__.py
index 4b5357d..968484b 100644
--- a/scompose/client/__init__.py
+++ b/scompose/client/__init__.py
@@ -54,7 +54,8 @@ def get_parser():
         "-f",
         dest="file",
         help="specify compose file (default singularity-compose.yml)",
-        default="singularity-compose.yml",
+        action="append",
+        default=[],
     )
 
     parser.add_argument(
@@ -231,6 +232,11 @@ def show_help(return_code=0):
         print(scompose.__version__)
         sys.exit(0)
 
+    # argparse inherits a funny behaviour that appends default values to the dest value whether you've specified a value
+    # or not. The bug/behaviour is documented here: https://bugs.python.org/issue16399
+    if len(args.file) == 0:
+        args.file = ["singularity-compose.yml"]
+
     # Does the user want a shell?
     if args.command == "build":
         from scompose.client.build import main
diff --git a/scompose/project/project.py b/scompose/project/project.py
index be6feee..b5388eb 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -67,8 +67,8 @@ def set_filename(self, filename):
         ==========
         filename: the singularity-compose.yml file to use
         """
-        self.filename = filename or "singularity-compose.yml"
-        self.working_dir = os.path.dirname(os.path.abspath(self.filename))
+        self.filename = filename
+        self.working_dir = os.getcwd()
 
     def set_name(self, name):
         """
@@ -80,11 +80,9 @@ def set_name(self, name):
         ==========
         name: if a customize name is provided, use it
         """
-        pwd = os.path.basename(os.path.dirname(os.path.abspath(self.filename)))
-        self.name = (name or pwd).lower()
+        self.name = (name or self.working_dir).lower()
 
     # Listing
-
     def ps(self):
         """
         Ps will print a table of instances, including pids and names.
@@ -141,7 +139,6 @@ def get_instance(self, name):
         return instance
 
     # Loading Functions
-
     def get_already_running(self):
         """
         Get already running instances.
@@ -163,15 +160,59 @@ def load(self):
         Load a singularity-compose.yml recipe, and validate it.
         """
 
-        if not os.path.exists(self.filename):
-            bot.error("%s does not exist." % self.filename)
-            sys.exit(1)
-
         try:
-            self.config = read_yaml(self.filename, quiet=True)
+            yaml_files = []
+
+            for f in self.filename:
+                # ensure file exists
+                if not os.path.exists(f):
+                    bot.error("%s does not exist." % f)
+                    sys.exit(1)
+                # read yaml file
+                yaml_files.append(read_yaml(f, quiet=True))
+
+            # merge/override yaml properties where applicable
+            self.config = self.deep_merge(yaml_files)
         except:  # ParserError
             bot.exit("Cannot parse %s, invalid yaml." % self.filename)
 
+    def deep_merge(self, yaml_files):
+        """merge singularity-compose.yml files into a single dict"""
+        if len(yaml_files) == 1:
+            # nothing to merge as the user specified a single file
+            return yaml_files[0]
+
+        base_yaml = None
+        for idx, item in enumerate(yaml_files):
+            if idx == 0:
+                base_yaml = item
+            else:
+                base_yaml = self.merge(base_yaml, item)
+
+        return base_yaml
+
+    def merge(self, a, b):
+        """merge dict b into a"""
+        for key in b:
+            if key in a:
+                # merge dicts recursively
+                if isinstance(a[key], dict) and isinstance(b[key], dict):
+                    a[key] = self.merge(a[key], b[key])
+
+                # if types are equal, b takes precedence
+                elif isinstance(a[key], type(b[key])):
+                    a[key] = b[key]
+
+                # if nothing matches then this means a conflict of types which should exist in the first place
+                else:
+                    bot.error(
+                        "key %s has property type mismatch in different files." % key
+                    )
+                    sys.exit(1)
+            else:
+                a[key] = b[key]
+        return a
+
     def parse(self):
         """
         Parse a loaded config

From e7e669440702e93c83de0f4b2b58b0f2d5a65e6f Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Tue, 5 Oct 2021 20:47:52 +0000
Subject: [PATCH 09/26] fix typo

---
 scompose/project/project.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scompose/project/project.py b/scompose/project/project.py
index b5388eb..14bb87f 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -203,7 +203,7 @@ def merge(self, a, b):
                 elif isinstance(a[key], type(b[key])):
                     a[key] = b[key]
 
-                # if nothing matches then this means a conflict of types which should exist in the first place
+                # if nothing matches then this means a conflict of types which shouldn't exist in the first place
                 else:
                     bot.error(
                         "key %s has property type mismatch in different files." % key

From e1f5bf632db1b93ab907d053f2742d06392ae3fc Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Tue, 5 Oct 2021 22:47:07 +0000
Subject: [PATCH 10/26] implement some changes suggested during code review

---
 scompose/project/project.py | 70 +++++++------------------------------
 scompose/utils/__init__.py  | 49 ++++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 58 deletions(-)

diff --git a/scompose/project/project.py b/scompose/project/project.py
index 14bb87f..4884014 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -10,7 +10,7 @@
 
 from scompose.templates import get_template
 from scompose.logger import bot
-from scompose.utils import read_yaml, read_file, write_file
+from scompose.utils import read_file, write_file, build_interpolated_config
 from spython.main import get_client
 from .instance import Instance
 from ipaddress import IPv4Network
@@ -67,7 +67,14 @@ def set_filename(self, filename):
         ==========
         filename: the singularity-compose.yml file to use
         """
-        self.filename = filename
+        default_value = ["singularity-compose.yml"]
+        if filename is None:
+            self.filenames = default_value
+        elif isinstance(filename, list):
+            self.filenames = filename or default_value
+        else:
+            self.filenames = [filename]
+
         self.working_dir = os.getcwd()
 
     def set_name(self, name):
@@ -156,62 +163,9 @@ def get_already_running(self):
         return {x["instance"]: x for x in instances}
 
     def load(self):
-        """
-        Load a singularity-compose.yml recipe, and validate it.
-        """
-
-        try:
-            yaml_files = []
-
-            for f in self.filename:
-                # ensure file exists
-                if not os.path.exists(f):
-                    bot.error("%s does not exist." % f)
-                    sys.exit(1)
-                # read yaml file
-                yaml_files.append(read_yaml(f, quiet=True))
-
-            # merge/override yaml properties where applicable
-            self.config = self.deep_merge(yaml_files)
-        except:  # ParserError
-            bot.exit("Cannot parse %s, invalid yaml." % self.filename)
-
-    def deep_merge(self, yaml_files):
-        """merge singularity-compose.yml files into a single dict"""
-        if len(yaml_files) == 1:
-            # nothing to merge as the user specified a single file
-            return yaml_files[0]
-
-        base_yaml = None
-        for idx, item in enumerate(yaml_files):
-            if idx == 0:
-                base_yaml = item
-            else:
-                base_yaml = self.merge(base_yaml, item)
-
-        return base_yaml
-
-    def merge(self, a, b):
-        """merge dict b into a"""
-        for key in b:
-            if key in a:
-                # merge dicts recursively
-                if isinstance(a[key], dict) and isinstance(b[key], dict):
-                    a[key] = self.merge(a[key], b[key])
-
-                # if types are equal, b takes precedence
-                elif isinstance(a[key], type(b[key])):
-                    a[key] = b[key]
-
-                # if nothing matches then this means a conflict of types which shouldn't exist in the first place
-                else:
-                    bot.error(
-                        "key %s has property type mismatch in different files." % key
-                    )
-                    sys.exit(1)
-            else:
-                a[key] = b[key]
-        return a
+        """load a singularity-compose.yml recipe, and validate it."""
+        # merge/override yaml properties where applicable
+        self.config = build_interpolated_config(self.filenames)
 
     def parse(self):
         """
diff --git a/scompose/utils/__init__.py b/scompose/utils/__init__.py
index 5e886dd..eadd926 100644
--- a/scompose/utils/__init__.py
+++ b/scompose/utils/__init__.py
@@ -150,6 +150,55 @@ def _read_yaml(section, quiet=False):
     return metadata
 
 
+def build_interpolated_config(file_list):
+    yaml_files = []
+    for f in file_list:
+        try:
+            # ensure file exists
+            if not os.path.exists(f):
+                print("%s does not exist." % f)
+                sys.exit(1)
+            # read yaml file
+            yaml_files.append(read_yaml(f, quiet=True))
+        except:  # ParserError
+            print("Cannot parse %s, invalid yaml." % f)
+            sys.exit(1)
+
+    # merge/override yaml properties where applicable
+    return _deep_merge(yaml_files)
+
+
+def _deep_merge(yaml_files):
+    """merge yaml files into a single dict"""
+    base_yaml = None
+    for idx, item in enumerate(yaml_files):
+        if idx == 0:
+            base_yaml = item
+        else:
+            base_yaml = _merge(base_yaml, item)
+
+    return base_yaml
+
+
+def _merge(self, a, b):
+    """merge dict b into a"""
+    for key in b:
+        if key in a:
+            # merge dicts recursively
+            if isinstance(a[key], dict) and isinstance(b[key], dict):
+                a[key] = self.merge(a[key], b[key])
+            # if types are equal, b takes precedence
+            elif isinstance(a[key], type(b[key])):
+                a[key] = b[key]
+            # if nothing matches then this means a conflict of types which shouldn't exist in the first place
+            else:
+                print("key '%s': type mismatch in different files." % key)
+                sys.exit(1)
+        else:
+            a[key] = b[key]
+    return a
+
+
 # Json
 
 

From 14996abb7eefa79862436424364fc5b13bc8ea1f Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Wed, 6 Oct 2021 03:54:33 +0000
Subject: [PATCH 11/26] add tests for the config override feature

---
 .../config_override/singularity-compose-1.yml |   8 ++
 .../config_override/singularity-compose-2.yml |  12 ++
 scompose/tests/test_utils.py                  | 106 ++++++++++++++++++
 scompose/utils/__init__.py                    |   4 +-
 4 files changed, 128 insertions(+), 2 deletions(-)
 create mode 100644 scompose/tests/configs/config_override/singularity-compose-1.yml
 create mode 100644 scompose/tests/configs/config_override/singularity-compose-2.yml

diff --git a/scompose/tests/configs/config_override/singularity-compose-1.yml b/scompose/tests/configs/config_override/singularity-compose-1.yml
new file mode 100644
index 0000000..f4b860c
--- /dev/null
+++ b/scompose/tests/configs/config_override/singularity-compose-1.yml
@@ -0,0 +1,8 @@
+version: "2.0"
+instances:
+  echo:
+    build:
+      context: .
+      recipe: Singularity
+    start:
+      args: "arg0 arg1 arg2"
diff --git a/scompose/tests/configs/config_override/singularity-compose-2.yml b/scompose/tests/configs/config_override/singularity-compose-2.yml
new file mode 100644
index 0000000..139e7b7
--- /dev/null
+++ b/scompose/tests/configs/config_override/singularity-compose-2.yml
@@ -0,0 +1,12 @@
+version: "2.0"
+instances:
+  echo:
+    start:
+      options:
+        - fakeroot
+      args: "arg0 arg1"
+
+  hello:
+    image: from_the_other_side.sif
+    start:
+      args: "how are you?"
diff --git a/scompose/tests/test_utils.py b/scompose/tests/test_utils.py
index e2f45d4..02637d9 100644
--- a/scompose/tests/test_utils.py
+++ b/scompose/tests/test_utils.py
@@ -9,6 +9,8 @@
 import os
 import pytest
 
+here = os.path.dirname(os.path.abspath(__file__))
+
 
 def test_write_read_files(tmp_path):
     """test_write_read_files will test the functions write_file and read_file"""
@@ -88,3 +90,107 @@ def test_print_json():
 
     result = print_json({1: 1})
     assert result == '{\n    "1": 1\n}'
+
+
+def test_merge():
+    print("Testing utils._merge")
+    from scompose.utils import _merge
+
+    # No override
+    a = {"a": 123}
+    b = {"b": 456}
+    assert _merge(a, b) == {"a": 123, "b": 456}
+
+    # Override + merge
+    a = {"a": 123}
+    b = {"b": 456, "a": 789}
+    assert _merge(a, b) == {"a": 789, "b": 456}
+
+    # Override only
+    a = {"a": 123}
+    b = {"a": 789}
+    assert _merge(a, b) == {"a": 789}
+
+    # Dict merge
+    a = {"a": 123, "b": {"c": "d"}}
+    b = {"b": {"e": "f"}}
+    assert _merge(a, b) == {"a": 123, "b": {"c": "d", "e": "f"}}
+
+    # Dict merge + key override
+    a = {"a": 123, "b": {"c": "d"}}
+    b = {"b": {"c": "f"}}
+    assert _merge(a, b) == {"a": 123, "b": {"c": "f"}}
+
+
+def test_deep_merge():
+    print("Testing utils._deep_merge")
+    from scompose.utils import _deep_merge, read_yaml
+    config_override = os.path.join(here, "configs", "config_override")
+
+    # single file
+    yaml_files = [
+        read_yaml(
+            os.path.join(config_override, "singularity-compose-1.yml"), quiet=True
+        )
+    ]
+    ret = _deep_merge(yaml_files)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1 arg2"},
+        }
+    }
+
+    # multiple files
+    yaml_files = [
+        read_yaml(
+            os.path.join(config_override, "singularity-compose-1.yml"), quiet=True
+        ),
+        read_yaml(
+            os.path.join(config_override, "singularity-compose-2.yml"), quiet=True
+        ),
+    ]
+    ret = _deep_merge(yaml_files)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1", "options": ["fakeroot"]},
+        },
+        "hello": {
+            "image": "from_the_other_side.sif",
+            "start": {"args": "how are you?"},
+        },
+    }
+
+
+def test_build_interpolated_config():
+    print("Testing utils.build_interpolated_config")
+    from scompose.utils import build_interpolated_config
+    config_override = os.path.join(here, "configs", "config_override")
+
+    # single file
+    file_list = [os.path.join(config_override, "singularity-compose-1.yml")]
+    ret = build_interpolated_config(file_list)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1 arg2"},
+        }
+    }
+
+    # multiple files
+    file_list = [
+            os.path.join(config_override, "singularity-compose-1.yml"),
+            os.path.join(config_override, "singularity-compose-2.yml"),
+    ]
+    ret = build_interpolated_config(file_list)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1", "options": ["fakeroot"]},
+        },
+        "hello": {
+            "image": "from_the_other_side.sif",
+            "start": {"args": "how are you?"},
+        },
+    }
diff --git a/scompose/utils/__init__.py b/scompose/utils/__init__.py
index eadd926..b52b940 100644
--- a/scompose/utils/__init__.py
+++ b/scompose/utils/__init__.py
@@ -180,13 +180,13 @@ def _deep_merge(yaml_files):
     return base_yaml
 
 
-def _merge(self, a, b):
+def _merge(a, b):
     """merge dict b into a"""
     for key in b:
         if key in a:
             # merge dicts recursively
             if isinstance(a[key], dict) and isinstance(b[key], dict):
-                a[key] = self.merge(a[key], b[key])
+                a[key] = _merge(a[key], b[key])
             # if types are equal, b takes precedence
             elif isinstance(a[key], type(b[key])):
                 a[key] = b[key]

From 0fa847731b34107612f2564180a1ff2ac85f543a Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Wed, 6 Oct 2021 03:56:05 +0000
Subject: [PATCH 12/26] good old black formatting that I keep forgetting :)

---
 scompose/tests/test_utils.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scompose/tests/test_utils.py b/scompose/tests/test_utils.py
index 02637d9..5eb5b52 100644
--- a/scompose/tests/test_utils.py
+++ b/scompose/tests/test_utils.py
@@ -125,6 +125,7 @@ def test_merge():
 def test_deep_merge():
     print("Testing utils._deep_merge")
     from scompose.utils import _deep_merge, read_yaml
+
     config_override = os.path.join(here, "configs", "config_override")
 
     # single file
@@ -166,6 +167,7 @@ def test_deep_merge():
 def test_build_interpolated_config():
     print("Testing utils.build_interpolated_config")
     from scompose.utils import build_interpolated_config
+
     config_override = os.path.join(here, "configs", "config_override")
 
     # single file
@@ -180,8 +182,8 @@ def test_build_interpolated_config():
 
     # multiple files
     file_list = [
-            os.path.join(config_override, "singularity-compose-1.yml"),
-            os.path.join(config_override, "singularity-compose-2.yml"),
+        os.path.join(config_override, "singularity-compose-1.yml"),
+        os.path.join(config_override, "singularity-compose-2.yml"),
     ]
     ret = build_interpolated_config(file_list)
     assert ret["instances"] == {

From 35388e912dacbd1d11b0615bc0ab05900fdd8b18 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Wed, 6 Oct 2021 04:31:15 +0000
Subject: [PATCH 13/26] add docs

---
 docs/commands.md            | 128 ++++++++++++++++++++++++++++++++----
 scompose/client/__init__.py |   2 +-
 2 files changed, 118 insertions(+), 12 deletions(-)

diff --git a/docs/commands.md b/docs/commands.md
index 13ebc8b..c877093 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -3,7 +3,7 @@
 The following commands are currently supported. Remember, you must be in the 
 present working directory of the compose file to reference the correct instances.
 
-## Check
+## check
 
 To do a sanity check of your singularity-compose.yml, you can use `singularity-compose check`
 
@@ -14,7 +14,7 @@ singularity-compose.yml is valid.
 This will eventually be extended to allow checking for combined files, which
 is under development.
 
-## Build
+## build
 
 Build will either build a container recipe, or pull a container to the
 instance folder. In both cases, it's named after the instance so we can
@@ -32,8 +32,7 @@ If the build requires sudo (if you've defined sections in the config that warran
 setting up networking with sudo) the build will instead give you an instruction
 to run with sudo.
 
-
-## Up
+## up
 
 If you want to both build and bring them up, you can use "up." Note that for
 builds that require sudo, this will still stop and ask you to build with sudo.
@@ -64,7 +63,7 @@ $ singularity-compose up --no-resolv
 Creating app
 ```
 
-## Create
+## create
 
 Given that you have built your containers with `singularity-compose build`,
 you can create your instances as follows:
@@ -105,7 +104,7 @@ INSTANCES  NAME PID     IMAGE
 3         nginx	6543	nginx.sif
 ```
 
-## Shell
+## shell
 
 It's sometimes helpful to peek inside a running instance, either to look at permissions,
 inspect binds, or manually test running something.
@@ -116,7 +115,7 @@ $ singularity-compose shell app
 Singularity app.sif:~/Documents/Dropbox/Code/singularity/singularity-compose-example> 
 ```
 
-## Exec
+## exec
 
 You can easily execute a command to a running instance:
 
@@ -146,7 +145,7 @@ usr
 var
 ```
 
-## Run
+## run
 
 If a container has a `%runscript` section (or a Docker entrypoint/cmd that 
 was converted to one), you can run that script easily:
@@ -159,7 +158,7 @@ If your container didn't have any kind of runscript, the startscript
 will be used instead.
 
 
-## Down
+## down
 
 You can bring one or more instances down (meaning, stopping them) by doing:
 
@@ -184,7 +183,7 @@ in order to kill instances after the specified number of seconds:
 singularity-compose down -t 100
 ```
 
-## Logs
+## logs
 
 You can of course view logs for all instances, or just specific named ones:
 
@@ -202,7 +201,7 @@ nginx: [emerg] host not found in upstream "uwsgi" in /etc/nginx/conf.d/default.c
 nginx: [emerg] host not found in upstream "uwsgi" in /etc/nginx/conf.d/default.conf:22
 ```
 
-## Config
+## config
 
 You can load and validate the configuration file (singularity-compose.yml) and
 print it to the screen as follows:
@@ -257,4 +256,111 @@ $ singularity-compose config
 }
 ```
 
+#Global arguments
+
+The following arguments are supported for all commands available.
+
+## debug
+
+Set logging verbosity to debug.
+
+```bash
+singularity-compose --debug version
+```
+
+This is equivalent to passing `--log-level=DEBUG` to the CLI.
+
+```bash
+singularity-compose --log-level='DEBUG' version
+```
+
+## log_level
+
+Change logging verbosity. Accepted values are: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`
+
+```bash
+singularity-compose --log-level='INFO' version
+```
+
+## file
+
+Specify the location of a Compose configuration file
+
+Default value: `singularity-compose.yml`
+
+Aliases `--file`, `-f`. 
+
+You can supply multiple `-f` configuration files. When you supply multiple files, `singularity-compose`
+ combines them into a single configuration. It builds the configuration in the order you supply the
+files. Subsequent files override and add to their predecessors.
+
+For example consider this command line:
+
+```bash
+singularity-compose -f singularity-compose.yml -f singularity-compose.dev.yml up
+```
+
+The `singularity-compose.yml` file might specify a `webapp` instance:
+
+```yaml
+instances:
+  webapp:
+    image: webapp.sif
+    start:
+      args: "start-daemon"
+    port:
+      - "80:80"
+    volumes:
+      - /mnt/shared_drive/folder:/webapp/data
+```
+
+if the `singularity-compose.dev.yml` also specifies this same service, any matching fields override 
+the previous files.
+
+```yaml
+instances:
+  webapp:
+    start:
+      args: "start-daemon -debug"
+    volumes:
+      - /home/user/folder:/webapp/data
+```
+
+The result of the examples above would be translated in runtime to:
+
+```yaml
+instances:
+  webapp:
+    image: webapp.sif
+    start:
+      args: "start-daemon -debug"
+    port:
+      - "80:80"
+    volumes:
+      - /home/user/folder:/webapp/data
+```
+
+## project-name
+
+Specify project name.
+
+Default value: `$PWD`
+
+Aliases `--project-name`, `-p`. 
+
+```bash
+singularity-compose --project-name 'my_cool_project' up
+```
+
+## project-directory
+
+Specify project working directory
+
+Default value: compose file location
+
+
+```bash
+singularity-compose --project-directory /home/user/myfolder up
+```
+
 [home](/README.md#singularity-compose)
diff --git a/scompose/client/__init__.py b/scompose/client/__init__.py
index 968484b..4d26984 100644
--- a/scompose/client/__init__.py
+++ b/scompose/client/__init__.py
@@ -53,7 +53,7 @@ def get_parser():
         "--file",
         "-f",
         dest="file",
-        help="specify compose file (default singularity-compose.yml)",
+        help="specify compose file (default singularity-compose.yml). It can be used multiple times",
         action="append",
         default=[],
     )

From 6f9544fca19cf89c42cbea0ca0b048b346ec2e49 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Wed, 6 Oct 2021 21:04:39 +0000
Subject: [PATCH 14/26] tweak check group so it takes into account multiple
 files instead

---
 scompose/client/check.py    | 13 +++++++------
 scompose/project/config.py  | 30 +++++++-----------------------
 scompose/project/project.py | 13 +++----------
 3 files changed, 17 insertions(+), 39 deletions(-)

diff --git a/scompose/client/check.py b/scompose/client/check.py
index ee9f508..9dffb2a 100644
--- a/scompose/client/check.py
+++ b/scompose/client/check.py
@@ -13,12 +13,13 @@
 
 
 def main(args, parser, extra):
-    """validate a singularity-compose.yml for correctness.
+    """validate compose files for correctness.
 
     Eventually this will also have a --preview flag to show combined configs.
     """
-    result = validate_config(args.file)
-    if not result:
-        bot.info("%s is valid." % args.file)
-    else:
-        bot.exit("%s is not valid." % args.file)
+    for f in args.file:
+        result = validate_config(f)
+        if not result:
+            bot.info("%s is valid." % f)
+        else:
+            bot.exit("%s is not valid." % f)
diff --git a/scompose/project/config.py b/scompose/project/config.py
index 1ea2022..e34d181 100644
--- a/scompose/project/config.py
+++ b/scompose/project/config.py
@@ -8,7 +8,7 @@
 
 """
 
-import os
+import sys
 from scompose.utils import read_yaml
 
 # We don't require jsonschema, so catch import error and alert user
@@ -46,42 +46,29 @@ def validate_config(filepath):
 
 instance_network = {
     "type": "object",
-    "properties": {
-        "allocate_ip": {"type": "boolean"},
-        "enable": {"type": "boolean"},
-    },
+    "properties": {"allocate_ip": {"type": "boolean"}, "enable": {"type": "boolean"},},
 }
 
 
 instance_start = {
     "type": "object",
-    "properties": {
-        "args": {"type": ["string", "array"]},
-        "options": string_list,
-    },
+    "properties": {"args": {"type": ["string", "array"]}, "options": string_list,},
 }
 
 instance_run = {
     "type": "object",
-    "properties": {
-        "args": {"type": ["string", "array"]},
-        "options": string_list,
-    },
+    "properties": {"args": {"type": ["string", "array"]}, "options": string_list,},
 }
 
 instance_post = {
     "type": "object",
-    "properties": {
-        "commands": string_list,
-    },
+    "properties": {"commands": string_list,},
 }
 
 instance_exec = {
     "type": "object",
     "properties": {"options": string_list, "command": {"type": "string"}},
-    "required": [
-        "command",
-    ],
+    "required": ["command",],
 }
 
 # A single instance
@@ -112,10 +99,7 @@ def validate_config(filepath):
     "$schema": schema_url,
     "title": "Singularity Compose Schema",
     "type": "object",
-    "required": [
-        "version",
-        "instances",
-    ],
+    "required": ["version", "instances",],
     "properties": properties,
     "additionalProperties": False,
 }
diff --git a/scompose/project/project.py b/scompose/project/project.py
index 4884014..a3742f8 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -65,7 +65,7 @@ def set_filename(self, filename):
 
         Parameters
         ==========
-        filename: the singularity-compose.yml file to use
+        filename: the singularity-compose.yml file to use. This can be a str or a list of str
         """
         default_value = ["singularity-compose.yml"]
         if filename is None:
@@ -450,11 +450,7 @@ def create(
         return self._create(names, writable_tmpfs=writable_tmpfs, no_resolv=no_resolv)
 
     def up(
-        self,
-        names=None,
-        writable_tmpfs=True,
-        bridge="10.22.0.0/16",
-        no_resolv=False,
+        self, names=None, writable_tmpfs=True, bridge="10.22.0.0/16", no_resolv=False,
     ):
 
         """
@@ -463,10 +459,7 @@ def up(
         This will build before if a container binary does not exist.
         """
         return self._create(
-            names,
-            command="up",
-            writable_tmpfs=writable_tmpfs,
-            no_resolv=no_resolv,
+            names, command="up", writable_tmpfs=writable_tmpfs, no_resolv=no_resolv,
         )
 
     def _create(

From 01a522ab5931c677acccd40dd7d251ba9a1dbbdc Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Wed, 6 Oct 2021 22:17:38 +0000
Subject: [PATCH 15/26] implement preview option

---
 scompose/client/__init__.py | 10 +++++++++-
 scompose/client/check.py    | 15 +++++++++++++--
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/scompose/client/__init__.py b/scompose/client/__init__.py
index 4d26984..7c7bf5e 100644
--- a/scompose/client/__init__.py
+++ b/scompose/client/__init__.py
@@ -103,6 +103,14 @@ def get_parser():
         "check", help="check or validate singularity-compose.yml"
     )
 
+    check.add_argument(
+        "--preview",
+        dest="preview",
+        help="print compose file(s) interpolated content",
+        default=False,
+        action="store_true",
+    )
+
     # Config
 
     config = subparsers.add_parser("config", help="Validate and view the compose file")
@@ -157,7 +165,7 @@ def get_parser():
 
     execute = subparsers.add_parser("exec", help="execute a command to an instance")
 
-    run = subparsers.add_parser("run", help="run an instance runscript.")
+    run = subparsers.add_parser("run", help="run an instance runscript")
 
     shell = subparsers.add_parser("shell", help="shell into an instance")
 
diff --git a/scompose/client/check.py b/scompose/client/check.py
index 9dffb2a..b9d39b2 100644
--- a/scompose/client/check.py
+++ b/scompose/client/check.py
@@ -10,16 +10,27 @@
 
 from scompose.project.config import validate_config
 from scompose.logger import bot
+from scompose.utils import build_interpolated_config, print_json
 
 
 def main(args, parser, extra):
-    """validate compose files for correctness.
+    """
+    Validate compose files for correctness.
 
-    Eventually this will also have a --preview flag to show combined configs.
+    CLI Arguments
+    ==========
+        --preview flag to show combined configs.
     """
+
+    # validate compose files
     for f in args.file:
         result = validate_config(f)
         if not result:
             bot.info("%s is valid." % f)
         else:
             bot.exit("%s is not valid." % f)
+
+    if args.preview:
+        # preview
+        config = build_interpolated_config(args.file)
+        print("Combined configs:\n %s" % print_json(config))

From 2d10fe3dcd9df1268b56e2edb14434c356191bc7 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Wed, 6 Oct 2021 22:24:15 +0000
Subject: [PATCH 16/26] update changelog and add docs

---
 CHANGELOG.md     |  3 +++
 docs/commands.md | 39 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f2ebec8..0b3b6bc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,7 +14,10 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pypi.
 
 ## [0.0.x](https://github.com/singularityhub/singularity-compose/tree/master) (0.0.x)
+ - 
  - adding jsonschema validation and check command (0.0.12)
+   - implement configuration override feature
+   - implement `--preview` argument for the `check` command 
  - add network->enable option on composer file (0.1.11)
  - add network->allocate_ip option on composer file (0.1.10)
  - version 2.0 of the spec with added fakeroot network, start, exec, and run options (0.1.0)
diff --git a/docs/commands.md b/docs/commands.md
index c877093..e2eccb8 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -10,9 +10,44 @@ To do a sanity check of your singularity-compose.yml, you can use `singularity-c
 ```bash
 $ singularity-compose check
 singularity-compose.yml is valid.
+
+$ singularity-compose -f singularity-compose.yml \ 
+          -f singularity-compose.override.yml check 
+singularity-compose.yml is valid.
+singularity-compose.override.yml is valid.
+```
+
+To view the combined compose files you can use `--preview`.
+
+```bash
+$ singularity-compose -f singularity-compose.yml \ 
+          -f singularity-compose.override.yml check  --preview
+singularity-compose.yml is valid.
+singularity-compose.override.yml is valid.
+Combined configs:
+{
+    "version": "2.0",
+    "instances": {
+        "cvatdb": {
+            "network": {
+                "enable": false
+            },
+            "volumes": [
+                "./recipes/postgres/env.sh:/.singularity.d/env/env.sh",
+                "./volumes/postgres/conf:/opt/bitnami/postgresql/conf",
+                "./volumes/postgres/tmp:/opt/bitnami/postgresql/tmp",
+                "/home/vagrant/postgres_data:/bitnami/postgresql"
+            ],
+            "build": {
+                "context": ".",
+                "recipe": "./recipes/postgres/main.def",
+                "options": [
+                    "fakeroot"
+                ]
+            }
+        },
+ }
 ```
-This will eventually be extended to allow checking for combined files, which
-is under development.
 
 ## build
 

From a40e8f43b67abf8c25daf3d0ce722c353a22560f Mon Sep 17 00:00:00 2001
From: Paulo Miguel Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Thu, 7 Oct 2021 13:18:47 +1300
Subject: [PATCH 17/26] Apply suggestions from code review

Co-authored-by: Vanessasaurus <814322+vsoch@users.noreply.github.com>
---
 docs/commands.md | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/commands.md b/docs/commands.md
index e2eccb8..7b20f94 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -291,7 +291,7 @@ $ singularity-compose config
 }
 ```
 
-#Global arguments
+# Global arguments
 
 The following arguments are supported for all commands available.
 
@@ -300,13 +300,13 @@ The following arguments are supported for all commands available.
 Set logging verbosity to debug.
 
 ```bash
-singularity-compose --debug version
+$ singularity-compose --debug version
 ```
 
 This is equivalent to passing `--log-level=DEBUG` to the CLI.
 
 ```bash
-singularity-compose --log-level='DEBUG' version
+$ singularity-compose --log-level='DEBUG' version
 ```
 
 ## log_level
@@ -314,7 +314,7 @@ singularity-compose --log-level='DEBUG' version
 Change logging verbosity. Accepted values are: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`
 
 ```bash
-singularity-compose --log-level='INFO' version
+$ singularity-compose --log-level='INFO' version
 ```
 
 ## file
@@ -329,10 +329,10 @@ You can supply multiple `-f` configuration files. When you supply multiple files
  combines them into a single configuration. It builds the configuration in the order you supply the
 files. Subsequent files override and add to their predecessors.
 
-For example consider this command line:
+For example consider this command:
 
 ```bash
-singularity-compose -f singularity-compose.yml -f singularity-compose.dev.yml up
+$ singularity-compose -f singularity-compose.yml -f singularity-compose.dev.yml up
 ```
 
 The `singularity-compose.yml` file might specify a `webapp` instance:
@@ -384,7 +384,7 @@ Default value: `$PWD`
 Aliases `--project-name`, `-p`. 
 
 ```bash
-singularity-compose --project-name 'my_cool_project' up
+$ singularity-compose --project-name 'my_cool_project' up
 ```
 
 ## project-directory
@@ -395,7 +395,7 @@ Default value: compose file location
 
 
 ```bash
-singularity-compose --project-directory /home/user/myfolder up
+$ singularity-compose --project-directory /home/user/myfolder up
 ```
 
 [home](/README.md#singularity-compose)

From 900d8ee3fe8c8f48c336f8b4e324a8e5b8d0339e Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Thu, 7 Oct 2021 00:31:37 +0000
Subject: [PATCH 18/26] move deep_merge stuff to a brand new module

---
 scompose/client/check.py     |  8 ++++--
 scompose/config/__init__.py  | 53 ++++++++++++++++++++++++++++++++++++
 scompose/project/project.py  |  5 ++--
 scompose/tests/test_utils.py | 11 ++++----
 scompose/utils/__init__.py   | 49 ---------------------------------
 5 files changed, 67 insertions(+), 59 deletions(-)
 create mode 100644 scompose/config/__init__.py

diff --git a/scompose/client/check.py b/scompose/client/check.py
index b9d39b2..eede230 100644
--- a/scompose/client/check.py
+++ b/scompose/client/check.py
@@ -10,7 +10,8 @@
 
 from scompose.project.config import validate_config
 from scompose.logger import bot
-from scompose.utils import build_interpolated_config, print_json
+from scompose.config import merge_config
+import yaml
 
 
 def main(args, parser, extra):
@@ -32,5 +33,6 @@ def main(args, parser, extra):
 
     if args.preview:
         # preview
-        config = build_interpolated_config(args.file)
-        print("Combined configs:\n %s" % print_json(config))
+        config = merge_config(args.file)
+        print("Combined configs:")
+        print(yaml.dump(config, sort_keys=False))
diff --git a/scompose/config/__init__.py b/scompose/config/__init__.py
new file mode 100644
index 0000000..0bca48b
--- /dev/null
+++ b/scompose/config/__init__.py
@@ -0,0 +1,53 @@
+import os
+import sys
+
+from scompose.utils import read_yaml
+
+
+def merge_config(file_list):
+    yaml_files = []
+    for f in file_list:
+        try:
+            # ensure file exists
+            if not os.path.exists(f):
+                print("%s does not exist." % f)
+                sys.exit(1)
+            # read yaml file
+            yaml_files.append(read_yaml(f, quiet=True))
+        except:  # ParserError
+            print("Cannot parse %s, invalid yaml." % f)
+            sys.exit(1)
+
+    # merge/override yaml properties where applicable
+    return _deep_merge(yaml_files)
+
+
+def _deep_merge(yaml_files):
+    """merge yaml files into a single dict"""
+    base_yaml = None
+    for idx, item in enumerate(yaml_files):
+        if idx == 0:
+            base_yaml = item
+        else:
+            base_yaml = _merge(base_yaml, item)
+
+    return base_yaml
+
+
+def _merge(a, b):
+    """merge dict b into a"""
+    for key in b:
+        if key in a:
+            # merge dicts recursively
+            if isinstance(a[key], dict) and isinstance(b[key], dict):
+                a[key] = _merge(a[key], b[key])
+            # if types are equal, b takes precedence
+            elif isinstance(a[key], type(b[key])):
+                a[key] = b[key]
+            # if nothing matches then this means a conflict of types which shouldn't exist in the first place
+            else:
+                print("key '%s': type mismatch in different files." % key)
+                sys.exit(1)
+        else:
+            a[key] = b[key]
+    return a
\ No newline at end of file
diff --git a/scompose/project/project.py b/scompose/project/project.py
index a3742f8..7ccc768 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -10,7 +10,8 @@
 
 from scompose.templates import get_template
 from scompose.logger import bot
-from scompose.utils import read_file, write_file, build_interpolated_config
+from scompose.utils import read_file, write_file
+from ..config import merge_config
 from spython.main import get_client
 from .instance import Instance
 from ipaddress import IPv4Network
@@ -165,7 +166,7 @@ def get_already_running(self):
     def load(self):
         """load a singularity-compose.yml recipe, and validate it."""
         # merge/override yaml properties where applicable
-        self.config = build_interpolated_config(self.filenames)
+        self.config = merge_config(self.filenames)
 
     def parse(self):
         """
diff --git a/scompose/tests/test_utils.py b/scompose/tests/test_utils.py
index 5eb5b52..3967567 100644
--- a/scompose/tests/test_utils.py
+++ b/scompose/tests/test_utils.py
@@ -94,7 +94,7 @@ def test_print_json():
 
 def test_merge():
     print("Testing utils._merge")
-    from scompose.utils import _merge
+    from scompose.config import _merge
 
     # No override
     a = {"a": 123}
@@ -124,7 +124,8 @@ def test_merge():
 
 def test_deep_merge():
     print("Testing utils._deep_merge")
-    from scompose.utils import _deep_merge, read_yaml
+    from scompose.utils import read_yaml
+    from scompose.config import _deep_merge
 
     config_override = os.path.join(here, "configs", "config_override")
 
@@ -166,13 +167,13 @@ def test_deep_merge():
 
 def test_build_interpolated_config():
     print("Testing utils.build_interpolated_config")
-    from scompose.utils import build_interpolated_config
+    from scompose.config import merge_config
 
     config_override = os.path.join(here, "configs", "config_override")
 
     # single file
     file_list = [os.path.join(config_override, "singularity-compose-1.yml")]
-    ret = build_interpolated_config(file_list)
+    ret = merge_config(file_list)
     assert ret["instances"] == {
         "echo": {
             "build": {"context": ".", "recipe": "Singularity"},
@@ -185,7 +186,7 @@ def test_build_interpolated_config():
         os.path.join(config_override, "singularity-compose-1.yml"),
         os.path.join(config_override, "singularity-compose-2.yml"),
     ]
-    ret = build_interpolated_config(file_list)
+    ret = merge_config(file_list)
     assert ret["instances"] == {
         "echo": {
             "build": {"context": ".", "recipe": "Singularity"},
diff --git a/scompose/utils/__init__.py b/scompose/utils/__init__.py
index b52b940..5e886dd 100644
--- a/scompose/utils/__init__.py
+++ b/scompose/utils/__init__.py
@@ -150,55 +150,6 @@ def _read_yaml(section, quiet=False):
     return metadata
 
 
-def build_interpolated_config(file_list):
-    yaml_files = []
-    for f in file_list:
-        try:
-            # ensure file exists
-            if not os.path.exists(f):
-                print("%s does not exist." % f)
-                sys.exit(1)
-            # read yaml file
-            yaml_files.append(read_yaml(f, quiet=True))
-        except:  # ParserError
-            print("Cannot parse %s, invalid yaml." % f)
-            sys.exit(1)
-
-    # merge/override yaml properties where applicable
-    return _deep_merge(yaml_files)
-
-
-def _deep_merge(yaml_files):
-    """merge yaml files into a single dict"""
-    base_yaml = None
-    for idx, item in enumerate(yaml_files):
-        if idx == 0:
-            base_yaml = item
-        else:
-            base_yaml = _merge(base_yaml, item)
-
-    return base_yaml
-
-
-def _merge(a, b):
-    """merge dict b into a"""
-    for key in b:
-        if key in a:
-            # merge dicts recursively
-            if isinstance(a[key], dict) and isinstance(b[key], dict):
-                a[key] = _merge(a[key], b[key])
-            # if types are equal, b takes precedence
-            elif isinstance(a[key], type(b[key])):
-                a[key] = b[key]
-            # if nothing matches then this means a conflict of types which shouldn't exist in the first place
-            else:
-                print("key '%s': type mismatch in different files." % key)
-                sys.exit(1)
-        else:
-            a[key] = b[key]
-    return a
-
-
 # Json
 
 

From 949b088a601a61bad07cc1bdfe91516099bd3111 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Thu, 7 Oct 2021 00:32:56 +0000
Subject: [PATCH 19/26] black formatting

---
 scompose/config/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scompose/config/__init__.py b/scompose/config/__init__.py
index 0bca48b..cd7a3d6 100644
--- a/scompose/config/__init__.py
+++ b/scompose/config/__init__.py
@@ -50,4 +50,4 @@ def _merge(a, b):
                 sys.exit(1)
         else:
             a[key] = b[key]
-    return a
\ No newline at end of file
+    return a

From 86c0905acbbee266449db24d3a79744b9f8c9963 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Thu, 7 Oct 2021 01:26:05 +0000
Subject: [PATCH 20/26] remove extra bullet point

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0b3b6bc..39195a4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,7 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pypi.
 
 ## [0.0.x](https://github.com/singularityhub/singularity-compose/tree/master) (0.0.x)
- - 
+ 
  - adding jsonschema validation and check command (0.0.12)
    - implement configuration override feature
    - implement `--preview` argument for the `check` command 

From acea351a25cac52516c0cd6a397837e11e02c4ea Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Thu, 7 Oct 2021 02:03:41 +0000
Subject: [PATCH 21/26] implement code review changes

---
 docs/commands.md                              |  45 +++----
 scompose/client/check.py                      |  17 ++-
 .../singularity-compose-1.yml                 |   0
 .../singularity-compose-2.yml                 |   0
 scompose/tests/test_config.py                 | 118 ++++++++++++++++++
 scompose/tests/test_utils.py                  | 107 ----------------
 6 files changed, 146 insertions(+), 141 deletions(-)
 rename scompose/tests/configs/{config_override => config_merge}/singularity-compose-1.yml (100%)
 rename scompose/tests/configs/{config_override => config_merge}/singularity-compose-2.yml (100%)
 create mode 100644 scompose/tests/test_config.py

diff --git a/docs/commands.md b/docs/commands.md
index 7b20f94..f1e3694 100644
--- a/docs/commands.md
+++ b/docs/commands.md
@@ -22,31 +22,26 @@ To view the combined compose files you can use `--preview`.
 ```bash
 $ singularity-compose -f singularity-compose.yml \ 
           -f singularity-compose.override.yml check  --preview
-singularity-compose.yml is valid.
-singularity-compose.override.yml is valid.
-Combined configs:
-{
-    "version": "2.0",
-    "instances": {
-        "cvatdb": {
-            "network": {
-                "enable": false
-            },
-            "volumes": [
-                "./recipes/postgres/env.sh:/.singularity.d/env/env.sh",
-                "./volumes/postgres/conf:/opt/bitnami/postgresql/conf",
-                "./volumes/postgres/tmp:/opt/bitnami/postgresql/tmp",
-                "/home/vagrant/postgres_data:/bitnami/postgresql"
-            ],
-            "build": {
-                "context": ".",
-                "recipe": "./recipes/postgres/main.def",
-                "options": [
-                    "fakeroot"
-                ]
-            }
-        },
- }
+          
+version: '2.0'
+instances:
+  cvatdb:
+    start:
+      options:
+      - containall
+    network:
+      enable: false
+    volumes:
+    - ./recipes/postgres/env.sh:/.singularity.d/env/env.sh
+    - ./volumes/postgres/conf:/opt/bitnami/postgresql/conf
+    - ./volumes/postgres/tmp:/opt/bitnami/postgresql/tmp
+    - /home/vagrant/postgres_data:/bitnami/postgresql
+    build:
+      context: .
+      recipe: ./recipes/postgres/main.def
+      options:
+      - fakeroot
+
 ```
 
 ## build
diff --git a/scompose/client/check.py b/scompose/client/check.py
index eede230..55ffaef 100644
--- a/scompose/client/check.py
+++ b/scompose/client/check.py
@@ -23,16 +23,15 @@ def main(args, parser, extra):
         --preview flag to show combined configs.
     """
 
-    # validate compose files
-    for f in args.file:
-        result = validate_config(f)
-        if not result:
-            bot.info("%s is valid." % f)
-        else:
-            bot.exit("%s is not valid." % f)
-
     if args.preview:
         # preview
         config = merge_config(args.file)
-        print("Combined configs:")
         print(yaml.dump(config, sort_keys=False))
+    else:
+        # validate compose files
+        for f in args.file:
+            result = validate_config(f)
+            if not result:
+                bot.info("%s is valid." % f)
+            else:
+                bot.exit("%s is not valid." % f)
diff --git a/scompose/tests/configs/config_override/singularity-compose-1.yml b/scompose/tests/configs/config_merge/singularity-compose-1.yml
similarity index 100%
rename from scompose/tests/configs/config_override/singularity-compose-1.yml
rename to scompose/tests/configs/config_merge/singularity-compose-1.yml
diff --git a/scompose/tests/configs/config_override/singularity-compose-2.yml b/scompose/tests/configs/config_merge/singularity-compose-2.yml
similarity index 100%
rename from scompose/tests/configs/config_override/singularity-compose-2.yml
rename to scompose/tests/configs/config_merge/singularity-compose-2.yml
diff --git a/scompose/tests/test_config.py b/scompose/tests/test_config.py
new file mode 100644
index 0000000..6351d80
--- /dev/null
+++ b/scompose/tests/test_config.py
@@ -0,0 +1,118 @@
+#!/usr/bin/python
+
+# Copyright (C) 2017-2021 Vanessa Sochat.
+
+# This Source Code Form is subject to the terms of the
+# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+
+def test_merge():
+    print("Testing utils._merge")
+    from scompose.config import _merge
+
+    # No override
+    a = {"a": 123}
+    b = {"b": 456}
+    assert _merge(a, b) == {"a": 123, "b": 456}
+
+    # Override + merge
+    a = {"a": 123}
+    b = {"b": 456, "a": 789}
+    assert _merge(a, b) == {"a": 789, "b": 456}
+
+    # Override only
+    a = {"a": 123}
+    b = {"a": 789}
+    assert _merge(a, b) == {"a": 789}
+
+    # Dict merge
+    a = {"a": 123, "b": {"c": "d"}}
+    b = {"b": {"e": "f"}}
+    assert _merge(a, b) == {"a": 123, "b": {"c": "d", "e": "f"}}
+
+    # Dict merge + key override
+    a = {"a": 123, "b": {"c": "d"}}
+    b = {"b": {"c": "f"}}
+    assert _merge(a, b) == {"a": 123, "b": {"c": "f"}}
+
+
+def test_deep_merge():
+    print("Testing utils._deep_merge")
+    from scompose.utils import read_yaml
+    from scompose.config import _deep_merge
+
+    config_override = os.path.join(here, "configs", "config_merge")
+
+    # single file
+    yaml_files = [
+        read_yaml(
+            os.path.join(config_override, "singularity-compose-1.yml"), quiet=True
+        )
+    ]
+    ret = _deep_merge(yaml_files)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1 arg2"},
+        }
+    }
+
+    # multiple files
+    yaml_files = [
+        read_yaml(
+            os.path.join(config_override, "singularity-compose-1.yml"), quiet=True
+        ),
+        read_yaml(
+            os.path.join(config_override, "singularity-compose-2.yml"), quiet=True
+        ),
+    ]
+    ret = _deep_merge(yaml_files)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1", "options": ["fakeroot"]},
+        },
+        "hello": {
+            "image": "from_the_other_side.sif",
+            "start": {"args": "how are you?"},
+        },
+    }
+
+
+def test_merge_config():
+    print("Testing utils.build_interpolated_config")
+    from scompose.config import merge_config
+
+    config_override = os.path.join(here, "configs", "config_merge")
+
+    # single file
+    file_list = [os.path.join(config_override, "singularity-compose-1.yml")]
+    ret = merge_config(file_list)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1 arg2"},
+        }
+    }
+
+    # multiple files
+    file_list = [
+        os.path.join(config_override, "singularity-compose-1.yml"),
+        os.path.join(config_override, "singularity-compose-2.yml"),
+    ]
+    ret = merge_config(file_list)
+    assert ret["instances"] == {
+        "echo": {
+            "build": {"context": ".", "recipe": "Singularity"},
+            "start": {"args": "arg0 arg1", "options": ["fakeroot"]},
+        },
+        "hello": {
+            "image": "from_the_other_side.sif",
+            "start": {"args": "how are you?"},
+        },
+    }
diff --git a/scompose/tests/test_utils.py b/scompose/tests/test_utils.py
index 3967567..6b2efd5 100644
--- a/scompose/tests/test_utils.py
+++ b/scompose/tests/test_utils.py
@@ -90,110 +90,3 @@ def test_print_json():
 
     result = print_json({1: 1})
     assert result == '{\n    "1": 1\n}'
-
-
-def test_merge():
-    print("Testing utils._merge")
-    from scompose.config import _merge
-
-    # No override
-    a = {"a": 123}
-    b = {"b": 456}
-    assert _merge(a, b) == {"a": 123, "b": 456}
-
-    # Override + merge
-    a = {"a": 123}
-    b = {"b": 456, "a": 789}
-    assert _merge(a, b) == {"a": 789, "b": 456}
-
-    # Override only
-    a = {"a": 123}
-    b = {"a": 789}
-    assert _merge(a, b) == {"a": 789}
-
-    # Dict merge
-    a = {"a": 123, "b": {"c": "d"}}
-    b = {"b": {"e": "f"}}
-    assert _merge(a, b) == {"a": 123, "b": {"c": "d", "e": "f"}}
-
-    # Dict merge + key override
-    a = {"a": 123, "b": {"c": "d"}}
-    b = {"b": {"c": "f"}}
-    assert _merge(a, b) == {"a": 123, "b": {"c": "f"}}
-
-
-def test_deep_merge():
-    print("Testing utils._deep_merge")
-    from scompose.utils import read_yaml
-    from scompose.config import _deep_merge
-
-    config_override = os.path.join(here, "configs", "config_override")
-
-    # single file
-    yaml_files = [
-        read_yaml(
-            os.path.join(config_override, "singularity-compose-1.yml"), quiet=True
-        )
-    ]
-    ret = _deep_merge(yaml_files)
-    assert ret["instances"] == {
-        "echo": {
-            "build": {"context": ".", "recipe": "Singularity"},
-            "start": {"args": "arg0 arg1 arg2"},
-        }
-    }
-
-    # multiple files
-    yaml_files = [
-        read_yaml(
-            os.path.join(config_override, "singularity-compose-1.yml"), quiet=True
-        ),
-        read_yaml(
-            os.path.join(config_override, "singularity-compose-2.yml"), quiet=True
-        ),
-    ]
-    ret = _deep_merge(yaml_files)
-    assert ret["instances"] == {
-        "echo": {
-            "build": {"context": ".", "recipe": "Singularity"},
-            "start": {"args": "arg0 arg1", "options": ["fakeroot"]},
-        },
-        "hello": {
-            "image": "from_the_other_side.sif",
-            "start": {"args": "how are you?"},
-        },
-    }
-
-
-def test_build_interpolated_config():
-    print("Testing utils.build_interpolated_config")
-    from scompose.config import merge_config
-
-    config_override = os.path.join(here, "configs", "config_override")
-
-    # single file
-    file_list = [os.path.join(config_override, "singularity-compose-1.yml")]
-    ret = merge_config(file_list)
-    assert ret["instances"] == {
-        "echo": {
-            "build": {"context": ".", "recipe": "Singularity"},
-            "start": {"args": "arg0 arg1 arg2"},
-        }
-    }
-
-    # multiple files
-    file_list = [
-        os.path.join(config_override, "singularity-compose-1.yml"),
-        os.path.join(config_override, "singularity-compose-2.yml"),
-    ]
-    ret = merge_config(file_list)
-    assert ret["instances"] == {
-        "echo": {
-            "build": {"context": ".", "recipe": "Singularity"},
-            "start": {"args": "arg0 arg1", "options": ["fakeroot"]},
-        },
-        "hello": {
-            "image": "from_the_other_side.sif",
-            "start": {"args": "how are you?"},
-        },
-    }

From 22dd789cbbf85b5812b8ad16905d994d549b6d3e Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Thu, 7 Oct 2021 02:11:27 +0000
Subject: [PATCH 22/26] implement code review changes

---
 scompose/client/check.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/scompose/client/check.py b/scompose/client/check.py
index 55ffaef..96454bc 100644
--- a/scompose/client/check.py
+++ b/scompose/client/check.py
@@ -23,15 +23,16 @@ def main(args, parser, extra):
         --preview flag to show combined configs.
     """
 
+    # validate compose files
+    for f in args.file:
+        result = validate_config(f)
+        if not result and not args.preview:
+            bot.info("%s is valid." % f)
+        elif result:
+            bot.exit("%s is not valid." % f)
+
     if args.preview:
         # preview
         config = merge_config(args.file)
         print(yaml.dump(config, sort_keys=False))
-    else:
-        # validate compose files
-        for f in args.file:
-            result = validate_config(f)
-            if not result:
-                bot.info("%s is valid." % f)
-            else:
-                bot.exit("%s is not valid." % f)
+

From 37f9e7ed1ce2dc20aa1eb5af9692633d207d54a9 Mon Sep 17 00:00:00 2001
From: Paulo Almeida <paulo.miguel.almeida.rodenas@gmail.com>
Date: Thu, 7 Oct 2021 02:12:43 +0000
Subject: [PATCH 23/26] bitten by the black formatting (again)

---
 scompose/client/check.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scompose/client/check.py b/scompose/client/check.py
index 96454bc..65bc6b7 100644
--- a/scompose/client/check.py
+++ b/scompose/client/check.py
@@ -35,4 +35,3 @@ def main(args, parser, extra):
         # preview
         config = merge_config(args.file)
         print(yaml.dump(config, sort_keys=False))
-

From 2d75868ec947a9820fc4e9b66d8b5d204be33bd0 Mon Sep 17 00:00:00 2001
From: vsoch <vsoch@users.noreply.github.com>
Date: Wed, 6 Oct 2021 20:40:43 -0600
Subject: [PATCH 24/26] move config functions into proper module

Signed-off-by: vsoch <vsoch@users.noreply.github.com>
---
 scompose/client/check.py                      |  3 +-
 scompose/config/__init__.py                   | 26 +++++++++-
 .../{project/config.py => config/schema.py}   | 47 +++++++++----------
 scompose/project/project.py                   | 11 ++++-
 setup.py                                      |  5 +-
 5 files changed, 60 insertions(+), 32 deletions(-)
 rename scompose/{project/config.py => config/schema.py} (67%)

diff --git a/scompose/client/check.py b/scompose/client/check.py
index 65bc6b7..1f92a79 100644
--- a/scompose/client/check.py
+++ b/scompose/client/check.py
@@ -8,9 +8,8 @@
 
 """
 
-from scompose.project.config import validate_config
 from scompose.logger import bot
-from scompose.config import merge_config
+from scompose.config import validate_config, merge_config
 import yaml
 
 
diff --git a/scompose/config/__init__.py b/scompose/config/__init__.py
index cd7a3d6..db77a10 100644
--- a/scompose/config/__init__.py
+++ b/scompose/config/__init__.py
@@ -3,8 +3,26 @@
 
 from scompose.utils import read_yaml
 
+# We don't require jsonschema, so catch import error and alert user
+try:
+    from jsonschema import validate
+except ImportError as e:
+    msg = "pip install jsonschema"
+    sys.exit("jsonschema is required for checking and validation: %s\n %s" % (e, msg))
+
+
+def validate_config(filepath):
+    """
+    Validate a singularity-compose.yaml file.
+    """
+    cfg = read_yaml(filepath, quiet=True)
+    return validate(cfg, compose_schema)
+
 
 def merge_config(file_list):
+    """
+    Given one or more config files, merge into one
+    """
     yaml_files = []
     for f in file_list:
         try:
@@ -23,7 +41,9 @@ def merge_config(file_list):
 
 
 def _deep_merge(yaml_files):
-    """merge yaml files into a single dict"""
+    """
+    Merge yaml files into a single dict
+    """
     base_yaml = None
     for idx, item in enumerate(yaml_files):
         if idx == 0:
@@ -35,7 +55,9 @@ def _deep_merge(yaml_files):
 
 
 def _merge(a, b):
-    """merge dict b into a"""
+    """
+    Merge dict b into a
+    """
     for key in b:
         if key in a:
             # merge dicts recursively
diff --git a/scompose/project/config.py b/scompose/config/schema.py
similarity index 67%
rename from scompose/project/config.py
rename to scompose/config/schema.py
index e34d181..8849c11 100644
--- a/scompose/project/config.py
+++ b/scompose/config/schema.py
@@ -8,25 +8,6 @@
 
 """
 
-import sys
-from scompose.utils import read_yaml
-
-# We don't require jsonschema, so catch import error and alert user
-try:
-    from jsonschema import validate
-except ImportError as e:
-    msg = "pip install jsonschema"
-    sys.exit("jsonschema is required for checking and validation: %s\n %s" % (e, msg))
-
-
-def validate_config(filepath):
-    """
-    Validate a singularity-compose.yaml file.
-    """
-    cfg = read_yaml(filepath, quiet=True)
-    return validate(cfg, compose_schema)
-
-
 ## Singularity Compose Schema
 
 schema_url = "https://json-schema.org/draft-07/schema/#"
@@ -46,29 +27,42 @@ def validate_config(filepath):
 
 instance_network = {
     "type": "object",
-    "properties": {"allocate_ip": {"type": "boolean"}, "enable": {"type": "boolean"},},
+    "properties": {
+        "allocate_ip": {"type": "boolean"},
+        "enable": {"type": "boolean"},
+    },
 }
 
 
 instance_start = {
     "type": "object",
-    "properties": {"args": {"type": ["string", "array"]}, "options": string_list,},
+    "properties": {
+        "args": {"type": ["string", "array"]},
+        "options": string_list,
+    },
 }
 
 instance_run = {
     "type": "object",
-    "properties": {"args": {"type": ["string", "array"]}, "options": string_list,},
+    "properties": {
+        "args": {"type": ["string", "array"]},
+        "options": string_list,
+    },
 }
 
 instance_post = {
     "type": "object",
-    "properties": {"commands": string_list,},
+    "properties": {
+        "commands": string_list,
+    },
 }
 
 instance_exec = {
     "type": "object",
     "properties": {"options": string_list, "command": {"type": "string"}},
-    "required": ["command",],
+    "required": [
+        "command",
+    ],
 }
 
 # A single instance
@@ -99,7 +93,10 @@ def validate_config(filepath):
     "$schema": schema_url,
     "title": "Singularity Compose Schema",
     "type": "object",
-    "required": ["version", "instances",],
+    "required": [
+        "version",
+        "instances",
+    ],
     "properties": properties,
     "additionalProperties": False,
 }
diff --git a/scompose/project/project.py b/scompose/project/project.py
index 7ccc768..4f39977 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -451,7 +451,11 @@ def create(
         return self._create(names, writable_tmpfs=writable_tmpfs, no_resolv=no_resolv)
 
     def up(
-        self, names=None, writable_tmpfs=True, bridge="10.22.0.0/16", no_resolv=False,
+        self,
+        names=None,
+        writable_tmpfs=True,
+        bridge="10.22.0.0/16",
+        no_resolv=False,
     ):
 
         """
@@ -460,7 +464,10 @@ def up(
         This will build before if a container binary does not exist.
         """
         return self._create(
-            names, command="up", writable_tmpfs=writable_tmpfs, no_resolv=no_resolv,
+            names,
+            command="up",
+            writable_tmpfs=writable_tmpfs,
+            no_resolv=no_resolv,
         )
 
     def _create(
diff --git a/setup.py b/setup.py
index f304bb0..279ec56 100644
--- a/setup.py
+++ b/setup.py
@@ -90,7 +90,10 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"):
         install_requires=INSTALL_REQUIRES,
         setup_requires=["pytest-runner"],
         tests_require=TESTS_REQUIRES,
-        extras_require={"all": [INSTALL_REQUIRES_ALL], "checks": [INSTALL_REQUIRES_CHECKS]},
+        extras_require={
+            "all": [INSTALL_REQUIRES_ALL],
+            "checks": [INSTALL_REQUIRES_CHECKS],
+        },
         classifiers=[
             "Intended Audience :: Science/Research",
             "Intended Audience :: Developers",

From 779456ad4e42fa7c87d73c9b297ed385435ea37c Mon Sep 17 00:00:00 2001
From: vsoch <vsoch@users.noreply.github.com>
Date: Wed, 6 Oct 2021 20:48:10 -0600
Subject: [PATCH 25/26] trying to pin black to uptodate version

and moving validate.py into its own file so we do not always import

Signed-off-by: vsoch <vsoch@users.noreply.github.com>
---
 .github/workflows/main.yml  |  5 +++--
 scompose/client/check.py    |  3 ++-
 scompose/config/__init__.py | 25 ++++++++++---------------
 scompose/config/schema.py   | 21 +++++++++++++++++++++
 scompose/project/project.py |  1 -
 5 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a6bdd39..3a4e80b 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -14,12 +14,13 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Setup black environment
-        run: conda create --quiet --name black black pyflakes
+        run: conda create --quiet --name black pyflakes
 
       - name: Check formatting with black
         run: |
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate black
+          pip install black==21.6b0
           black --check scompose
 
       - name: Check imports with pyflakes
@@ -28,4 +29,4 @@ jobs:
           source activate black
           pyflakes scompose/utils
           # Will have some issues
-          pyflakes scompose/client scompose/project || true
+          pyflakes scompose/client scompose/project scompose/config || true
diff --git a/scompose/client/check.py b/scompose/client/check.py
index 1f92a79..f1408de 100644
--- a/scompose/client/check.py
+++ b/scompose/client/check.py
@@ -9,7 +9,8 @@
 """
 
 from scompose.logger import bot
-from scompose.config import validate_config, merge_config
+from scompose.config import merge_config
+from scompose.config.validate import validate_config
 import yaml
 
 
diff --git a/scompose/config/__init__.py b/scompose/config/__init__.py
index db77a10..83c575c 100644
--- a/scompose/config/__init__.py
+++ b/scompose/config/__init__.py
@@ -1,22 +1,17 @@
-import os
-import sys
+"""
 
-from scompose.utils import read_yaml
+Copyright (C) 2019-2021 Vanessa Sochat.
 
-# We don't require jsonschema, so catch import error and alert user
-try:
-    from jsonschema import validate
-except ImportError as e:
-    msg = "pip install jsonschema"
-    sys.exit("jsonschema is required for checking and validation: %s\n %s" % (e, msg))
+This Source Code Form is subject to the terms of the
+Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed
+with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+"""
 
-def validate_config(filepath):
-    """
-    Validate a singularity-compose.yaml file.
-    """
-    cfg = read_yaml(filepath, quiet=True)
-    return validate(cfg, compose_schema)
+import os
+import sys
+
+from scompose.utils import read_yaml
 
 
 def merge_config(file_list):
diff --git a/scompose/config/schema.py b/scompose/config/schema.py
index 8849c11..d6c8ded 100644
--- a/scompose/config/schema.py
+++ b/scompose/config/schema.py
@@ -8,6 +8,27 @@
 
 """
 
+import os
+import sys
+
+from scompose.utils import read_yaml
+
+# We don't require jsonschema, so catch import error and alert user
+try:
+    from jsonschema import validate
+except ImportError as e:
+    msg = "pip install jsonschema"
+    sys.exit("jsonschema is required for checking and validation: %s\n %s" % (e, msg))
+
+
+def validate_config(filepath):
+    """
+    Validate a singularity-compose.yaml file.
+    """
+    cfg = read_yaml(filepath, quiet=True)
+    return validate(cfg, compose_schema)
+
+
 ## Singularity Compose Schema
 
 schema_url = "https://json-schema.org/draft-07/schema/#"
diff --git a/scompose/project/project.py b/scompose/project/project.py
index 4f39977..b07f0bf 100644
--- a/scompose/project/project.py
+++ b/scompose/project/project.py
@@ -19,7 +19,6 @@
 import os
 import re
 import subprocess
-import sys
 
 
 class Project(object):

From 352ee85b81594643ea717e2eab3063a338682b3b Mon Sep 17 00:00:00 2001
From: vsoch <vsoch@users.noreply.github.com>
Date: Wed, 6 Oct 2021 20:53:58 -0600
Subject: [PATCH 26/26] use bot.exit over print and sys.exit  (which combines
 the two)

Signed-off-by: vsoch <vsoch@users.noreply.github.com>
---
 scompose/config/__init__.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/scompose/config/__init__.py b/scompose/config/__init__.py
index 83c575c..0937216 100644
--- a/scompose/config/__init__.py
+++ b/scompose/config/__init__.py
@@ -9,8 +9,8 @@
 """
 
 import os
-import sys
 
+from scompose.logger import bot
 from scompose.utils import read_yaml
 
 
@@ -23,13 +23,12 @@ def merge_config(file_list):
         try:
             # ensure file exists
             if not os.path.exists(f):
-                print("%s does not exist." % f)
-                sys.exit(1)
+                bot.exit("%s does not exist." % f)
+
             # read yaml file
             yaml_files.append(read_yaml(f, quiet=True))
         except:  # ParserError
-            print("Cannot parse %s, invalid yaml." % f)
-            sys.exit(1)
+            bot.exit("Cannot parse %s, invalid yaml." % f)
 
     # merge/override yaml properties where applicable
     return _deep_merge(yaml_files)
@@ -63,8 +62,7 @@ def _merge(a, b):
                 a[key] = b[key]
             # if nothing matches then this means a conflict of types which shouldn't exist in the first place
             else:
-                print("key '%s': type mismatch in different files." % key)
-                sys.exit(1)
+                bot.exit("key '%s': type mismatch in different files." % key)
         else:
             a[key] = b[key]
     return a