Merge pull request #154 from eyecan-ai/develop

Develop

.coveragerc

@@ -28,3 +28,6 @@ exclude_lines =
 
     # SamplesSequence stubs are not run
     @samples_sequence_stub
+
+    # Deprecated classes or functions are not run
+    @deprecated

pipelime/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = "Eyecan.ai"
 __email__ = "[email protected]"
-__version__ = "1.9.0"
+__version__ = "1.9.1"

pipelime/choixe/utils/io.py

@@ -111,9 +111,9 @@ def get_temp_dirs() -> Dict[str, List[str]]:
 
         user_dirs: Dict[str, List[str]] = {}
         for d in iglob(str(PipelimeTmp.base_dir() / (PipelimeTmp.prefix() + "*"))):
-            if os.path.isdir(d):
+            if os.path.isdir(d):  # pragma: no branch
                 names = os.path.basename(d).split("-")
-                if len(names) >= 3:
+                if len(names) >= 3:  # pragma: no branch
                     user_dirs.setdefault(names[2], []).append(d)
         return user_dirs
 
@@ -135,7 +135,8 @@ def _rmtree(cls, name):
         import shutil
 
         def onerror(func, path, exc_info):
-            if issubclass(exc_info[0], PermissionError):
+            # this code comes from TemporaryDirectory
+            if issubclass(exc_info[0], PermissionError):  # pragma: no cover
 
                 def resetperms(path):
                     try:

pipelime/cli/main.py

@@ -57,7 +57,30 @@ def _purge(self, data):
         return data
 
 
-def _complete_yaml(incomplete: str):
+def _set_logger_level(verbose: int):
+    """Set the logger level if not forced by the user."""
+    import os
+    import sys
+
+    from loguru import logger
+
+    if "LOGURU_LEVEL" not in os.environ:
+        if verbose == 0:
+            level = "WARNING"
+        elif verbose == 1:
+            level = "INFO"
+        elif verbose == 2:
+            level = "DEBUG"
+        else:
+            level = "TRACE"
+        logger.remove()
+        logger.add(sys.stderr, level=level)
+
+        # set the level in the environment so that it is inherited by subprocesses
+        os.environ["LOGURU_LEVEL"] = level
+
+
+def _complete_yaml(incomplete: str):  # pragma: no cover
     for v in Path(".").glob(f"{incomplete}*.yaml"):
         yield str(v)
 
@@ -468,6 +491,8 @@ def pl_main(
     TRUE boolean values. Use `false` or `true` to explicitly set a boolean
     and `none`/`null`/`nul` to enforce `None`.
     """
+    _set_logger_level(verbose)
+
     PipelimeSymbolsHelper.set_extra_modules(extra_modules)
 
     if command_args is None:

pipelime/commands/interfaces.py

@@ -819,7 +819,7 @@ class Interval(PydanticFieldWithDefaultMixin, pyd.BaseModel, extra="forbid"):
     """
 
     _default_type_description: t.ClassVar[t.Optional[str]] = "An interval of indexes."
-    _compact_form: t.ClassVar[t.Optional[str]] = "<start>[:<stop>]"
+    _compact_form: t.ClassVar[t.Optional[str]] = "[<start>][:<stop>]"
 
     start: t.Optional[int] = pyd.Field(
         None,
@@ -872,7 +872,7 @@ class ExtendedInterval(Interval):
     ``step`` indices or a string ``start:stop:step``.
     """
 
-    _compact_form: t.ClassVar[t.Optional[str]] = "<start>[:<stop>[:<step>]]"
+    _compact_form: t.ClassVar[t.Optional[str]] = "[<start>][:<stop>][:<step>]"
 
     step: t.Optional[int] = pyd.Field(
         None, description="The slice step, defaults to 1 (can be negative)."

pipelime/commands/shell.py

@@ -50,8 +50,9 @@ def command_name(self) -> str:
 
     def _to_command_chunk(self, key: str, value: Any) -> str:
         cmd = ""
-        if isinstance(value, Sequence):
-            cmd += f" --{key} {value[0]} {self._to_command_chunk(key, value[1:])}"
+        if isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
+            if value:
+                cmd += f" --{key} {value[0]} {self._to_command_chunk(key, value[1:])}"
         elif isinstance(value, Mapping):
             raise NotImplementedError("Mapping values are not supported")
         elif isinstance(value, bool):
@@ -68,8 +69,7 @@ def run(self) -> None:
         cmd = cmd.format(**args)
 
         for key in set(args.keys()).difference(fields):
-            value = args[key]
-            cmd += self._to_command_chunk(key, value)
+            cmd += self._to_command_chunk(key, args[key])
 
         with self.create_task(1, "Executing") as t:
             subprocess.run(cmd, shell=True)

pipelime/commands/split_ops.py

@@ -202,7 +202,7 @@ def run(self):
             range(len(split_sizes)), split_sizes, splits
         ):
             split_stop = split_start + split_length  # type: ignore
-            if split.output is not None:
+            if split.output is not None:  # pragma: no branch
                 seq = reader[split_start:split_stop]
                 seq = split.output.append_writer(seq)
                 self.grabber.grab_all(
@@ -376,7 +376,7 @@ def _value_to_str(self, value):
 
             def __call__(self, x: Sample):
                 value = x.deep_get(self._value_key)
-                if value is not None:
+                if value is not None:  # pragma: no branch
                     value = self._value_to_str(value)
                     self._groups.setdefault(value, []).append(
                         int(x[self._idx_key]())  # type: ignore

pipelime/commands/tempman.py

@@ -102,7 +102,7 @@ def run(self):
 
         to_delete = self._folders_to_delete()
 
-        if not self.force and to_delete:
+        if not self.force and to_delete:  # pragma: no cover
             table, total_size = self._paths_table(to_delete)
             table.title = f"\nTotal clean up: {self._human_size(total_size)}"
             rprint(table)
@@ -251,7 +251,7 @@ def _traverse(self, root_path: str, update_fn: t.Callable, total_init):
                 fp = os.path.join(dirpath, f)
                 try:
                     stat = os.stat(fp)
-                except OSError:
+                except OSError:  # pragma: no cover
                     continue
 
                 if stat.st_ino in seen:

pipelime/piper/checkpoint.py

@@ -164,6 +164,7 @@ def add_asset(
 
             if self.try_link:
                 if source.is_dir():
+                    target.mkdir(parents=True, exist_ok=True)
                     for parent_dir, dirs, files in os.walk(source):
                         subdir = os.path.relpath(parent_dir, source)
                         trg_parent = os.path.join(target, subdir)
@@ -174,12 +175,12 @@ def add_asset(
                             trgf = os.path.join(trg_parent, f)
                             try:
                                 os.link(srcf, trgf)
-                            except Exception:
+                            except Exception:  # pragma: no cover
                                 shutil.copy2(srcf, trgf)
                 else:
                     try:
                         os.link(source, target)
-                    except Exception:
+                    except Exception:  # pragma: no cover
                         shutil.copy2(source, target)
             elif source.is_dir():
                 shutil.copytree(source, target)

pipelime/piper/drawing/mermaid.py

@@ -16,7 +16,7 @@
 )
 
 
-class MermaidNodesGraphDrawer(NodesGraphDrawer):
+class MermaidNodesGraphDrawer(NodesGraphDrawer):  # pragma: no cover
     PRIVATE_CHARS = ["@"]
     SUBSTITUTION_CHAR = "_"
     OPERATION_NAME = "operation"

pipelime/piper/executors/base.py

@@ -14,6 +14,7 @@
 )
 from pipelime.piper.progress.tracker.base import TrackedTask
 from pipelime.piper import PipelimeCommand
+from pipelime.utils.deprecated import deprecated
 
 
 class NodesGraphExecutor(ABC):
@@ -155,6 +156,7 @@ def exec(
         """
 
 
+@deprecated("Use a DirectTrackCallback instead, see RunCommandBase")
 class WatcherNodesGraphExecutor(NodesGraphExecutor):
     """Decorator for `NodesGraphExecutor` that wraps an existing executor.
 

pipelime/piper/executors/factory.py

@@ -27,7 +27,7 @@ def get_executor(
         )
         executor.task = task
 
-        if watch:
+        if watch:  # pragma: no cover
             executor = WatcherNodesGraphExecutor(
                 executor, watch if isinstance(watch, (str, Path)) else None
             )

pipelime/utils/deprecated.py

@@ -0,0 +1,51 @@
+import functools
+import inspect
+import warnings
+
+
+def deprecated(reason):  # pragma: no cover
+    """Decorator to mark classes and functions as deprecated.
+    A warning will be emitted when the function is used or the class is created.
+    """
+
+    def _helper(wrapped, extra):
+        msg = (
+            "Call to deprecated "
+            f"{'class' if inspect.isclass(wrapped) else 'function'} "
+            f"{wrapped.__name__}{extra}"
+        )
+
+        def show_warning():
+            warnings.simplefilter("always", DeprecationWarning)
+            warnings.warn(
+                msg,
+                category=DeprecationWarning,
+                stacklevel=3,
+            )
+            warnings.simplefilter("default", DeprecationWarning)
+
+        if inspect.isclass(wrapped):
+            old_new1 = wrapped.__new__
+
+            def wrapped_new(cls, *args, **kwargs):
+                show_warning()
+                if old_new1 is object.__new__:
+                    return old_new1(cls)  # type: ignore
+                return old_new1(cls, *args, **kwargs)
+
+            wrapped.__new__ = staticmethod(functools.wraps(old_new1)(wrapped_new))  # type: ignore
+
+            return wrapped
+        else:
+
+            @functools.wraps(wrapped)
+            def wrapper(*args, **kwargs):
+                show_warning()
+                return wrapped(*args, **kwargs)
+
+            return wrapper
+
+    if isinstance(reason, (str, bytes)):
+        return functools.partial(_helper, extra=f" ({reason})")
+    else:
+        return _helper(reason, "")

tests/__init__.py

@@ -47,3 +47,9 @@ def samples_equal(s1, s2):
                 assert TestUtils.numpy_eq(v1(), v2())
             else:
                 assert v1() == v2()
+
+    @staticmethod
+    def sequences_equal(s1, s2):
+        assert len(s1) == len(s2)
+        for x1, x2 in zip(s1, s2):
+            TestAssert.samples_equal(x1, x2)

tests/conftest.py

@@ -61,12 +61,17 @@ def ckpt_dag(data_folder: Path) -> Path:
     return data_folder / "cli" / "ckpt_dag.py"
 
 
+@pytest.fixture(scope="session")
+def shell_cmd(data_folder: Path) -> Path:
+    return data_folder / "cli" / "shell_cmd.py"
+
+
 @pytest.fixture(scope="session")
 def minimnist_dataset(datasets_folder: Path) -> dict:
     return {
         "path": datasets_folder / "underfolder_minimnist",
         "root_keys": ["cfg", "numbers", "pose"],
-        "item_keys": ["image", "label", "mask", "metadata", "points"],
+        "item_keys": ["image", "label", "mask", "metadata", "values", "points"],
         "item_types": {
             "cfg": "YamlMetadataItem",
             "numbers": "TxtNumpyItem",
@@ -75,6 +80,7 @@ def minimnist_dataset(datasets_folder: Path) -> dict:
             "label": "TxtNumpyItem",
             "mask": "PngImageItem",
             "metadata": "JsonMetadataItem",
+            "values": "YamlMetadataItem",
             "points": "TxtNumpyItem",
         },
         "image_keys": ["image", "mask"],

tests/pipelime/cli/test_base.py

@@ -1,4 +1,5 @@
 import os
+import traceback
 from typing import Any, List
 
 import pytest
@@ -25,8 +26,15 @@ def _base_launch(self, args: list, exit_code=0, exc=None, user_input: str = ""):
         result = runner.invoke(_create_typer_app(), args, input=user_input)
 
         print("*** CLI COMMAND OUTPUT ***", result.output, sep="\n")
-        print("EXCEPTION:", type(result.exception))
-        print(result.exception)
+        if result.stderr_bytes is not None:
+            print("*** CLI COMMAND STDERR ***", result.stderr, sep="\n")
+        if result.exc_info is not None:
+            print("*** CLI COMMAND EXCEPTION ***")
+            traceback.print_exception(
+                result.exc_info[0],
+                value=result.exc_info[1],
+                tb=result.exc_info[2],
+            )
         print("EXIT CODE:", result.exit_code)
 
         assert result.exit_code == exit_code
@@ -92,7 +100,8 @@ def _check(result, *, in_modules=[], not_in_modules=[]):
         assert "Type" in result.output
         assert "Default" in result.output
 
-    def test_run(self, minimnist_dataset, tmp_path):
+    @pytest.mark.parametrize("verbose", ["", "-v", "-vv", "-vvv", "-vvvv"])
+    def test_run(self, minimnist_dataset, verbose, tmp_path):
         from pathlib import Path
         from typing import Sequence
 
@@ -114,6 +123,9 @@ def test_run(self, minimnist_dataset, tmp_path):
             "10",
         ]
 
+        if verbose:
+            args.append(verbose)
+
         self._base_launch(args)
 
         outreader = SamplesSequence.from_underfolder(outpath)  # type: ignore

tests/pipelime/commands/test_general_base.py

@@ -47,6 +47,11 @@ class TestGeneralCommandsBase:
             "is_optional": False,
             "is_shared": False,
         },
+        "values": {
+            "class_path": "YamlMetadataItem",
+            "is_optional": False,
+            "is_shared": False,
+        },
         "points": {
             "class_path": "TxtNumpyItem",
             "is_optional": False,