storage tree command

CHANGELOG.D/1435.feature

@@ -0,0 +1 @@
+Implement `neuro storage tree` command for displaying the directory tree on storage.

README.md

@@ -34,6 +34,7 @@
 		* [neuro storage rm](#neuro-storage-rm)
 		* [neuro storage mkdir](#neuro-storage-mkdir)
 		* [neuro storage mv](#neuro-storage-mv)
+		* [neuro storage tree](#neuro-storage-tree)
 		* [neuro storage load](#neuro-storage-load)
 	* [neuro image](#neuro-image)
 		* [neuro image ls](#neuro-image-ls)
@@ -847,6 +848,7 @@ Name | Description|
 | _[neuro storage rm](#neuro-storage-rm)_| Remove files or directories |
 | _[neuro storage mkdir](#neuro-storage-mkdir)_| Make directories |
 | _[neuro storage mv](#neuro-storage-mv)_| Move or rename files and directories |
+| _[neuro storage tree](#neuro-storage-tree)_| List contents of directories in a tree-like format |
 | _[neuro storage load](#neuro-storage-load)_| Copy files and directories using MinIO \(EXPERIMENTAL) |
 
 
@@ -923,11 +925,11 @@ neuro storage ls [OPTIONS] [PATHS]...
 
 Name | Description|
 |----|------------|
+|_\-a, --all_|do not ignore entries starting with .|
+|_\-d, --directory_|list directories themselves, not their contents.|
 |_\-h, --human-readable_|with -l print human readable sizes \(e.g., 2K, 540M).|
 |_-l_|use a long listing format.|
 |_--sort \[name | size | time]_|sort by given field, default is name.|
-|_\-d, --directory_|list directories themselves, not their contents.|
-|_\-a, --all_|do not ignore entries starting with .|
 |_--help_|Show this message and exit.|
 
 
@@ -1050,6 +1052,29 @@ Name | Description|
 
 
 
+### neuro storage tree
+
+List contents of directories in a tree-like format.<br/><br/>Tree is a recursive directory listing program that produces a depth indented<br/>listing of files, which is colorized ala dircolors if the LS_COLORS<br/>environment variable is set and output is to tty.  With no arguments, tree<br/>lists the files in the storage: directory.  When directory arguments are<br/>given, tree lists all the files and/or directories found in the given<br/>directories each in turn.  Upon completion of listing all files/directories<br/>found, tree returns the total number of files and/or directories listed.<br/><br/>By default PATH is equal user's home dir \(storage:)
+
+**Usage:**
+
+```bash
+neuro storage tree [OPTIONS] [PATH]
+```
+
+**Options:**
+
+Name | Description|
+|----|------------|
+|_\-a, --all_|do not ignore entries starting with .|
+|_\-h, --human-readable_|Print the size in a more human readable way.|
+|_\-s, --size_|Print the size in bytes of each file.|
+|_--sort \[name &#124; size &#124; time]_|sort by given field, default is name|
+|_--help_|Show this message and exit.|
+
+
+
+
 ### neuro storage load
 
 Copy files and directories using MinIO \(EXPERIMENTAL).<br/><br/>Same as "cp", but uses MinIO and the Amazon S3 protocol.<br/>\(DEPRECATED)
@@ -2105,11 +2130,11 @@ neuro ls [OPTIONS] [PATHS]...
 
 Name | Description|
 |----|------------|
+|_\-a, --all_|do not ignore entries starting with .|
+|_\-d, --directory_|list directories themselves, not their contents.|
 |_\-h, --human-readable_|with -l print human readable sizes \(e.g., 2K, 540M).|
 |_-l_|use a long listing format.|
 |_--sort \[name &#124; size &#124; time]_|sort by given field, default is name.|
-|_\-d, --directory_|list directories themselves, not their contents.|
-|_\-a, --all_|do not ignore entries starting with .|
 |_--help_|Show this message and exit.|
 
 

neuromation/cli/formatters/storage.py

@@ -1,8 +1,11 @@
 import abc
+import contextlib
 import enum
 import operator
 import os
+import sys
 import time
+from dataclasses import dataclass
 from fnmatch import fnmatch
 from math import ceil
 from time import monotonic
@@ -808,3 +811,95 @@ def flush(self) -> None:
         self.first_line = len(self.lines)
         self.last_line = 0
         self.last_update_time = self.time_factory()
+
+
+@dataclass(frozen=True)
+class Tree:
+    name: str
+    size: int
+    folders: Sequence["Tree"]
+    files: Sequence[FileStatus]
+
+
+class TreeFormatter:
+    ANSI_DELIMS = ["├", "└", "─", "│"]
+    SIMPLE_DELIMS = ["+", "+", "-", "|"]
+
+    def __init__(
+        self, *, color: bool, size: bool, human_readable: bool, sort: str
+    ) -> None:
+        self._ident: List[bool] = []
+        self._numdirs = 0
+        self._numfiles = 0
+        self._painter = get_painter(color, quote=True)
+        if sys.platform != "win32":
+            self._delims = self.ANSI_DELIMS
+        else:
+            self._delims = self.SIMPLE_DELIMS
+        if human_readable:
+            self._size_func = self._human_readable
+        elif size:
+            self._size_func = self._size
+        else:
+            self._size_func = self._none
+        self._key = FilesSorter(sort).key()
+
+    def __call__(self, tree: Tree) -> List[str]:
+        ret = self.listdir(tree)
+        ret.append("")
+        ret.append(f"{self._numdirs} directories, {self._numfiles} files")
+        return ret
+
+    def listdir(self, tree: Tree) -> List[str]:
+        ret = []
+        items = sorted(tree.folders + tree.files, key=self._key)  # type: ignore
+        ret.append(
+            self.pre()
+            + self._size_func(tree.size)
+            + self._painter.paint(tree.name, FileStatusType.DIRECTORY)
+        )
+        for num, item in enumerate(items):
+            if isinstance(item, Tree):
+                self._numdirs += 1
+                with self.ident(num == len(items) - 1):
+                    ret.extend(self.listdir(item))
+            else:
+                self._numfiles += 1
+                with self.ident(num == len(items) - 1):
+                    ret.append(
+                        self.pre()
+                        + self._size_func(item.size)
+                        + self._painter.paint(item.name, FileStatusType.FILE)
+                    )
+        return ret
+
+    def pre(self) -> str:
+        ret = []
+        for last in self._ident[:-1]:
+            if last:
+                ret.append(" " * 4)
+            else:
+                ret.append(self._delims[3] + " " * 3)
+        if self._ident:
+            last = self._ident[-1]
+            ret.append(self._delims[1] if last else self._delims[0])
+            ret.append(self._delims[2] * 2)
+            ret.append(" ")
+        return "".join(ret)
+
+    @contextlib.contextmanager
+    def ident(self, last: bool) -> Iterator[None]:
+        self._ident.append(last)
+        try:
+            yield
+        finally:
+            self._ident.pop()
+
+    def _size(self, size: int) -> str:
+        return f"[{size:>11}]  "
+
+    def _human_readable(self, size: int) -> str:
+        return f"[{format_size(size):>7}]  "
+
+    def _none(self, size: int) -> str:
+        return ""

neuromation/cli/storage.py

@@ -1,4 +1,5 @@
 import asyncio
+import dataclasses
 import glob as globmodule  # avoid conflict with subcommand "glob"
 import logging
 import os
@@ -27,12 +28,14 @@
 from neuromation.api.url_utils import _extract_path
 
 from .const import EX_OSFILE
-from .formatters import (
+from .formatters.jobs import JobStartProgress
+from .formatters.storage import (
     BaseFilesFormatter,
     FilesSorter,
-    JobStartProgress,
     LongFilesFormatter,
     SimpleFilesFormatter,
+    Tree,
+    TreeFormatter,
     VerticalColumnsFilesFormatter,
     create_storage_progress,
     get_painter,
@@ -108,6 +111,19 @@ async def rm(root: Root, paths: Sequence[str], recursive: bool, glob: bool) -> N
 
 @command()
 @argument("paths", nargs=-1)
+@option(
+    "-a",
+    "--all",
+    "show_all",
+    is_flag=True,
+    help="do not ignore entries starting with .",
+)
+@option(
+    "-d",
+    "--directory",
+    is_flag=True,
+    help="list directories themselves, not their contents.",
+)
 @option(
     "--human-readable",
     "-h",
@@ -121,19 +137,6 @@ async def rm(root: Root, paths: Sequence[str], recursive: bool, glob: bool) -> N
     default="name",
     help="sort by given field, default is name.",
 )
-@option(
-    "-d",
-    "--directory",
-    is_flag=True,
-    help="list directories themselves, not their contents.",
-)
-@option(
-    "-a",
-    "--all",
-    "show_all",
-    is_flag=True,
-    help="do not ignore entries starting with .",
-)
 async def ls(
     root: Root,
     paths: Sequence[str],
@@ -835,6 +838,67 @@ async def mv(
         sys.exit(EX_OSFILE)
 
 
+@command()
+@click.argument("path", required=False)
+@option(
+    "-a",
+    "--all",
+    "show_all",
+    is_flag=True,
+    help="do not ignore entries starting with .",
+)
+@option(
+    "--human-readable",
+    "-h",
+    is_flag=True,
+    help="Print the size in a more human readable way.",
+)
+@option(
+    "--size", "-s", is_flag=True, help="Print the size in bytes of each file.",
+)
+@option(
+    "--sort",
+    type=click.Choice(["name", "size", "time"]),
+    default="name",
+    help="sort by given field, default is name",
+)
+async def tree(
+    root: Root, path: str, size: bool, human_readable: bool, sort: str, show_all: bool
+) -> None:
+    """List contents of directories in a tree-like format.
+
+    Tree is a recursive directory listing program that produces a depth indented listing
+    of files, which is colorized ala dircolors if the LS_COLORS environment variable is
+    set and output is to tty.  With no arguments, tree lists the files in the storage:
+    directory.  When directory arguments are given, tree lists all the files and/or
+    directories found in the given directories each in turn.  Upon completion of listing
+    all files/directories found, tree returns the total number of files and/or
+    directories listed.
+
+    By default PATH is equal user's home dir (storage:)
+
+    """
+    if not path:
+        path = "storage:"
+    uri = parse_file_resource(path, root)
+
+    errors = False
+    try:
+        tree = await fetch_tree(root.client, uri, show_all)
+        tree = dataclasses.replace(tree, name=str(path))
+    except (OSError, ResourceNotFound) as error:
+        log.error(f"cannot fetch tree for {uri}: {error}")
+        errors = True
+    else:
+        formatter = TreeFormatter(
+            color=root.color, size=size, human_readable=human_readable, sort=sort
+        )
+        pager_maybe(formatter(tree), root.tty, root.terminal_size)
+
+    if errors:
+        sys.exit(EX_OSFILE)
+
+
 async def _expand(
     paths: Sequence[str], root: Root, glob: bool, allow_file: bool = False
 ) -> List[URL]:
@@ -879,6 +943,7 @@ async def _is_dir(root: Root, uri: URL) -> bool:
 storage.add_command(rm)
 storage.add_command(mkdir)
 storage.add_command(mv)
+storage.add_command(tree)
 storage.add_command(load)
 
 
@@ -897,3 +962,27 @@ async def calc_filters(
             else:
                 ret.append((True, flt))
     return tuple(ret)
+
+
+async def fetch_tree(client: Client, uri: URL, show_all: bool) -> Tree:
+    loop = asyncio.get_event_loop()
+    folders = []
+    files = []
+    tasks = []
+    size = 0
+    items = await client.storage.ls(uri)
+    for item in items:
+        if not show_all and item.name.startswith("."):
+            continue
+        if item.is_dir():
+            tasks.append(
+                loop.create_task(fetch_tree(client, uri / item.name, show_all))
+            )
+        else:
+            files.append(item)
+            size += item.size
+    for task in tasks:
+        subtree = await task
+        folders.append(subtree)
+        size += subtree.size
+    return Tree(uri.name, size, folders, files)

neuromation/cli/utils.py

@@ -542,7 +542,7 @@ def do_deprecated_quiet(
 
 
 def format_size(value: float) -> str:
-    return humanize.naturalsize(value, gnu=True, format="%.4g")
+    return humanize.naturalsize(value, gnu=True)
 
 
 def pager_maybe(

tests/cli/formatters/test_admin_formatters.py

@@ -141,8 +141,8 @@ def test_cluster_with_cloud_provider_with_minimum_node_pool_properties_list(
               \x1b[1mStatus: \x1b[0mDeployed
               \x1b[1mNode pools:\x1b[0m
                 Machine       CPU  Memory                   GPU  Size
-                n1-highmem-8  7.0     45G                           2
-                n1-highmem-8  7.0     45G  1 x nvidia-tesla-k80     2"""  # noqa: E501, ignore line length
+                n1-highmem-8  7.0   45.0G                           2
+                n1-highmem-8  7.0   45.0G  1 x nvidia-tesla-k80     2"""  # noqa: E501, ignore line length
         )
         assert "\n".join(formatter(clusters)) == expected_out
 
@@ -176,7 +176,7 @@ def test_cluster_with_cloud_provider_with_maximum_node_pool_properties_list(
               \x1b[1mRegion: \x1b[0mus-central1
               \x1b[1mNode pools:\x1b[0m
                 Machine       CPU  Memory  Preemptible  GPU  TPU  Min  Max  Idle
-                n1-highmem-8  7.0     45G      {self._yes}           {self._yes}    1    2     1
-                n1-highmem-8  7.0     45G       {self._no}            {self._no}    1    2     0"""  # noqa: E501, ignore line length
+                n1-highmem-8  7.0   45.0G      {self._yes}           {self._yes}    1    2     1
+                n1-highmem-8  7.0   45.0G       {self._no}            {self._no}    1    2     0"""  # noqa: E501, ignore line length
         )
         assert "\n".join(formatter(clusters)) == expected_out