diff --git a/CHANGELOG.D/1435.feature b/CHANGELOG.D/1435.feature new file mode 100644 index 000000000..e00d1106c --- /dev/null +++ b/CHANGELOG.D/1435.feature @@ -0,0 +1 @@ +Implement `neuro storage tree` command for displaying the directory tree on storage. \ No newline at end of file diff --git a/README.md b/README.md index c99cdc0dd..dba8610ec 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ * [neuro storage rm](#neuro-storage-rm) * [neuro storage mkdir](#neuro-storage-mkdir) * [neuro storage mv](#neuro-storage-mv) + * [neuro storage tree](#neuro-storage-tree) * [neuro storage load](#neuro-storage-load) * [neuro image](#neuro-image) * [neuro image ls](#neuro-image-ls) @@ -847,6 +848,7 @@ Name | Description| | _[neuro storage rm](#neuro-storage-rm)_| Remove files or directories | | _[neuro storage mkdir](#neuro-storage-mkdir)_| Make directories | | _[neuro storage mv](#neuro-storage-mv)_| Move or rename files and directories | +| _[neuro storage tree](#neuro-storage-tree)_| List contents of directories in a tree-like format | | _[neuro storage load](#neuro-storage-load)_| Copy files and directories using MinIO \(EXPERIMENTAL) | @@ -923,11 +925,11 @@ neuro storage ls [OPTIONS] [PATHS]... Name | Description| |----|------------| +|_\-a, --all_|do not ignore entries starting with .| +|_\-d, --directory_|list directories themselves, not their contents.| |_\-h, --human-readable_|with -l print human readable sizes \(e.g., 2K, 540M).| |_-l_|use a long listing format.| |_--sort \[name | size | time]_|sort by given field, default is name.| -|_\-d, --directory_|list directories themselves, not their contents.| -|_\-a, --all_|do not ignore entries starting with .| |_--help_|Show this message and exit.| @@ -1050,6 +1052,29 @@ Name | Description| +### neuro storage tree + +List contents of directories in a tree-like format.

Tree is a recursive directory listing program that produces a depth indented
listing of files, which is colorized ala dircolors if the LS_COLORS
environment variable is set and output is to tty. With no arguments, tree
lists the files in the storage: directory. When directory arguments are
given, tree lists all the files and/or directories found in the given
directories each in turn. Upon completion of listing all files/directories
found, tree returns the total number of files and/or directories listed.

By default PATH is equal user's home dir \(storage:) + +**Usage:** + +```bash +neuro storage tree [OPTIONS] [PATH] +``` + +**Options:** + +Name | Description| +|----|------------| +|_\-a, --all_|do not ignore entries starting with .| +|_\-h, --human-readable_|Print the size in a more human readable way.| +|_\-s, --size_|Print the size in bytes of each file.| +|_--sort \[name | size | time]_|sort by given field, default is name| +|_--help_|Show this message and exit.| + + + + ### neuro storage load Copy files and directories using MinIO \(EXPERIMENTAL).

Same as "cp", but uses MinIO and the Amazon S3 protocol.
\(DEPRECATED) @@ -2105,11 +2130,11 @@ neuro ls [OPTIONS] [PATHS]... Name | Description| |----|------------| +|_\-a, --all_|do not ignore entries starting with .| +|_\-d, --directory_|list directories themselves, not their contents.| |_\-h, --human-readable_|with -l print human readable sizes \(e.g., 2K, 540M).| |_-l_|use a long listing format.| |_--sort \[name | size | time]_|sort by given field, default is name.| -|_\-d, --directory_|list directories themselves, not their contents.| -|_\-a, --all_|do not ignore entries starting with .| |_--help_|Show this message and exit.| diff --git a/neuromation/cli/formatters/storage.py b/neuromation/cli/formatters/storage.py index 17b74efb2..f5f16ba20 100644 --- a/neuromation/cli/formatters/storage.py +++ b/neuromation/cli/formatters/storage.py @@ -1,8 +1,11 @@ import abc +import contextlib import enum import operator import os +import sys import time +from dataclasses import dataclass from fnmatch import fnmatch from math import ceil from time import monotonic @@ -808,3 +811,95 @@ def flush(self) -> None: self.first_line = len(self.lines) self.last_line = 0 self.last_update_time = self.time_factory() + + +@dataclass(frozen=True) +class Tree: + name: str + size: int + folders: Sequence["Tree"] + files: Sequence[FileStatus] + + +class TreeFormatter: + ANSI_DELIMS = ["├", "└", "─", "│"] + SIMPLE_DELIMS = ["+", "+", "-", "|"] + + def __init__( + self, *, color: bool, size: bool, human_readable: bool, sort: str + ) -> None: + self._ident: List[bool] = [] + self._numdirs = 0 + self._numfiles = 0 + self._painter = get_painter(color, quote=True) + if sys.platform != "win32": + self._delims = self.ANSI_DELIMS + else: + self._delims = self.SIMPLE_DELIMS + if human_readable: + self._size_func = self._human_readable + elif size: + self._size_func = self._size + else: + self._size_func = self._none + self._key = FilesSorter(sort).key() + + def __call__(self, tree: Tree) -> List[str]: + ret = self.listdir(tree) + ret.append("") + ret.append(f"{self._numdirs} directories, {self._numfiles} files") + return ret + + def listdir(self, tree: Tree) -> List[str]: + ret = [] + items = sorted(tree.folders + tree.files, key=self._key) # type: ignore + ret.append( + self.pre() + + self._size_func(tree.size) + + self._painter.paint(tree.name, FileStatusType.DIRECTORY) + ) + for num, item in enumerate(items): + if isinstance(item, Tree): + self._numdirs += 1 + with self.ident(num == len(items) - 1): + ret.extend(self.listdir(item)) + else: + self._numfiles += 1 + with self.ident(num == len(items) - 1): + ret.append( + self.pre() + + self._size_func(item.size) + + self._painter.paint(item.name, FileStatusType.FILE) + ) + return ret + + def pre(self) -> str: + ret = [] + for last in self._ident[:-1]: + if last: + ret.append(" " * 4) + else: + ret.append(self._delims[3] + " " * 3) + if self._ident: + last = self._ident[-1] + ret.append(self._delims[1] if last else self._delims[0]) + ret.append(self._delims[2] * 2) + ret.append(" ") + return "".join(ret) + + @contextlib.contextmanager + def ident(self, last: bool) -> Iterator[None]: + self._ident.append(last) + try: + yield + finally: + self._ident.pop() + + def _size(self, size: int) -> str: + return f"[{size:>11}] " + + def _human_readable(self, size: int) -> str: + return f"[{format_size(size):>7}] " + + def _none(self, size: int) -> str: + return "" diff --git a/neuromation/cli/storage.py b/neuromation/cli/storage.py index 9e59967c3..91583246b 100644 --- a/neuromation/cli/storage.py +++ b/neuromation/cli/storage.py @@ -1,4 +1,5 @@ import asyncio +import dataclasses import glob as globmodule # avoid conflict with subcommand "glob" import logging import os @@ -27,12 +28,14 @@ from neuromation.api.url_utils import _extract_path from .const import EX_OSFILE -from .formatters import ( +from .formatters.jobs import JobStartProgress +from .formatters.storage import ( BaseFilesFormatter, FilesSorter, - JobStartProgress, LongFilesFormatter, SimpleFilesFormatter, + Tree, + TreeFormatter, VerticalColumnsFilesFormatter, create_storage_progress, get_painter, @@ -108,6 +111,19 @@ async def rm(root: Root, paths: Sequence[str], recursive: bool, glob: bool) -> N @command() @argument("paths", nargs=-1) +@option( + "-a", + "--all", + "show_all", + is_flag=True, + help="do not ignore entries starting with .", +) +@option( + "-d", + "--directory", + is_flag=True, + help="list directories themselves, not their contents.", +) @option( "--human-readable", "-h", @@ -121,19 +137,6 @@ async def rm(root: Root, paths: Sequence[str], recursive: bool, glob: bool) -> N default="name", help="sort by given field, default is name.", ) -@option( - "-d", - "--directory", - is_flag=True, - help="list directories themselves, not their contents.", -) -@option( - "-a", - "--all", - "show_all", - is_flag=True, - help="do not ignore entries starting with .", -) async def ls( root: Root, paths: Sequence[str], @@ -835,6 +838,67 @@ async def mv( sys.exit(EX_OSFILE) +@command() +@click.argument("path", required=False) +@option( + "-a", + "--all", + "show_all", + is_flag=True, + help="do not ignore entries starting with .", +) +@option( + "--human-readable", + "-h", + is_flag=True, + help="Print the size in a more human readable way.", +) +@option( + "--size", "-s", is_flag=True, help="Print the size in bytes of each file.", +) +@option( + "--sort", + type=click.Choice(["name", "size", "time"]), + default="name", + help="sort by given field, default is name", +) +async def tree( + root: Root, path: str, size: bool, human_readable: bool, sort: str, show_all: bool +) -> None: + """List contents of directories in a tree-like format. + + Tree is a recursive directory listing program that produces a depth indented listing + of files, which is colorized ala dircolors if the LS_COLORS environment variable is + set and output is to tty. With no arguments, tree lists the files in the storage: + directory. When directory arguments are given, tree lists all the files and/or + directories found in the given directories each in turn. Upon completion of listing + all files/directories found, tree returns the total number of files and/or + directories listed. + + By default PATH is equal user's home dir (storage:) + + """ + if not path: + path = "storage:" + uri = parse_file_resource(path, root) + + errors = False + try: + tree = await fetch_tree(root.client, uri, show_all) + tree = dataclasses.replace(tree, name=str(path)) + except (OSError, ResourceNotFound) as error: + log.error(f"cannot fetch tree for {uri}: {error}") + errors = True + else: + formatter = TreeFormatter( + color=root.color, size=size, human_readable=human_readable, sort=sort + ) + pager_maybe(formatter(tree), root.tty, root.terminal_size) + + if errors: + sys.exit(EX_OSFILE) + + async def _expand( paths: Sequence[str], root: Root, glob: bool, allow_file: bool = False ) -> List[URL]: @@ -879,6 +943,7 @@ async def _is_dir(root: Root, uri: URL) -> bool: storage.add_command(rm) storage.add_command(mkdir) storage.add_command(mv) +storage.add_command(tree) storage.add_command(load) @@ -897,3 +962,27 @@ async def calc_filters( else: ret.append((True, flt)) return tuple(ret) + + +async def fetch_tree(client: Client, uri: URL, show_all: bool) -> Tree: + loop = asyncio.get_event_loop() + folders = [] + files = [] + tasks = [] + size = 0 + items = await client.storage.ls(uri) + for item in items: + if not show_all and item.name.startswith("."): + continue + if item.is_dir(): + tasks.append( + loop.create_task(fetch_tree(client, uri / item.name, show_all)) + ) + else: + files.append(item) + size += item.size + for task in tasks: + subtree = await task + folders.append(subtree) + size += subtree.size + return Tree(uri.name, size, folders, files) diff --git a/neuromation/cli/utils.py b/neuromation/cli/utils.py index d49d27512..39b663747 100644 --- a/neuromation/cli/utils.py +++ b/neuromation/cli/utils.py @@ -542,7 +542,7 @@ def do_deprecated_quiet( def format_size(value: float) -> str: - return humanize.naturalsize(value, gnu=True, format="%.4g") + return humanize.naturalsize(value, gnu=True) def pager_maybe( diff --git a/tests/cli/formatters/test_admin_formatters.py b/tests/cli/formatters/test_admin_formatters.py index 2fb982f7c..c6f20548d 100644 --- a/tests/cli/formatters/test_admin_formatters.py +++ b/tests/cli/formatters/test_admin_formatters.py @@ -141,8 +141,8 @@ def test_cluster_with_cloud_provider_with_minimum_node_pool_properties_list( \x1b[1mStatus: \x1b[0mDeployed \x1b[1mNode pools:\x1b[0m Machine CPU Memory GPU Size - n1-highmem-8 7.0 45G 2 - n1-highmem-8 7.0 45G 1 x nvidia-tesla-k80 2""" # noqa: E501, ignore line length + n1-highmem-8 7.0 45.0G 2 + n1-highmem-8 7.0 45.0G 1 x nvidia-tesla-k80 2""" # noqa: E501, ignore line length ) assert "\n".join(formatter(clusters)) == expected_out @@ -176,7 +176,7 @@ def test_cluster_with_cloud_provider_with_maximum_node_pool_properties_list( \x1b[1mRegion: \x1b[0mus-central1 \x1b[1mNode pools:\x1b[0m Machine CPU Memory Preemptible GPU TPU Min Max Idle - n1-highmem-8 7.0 45G {self._yes} {self._yes} 1 2 1 - n1-highmem-8 7.0 45G {self._no} {self._no} 1 2 0""" # noqa: E501, ignore line length + n1-highmem-8 7.0 45.0G {self._yes} {self._yes} 1 2 1 + n1-highmem-8 7.0 45.0G {self._no} {self._no} 1 2 0""" # noqa: E501, ignore line length ) assert "\n".join(formatter(clusters)) == expected_out diff --git a/tests/cli/formatters/test_config_formatters.py b/tests/cli/formatters/test_config_formatters.py index 3f613c52d..a17c86536 100644 --- a/tests/cli/formatters/test_config_formatters.py +++ b/tests/cli/formatters/test_config_formatters.py @@ -38,10 +38,10 @@ async def test_output(self, root: Root) -> None: Docker Registry URL: https://registry-dev.neu.ro Resource Presets: Name #CPU Memory Preemptible GPU - gpu-small 7 30G {no} 1 x nvidia-tesla-k80 - gpu-large 7 60G {no} 1 x nvidia-tesla-v100 - cpu-small 7 2G {no} - cpu-large 7 14G {no}""" + gpu-small 7 30.0G {no} 1 x nvidia-tesla-k80 + gpu-large 7 60.0G {no} 1 x nvidia-tesla-v100 + cpu-small 7 2.0G {no} + cpu-large 7 14.0G {no}""" ) async def test_output_for_tpu_presets( @@ -89,13 +89,13 @@ async def test_output_for_tpu_presets( Docker Registry URL: https://registry-dev.neu.ro Resource Presets: Name #CPU Memory Preemptible GPU TPU - gpu-small 7 30G {no} 1 x nvidia-tesla-k80 - gpu-large 7 60G {no} 1 x nvidia-tesla-v100 - cpu-small 7 2G {no} - cpu-large 7 14G {no} - cpu-large-p 7 14G {yes} - tpu-small 2 2G {no} v3-8/1.14 - hybrid 4 30G {no} 2 x nvidia-tesla-v100 v3-64/1.14""" # noqa: E501, ignore line length + gpu-small 7 30.0G {no} 1 x nvidia-tesla-k80 + gpu-large 7 60.0G {no} 1 x nvidia-tesla-v100 + cpu-small 7 2.0G {no} + cpu-large 7 14.0G {no} + cpu-large-p 7 14.0G {yes} + tpu-small 2 2.0G {no} v3-8/1.14 + hybrid 4 30.0G {no} 2 x nvidia-tesla-v100 v3-64/1.14""" # noqa: E501, ignore line length ) diff --git a/tests/cli/formatters/test_jobs_formatters.py b/tests/cli/formatters/test_jobs_formatters.py index cf5063e21..9b8394e55 100644 --- a/tests/cli/formatters/test_jobs_formatters.py +++ b/tests/cli/formatters/test_jobs_formatters.py @@ -1172,7 +1172,7 @@ def test_tiny_container(self) -> None: resource_formatter = ResourcesFormatter() assert ( resource_formatter(resources) == "Resources:\n" - " Memory: 16M\n" + " Memory: 16.0M\n" " CPU: 0.1" ) @@ -1189,7 +1189,7 @@ def test_gpu_container(self) -> None: resource_formatter = ResourcesFormatter() assert ( resource_formatter(resources) == "Resources:\n" - " Memory: 1G\n" + " Memory: 1.0G\n" " CPU: 2.0\n" " GPU: 1.0 x nvidia-tesla-p4" ) @@ -1207,7 +1207,7 @@ def test_shm_container(self) -> None: resource_formatter = ResourcesFormatter() assert ( resource_formatter(resources) == "Resources:\n" - " Memory: 16M\n" + " Memory: 16.0M\n" " CPU: 0.1\n" " Additional: Extended SHM space" ) @@ -1225,7 +1225,7 @@ def test_tpu_container(self) -> None: resource_formatter = ResourcesFormatter() assert ( resource_formatter(resources=resources) == "Resources:\n" - " Memory: 16M\n" + " Memory: 16.0M\n" " CPU: 0.1\n" " TPU: v2-8/1.14\n" " Additional: Extended SHM space" diff --git a/tests/cli/formatters/test_storage_formatters.py b/tests/cli/formatters/test_storage_formatters.py index 74257d7ed..a86827c2c 100644 --- a/tests/cli/formatters/test_storage_formatters.py +++ b/tests/cli/formatters/test_storage_formatters.py @@ -398,11 +398,11 @@ def test_long_formatter(self) -> None: formatter = LongFilesFormatter(human_readable=True, color=False) assert list(formatter(self.files_and_folders)) == [ - "-r 2K 2018-01-01 03:00:00 File1", - "-r 1K 2018-10-10 13:10:10 File2", - "-r 1000K 2019-02-02 05:02:02 File3 with space", - "dm 0 2017-03-03 06:03:03 Folder1", - "dm 0 2017-03-03 06:03:02 1Folder with space", + "-r 2.0K 2018-01-01 03:00:00 File1", + "-r 1.0K 2018-10-10 13:10:10 File2", + "-r 1000.0K 2019-02-02 05:02:02 File3 with space", + "dm 0 2017-03-03 06:03:03 Folder1", + "dm 0 2017-03-03 06:03:02 1Folder with space", ] def test_column_formatter(self) -> None: diff --git a/tests/e2e/test_e2e_storage.py b/tests/e2e/test_e2e_storage.py index e0bbb6725..7c6e3211e 100644 --- a/tests/e2e/test_e2e_storage.py +++ b/tests/e2e/test_e2e_storage.py @@ -1,6 +1,8 @@ import errno import os import subprocess +import sys +import textwrap from pathlib import Path, PurePath from typing import Tuple @@ -8,6 +10,7 @@ from yarl import URL from neuromation.cli.const import EX_OSFILE +from neuromation.cli.formatters.storage import TreeFormatter from tests.e2e import Helper from tests.e2e.utils import FILE_SIZE_B @@ -661,3 +664,36 @@ def test_e2e_ls_show_hidden(tmp_path: Path, helper: Helper) -> None: captured = helper.run_cli(["storage", "ls", "--all", helper.tmpstorage + "/folder"]) assert captured.out.splitlines() == [".bar", "foo"] + + +@pytest.mark.e2e +def test_tree(helper: Helper, data: _Data, tmp_path: Path) -> None: + folder = tmp_path / "folder" + folder.mkdir() + (folder / "foo").write_bytes(b"foo") + (folder / "bar").write_bytes(b"bar") + subfolder = folder / "folder" + subfolder.mkdir() + (subfolder / "baz").write_bytes(b"baz") + + helper.run_cli(["storage", "cp", "-r", folder.as_uri(), helper.tmpstorage]) + + capture = helper.run_cli(["storage", "tree", helper.tmpstorage]) + assert capture.err == "" + + expected = textwrap.dedent( + f"""\ + '{helper.tmpstorage}' + ├── 'bar' + ├── 'folder' + │ └── 'baz' + └── 'foo' + + 1 directories, 3 files""" + ) + if sys.platform == "win32": + trans = str.maketrans( + "".join(TreeFormatter.ANSI_DELIMS), "".join(TreeFormatter.SIMPLE_DELIMS) + ) + expected = expected.translate(trans) + assert capture.out == expected