From c650903ae15edcf5d1f21d950583cdef9a90ba24 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 31 Jul 2019 14:05:28 +0300 Subject: [PATCH] New file progress (#933) --- CHANGELOG.D/933.feature | 1 + README.md | 4 +- neuromation/api/__init__.py | 12 +- neuromation/api/abc.py | 18 +- neuromation/api/storage.py | 43 ++- neuromation/cli/formatters/images.py | 2 +- neuromation/cli/formatters/jobs.py | 6 +- neuromation/cli/formatters/storage.py | 254 +++++++++++----- neuromation/cli/printer.py | 14 +- neuromation/cli/storage.py | 15 +- tests/cli/test_formatters.py | 66 +++++ tests/cli/test_printer.py | 8 +- tests/cli/test_storage_progress.py | 404 ++++++++++++++++++++++++-- 13 files changed, 709 insertions(+), 138 deletions(-) create mode 100644 CHANGELOG.D/933.feature diff --git a/CHANGELOG.D/933.feature b/CHANGELOG.D/933.feature new file mode 100644 index 000000000..7bbec9a13 --- /dev/null +++ b/CHANGELOG.D/933.feature @@ -0,0 +1 @@ +Show progress for `neuro cp` by default. \ No newline at end of file diff --git a/README.md b/README.md index 9cc9584dd..eeb6c440e 100644 --- a/README.md +++ b/README.md @@ -582,7 +582,7 @@ Name | Description| |_\--glob / --no-glob_|Expand glob patterns in SOURCES with explicit scheme \[default: True]| |_\-t, --target-directory DIRECTORY_|Copy all SOURCES into DIRECTORY| |_\-T, --no-target-directory_|Treat DESTINATION as a normal file| -|_\-p, --progress_|Show progress, off by default| +|_\-p, --progress / -P, --no-progress_|Show progress, on by default| |_--help_|Show this message and exit.| @@ -1647,7 +1647,7 @@ Name | Description| |_\--glob / --no-glob_|Expand glob patterns in SOURCES with explicit scheme \[default: True]| |_\-t, --target-directory DIRECTORY_|Copy all SOURCES into DIRECTORY| |_\-T, --no-target-directory_|Treat DESTINATION as a normal file| -|_\-p, --progress_|Show progress, off by default| +|_\-p, --progress / -P, --no-progress_|Show progress, on by default| |_--help_|Show this message and exit.| diff --git a/neuromation/api/__init__.py b/neuromation/api/__init__.py index 9e0b18fe2..b03954d85 100644 --- a/neuromation/api/__init__.py +++ b/neuromation/api/__init__.py @@ -6,10 +6,12 @@ from .abc import ( AbstractDockerImageProgress, - AbstractStorageProgress, + AbstractFileProgress, + AbstractRecursiveFileProgress, StorageProgressComplete, + StorageProgressEnterDir, StorageProgressFail, - StorageProgressMkdir, + StorageProgressLeaveDir, StorageProgressStart, StorageProgressStep, ) @@ -70,13 +72,15 @@ "AuthError", "AuthenticationError", "AuthorizationError", - "AbstractStorageProgress", + "AbstractFileProgress", + "AbstractRecursiveFileProgress", "AbstractDockerImageProgress", "StorageProgressStart", "StorageProgressComplete", "StorageProgressStep", - "StorageProgressMkdir", "StorageProgressFail", + "StorageProgressEnterDir", + "StorageProgressLeaveDir", "RemoteImage", "LocalImage", "Factory", diff --git a/neuromation/api/abc.py b/neuromation/api/abc.py index 17f98ab56..2ce85748a 100644 --- a/neuromation/api/abc.py +++ b/neuromation/api/abc.py @@ -27,7 +27,13 @@ class StorageProgressStep: @dataclass -class StorageProgressMkdir: +class StorageProgressEnterDir: + src: URL + dst: URL + + +@dataclass +class StorageProgressLeaveDir: src: URL dst: URL @@ -39,7 +45,7 @@ class StorageProgressFail: message: str -class AbstractStorageProgress(abc.ABC): +class AbstractFileProgress(abc.ABC): # design note: # dataclasses used instead of direct passing parameters # because a dataclass is forward-compatible @@ -58,8 +64,14 @@ def complete(self, data: StorageProgressComplete) -> None: def step(self, data: StorageProgressStep) -> None: pass # pragma: no cover + +class AbstractRecursiveFileProgress(AbstractFileProgress): + @abc.abstractmethod + def enter(self, data: StorageProgressEnterDir) -> None: + pass # pragma: no cover + @abc.abstractmethod - def mkdir(self, data: StorageProgressMkdir) -> None: + def leave(self, data: StorageProgressLeaveDir) -> None: pass # pragma: no cover @abc.abstractmethod diff --git a/neuromation/api/storage.py b/neuromation/api/storage.py index c8c850022..5ab8eb8e0 100644 --- a/neuromation/api/storage.py +++ b/neuromation/api/storage.py @@ -12,10 +12,12 @@ from yarl import URL from .abc import ( - AbstractStorageProgress, + AbstractFileProgress, + AbstractRecursiveFileProgress, StorageProgressComplete, + StorageProgressEnterDir, StorageProgressFail, - StorageProgressMkdir, + StorageProgressLeaveDir, StorageProgressStart, StorageProgressStep, ) @@ -241,7 +243,7 @@ async def mv(self, src: URL, dst: URL) -> None: # high-level helpers async def _iterate_file( - self, src: Path, dst: URL, *, progress: AbstractStorageProgress + self, src: Path, dst: URL, *, progress: AbstractFileProgress ) -> AsyncIterator[bytes]: loop = asyncio.get_event_loop() src_url = URL(src.as_uri()) @@ -258,7 +260,7 @@ async def _iterate_file( progress.complete(StorageProgressComplete(src_url, dst, size)) async def upload_file( - self, src: URL, dst: URL, *, progress: Optional[AbstractStorageProgress] = None + self, src: URL, dst: URL, *, progress: Optional[AbstractFileProgress] = None ) -> None: if progress is None: progress = _DummyProgress() @@ -297,7 +299,11 @@ async def upload_file( await self.create(dst, self._iterate_file(path, dst, progress=progress)) async def upload_dir( - self, src: URL, dst: URL, *, progress: Optional[AbstractStorageProgress] = None + self, + src: URL, + dst: URL, + *, + progress: Optional[AbstractRecursiveFileProgress] = None, ) -> None: if progress is None: progress = _DummyProgress() @@ -314,8 +320,9 @@ async def upload_dir( raise NotADirectoryError(errno.ENOTDIR, "Not a directory", str(dst)) except ResourceNotFound: await self.mkdirs(dst) - progress.mkdir(StorageProgressMkdir(src, dst)) - for child in path.iterdir(): + progress.enter(StorageProgressEnterDir(src, dst)) + folder = sorted(path.iterdir(), key=lambda item: (item.is_dir(), item.name)) + for child in folder: if child.is_file(): await self.upload_file( src / child.name, dst / child.name, progress=progress @@ -335,9 +342,10 @@ async def upload_dir( f"Cannot upload {child}, not regular file/directory", ) ) # pragma: no cover + progress.leave(StorageProgressLeaveDir(src, dst)) async def download_file( - self, src: URL, dst: URL, *, progress: Optional[AbstractStorageProgress] = None + self, src: URL, dst: URL, *, progress: Optional[AbstractFileProgress] = None ) -> None: if progress is None: progress = _DummyProgress() @@ -359,7 +367,11 @@ async def download_file( progress.complete(StorageProgressComplete(src, dst, size)) async def download_dir( - self, src: URL, dst: URL, *, progress: Optional[AbstractStorageProgress] = None + self, + src: URL, + dst: URL, + *, + progress: Optional[AbstractRecursiveFileProgress] = None, ) -> None: if progress is None: progress = _DummyProgress() @@ -367,8 +379,9 @@ async def download_dir( dst = normalize_local_path_uri(dst) path = _extract_path(dst) path.mkdir(parents=True, exist_ok=True) - progress.mkdir(StorageProgressMkdir(src, dst)) - for child in await self.ls(src): + progress.enter(StorageProgressEnterDir(src, dst)) + folder = sorted(await self.ls(src), key=lambda item: (item.is_dir(), item.name)) + for child in folder: if child.is_file(): await self.download_file( src / child.name, dst / child.name, progress=progress @@ -385,6 +398,7 @@ async def download_dir( f"Cannot download {child}, not regular file/directory", ) ) # pragma: no cover + progress.leave(StorageProgressLeaveDir(src, dst)) _magic_check = re.compile("(?:[*?[])") @@ -412,7 +426,7 @@ def _file_status_from_api(values: Dict[str, Any]) -> FileStatus: ) -class _DummyProgress(AbstractStorageProgress): +class _DummyProgress(AbstractRecursiveFileProgress): def start(self, data: StorageProgressStart) -> None: pass @@ -422,7 +436,10 @@ def complete(self, data: StorageProgressComplete) -> None: def step(self, data: StorageProgressStep) -> None: pass - def mkdir(self, data: StorageProgressMkdir) -> None: + def enter(self, data: StorageProgressEnterDir) -> None: + pass + + def leave(self, data: StorageProgressLeaveDir) -> None: pass def fail(self, data: StorageProgressFail) -> None: diff --git a/neuromation/cli/formatters/images.py b/neuromation/cli/formatters/images.py index 25af3752c..5aa4323a3 100644 --- a/neuromation/cli/formatters/images.py +++ b/neuromation/cli/formatters/images.py @@ -60,8 +60,8 @@ def progress(self, message: str, layer_id: str) -> None: lineno = self._mapping[layer_id] self._printer.print(message, lineno) else: - self._printer.print(message) self._mapping[layer_id] = self._printer.total_lines + self._printer.print(message) else: self._printer.print(message) diff --git a/neuromation/cli/formatters/jobs.py b/neuromation/cli/formatters/jobs.py index 46ed4c0c6..d18471acd 100644 --- a/neuromation/cli/formatters/jobs.py +++ b/neuromation/cli/formatters/jobs.py @@ -1,10 +1,10 @@ import abc import datetime import itertools +import sys import time from dataclasses import dataclass from math import floor -from sys import platform from typing import Iterable, Iterator, List, Mapping import humanize @@ -327,7 +327,7 @@ def __init__(self, color: bool): self._time = time.time() self._color = color self._prev = "" - if platform == "win32": + if sys.platform == "win32": self._spinner = itertools.cycle("-\\|/") else: self._spinner = itertools.cycle("◢◣◤◥") @@ -351,8 +351,8 @@ def __call__(self, job: JobDescription) -> None: if self._prev: self._printer.print(self._prev, lineno=self._lineno) self._prev = msg - self._printer.print(msg) self._lineno = self._printer.total_lines + self._printer.print(msg) else: self._printer.print( f"{msg} {next(self._spinner)} [{dt:.1f} sec]", lineno=self._lineno diff --git a/neuromation/cli/formatters/storage.py b/neuromation/cli/formatters/storage.py index 40408d4ed..a7f040044 100644 --- a/neuromation/cli/formatters/storage.py +++ b/neuromation/cli/formatters/storage.py @@ -5,7 +5,7 @@ import time from fnmatch import fnmatch from math import ceil -from typing import Any, Dict, Iterator, List, Sequence +from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple import click import humanize @@ -13,17 +13,20 @@ from yarl import URL from neuromation.api import ( - AbstractStorageProgress, + AbstractRecursiveFileProgress, Action, FileStatus, FileStatusType, StorageProgressComplete, + StorageProgressEnterDir, StorageProgressFail, - StorageProgressMkdir, + StorageProgressLeaveDir, StorageProgressStart, StorageProgressStep, ) from neuromation.api.url_utils import _extract_path +from neuromation.cli.printer import TTYPrinter +from neuromation.cli.root import Root RECENT_TIME_DELTA = 365 * 24 * 60 * 60 / 2 @@ -106,9 +109,10 @@ def paint(self, label: str, type: FileStatusType) -> str: class GnuPainter(BasePainter): - def __init__(self, ls_colors: str): + def __init__(self, ls_colors: str, *, underline: bool = False): self._defaults() self._parse_ls_colors(ls_colors) + self._underline = underline def _defaults(self) -> None: self.color_indicator: Dict[GnuIndicators, str] = { @@ -341,16 +345,28 @@ def paint(self, label: str, type: FileStatusType) -> str: color = value break if color: + if self._underline: + underline = ( + self.color_indicator[GnuIndicators.LEFT] + + "4" + + self.color_indicator[GnuIndicators.RIGHT] + ) + else: + underline = "" return ( self.color_indicator[GnuIndicators.LEFT] + color + self.color_indicator[GnuIndicators.RIGHT] + + underline + label + self.color_indicator[GnuIndicators.LEFT] + self.color_indicator[GnuIndicators.RESET] + self.color_indicator[GnuIndicators.RIGHT] ) - return label + if self._underline: + return style(label, underline=self._underline) + else: + return label class BSDAttributes(enum.Enum): @@ -368,7 +384,8 @@ class BSDAttributes(enum.Enum): class BSDPainter(BasePainter): - def __init__(self, lscolors: str): + def __init__(self, lscolors: str, *, underline: bool = False): + self._underline = underline self._parse_lscolors(lscolors) def _parse_lscolors(self, lscolors: str) -> None: @@ -402,8 +419,14 @@ def paint(self, label: str, type: FileStatusType) -> str: bold = True if color[1] in char_to_color.keys(): bg = char_to_color[color[1]] - if fg or bg or bold: - return style(label, fg=fg, bg=bg, bold=bold) + if self._underline: + underline: Optional[bool] = True + else: + underline = None + if fg or bg or bold or underline: + return style(label, fg=fg, bg=bg, bold=bold, underline=underline) + if self._underline: + return style(label, underline=self._underline) return label @@ -411,10 +434,10 @@ def get_painter(color: bool, *, quote: bool = False) -> BasePainter: if color: ls_colors = os.getenv("LS_COLORS") if ls_colors: - return GnuPainter(ls_colors) + return GnuPainter(ls_colors, underline=quote) lscolors = os.getenv("LSCOLORS") if lscolors: - return BSDPainter(lscolors) + return BSDPainter(lscolors, underline=quote) if quote: return QuotedPainter() else: @@ -539,17 +562,20 @@ def key(self) -> Any: # progress indicator -def create_storage_progress( - color: bool, show_progress: bool, verbose: bool -) -> AbstractStorageProgress: +class BaseStorageProgress(AbstractRecursiveFileProgress): + @abc.abstractmethod + def begin(self, src: URL, dst: URL) -> None: # pragma: no cover + pass + + +def create_storage_progress(root: Root, show_progress: bool) -> BaseStorageProgress: if show_progress: - return StandardPrintPercentOnly(color) - if verbose: - return NoPercentPrinter(color) - return QuietPrinter(color) + return TTYProgress(root) + else: + return StreamProgress(root) -def fmt_url(url: URL) -> str: +def format_url(url: URL) -> str: if url.scheme == "file": path = _extract_path(url) return str(path) @@ -557,89 +583,179 @@ def fmt_url(url: URL) -> str: return str(url) -class QuietPrinter(AbstractStorageProgress): - def __init__(self, color: bool): - self.painter = get_painter(color, quote=True) +class StreamProgress(BaseStorageProgress): + def __init__(self, root: Root) -> None: + self.painter = get_painter(root.color, quote=True) + self.verbose = root.verbosity > 0 + + def fmt_url(self, url: URL, type: FileStatusType) -> str: + label = format_url(url) + return self.painter.paint(label, type) + + def begin(self, src: URL, dst: URL) -> None: + if self.verbose: + src_label = self.fmt_url(src, FileStatusType.DIRECTORY) + dst_label = self.fmt_url(dst, FileStatusType.DIRECTORY) + click.echo(f"Copy {src_label} -> {dst_label}") def start(self, data: StorageProgressStart) -> None: pass def complete(self, data: StorageProgressComplete) -> None: - pass + if not self.verbose: + return + src = self.fmt_url(data.src, FileStatusType.FILE) + dst = self.fmt_url(data.dst, FileStatusType.FILE) + click.echo(f"{src} -> {dst}") def step(self, data: StorageProgressStep) -> None: pass - def mkdir(self, data: StorageProgressMkdir) -> None: + def enter(self, data: StorageProgressEnterDir) -> None: + if not self.verbose: + return + src = self.fmt_url(data.src, FileStatusType.FILE) + dst = self.fmt_url(data.dst, FileStatusType.FILE) + click.echo(f"{src} -> {dst}") + + def leave(self, data: StorageProgressLeaveDir) -> None: pass def fail(self, data: StorageProgressFail) -> None: - src = self.painter.paint(fmt_url(data.src), FileStatusType.FILE) - dst = self.painter.paint(fmt_url(data.dst), FileStatusType.FILE) + src = self.fmt_url(data.src, FileStatusType.FILE) + dst = self.fmt_url(data.dst, FileStatusType.FILE) click.echo( click.style("Failure:", fg="red") + f" {src} -> {dst} [{data.message}]", err=True, ) -class NoPercentPrinter(AbstractStorageProgress): - def __init__(self, color: bool): - self.painter = get_painter(color, quote=True) +class TTYProgress(BaseStorageProgress): + HEIGHT = 10 - def start(self, data: StorageProgressStart) -> None: - pass + def __init__(self, root: Root) -> None: + self.painter = get_painter(root.color, quote=True) + self.printer = TTYPrinter() + self.half_width = (root.terminal_size[0] - 10) // 2 + self.full_width = root.terminal_size[0] - 20 + self.lines: List[Tuple[bool, str]] = [] + self.dir_stack: List[str] = [] + self.verbose = root.verbosity > 0 - def complete(self, data: StorageProgressComplete) -> None: - src = self.painter.paint(fmt_url(data.src), FileStatusType.FILE) - dst = self.painter.paint(fmt_url(data.dst), FileStatusType.FILE) - click.echo(f"{src} -> {dst}") - - def step(self, data: StorageProgressStep) -> None: - pass - - def mkdir(self, data: StorageProgressMkdir) -> None: - src = self.painter.paint(fmt_url(data.src), FileStatusType.FILE) - dst = self.painter.paint(fmt_url(data.dst), FileStatusType.FILE) - click.echo(f"{src} -> {dst}") - - def fail(self, data: StorageProgressFail) -> None: - src = self.painter.paint(fmt_url(data.src), FileStatusType.FILE) - dst = self.painter.paint(fmt_url(data.dst), FileStatusType.FILE) - click.echo( - click.style("Failure:", fg="red") + f" {src} -> {dst} [{data.message}]", - err=True, - ) + def fmt_url(self, url: URL, type: FileStatusType, *, half: bool) -> str: + label = str(url) + if half: + width = self.half_width + else: + width = self.full_width + while len(label) > width: + parts = list(url.parts) + if len(parts) < 2: + break + if parts[0] == "/": + if len(parts) < 3: + slash = "/" + break + slash, first, second, *last = parts + if first == "...": + if last: + parts = ["..."] + last + else: + break + else: + parts = ["...", second] + last + else: + slash = "" + # len(parts) > 1 always + first, second, *last = parts + if first == "...": + if last: + parts = ["..."] + last + else: + break + else: + parts = ["...", second] + last + if url.host or slash: + pre = f"//{url.host or ''}{slash}" + else: + pre = "" + url = URL(f"{url.scheme}:{pre}{'/'.join(parts)}") + label = str(url) + return self.fmt_str(label, type) + + def fmt_str(self, label: str, type: FileStatusType) -> str: + return self.painter.paint(label, type) + + def fmt_size(self, size: int) -> str: + return humanize.naturalsize(size, gnu=True) + + def begin(self, src: URL, dst: URL) -> None: + if self.verbose: + click.echo("Copy") + click.echo(self.fmt_str(str(src), FileStatusType.DIRECTORY)) + click.echo("=>") + click.echo(self.fmt_str(str(dst), FileStatusType.DIRECTORY)) + else: + src_label = self.fmt_url(src, FileStatusType.DIRECTORY, half=True) + dst_label = self.fmt_url(dst, FileStatusType.DIRECTORY, half=True) + click.echo(f"Copy {src_label} => {dst_label}") + def enter(self, data: StorageProgressEnterDir) -> None: + src = self.fmt_url(data.src, FileStatusType.DIRECTORY, half=False) + self.dir_stack.append(src) + self.append(f"{src}", is_dir=True) -class StandardPrintPercentOnly(AbstractStorageProgress): - def __init__(self, color: bool): - self.painter = get_painter(color, quote=True) + def leave(self, data: StorageProgressLeaveDir) -> None: + del self.dir_stack[-1] + if self.dir_stack: + self.append(f"{self.dir_stack[-1]}", is_dir=True) def start(self, data: StorageProgressStart) -> None: - src = self.painter.paint(fmt_url(data.src), FileStatusType.FILE) - dst = self.painter.paint(fmt_url(data.dst), FileStatusType.FILE) - click.echo(f"Start copying {src} -> {dst}.") + src = self.fmt_str(data.src.name, FileStatusType.FILE) + progress = 0 + current = self.fmt_size(0) + total = self.fmt_size(data.size) + self.append(f"{src} [{progress:.2f}%] {current} of {total}") def complete(self, data: StorageProgressComplete) -> None: - src = self.painter.paint(fmt_url(data.src), FileStatusType.FILE) - dst = self.painter.paint(fmt_url(data.dst), FileStatusType.FILE) - click.echo(f"\rFile {src} -> {dst} copying completed.") + src = self.fmt_str(data.src.name, FileStatusType.FILE) + total = self.fmt_size(data.size) + self.replace(f"{src} {total}") def step(self, data: StorageProgressStep) -> None: - src = self.painter.paint(fmt_url(data.src), FileStatusType.FILE) - dst = self.painter.paint(fmt_url(data.dst), FileStatusType.FILE) + src = self.fmt_str(data.src.name, FileStatusType.FILE) progress = (100 * data.current) / data.size - click.echo(f"\r{src} -> {dst}: {progress:.2f}%.", nl=False) - - def mkdir(self, data: StorageProgressMkdir) -> None: - src = self.painter.paint(fmt_url(data.src), FileStatusType.FILE) - dst = self.painter.paint(fmt_url(data.dst), FileStatusType.FILE) - click.echo(f"Copy directory {src} -> {dst}.") + current = self.fmt_size(data.current) + total = self.fmt_size(data.size) + self.replace(f"{src} [{progress:.2f}%] {current} of {total}") def fail(self, data: StorageProgressFail) -> None: - src = self.painter.paint(fmt_url(data.src), FileStatusType.FILE) - dst = self.painter.paint(fmt_url(data.dst), FileStatusType.FILE) + src = self.fmt_str(str(data.src), FileStatusType.FILE) + dst = self.fmt_str(str(data.dst), FileStatusType.FILE) click.echo( click.style("Failure:", fg="red") + f" {src} -> {dst} [{data.message}]", err=True, ) + # clear lines to sync with writing to stderr + self.lines = [] + + def append(self, msg: str, is_dir: bool = False) -> None: + self.lines.append((is_dir, msg)) + if len(self.lines) > self.HEIGHT: + if not self.lines[0][0]: + # top line is not a dir, drop it. + del self.lines[0] + else: + if any(line[0] for line in self.lines[1:]): + # there are folder lines below + del self.lines[0] + else: + # there is only top folder line, drop next file line + del self.lines[1] + for lineno, line in enumerate(self.lines): + self.printer.print(line[1], lineno) + + def replace(self, msg: str) -> None: + # replace last line + self.lines[-1] = (False, msg) + self.printer.print(msg, len(self.lines) - 1) diff --git a/neuromation/cli/printer.py b/neuromation/cli/printer.py index 75e52bcad..08ca036e1 100644 --- a/neuromation/cli/printer.py +++ b/neuromation/cli/printer.py @@ -1,7 +1,6 @@ import abc from os import linesep from time import time -from typing import Optional import click @@ -43,17 +42,16 @@ def __init__(self) -> None: def total_lines(self) -> int: return self._total_lines - def print(self, text: str, lineno: Optional[int] = None) -> str: + def print(self, text: str, lineno: int = -1) -> str: """ Print given text on specified line - If lineno is not passed then text will be printed on latest line + If lineno is not passed then text will be printed on latest line """ - assert lineno is None or lineno > 0 - if not lineno: - lineno = self._total_lines + 1 + if lineno < 0: + lineno = self._total_lines commands = [] - diff = self._total_lines - lineno + 1 + diff = self._total_lines - lineno clear_tail = False if diff > 0: @@ -73,7 +71,7 @@ def print(self, text: str, lineno: Optional[int] = None) -> str: commands.append(CURSOR_DOWN.format(diff - 1)) message = "".join(commands) - self._total_lines = max(self._total_lines, lineno) + self._total_lines = max(self._total_lines, lineno + 1) self._print(message) return message diff --git a/neuromation/cli/storage.py b/neuromation/cli/storage.py index 570c63ce5..84322b513 100644 --- a/neuromation/cli/storage.py +++ b/neuromation/cli/storage.py @@ -181,7 +181,13 @@ async def glob(root: Root, patterns: Sequence[str]) -> None: is_flag=True, help="Treat DESTINATION as a normal file", ) -@click.option("-p", "--progress", is_flag=True, help="Show progress, off by default") +@click.option( + "-p/-P", + "--progress/--no-progress", + is_flag=True, + default=True, + help="Show progress, on by default", +) @async_cmd() async def cp( root: Root, @@ -262,14 +268,15 @@ async def cp( if no_target_directory and len(srcs) > 1: raise click.UsageError(f"Extra operand after {str(srcs[1])!r}") - tty = root.tty and progress + show_progress = root.tty and progress for src in srcs: if target_dir: dst = target_dir / src.name assert dst - progress_obj = create_storage_progress(root.color, tty, root.verbosity > 0) + progress_obj = create_storage_progress(root, show_progress) + progress_obj.begin(src, dst) if src.scheme == "file" and dst.scheme == "storage": if recursive: @@ -672,7 +679,7 @@ async def _expand( uri = parse_file_resource(path, root) if root.verbosity > 0: painter = get_painter(root.color, quote=True) - curi = painter.paint(str(str), FileStatusType.FILE) + curi = painter.paint(str(uri), FileStatusType.FILE) click.echo(f"Expand {curi}") uri_path = str(_extract_path(uri)) if glob and globmodule.has_magic(uri_path): diff --git a/tests/cli/test_formatters.py b/tests/cli/test_formatters.py index 717b364b3..e8f258a78 100644 --- a/tests/cli/test_formatters.py +++ b/tests/cli/test_formatters.py @@ -979,6 +979,41 @@ def test_coloring(self) -> None: assert painter.paint(file.name, file.type) == "\x1b[0;46mtest.txt\x1b[0m" assert painter.paint(folder.name, folder.type) == "\x1b[01;34mtmp\x1b[0m" + def test_coloring_underline(self) -> None: + file = FileStatus( + "test.txt", + 1024, + FileStatusType.FILE, + int(time.mktime(time.strptime("2018-01-01 03:00:00", "%Y-%m-%d %H:%M:%S"))), + "read", + ) + folder = FileStatus( + "tmp", + 0, + FileStatusType.DIRECTORY, + int(time.mktime(time.strptime("2018-01-01 03:00:00", "%Y-%m-%d %H:%M:%S"))), + "write", + ) + painter = GnuPainter("di=32;41:fi=0;44:no=0;46", underline=True) + assert painter.paint(file.name, file.type) == "\x1b[0;44m\x1b[4mtest.txt\x1b[0m" + assert painter.paint(folder.name, folder.type) == "\x1b[32;41m\x1b[4mtmp\x1b[0m" + + painter = GnuPainter("di=32;41:no=0;46", underline=True) + assert painter.paint(file.name, file.type) == "\x1b[0;46m\x1b[4mtest.txt\x1b[0m" + assert painter.paint(folder.name, folder.type) == "\x1b[32;41m\x1b[4mtmp\x1b[0m" + + painter = GnuPainter("no=0;46", underline=True) + assert painter.paint(file.name, file.type) == "\x1b[0;46m\x1b[4mtest.txt\x1b[0m" + assert painter.paint(folder.name, folder.type) == "\x1b[01;34m\x1b[4mtmp\x1b[0m" + + painter = GnuPainter("*.text=0;46", underline=True) + assert painter.paint(file.name, file.type) == "\x1b[4mtest.txt\x1b[0m" + assert painter.paint(folder.name, folder.type) == "\x1b[01;34m\x1b[4mtmp\x1b[0m" + + painter = GnuPainter("*.txt=0;46", underline=True) + assert painter.paint(file.name, file.type) == "\x1b[0;46m\x1b[4mtest.txt\x1b[0m" + assert painter.paint(folder.name, folder.type) == "\x1b[01;34m\x1b[4mtmp\x1b[0m" + class TestBSDPainter: def test_color_parsing(self) -> None: @@ -1010,6 +1045,37 @@ def test_coloring(self) -> None: "tmp", fg="blue", bg="black", bold=True ) + def test_coloring_underline(self) -> None: + file = FileStatus( + "test.txt", + 1024, + FileStatusType.FILE, + int(time.mktime(time.strptime("2018-01-01 03:00:00", "%Y-%m-%d %H:%M:%S"))), + "read", + ) + folder = FileStatus( + "tmp", + 0, + FileStatusType.DIRECTORY, + int(time.mktime(time.strptime("2018-01-01 03:00:00", "%Y-%m-%d %H:%M:%S"))), + "write", + ) + painter = BSDPainter("exfxcxdxbxegedabagacad", underline=True) + assert painter.paint(file.name, file.type) == click.style( + "test.txt", underline=True + ) + assert painter.paint(folder.name, folder.type) == click.style( + "tmp", fg="blue", underline=True + ) + + painter = BSDPainter("Eafxcxdxbxegedabagacad", underline=True) + assert painter.paint(file.name, file.type) == click.style( + "test.txt", underline=True + ) + assert painter.paint(folder.name, folder.type) == click.style( + "tmp", fg="blue", bg="black", bold=True, underline=True + ) + class TestPainterFactory: def test_detection(self, monkeypatch: Any) -> None: diff --git a/tests/cli/test_printer.py b/tests/cli/test_printer.py index 53cb1c455..f1794db39 100644 --- a/tests/cli/test_printer.py +++ b/tests/cli/test_printer.py @@ -95,13 +95,13 @@ def test_message_lineno(self, printer: TTYPrinter, capfd: Any) -> None: assert printer.total_lines == 0 printer.print("message1") assert printer.total_lines == 1 - printer.print("message1-replace", 1) + printer.print("message1-replace", 0) assert printer.total_lines == 1 - printer.print("message3", 3) + printer.print("message3", 2) assert printer.total_lines == 3 - printer.print("message7", 7) + printer.print("message7", 6) assert printer.total_lines == 7 - printer.print("message2", 2) + printer.print("message2", 1) assert printer.total_lines == 7 printer.close() out, err = capfd.readouterr() diff --git a/tests/cli/test_storage_progress.py b/tests/cli/test_storage_progress.py index e75547105..857141615 100644 --- a/tests/cli/test_storage_progress.py +++ b/tests/cli/test_storage_progress.py @@ -1,76 +1,140 @@ import sys -from typing import Any +from pathlib import Path +from typing import Any, List +from unittest import mock +import click from yarl import URL from neuromation.api import ( + FileStatusType, StorageProgressComplete, + StorageProgressEnterDir, StorageProgressFail, - StorageProgressMkdir, + StorageProgressLeaveDir, StorageProgressStart, StorageProgressStep, ) from neuromation.cli.formatters import create_storage_progress from neuromation.cli.formatters.storage import ( - NoPercentPrinter, - QuietPrinter, - StandardPrintPercentOnly, + BaseStorageProgress, + StreamProgress, + TTYProgress, + format_url, ) +from neuromation.cli.root import Root + + +def unstyle(report: BaseStorageProgress) -> List[str]: + assert isinstance(report, TTYProgress) + return [click.unstyle(line) for (is_dir, line) in report.lines] + + +def test_format_url_storage() -> None: + u = URL("storage://asvetlov/folder") + assert format_url(u) == "storage://asvetlov/folder" + + +def test_format_url_file() -> None: + u = URL("file:///asvetlov/folder") + if sys.platform == "win32": + expected = "\\asvetlov\\folder" + else: + expected = "/asvetlov/folder" + assert format_url(u) == expected + + +def make_root(color: bool, tty: bool, verbose: bool) -> Root: + return Root(color, tty, (80, 25), True, 60, Path("~/.nmrc"), verbosity=int(verbose)) def test_progress_factory_none() -> None: - progress = create_storage_progress(False, False, False) - assert isinstance(progress, QuietPrinter) + progress = create_storage_progress(make_root(False, False, False), False) + assert isinstance(progress, StreamProgress) def test_progress_factory_verbose() -> None: - progress = create_storage_progress(False, False, True) - assert isinstance(progress, NoPercentPrinter) + progress = create_storage_progress(make_root(False, False, False), False) + assert isinstance(progress, StreamProgress) def test_progress_factory_percent() -> None: - progress = create_storage_progress(False, True, False) - assert isinstance(progress, StandardPrintPercentOnly) + progress = create_storage_progress(make_root(False, False, False), True) + assert isinstance(progress, TTYProgress) -def test_simple_progress(capsys: Any) -> None: - report = create_storage_progress(False, True, False) +def test_quiet_stream_progress(capsys: Any) -> None: + report = create_storage_progress(make_root(False, False, False), False) src = URL("file:///abc") - src_str = "/abc" if not sys.platform == "win32" else "\\abc" dst = URL("storage:xyz") - dst_str = "storage:xyz" + + report.begin(src, dst) + captured = capsys.readouterr() + assert captured.out == f"" + + report.enter(StorageProgressEnterDir(src, dst)) + captured = capsys.readouterr() + assert captured.out == f"" report.start(StorageProgressStart(src, dst, 600)) captured = capsys.readouterr() - assert captured.out == f"Start copying '{src_str}' -> '{dst_str}'.\n" + assert captured.out == f"" report.step(StorageProgressStep(src, dst, 300, 600)) captured = capsys.readouterr() - assert captured.out == f"\r'{src_str}' -> '{dst_str}': 50.00%." + assert captured.out == "" report.step(StorageProgressStep(src, dst, 400, 600)) captured = capsys.readouterr() - assert captured.out == f"\r'{src_str}' -> '{dst_str}': 66.67%." + assert captured.out == "" report.complete(StorageProgressComplete(src, dst, 600)) captured = capsys.readouterr() - assert captured.out == f"\rFile '{src_str}' -> '{dst_str}' copying completed.\n" + assert captured.out == f"" + + report.leave(StorageProgressLeaveDir(src, dst)) + captured = capsys.readouterr() + assert captured.out == f"" -def test_mkdir(capsys: Any) -> None: - report = create_storage_progress(False, True, False) +def test_stream_progress(capsys: Any) -> None: + report = create_storage_progress(make_root(False, False, True), False) src = URL("file:///abc") src_str = "/abc" if not sys.platform == "win32" else "\\abc" dst = URL("storage:xyz") dst_str = "storage:xyz" - report.mkdir(StorageProgressMkdir(src, dst)) + report.begin(src, dst) + captured = capsys.readouterr() + assert captured.out == f"Copy '{src_str}' -> '{dst_str}'\n" + + report.enter(StorageProgressEnterDir(src, dst)) + captured = capsys.readouterr() + assert captured.out == f"'{src_str}' -> '{dst_str}'\n" + + report.start(StorageProgressStart(src, dst, 600)) + captured = capsys.readouterr() + assert captured.out == f"" + + report.step(StorageProgressStep(src, dst, 300, 600)) + captured = capsys.readouterr() + assert captured.out == "" + + report.step(StorageProgressStep(src, dst, 400, 600)) + captured = capsys.readouterr() + assert captured.out == "" + + report.complete(StorageProgressComplete(src, dst, 600)) + captured = capsys.readouterr() + assert captured.out == f"'{src_str}' -> '{dst_str}'\n" + + report.leave(StorageProgressLeaveDir(src, dst)) captured = capsys.readouterr() - assert captured.out == f"Copy directory '{src_str}' -> '{dst_str}'.\n" + assert captured.out == f"" -def test_fail1(capsys: Any) -> None: - report = create_storage_progress(False, True, False) +def test_stream_fail1(capsys: Any) -> None: + report = create_storage_progress(make_root(False, True, False), False) src = URL("file:///abc") src_str = "/abc" if not sys.platform == "win32" else "\\abc" dst = URL("storage:xyz") @@ -80,8 +144,8 @@ def test_fail1(capsys: Any) -> None: assert captured.err == f"Failure: '{src_str}' -> 'storage:xyz' [error]\n" -def test_fail2(capsys: Any) -> None: - report = create_storage_progress(False, False, True) +def test_stream_fail2(capsys: Any) -> None: + report = create_storage_progress(make_root(False, True, False), False) src = URL("file:///abc") src_str = "/abc" if not sys.platform == "win32" else "\\abc" dst = URL("storage:xyz") @@ -89,3 +153,289 @@ def test_fail2(capsys: Any) -> None: report.fail(StorageProgressFail(src, dst, "error")) captured = capsys.readouterr() assert captured.err == f"Failure: '{src_str}' -> 'storage:xyz' [error]\n" + + +def test_tty_progress(capsys: Any) -> None: + report = create_storage_progress(make_root(False, True, False), True) + src = URL("file:///abc") + dst = URL("storage:xyz") + src_f = URL("file:///abc/file.txt") + dst_f = URL("storage:xyz/file.txt") + + report.begin(src, dst) + captured = capsys.readouterr() + assert captured.out == f"Copy 'file:///abc' => 'storage:xyz'\n" + + report.enter(StorageProgressEnterDir(src, dst)) + assert unstyle(report) == ["'file:///abc'"] + + report.start(StorageProgressStart(src_f, dst_f, 600)) + assert unstyle(report) == ["'file:///abc'", "'file.txt' [0.00%] 0B of 600B"] + + report.step(StorageProgressStep(src_f, dst_f, 300, 600)) + assert unstyle(report) == ["'file:///abc'", "'file.txt' [50.00%] 300B of 600B"] + + report.step(StorageProgressStep(src_f, dst_f, 400, 600)) + assert unstyle(report) == ["'file:///abc'", "'file.txt' [66.67%] 400B of 600B"] + + report.complete(StorageProgressComplete(src_f, dst_f, 600)) + assert unstyle(report) == ["'file:///abc'", "'file.txt' 600B"] + + report.leave(StorageProgressLeaveDir(src, dst)) + assert unstyle(report) == ["'file:///abc'", "'file.txt' 600B"] + + +def test_tty_verbose(capsys: Any) -> None: + report = create_storage_progress(make_root(False, True, True), True) + src = URL("file:///abc") + dst = URL("storage:xyz") + + report.begin(src, dst) + captured = capsys.readouterr() + assert captured.out == f"Copy\n'file:///abc'\n=>\n'storage:xyz'\n" + + +def test_tty_nested() -> None: + report = create_storage_progress(make_root(False, True, False), True) + src = URL("file:///abc") + dst = URL("storage:xyz") + src_f = URL("file:///abc/file.txt") + dst_f = URL("storage:xyz/file.txt") + src2 = URL("file:///abc/cde") + dst2 = URL("storage:xyz/cde") + src2_f = URL("file:///abc/cde/file.txt") + dst2_f = URL("storage:xyz/cde/file.txt") + + report.enter(StorageProgressEnterDir(src, dst)) + assert unstyle(report) == ["'file:///abc'"] + + report.start(StorageProgressStart(src_f, dst_f, 600)) + assert unstyle(report) == ["'file:///abc'", "'file.txt' [0.00%] 0B of 600B"] + + report.step(StorageProgressStep(src_f, dst_f, 300, 600)) + assert unstyle(report) == ["'file:///abc'", "'file.txt' [50.00%] 300B of 600B"] + + report.step(StorageProgressStep(src_f, dst_f, 400, 600)) + assert unstyle(report) == ["'file:///abc'", "'file.txt' [66.67%] 400B of 600B"] + + report.complete(StorageProgressComplete(src_f, dst_f, 600)) + assert unstyle(report) == ["'file:///abc'", "'file.txt' 600B"] + + report.enter(StorageProgressEnterDir(src2, dst2)) + assert unstyle(report) == ["'file:///abc'", "'file.txt' 600B", "'file:///abc/cde'"] + + report.start(StorageProgressStart(src2_f, dst2_f, 800)) + assert unstyle(report) == [ + "'file:///abc'", + "'file.txt' 600B", + "'file:///abc/cde'", + "'file.txt' [0.00%] 0B of 800B", + ] + + report.step(StorageProgressStep(src2_f, dst2_f, 300, 800)) + assert unstyle(report) == [ + "'file:///abc'", + "'file.txt' 600B", + "'file:///abc/cde'", + "'file.txt' [37.50%] 300B of 800B", + ] + + report.complete(StorageProgressComplete(src2_f, dst_f, 800)) + assert unstyle(report) == [ + "'file:///abc'", + "'file.txt' 600B", + "'file:///abc/cde'", + "'file.txt' 800B", + ] + + report.leave(StorageProgressLeaveDir(src2, dst2)) + assert unstyle(report) == [ + "'file:///abc'", + "'file.txt' 600B", + "'file:///abc/cde'", + "'file.txt' 800B", + "'file:///abc'", + ] + + report.leave(StorageProgressLeaveDir(src, dst)) + assert unstyle(report) == [ + "'file:///abc'", + "'file.txt' 600B", + "'file:///abc/cde'", + "'file.txt' 800B", + "'file:///abc'", + ] + + +def test_fail_tty(capsys: Any) -> None: + report = create_storage_progress(make_root(False, True, False), True) + src = URL("file:///abc") + dst = URL("storage:xyz") + + report.fail(StorageProgressFail(src, dst, "error")) + captured = capsys.readouterr() + assert captured.err == f"Failure: 'file:///abc' -> 'storage:xyz' [error]\n" + + +def test_tty_fmt_url() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL("storage://andrew/folder/file.txt") + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=True)) + == "'storage://andrew/folder/file.txt'" + ) + + +def test_tty_fmt_storage_url_over_half() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL("storage://andrew/folder0/folder1/file.txt") + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=True)) + == "'storage://andrew/.../file.txt'" + ) + + +def test_tty_fmt_storage_url_over_full() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL( + "storage://andrew/" + + "/".join("folder" + str(i) for i in range(5)) + + "/file.txt" + ) + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=False)) + == "'storage://andrew/.../folder2/folder3/folder4/file.txt'" + ) + + +def test_tty_fmt_url_over_half_single_segment() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL("file://" + "a" * 40) + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=True)) + == "'file://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'" + ) + + +def test_tty_fmt_url_over_half_single_segment2() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL("file:///" + "a" * 40) + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=True)) + == "'file:///aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'" + ) + + +def test_tty_fmt_url_over_half_long_segment() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL("file:///andrew/" + "a" * 30) + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=True)) + == "'file:///.../aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'" + ) + + +def test_tty_fmt_file_url_over_half() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL("file:///andrew/folder0/folder1/file.txt") + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=True)) + == "'file:///.../folder1/file.txt'" + ) + + +def test_tty_fmt_file_url_over_full() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL( + "file:///andrew/" + "/".join("folder" + str(i) for i in range(5)) + "/file.txt" + ) + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=False)) + == "'file:///.../folder0/folder1/folder2/folder3/folder4/file.txt'" + ) + + +def test_tty_fmt_url_relative_over() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL("storage:folder1/folder2/folder3/folder4/folder5") + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=True)) + == "'storage:.../folder3/folder4/folder5'" + ) + + +def test_tty_fmt_url_relative_over_long_2_segments() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL("storage:folder/" + "a" * 30) + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=True)) + == "'storage:.../aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'" + ) + + +def test_tty_fmt_url_relative_over_single_segment() -> None: + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + url = URL("storage:" + "a" * 35) + assert ( + click.unstyle(report.fmt_url(url, FileStatusType.FILE, half=True)) + == "'storage:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'" + ) + + +def test_tty_append_files() -> None: + with mock.patch.object(TTYProgress, "HEIGHT", 3): + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + mock.patch.object(report, "HEIGHT", 3) + assert report.lines == [] + report.append("a") + assert report.lines == [(False, "a")] + report.append("b") + assert report.lines == [(False, "a"), (False, "b")] + report.append("c") + assert report.lines == [(False, "a"), (False, "b"), (False, "c")] + report.append("d") + assert report.lines == [(False, "b"), (False, "c"), (False, "d")] + + +def test_tty_append_dir() -> None: + with mock.patch.object(TTYProgress, "HEIGHT", 3): + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + mock.patch.object(report, "HEIGHT", 3) + assert report.lines == [] + report.append("a", is_dir=True) + assert report.lines == [(True, "a")] + report.append("b") + assert report.lines == [(True, "a"), (False, "b")] + report.append("c") + assert report.lines == [(True, "a"), (False, "b"), (False, "c")] + report.append("d") + assert report.lines == [(True, "a"), (False, "c"), (False, "d")] + + +def test_tty_append_second_dir() -> None: + with mock.patch.object(TTYProgress, "HEIGHT", 3): + report = create_storage_progress(make_root(False, True, False), True) + assert isinstance(report, TTYProgress) + mock.patch.object(report, "HEIGHT", 3) + assert report.lines == [] + report.append("a", is_dir=True) + assert report.lines == [(True, "a")] + report.append("b") + assert report.lines == [(True, "a"), (False, "b")] + report.append("c", is_dir=True) + assert report.lines == [(True, "a"), (False, "b"), (True, "c")] + report.append("d") + assert report.lines == [(False, "b"), (True, "c"), (False, "d")]