diff --git a/python/.coveragerc b/python/.coveragerc index 0d8c9d4a8..725371c9b 100644 --- a/python/.coveragerc +++ b/python/.coveragerc @@ -1,6 +1,3 @@ [run] omit = tests/* - neuromation/logging/colors.py - neuromation/logging/formatter.py - neuromation/cli/main.py - neuromation/build-tools/cli-help-generator.py + build-tools/* diff --git a/python/CHANGELOG.D/439.feature b/python/CHANGELOG.D/439.feature new file mode 100644 index 000000000..cf9d9fa9d --- /dev/null +++ b/python/CHANGELOG.D/439.feature @@ -0,0 +1 @@ +Add top-level aliases for the most frequent commands. \ No newline at end of file diff --git a/python/README.md b/python/README.md index c4ae1f825..d096357eb 100644 --- a/python/README.md +++ b/python/README.md @@ -10,29 +10,46 @@ * [neuro config auth](#neuro-config-auth) * [neuro config forget](#neuro-config-forget) * [neuro config id_rsa](#neuro-config-id_rsa) + * [neuro config login](#neuro-config-login) + * [neuro config logout](#neuro-config-logout) * [neuro config show](#neuro-config-show) * [neuro config show-token](#neuro-config-show-token) * [neuro config url](#neuro-config-url) + * [neuro cp](#neuro-cp) + * [neuro exec](#neuro-exec) * [neuro help](#neuro-help) * [neuro image](#neuro-image) * [neuro image ls](#neuro-image-ls) * [neuro image pull](#neuro-image-pull) * [neuro image push](#neuro-image-push) + * [neuro images](#neuro-images) * [neuro job](#neuro-job) * [neuro job exec](#neuro-job-exec) * [neuro job kill](#neuro-job-kill) * [neuro job list](#neuro-job-list) + * [neuro job logs](#neuro-job-logs) + * [neuro job ls](#neuro-job-ls) * [neuro job monitor](#neuro-job-monitor) * [neuro job ssh](#neuro-job-ssh) * [neuro job status](#neuro-job-status) * [neuro job submit](#neuro-job-submit) * [neuro job top](#neuro-job-top) + * [neuro kill](#neuro-kill) * [neuro login](#neuro-login) * [neuro logout](#neuro-logout) + * [neuro logs](#neuro-logs) + * [neuro ls](#neuro-ls) + * [neuro mkdir](#neuro-mkdir) * [neuro model](#neuro-model) * [neuro model debug](#neuro-model-debug) * [neuro model train](#neuro-model-train) + * [neuro mv](#neuro-mv) + * [neuro ps](#neuro-ps) + * [neuro pull](#neuro-pull) + * [neuro push](#neuro-push) + * [neuro rm](#neuro-rm) * [neuro share](#neuro-share) + * [neuro status](#neuro-status) * [neuro storage](#neuro-storage) * [neuro storage cp](#neuro-storage-cp) * [neuro storage ls](#neuro-storage-ls) @@ -45,6 +62,8 @@ * [neuro store mkdir](#neuro-store-mkdir) * [neuro store mv](#neuro-store-mv) * [neuro store rm](#neuro-store-rm) + * [neuro submit](#neuro-submit) + * [neuro top](#neuro-top) * [Api](#Api) * [Contributing](#Contributing) @@ -59,7 +78,7 @@ Package ship command line tool called [_neuro_](#neuro). With [_neuro_](#neuro) # neuro - ▇ ◣
▇ ◥ ◣
◣ ◥ ▇
▇ ◣ ▇
▇ ◥ ◣ ▇
▇ ◥ ▇ Neuromation Platform
▇ ◣ ◥
◥ ◣ ▇ Deep network training,
◥ ▇ inference and datasets
◥ +Neuromation console. **Usage:** @@ -81,16 +100,56 @@ Name | Description| * _[neuro completion](#neuro-completion)_: Generates code to enable shell-completion. * _[neuro config](#neuro-config)_: Client configuration settings commands. -* _[neuro help](#neuro-help)_: Get help on a command -* _[neuro image](#neuro-image)_: Docker image operations +* _[neuro cp](#neuro-cp)_: Copy files and directories. + +Either SOURCE or DESTINATION should have storage:// scheme. If scheme is +omitted, file:// scheme is assumed. + +* _[neuro exec](#neuro-exec)_: Executes command in a running job. +* _[neuro help](#neuro-help)_: Get help on a command. +* _[neuro image](#neuro-image)_: Docker image operations. +* _[neuro images](#neuro-images)_: List user's images which are available for jobs. + +You will see here own and shared with you images * _[neuro job](#neuro-job)_: Job operations. +* _[neuro kill](#neuro-kill)_: Kill job(s). * _[neuro login](#neuro-login)_: Log into Neuromation Platform. * _[neuro logout](#neuro-logout)_: Log out. -* _[neuro model](#neuro-model)_: Model operations. +* _[neuro logs](#neuro-logs)_: Fetch the logs of a container. +* _[neuro ls](#neuro-ls)_: List directory contents. + +By default PATH is equal user`s home dir (storage:) +* _[neuro mkdir](#neuro-mkdir)_: Make directories. +* _[neuro model](#neuro-model)_: Model operations. (DEPRECATED) +* _[neuro mv](#neuro-mv)_: Move or rename files and directories. + +SOURCE must contain path to the file or directory existing on the storage, +and DESTINATION must contain the full path to the target file or directory. + +* _[neuro ps](#neuro-ps)_: List all jobs. + +* _[neuro pull](#neuro-pull)_: Pull an image from platform registry. + +Remote image name must be URL with image:// scheme. Image names can contain +tag. + +* _[neuro push](#neuro-push)_: Push an image to platform registry. + +Remote image must be URL with image:// scheme. Image names can contains tag. +If tags not specified 'latest' will be used as value. + +* _[neuro rm](#neuro-rm)_: Remove files or directories. + * _[neuro share](#neuro-share)_: Shares resource specified by URI to a USER with PERMISSION +* _[neuro status](#neuro-status)_: Display status of a job. * _[neuro storage](#neuro-storage)_: Storage operations. * _[neuro store](#neuro-store)_: Alias for storage (DEPRECATED) +* _[neuro submit](#neuro-submit)_: Start job using IMAGE. + +COMMANDS list will be passed as commands to model container. + +* _[neuro top](#neuro-top)_: Display real-time job telemetry. @@ -186,6 +245,8 @@ Name | Description| FILE is being used for accessing remote shell, remote debug. Note: this is temporal and going to be replaced in future by JWT token. +* _[neuro config login](#neuro-config-login)_: Log into Neuromation Platform. +* _[neuro config logout](#neuro-config-logout)_: Log out. * _[neuro config show](#neuro-config-show)_: Print current settings. * _[neuro config show-token](#neuro-config-show-token)_: Print current authorization token. * _[neuro config url](#neuro-config-url)_: Update settings with provided platform URL. @@ -251,6 +312,44 @@ Name | Description| +### neuro config login + +Log into Neuromation Platform. + +**Usage:** + +```bash +neuro config login [OPTIONS] [URL] +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + +### neuro config logout + +Log out. + +**Usage:** + +```bash +neuro config logout [OPTIONS] +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + ### neuro config show Print current settings. @@ -317,9 +416,66 @@ Name | Description| +## neuro cp + +Copy files and directories.

Either SOURCE or DESTINATION should have storage:// scheme. If scheme is
omitted, file:// scheme is assumed.
+ +**Usage:** + +```bash +neuro cp [OPTIONS] SOURCE DESTINATION +``` + +**Examples:** + +```bash + + +# copy local file ./foo into remote storage root +neuro storage cp ./foo storage:/// +neuro storage cp ./foo storage:/ + +# download remote file foo into local file foo with +# explicit file:// scheme set +neuro storage cp storage:///foo file:///foo + +``` + +**Options:** + +Name | Description| +|----|------------| +|_\-r, --recursive_|Recursive copy, off by default| +|_\-p, --progress_|Show progress, off by default| +|_--help_|Show this message and exit.| + + + + +## neuro exec + +Executes command in a running job. + +**Usage:** + +```bash +neuro exec [OPTIONS] ID CMD... +``` + +**Options:** + +Name | Description| +|----|------------| +|_\-t, --tty_|Allocate virtual tty. Useful for interactive jobs.| +|_\--no-key-check_|Disable host key checks. Should be used with caution.| +|_--help_|Show this message and exit.| + + + + ## neuro help -Get help on a command +Get help on a command. **Usage:** @@ -338,7 +494,7 @@ Name | Description| ## neuro image -Docker image operations +Docker image operations. **Usage:** @@ -451,6 +607,25 @@ Name | Description| +## neuro images + +List user's images which are available for jobs.

You will see here own and shared with you images + +**Usage:** + +```bash +neuro images [OPTIONS] +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + ## neuro job Job operations. @@ -471,19 +646,22 @@ Name | Description| **Commands:** * _[neuro job exec](#neuro-job-exec)_: Executes command in a running job. -* _[neuro job kill](#neuro-job-kill)_: Kill job(s) -* _[neuro job list](#neuro-job-list)_: List all jobs. +* _[neuro job kill](#neuro-job-kill)_: Kill job(s). +* _[neuro job list](#neuro-job-list)_: Alias for ls. (DEPRECATED) +* _[neuro job logs](#neuro-job-logs)_: Fetch the logs of a container. +* _[neuro job ls](#neuro-job-ls)_: List all jobs. + +* _[neuro job monitor](#neuro-job-monitor)_: Alias for logs. (DEPRECATED) +* _[neuro job ssh](#neuro-job-ssh)_: Starts ssh terminal connected to running job. -* _[neuro job monitor](#neuro-job-monitor)_: Monitor job output stream -* _[neuro job ssh](#neuro-job-ssh)_: Starts ssh terminal connected to running job. Job should be started with SSH -support enabled. +Job should be started with SSH support enabled. -* _[neuro job status](#neuro-job-status)_: Display status of a job +* _[neuro job status](#neuro-job-status)_: Display status of a job. * _[neuro job submit](#neuro-job-submit)_: Start job using IMAGE. COMMANDS list will be passed as commands to model container. -* _[neuro job top](#neuro-job-top)_: Display real-time job telemetry +* _[neuro job top](#neuro-job-top)_: Display real-time job telemetry. @@ -511,7 +689,7 @@ Name | Description| ### neuro job kill -Kill job\(s) +Kill job\(s). **Usage:** @@ -530,7 +708,7 @@ Name | Description| ### neuro job list -List all jobs.
+Alias for ls. \(DEPRECATED) **Usage:** @@ -538,6 +716,47 @@ List all jobs.
neuro job list [OPTIONS] ``` +**Options:** + +Name | Description| +|----|------------| +|_\-s, --status \[pending|running|succeeded|failed|all]_|Filter out job by status \(multiple option)| +|_\-d, --description DESCRIPTION_|Filter out job by job description \(exact match)| +|_\-q, --quiet_|| +|_--help_|Show this message and exit.| + + + + +### neuro job logs + +Fetch the logs of a container. + +**Usage:** + +```bash +neuro job logs [OPTIONS] ID +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + +### neuro job ls + +List all jobs.
+ +**Usage:** + +```bash +neuro job ls [OPTIONS] +``` + **Examples:** ```bash @@ -563,7 +782,7 @@ Name | Description| ### neuro job monitor -Monitor job output stream +Alias for logs. \(DEPRECATED) **Usage:** @@ -582,7 +801,7 @@ Name | Description| ### neuro job ssh -Starts ssh terminal connected to running job. Job should be started with SSH
support enabled.
+Starts ssh terminal connected to running job.

Job should be started with SSH support enabled.
**Usage:** @@ -595,7 +814,7 @@ neuro job ssh [OPTIONS] ID ```bash -neuro job ssh --user alfa --key ./my_docker_id_rsa job-abc-def-ghk +neuro job ssh --user alfa --key ./my_docker_id_rsa job-abc-def-ghk (DEPRECATED) ``` @@ -612,7 +831,7 @@ Name | Description| ### neuro job status -Display status of a job +Display status of a job. **Usage:** @@ -680,7 +899,7 @@ Name | Description| ### neuro job top -Display real-time job telemetry +Display real-time job telemetry. **Usage:** @@ -697,6 +916,25 @@ Name | Description| +## neuro kill + +Kill job\(s). + +**Usage:** + +```bash +neuro kill [OPTIONS] ID... +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + ## neuro login Log into Neuromation Platform. @@ -735,9 +973,66 @@ Name | Description| +## neuro logs + +Fetch the logs of a container. + +**Usage:** + +```bash +neuro logs [OPTIONS] ID +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + +## neuro ls + +List directory contents.

By default PATH is equal user`s home dir \(storage:) + +**Usage:** + +```bash +neuro ls [OPTIONS] [PATH] +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + +## neuro mkdir + +Make directories. + +**Usage:** + +```bash +neuro mkdir [OPTIONS] PATH +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + ## neuro model -Model operations. +Model operations. \(DEPRECATED) **Usage:** @@ -826,6 +1121,163 @@ Name | Description| +## neuro mv + +Move or rename files and directories.

SOURCE must contain path to the file or directory existing on the storage,
and DESTINATION must contain the full path to the target file or directory.
+ +**Usage:** + +```bash +neuro mv [OPTIONS] SOURCE DESTINATION +``` + +**Examples:** + +```bash + + +# move or rename remote file +neuro storage mv storage://{username}/foo.txt storage://{username}/bar.txt +neuro storage mv storage://{username}/foo.txt storage://~/bar/baz/foo.txt + +# move or rename remote directory +neuro storage mv storage://{username}/foo/ storage://{username}/bar/ +neuro storage mv storage://{username}/foo/ storage://{username}/bar/baz/foo/ + +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + +## neuro ps + +List all jobs.
+ +**Usage:** + +```bash +neuro ps [OPTIONS] +``` + +**Examples:** + +```bash + + +neuro job list --description="my favourite job" +neuro job list --status=all +neuro job list -s pending -s running -q + +``` + +**Options:** + +Name | Description| +|----|------------| +|_\-s, --status \[pending|running|succeeded|failed|all]_|Filter out job by status \(multiple option)| +|_\-d, --description DESCRIPTION_|Filter out job by job description \(exact match)| +|_\-q, --quiet_|| +|_--help_|Show this message and exit.| + + + + +## neuro pull + +Pull an image from platform registry.

Remote image name must be URL with image:// scheme. Image names can contain
tag.
+ +**Usage:** + +```bash +neuro pull [OPTIONS] IMAGE_NAME [LOCAL_IMAGE_NAME] +``` + +**Examples:** + +```bash + + +neuro image pull image:myimage +neuro image pull image://myfriend/alpine:shared +neuro image pull image://username/my-alpine:production alpine:from-registry + +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + +## neuro push + +Push an image to platform registry.

Remote image must be URL with image:// scheme. Image names can contains tag.
If tags not specified 'latest' will be used as value.
+ +**Usage:** + +```bash +neuro push [OPTIONS] IMAGE_NAME [REMOTE_IMAGE_NAME] +``` + +**Examples:** + +```bash + + +neuro image push myimage +neuro image push alpine:latest image:my-alpine:production +neuro image push alpine image://myfriend/alpine:shared + +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + +## neuro rm + +Remove files or directories.
+ +**Usage:** + +```bash +neuro rm [OPTIONS] PATH +``` + +**Examples:** + +```bash + + +neuro storage rm storage:///foo/bar/ +neuro storage rm storage:/foo/bar/ +neuro storage rm storage://{username}/foo/bar/ + +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + ## neuro share Shares resource specified by URI to a USER with PERMISSION
@@ -853,6 +1305,25 @@ Name | Description| +## neuro status + +Display status of a job. + +**Usage:** + +```bash +neuro status [OPTIONS] ID +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + ## neuro storage Storage operations. @@ -1207,6 +1678,74 @@ Name | Description| +## neuro submit + +Start job using IMAGE.

COMMANDS list will be passed as commands to model container.
+ +**Usage:** + +```bash +neuro submit [OPTIONS] IMAGE [CMD]... +``` + +**Examples:** + +```bash + + +# Starts a container pytorch:latest with two paths mounted. Directory /q1/ +# is mounted in read only mode to /qm directory within container. +# Directory /mod mounted to /mod directory in read-write mode. +neuro job submit --volume storage:/q1:/qm:ro --volume storage:/mod:/mod:rw pytorch:latest + +# Starts a container pytorch:latest with connection enabled to port 22 and +# sets PYTHONPATH environment value to /python. +# Please note that SSH server should be provided by container. +neuro job submit --env PYTHONPATH=/python --volume storage:/data/2018q1:/data:ro --ssh 22 pytorch:latest + +``` + +**Options:** + +Name | Description| +|----|------------| +|_\-g, --gpu NUMBER_|Number of GPUs to request \[default: 0]| +|_\--gpu-model MODEL_|GPU to use \[default: nvidia\-tesla-k80]| +|_\-c, --cpu NUMBER_|Number of CPUs to request \[default: 0.1]| +|_\-m, --memory AMOUNT_|Memory amount to request \[default: 1G]| +|_\-x, --extshm_|Request extended '/dev/shm' space| +|_--http INTEGER_|Enable HTTP port forwarding to container| +|_--ssh INTEGER_|Enable SSH port forwarding to container| +|_\--preemptible / --non-preemptible_|Run job on a lower-cost preemptible instance| +|_\-d, --description DESC_|Add optional description to the job| +|_\-q, --quiet_|Run command in quiet mode \(print only job id)| +|_--volume MOUNT_|Mounts directory from vault into container. Use multiple options to mount more than one volume| +|_\-e, --env VAR=VAL_|Set environment variable in container Use multiple options to define more than one variable| +|_\--env-file PATH_|File with environment variables to pass| +|_--help_|Show this message and exit.| + + + + +## neuro top + +Display real-time job telemetry. + +**Usage:** + +```bash +neuro top [OPTIONS] ID +``` + +**Options:** + +Name | Description| +|----|------------| +|_--help_|Show this message and exit.| + + + + # Api *TODO* diff --git a/python/neuromation/cli/completion.py b/python/neuromation/cli/completion.py index fae0db1d3..3be7285b1 100644 --- a/python/neuromation/cli/completion.py +++ b/python/neuromation/cli/completion.py @@ -2,14 +2,16 @@ import click +from .utils import group + CFG_FILE = {"bash": Path("~/.bashrc"), "zsh": Path("~/.zshrc")} SOURCE_CMD = {"bash": "source", "zsh": "source_zsh"} -ACTIVATION_TEMPLATE = 'eval "$(_NEURO_COMPLETE={shell} neuro)"' +ACTIVATION_TEMPLATE = 'eval "$(_NEURO_COMPLETE={cmd} neuro)"' -@click.group() +@group() def completion() -> None: """ Generates code to enable shell-completion. @@ -29,7 +31,7 @@ def generate(shell: str) -> None: Provide an instruction for shell completion generation. """ click.echo(f"Push the following line into your {CFG_FILE[shell]}") - click.echo(ACTIVATION_TEMPLATE.format(shell=shell)) + click.echo(ACTIVATION_TEMPLATE.format(cmd=SOURCE_CMD[shell])) @completion.command() @@ -46,5 +48,5 @@ def patch(shell: str) -> None: """ profile_file = CFG_FILE[shell].expanduser() with profile_file.open("a+") as profile: - profile.write(ACTIVATION_TEMPLATE.format(shell=shell)) + profile.write(ACTIVATION_TEMPLATE.format(cmd=SOURCE_CMD[shell])) profile.write("\n") diff --git a/python/neuromation/cli/config.py b/python/neuromation/cli/config.py index 659f339ae..6ee2db861 100644 --- a/python/neuromation/cli/config.py +++ b/python/neuromation/cli/config.py @@ -5,9 +5,10 @@ from .defaults import API_URL from .formatter import ConfigFormatter from .rc import Config +from .utils import group -@click.group() +@group() def config() -> None: """Client configuration settings commands.""" @@ -81,7 +82,7 @@ def forget() -> None: rc.ConfigFactory.forget_auth_token() -@click.command() +@config.command() @click.argument("url", required=False, default=API_URL, type=URL) def login(url: URL) -> None: """ @@ -91,7 +92,7 @@ def login(url: URL) -> None: click.echo(f"Logged into {url}") -@click.command() +@config.command() def logout() -> None: """ Log out. diff --git a/python/neuromation/cli/image.py b/python/neuromation/cli/image.py index bf8576995..2e2779128 100644 --- a/python/neuromation/cli/image.py +++ b/python/neuromation/cli/image.py @@ -8,13 +8,13 @@ from .command_spinner import SpinnerBase from .rc import Config -from .utils import run_async +from .utils import group, run_async -@click.group() +@group() def image() -> None: """ - Docker image operations + Docker image operations. """ diff --git a/python/neuromation/cli/job.py b/python/neuromation/cli/job.py index db1ab09f1..82bf0c01d 100644 --- a/python/neuromation/cli/job.py +++ b/python/neuromation/cli/job.py @@ -26,13 +26,13 @@ ) from .rc import Config from .ssh_utils import connect_ssh -from .utils import run_async +from .utils import alias, group, run_async log = logging.getLogger(__name__) -@click.group() +@group() def job() -> None: """ Job operations. @@ -220,7 +220,7 @@ async def exec( sys.exit(retcode) -@job.command() +@job.command(deprecated=True) @click.argument("id") @click.option( "--user", help="Container user name", default=JOB_SSH_USER, show_default=True @@ -231,6 +231,7 @@ async def exec( async def ssh(cfg: Config, id: str, user: str, key: str) -> None: """ Starts ssh terminal connected to running job. + Job should be started with SSH support enabled. Examples: @@ -248,9 +249,9 @@ async def ssh(cfg: Config, id: str, user: str, key: str) -> None: @click.argument("id") @click.pass_obj @run_async -async def monitor(cfg: Config, id: str) -> None: +async def logs(cfg: Config, id: str) -> None: """ - Monitor job output stream + Fetch the logs of a container. """ timeout = aiohttp.ClientTimeout( total=None, connect=None, sock_read=None, sock_connect=30 @@ -263,6 +264,9 @@ async def monitor(cfg: Config, id: str) -> None: click.echo(chunk.decode(errors="ignore"), nl=False) +job.add_command(alias(logs, "monitor")) + + @job.command() @click.option( "-s", @@ -280,9 +284,7 @@ async def monitor(cfg: Config, id: str) -> None: @click.option("-q", "--quiet", is_flag=True) @click.pass_obj @run_async -async def list( - cfg: Config, status: Sequence[str], description: str, quiet: bool -) -> None: +async def ls(cfg: Config, status: Sequence[str], description: str, quiet: bool) -> None: """ List all jobs. @@ -308,13 +310,16 @@ async def list( click.echo(formatter(jobs, statuses, description)) +job.add_command(alias(ls, "list")) + + @job.command() @click.argument("id") @click.pass_obj @run_async async def status(cfg: Config, id: str) -> None: """ - Display status of a job + Display status of a job. """ async with cfg.make_client() as client: res = await client.jobs.status(id) @@ -327,7 +332,7 @@ async def status(cfg: Config, id: str) -> None: @run_async async def top(cfg: Config, id: str) -> None: """ - Display real-time job telemetry + Display real-time job telemetry. """ formatter = JobTelemetryFormatter() async with cfg.make_client() as client: @@ -346,7 +351,7 @@ async def top(cfg: Config, id: str) -> None: @run_async async def kill(cfg: Config, id: Sequence[str]) -> None: """ - Kill job(s) + Kill job(s). """ errors = [] async with cfg.make_client() as client: diff --git a/python/neuromation/cli/main.py b/python/neuromation/cli/main.py index b9208c3a7..830a436e9 100644 --- a/python/neuromation/cli/main.py +++ b/python/neuromation/cli/main.py @@ -12,15 +12,8 @@ from neuromation.cli.rc import RCException from neuromation.logging import ConsoleWarningFormatter -from . import rc -from .completion import completion -from .config import config, login, logout -from .image import image -from .job import job -from .model import model -from .share import share -from .storage import storage -from .utils import DeprecatedGroup +from . import completion, config, image, job, model, rc, share, storage +from .utils import DeprecatedGroup, MainGroup, alias # For stream copying from file to http or from http to file @@ -61,7 +54,7 @@ def setup_console_handler( LOG_ERROR = log.error -@click.group(context_settings=dict(help_option_names=["-h", "--help"])) +@click.group(cls=MainGroup) @click.option("-v", "--verbose", count=True, type=int) @click.option("--show-traceback", is_flag=True) @click.version_option( @@ -70,18 +63,18 @@ def setup_console_handler( @click.pass_context def cli(ctx: click.Context, verbose: int, show_traceback: bool) -> None: """ - \b - ▇ ◣ - ▇ ◥ ◣ - ◣ ◥ ▇ - ▇ ◣ ▇ - ▇ ◥ ◣ ▇ - ▇ ◥ ▇ Neuromation Platform - ▇ ◣ ◥ - ◥ ◣ ▇ Deep network training, - ◥ ▇ inference and datasets - ◥ + Neuromation console. """ + # ▇ ◣ + # ▇ ◥ ◣ + # ◣ ◥ ▇ + # ▇ ◣ ▇ + # ▇ ◥ ◣ ▇ + # ▇ ◥ ▇ Neuromation Platform + # ▇ ◣ ◥ + # ◥ ◣ ▇ Deep network training, + # ◥ ▇ inference and datasets + # ◥ global LOG_ERROR if show_traceback: LOG_ERROR = log.exception @@ -95,7 +88,7 @@ def cli(ctx: click.Context, verbose: int, show_traceback: bool) -> None: @click.argument("command", nargs=-1) @click.pass_context def help(ctx: click.Context, command: Sequence[str]) -> None: - """Get help on a command""" + """Get help on a command.""" top_ctx = ctx while top_ctx.parent is not None: top_ctx = top_ctx.parent @@ -123,16 +116,31 @@ def help(ctx: click.Context, command: Sequence[str]) -> None: ctx.close() -cli.add_command(login) -cli.add_command(logout) -cli.add_command(config) -cli.add_command(storage) -cli.add_command(DeprecatedGroup(storage, name="store")) -cli.add_command(model) -cli.add_command(job) -cli.add_command(image) -cli.add_command(share) -cli.add_command(completion) +cli.add_command(config.config) +cli.add_command(config.login) +cli.add_command(config.logout) +cli.add_command(storage.storage) +cli.add_command(storage.rm) +cli.add_command(storage.ls) +cli.add_command(storage.cp) +cli.add_command(storage.mkdir) +cli.add_command(storage.mv) +cli.add_command(DeprecatedGroup(storage.storage, name="store")) +cli.add_command(model.model) +cli.add_command(job.job) +cli.add_command(job.submit) +cli.add_command(job.exec) +cli.add_command(job.logs) +cli.add_command(alias(job.ls, "ps", help=job.ls.help, deprecated=False)) +cli.add_command(job.status) +cli.add_command(job.top) +cli.add_command(job.kill) +cli.add_command(image.image) +cli.add_command(image.push) +cli.add_command(image.pull) +cli.add_command(alias(image.ls, "images", help=image.ls.help, deprecated=False)) +cli.add_command(share.share) +cli.add_command(completion.completion) def main(args: Optional[List[str]] = None) -> None: diff --git a/python/neuromation/cli/model.py b/python/neuromation/cli/model.py index 93da007a0..7411bdd03 100644 --- a/python/neuromation/cli/model.py +++ b/python/neuromation/cli/model.py @@ -18,13 +18,13 @@ from .formatter import JobFormatter from .rc import Config from .ssh_utils import remote_debug -from .utils import run_async +from .utils import group, run_async log = logging.getLogger(__name__) -@click.group() +@group(deprecated=True) def model() -> None: """ Model operations. diff --git a/python/neuromation/cli/share.py b/python/neuromation/cli/share.py index 9c24bd1fa..027649f54 100644 --- a/python/neuromation/cli/share.py +++ b/python/neuromation/cli/share.py @@ -4,10 +4,10 @@ from neuromation.client import Action, IllegalArgumentError, Permission from .rc import Config -from .utils import run_async +from .utils import command, run_async -@click.command() +@command() @click.argument("uri") @click.argument("user") @click.argument("permission", type=click.Choice(["read", "write", "manage"])) diff --git a/python/neuromation/cli/storage.py b/python/neuromation/cli/storage.py index 5f07478ce..cd83cef54 100644 --- a/python/neuromation/cli/storage.py +++ b/python/neuromation/cli/storage.py @@ -7,13 +7,13 @@ from .command_progress_report import ProgressBase from .formatter import StorageLsFormatter from .rc import Config -from .utils import run_async +from .utils import group, run_async log = logging.getLogger(__name__) -@click.group() +@group() def storage() -> None: """ Storage operations. diff --git a/python/neuromation/cli/utils.py b/python/neuromation/cli/utils.py index 6e3bfac83..aeba09bd1 100644 --- a/python/neuromation/cli/utils.py +++ b/python/neuromation/cli/utils.py @@ -1,5 +1,16 @@ from functools import wraps -from typing import Any, Awaitable, Callable, Iterable, Optional, TypeVar +from typing import ( + Any, + Awaitable, + Callable, + Iterable, + List, + Optional, + Sequence, + Tuple, + Type, + TypeVar, +) import click @@ -17,7 +28,79 @@ def wrapper(*args: Any, **kwargs: Any) -> _T: return wrapper -class DeprecatedGroup(click.MultiCommand): +class HelpFormatter(click.HelpFormatter): + def write_heading(self, heading: str) -> None: + self.write( + click.style( + "%*s%s:\n" % (self.current_indent, "", heading), + bold=True, + underline=True, + ) + ) + + +class Context(click.Context): + def make_formatter(self) -> click.HelpFormatter: + return HelpFormatter( + width=self.terminal_width, max_width=self.max_content_width + ) + + +class MakeContextMixin: + def make_context( + self, + info_name: str, + args: Sequence[str], + parent: Optional[click.Context] = None, + **extra: Any, + ) -> Context: + for key, value in self.context_settings.items(): # type: ignore + if key not in extra: + extra[key] = value + ctx = Context(self, info_name=info_name, parent=parent, **extra) # type: ignore + with ctx.scope(cleanup=False): + self.parse_args(ctx, args) # type: ignore + return ctx + + +class Command(MakeContextMixin, click.Command): + pass + + +def command( + name: Optional[str] = None, cls: Type[Command] = Command, **kwargs: Any +) -> Command: + return click.command(name=name, cls=cls, **kwargs) # type: ignore + + +class Group(MakeContextMixin, click.Group): + def command( + self, *args: Any, **kwargs: Any + ) -> Callable[[Callable[..., Any]], Command]: + def decorator(f: Callable[..., Any]) -> Command: + cmd = command(*args, **kwargs)(f) + self.add_command(cmd) + return cmd + + return decorator + + def group( + self, *args: Any, **kwargs: Any + ) -> Callable[[Callable[..., Any]], "Group"]: + def decorator(f: Callable[..., Any]) -> Group: + cmd = group(*args, **kwargs)(f) + self.add_command(cmd) + return cmd + + return decorator + + +def group(name: Optional[str] = None, **kwargs: Any) -> Group: + kwargs.setdefault("cls", Group) + return click.group(name=name, **kwargs) # type: ignore + + +class DeprecatedGroup(MakeContextMixin, click.MultiCommand): def __init__( self, origin: click.MultiCommand, name: Optional[str] = None, **attrs: Any ) -> None: @@ -31,3 +114,77 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Comma def list_commands(self, ctx: click.Context) -> Iterable[str]: return self.origin.list_commands(ctx) + + +class MainGroup(Group): + def _format_group( + self, + title: str, + grp: Sequence[Tuple[str, click.Command]], + formatter: click.HelpFormatter, + ) -> None: + # allow for 3 times the default spacing + if not grp: + return + + width = formatter.width + assert width is not None + limit = width - 6 - max(len(cmd[0]) for cmd in grp) + + rows = [] + for subcommand, cmd in grp: + help = cmd.get_short_help_str(limit) # type: ignore + rows.append((subcommand, help)) + + if rows: + with formatter.section(title): + formatter.write_dl(rows) + + def format_commands( + self, ctx: click.Context, formatter: click.HelpFormatter + ) -> None: + """Extra format methods for multi methods that adds all the commands + after the options. + """ + commands: List[Tuple[str, click.Command]] = [] + groups: List[Tuple[str, click.MultiCommand]] = [] + + for subcommand in self.list_commands(ctx): + cmd = self.get_command(ctx, subcommand) + # What is this, the tool lied about a command. Ignore it + if cmd is None: + continue + if cmd.hidden: # type: ignore + continue + + if isinstance(cmd, click.MultiCommand): + groups.append((subcommand, cmd)) + else: + commands.append((subcommand, cmd)) + + self._format_group("Command Groups", groups, formatter) + self._format_group("Commands", commands, formatter) + + +def alias( + origin: click.Command, + name: str, + *, + deprecated: bool = True, + help: Optional[str] = None, +) -> click.Command: + if help is None: + help = f"Alias for {origin.name}." + return Command( # type: ignore + name=name, + context_settings=origin.context_settings, + callback=origin.callback, + params=origin.params, + help=help, + epilog=origin.epilog, + short_help=origin.short_help, + options_metavar=origin.options_metavar, + add_help_option=origin.add_help_option, + hidden=origin.hidden, # type: ignore + deprecated=deprecated, + ) diff --git a/python/requirements/base.txt b/python/requirements/base.txt index 1533cff28..576b1df6e 100644 --- a/python/requirements/base.txt +++ b/python/requirements/base.txt @@ -7,5 +7,6 @@ python-dateutil==2.7.5 yarl==1.3.0 aiodocker>=0.14.0 click==7.0 +colorama==0.4.1 -e . diff --git a/python/setup.py b/python/setup.py index 0afa3c7a0..02e63fc00 100644 --- a/python/setup.py +++ b/python/setup.py @@ -30,6 +30,7 @@ "yarl>=1.3.0", "aiodocker>=0.14.0", "click>=7.0", + "colorama>=0.4", ], include_package_data=True, description="Neuromation Platform API client", diff --git a/python/tests/cli/test_click_utils.py b/python/tests/cli/test_click_utils.py new file mode 100644 index 000000000..ee9aa32be --- /dev/null +++ b/python/tests/cli/test_click_utils.py @@ -0,0 +1,171 @@ +from textwrap import dedent + +from click.testing import CliRunner + +from neuromation.cli.utils import DeprecatedGroup, MainGroup, command, group + + +def test_print(): + @group() + def sub_command(): + pass + + @command() + def plain_cmd(): + pass + + @group(cls=MainGroup) + def main(): + pass + + main.add_command(sub_command) + main.add_command(plain_cmd) + + runner = CliRunner() + result = runner.invoke(main, []) + assert result.exit_code == 0 + assert result.output == dedent( + """\ + Usage: main [OPTIONS] COMMAND [ARGS]... + + Options: + --help Show this message and exit. + + Command Groups: + sub-command + + Commands: + plain-cmd + """ + ) + + +def test_print_use_group_helpers(): + @group(cls=MainGroup) + def main(): + pass + + @main.group() + def sub_command(): + pass + + @main.command() + def plain_cmd(): + pass + + runner = CliRunner() + result = runner.invoke(main, []) + assert result.exit_code == 0 + assert result.output == dedent( + """\ + Usage: main [OPTIONS] COMMAND [ARGS]... + + Options: + --help Show this message and exit. + + Command Groups: + sub-command + + Commands: + plain-cmd + """ + ) + + +def test_print_hidden(): + @group() + def sub_command(): + pass + + @command(hidden=True) + def plain_cmd(): + pass + + @group(cls=MainGroup) + def main(): + pass + + main.add_command(sub_command) + main.add_command(plain_cmd) + + runner = CliRunner() + result = runner.invoke(main, []) + assert result.exit_code == 0 + assert result.output == dedent( + """\ + Usage: main [OPTIONS] COMMAND [ARGS]... + + Options: + --help Show this message and exit. + + Command Groups: + sub-command + """ + ) + + +def test_print_deprecated_group(): + @group() + def sub_command(): + """ + Sub-command. + """ + + @group(cls=MainGroup) + def main(): + pass + + main.add_command(sub_command) + main.add_command(DeprecatedGroup(sub_command, name="alias")) + + runner = CliRunner() + result = runner.invoke(main, []) + assert result.exit_code == 0 + assert result.output == dedent( + """\ + Usage: main [OPTIONS] COMMAND [ARGS]... + + Options: + --help Show this message and exit. + + Command Groups: + alias Alias for sub-command + sub-command Sub-command. + """ + ) + + +def test_print_deprecated_group_content(): + @group() + def sub_command(): + """ + Sub-command. + """ + + @sub_command.command() + def cmd(): + """Command.""" + + @group(cls=MainGroup) + def main(): + pass + + main.add_command(sub_command) + main.add_command(DeprecatedGroup(sub_command, name="alias")) + + runner = CliRunner() + result = runner.invoke(main, ["alias"]) + assert result.exit_code == 0 + assert result.output == dedent( + """\ + Usage: main alias [OPTIONS] COMMAND [ARGS]... + + Alias for sub-command (DEPRECATED) + + Options: + --help Show this message and exit. + + Commands: + cmd Command. + """ + ) diff --git a/python/tests/e2e/test_e2e_images.py b/python/tests/e2e/test_e2e_images.py index 9407fe2e2..7e04b9de9 100644 --- a/python/tests/e2e/test_e2e_images.py +++ b/python/tests/e2e/test_e2e_images.py @@ -96,6 +96,6 @@ def test_images_complete_lifecycle(run, image, tag, loop, docker): job_id = captured.out.strip() assert job_id.startswith("job-") wait_job_change_state_to(run, job_id, Status.SUCCEEDED, Status.FAILED) - captured = run(["job", "monitor", job_id]) + captured = run(["job", "logs", job_id]) assert not captured.err assert captured.out.strip() == tag diff --git a/python/tests/e2e/test_e2e_jobs.py b/python/tests/e2e/test_e2e_jobs.py index a391a086e..fcaa9dc21 100644 --- a/python/tests/e2e/test_e2e_jobs.py +++ b/python/tests/e2e/test_e2e_jobs.py @@ -23,7 +23,7 @@ @pytest.mark.e2e def test_job_lifecycle(run): # Remember original running jobs - captured = run(["job", "list", "--status", "running", "--status", "pending"]) + captured = run(["job", "ls", "--status", "running", "--status", "pending"]) store_out_list = captured.out.strip().split("\n")[1:] jobs_orig = [x.split("\t")[0] for x in store_out_list] @@ -53,7 +53,7 @@ def test_job_lifecycle(run): assert job_id not in jobs_orig # Check it is in a running,pending job list now - captured = run(["job", "list", "--status", "running", "--status", "pending"]) + captured = run(["job", "ls", "--status", "running", "--status", "pending"]) store_out_list = captured.out.strip().split("\n")[1:] jobs_updated = [x.split("\t")[0] for x in store_out_list] assert job_id in jobs_updated @@ -62,14 +62,14 @@ def test_job_lifecycle(run): wait_job_change_state_to(run, job_id, Status.RUNNING) # Check that it is in a running job list - captured = run(["job", "list", "--status", "running"]) + captured = run(["job", "ls", "--status", "running"]) store_out = captured.out.strip() assert job_id in store_out # Check that the command is in the list assert command in store_out # Check that no command is in the list if quite - captured = run(["job", "list", "--status", "running", "-q"]) + captured = run(["job", "ls", "--status", "running", "-q"]) store_out = captured.out.strip() assert job_id in store_out assert command not in store_out @@ -83,7 +83,7 @@ def test_job_lifecycle(run): wait_job_change_state_from(run, job_id, Status.RUNNING) # Check that it is not in a running job list anymore - captured = run(["job", "list", "--status", "running"]) + captured = run(["job", "ls", "--status", "running"]) store_out = captured.out.strip() assert job_id not in store_out @@ -91,7 +91,7 @@ def test_job_lifecycle(run): @pytest.mark.e2e def test_job_description(run): # Remember original running jobs - captured = run(["job", "list", "--status", "running", "--status", "pending"]) + captured = run(["job", "ls", "--status", "running", "--status", "pending"]) store_out_list = captured.out.strip().split("\n")[1:] jobs_orig = [x.split("\t")[0] for x in store_out_list] description = "Test description for a job" @@ -123,7 +123,7 @@ def test_job_description(run): assert job_id not in jobs_orig # Check it is in a running,pending job list now - captured = run(["job", "list", "--status", "running", "--status", "pending"]) + captured = run(["job", "ls", "--status", "running", "--status", "pending"]) store_out_list = captured.out.strip().split("\n")[1:] jobs_updated = [x.split("\t")[0] for x in store_out_list] assert job_id in jobs_updated @@ -132,7 +132,7 @@ def test_job_description(run): wait_job_change_state_to(run, job_id, Status.RUNNING, Status.FAILED) # Check that it is in a running job list - captured = run(["job", "list", "--status", "running"]) + captured = run(["job", "ls", "--status", "running"]) store_out = captured.out.strip() assert job_id in store_out # Check that description is in the list @@ -140,7 +140,7 @@ def test_job_description(run): assert command in store_out # Check that no description is in the list if quite - captured = run(["job", "list", "--status", "running", "-q"]) + captured = run(["job", "ls", "--status", "running", "-q"]) store_out = captured.out.strip() assert job_id in store_out assert description not in store_out @@ -155,7 +155,7 @@ def test_job_description(run): wait_job_change_state_from(run, job_id, Status.RUNNING) # Check that it is not in a running job list anymore - captured = run(["job", "list", "--status", "running"]) + captured = run(["job", "ls", "--status", "running"]) store_out = captured.out.strip() assert job_id not in store_out @@ -163,7 +163,7 @@ def test_job_description(run): @pytest.mark.e2e def test_unschedulable_job_lifecycle(run): # Remember original running jobs - captured = run(["job", "list", "--status", "running", "--status", "pending"]) + captured = run(["job", "ls", "--status", "running", "--status", "pending"]) store_out_list = captured.out.strip().split("\n")[1:] jobs_orig = [x.split("\t")[0] for x in store_out_list] @@ -193,7 +193,7 @@ def test_unschedulable_job_lifecycle(run): assert job_id not in jobs_orig # Check it is in a running,pending job list now - captured = run(["job", "list", "--status", "running", "--status", "pending"]) + captured = run(["job", "ls", "--status", "running", "--status", "pending"]) store_out_list = captured.out.strip().split("\n")[1:] jobs_updated = [x.split("\t")[0] for x in store_out_list] assert job_id in jobs_updated @@ -210,7 +210,7 @@ def test_unschedulable_job_lifecycle(run): wait_job_change_state_from(run, job_id, Status.RUNNING) # Check that it is not in a running job list anymore - captured = run(["job", "list", "--status", "running"]) + captured = run(["job", "ls", "--status", "running"]) store_out = captured.out.strip() assert job_id not in store_out @@ -218,7 +218,7 @@ def test_unschedulable_job_lifecycle(run): @pytest.mark.e2e def test_two_jobs_at_once(run): # Remember original running jobs - captured = run(["job", "list", "--status", "running", "--status", "pending"]) + captured = run(["job", "ls", "--status", "running", "--status", "pending"]) store_out_list = captured.out.strip().split("\n")[1:] jobs_orig = [x.split("\t")[0] for x in store_out_list] @@ -267,7 +267,7 @@ def test_two_jobs_at_once(run): assert second_job_id not in jobs_orig # Check it is in a running,pending job list now - captured = run(["job", "list", "--status", "running", "--status", "pending"]) + captured = run(["job", "ls", "--status", "running", "--status", "pending"]) store_out_list = captured.out.strip().split("\n")[1:] jobs_updated = [x.split("\t")[0] for x in store_out_list] assert first_job_id in jobs_updated @@ -278,7 +278,7 @@ def test_two_jobs_at_once(run): wait_job_change_state_to(run, second_job_id, Status.RUNNING, Status.FAILED) # Check that it is in a running job list - captured = run(["job", "list", "--status", "running"]) + captured = run(["job", "ls", "--status", "running"]) store_out = captured.out.strip() assert first_job_id in store_out assert second_job_id in store_out @@ -286,7 +286,7 @@ def test_two_jobs_at_once(run): assert command in store_out # Check that no command is in the list if quite - captured = run(["job", "list", "--status", "running", "-q"]) + captured = run(["job", "ls", "--status", "running", "-q"]) store_out = captured.out.strip() assert first_job_id in store_out assert second_job_id in store_out @@ -302,7 +302,7 @@ def test_two_jobs_at_once(run): wait_job_change_state_from(run, second_job_id, Status.RUNNING) # Check that it is not in a running job list anymore - captured = run(["job", "list", "--status", "running"]) + captured = run(["job", "ls", "--status", "running"]) store_out = captured.out.strip() assert first_job_id not in store_out assert first_job_id not in store_out