diff --git a/content/docs/command-reference/apply-remote.md b/content/docs/command-reference/apply-remote.md index 7288e853..f9eba28f 100644 --- a/content/docs/command-reference/apply-remote.md +++ b/content/docs/command-reference/apply-remote.md @@ -7,11 +7,15 @@ a MLEM object to `output` if provided. Otherwise, it will be printed to ## Synopsis ```usage -usage: mlem apply-remote [options] [subtype] data - -arguments: -[SUBTYPE] Type of client. Choices: ['http', 'rmq'] -DATA Path to dataset object [required] +usage: mlem apply-remote [-d ] [-p ] [--rev ] + [-o ] [--tp ] [-m ] + [--index] [--json] [-f ] [-h] + [ [client options] | --load ] + +Builtin clients: +- http +- rmq +- sagemaker ``` ## Description @@ -27,18 +31,19 @@ clients are `http` and `rmq` - which are used to launch requests against the ## Options -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `--rev TEXT`: Repo revision to use [default: (none)] -- `-o, --output TEXT`: Where to store the outputs. -- `--target-project, --tp TEXT`: Project to save target to [default: (none)] -- `-m, --method TEXT`: Which model method is to be applied [default: predict] -- `--index / --no-index`: Whether to index output in .mlem directory -- `--json`: Output as json -- `-l, --load TEXT`: File to load client config from -- `-c, --conf TEXT`: Options for client in format `field.name=value` -- `-f, --file_conf TEXT`: File with options for client in format +- `-d `, `--data ` - Path to MLEM dataset [required] +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `--rev ` - Repo revision to use [default: (none)] +- `-o `, `--output ` - Where to save model outputs +- `--tp `, `--target-project ` - Project to save target to [default: + (none)] +- `-m `, `--method ` - Which model method is to be applied [default: + predict] +- `--index` / `--no-index` - Whether to index output in .mlem directory +- `--json` - Output as json +- `-f `, `--file_conf ` - File with options for client in format `field.name=path_to_config` -- `-h, --help`: Show this message and exit. +- `-h`, `--help` - Show this message and exit. ## Example: Apply a locally hosted model to a local dataset diff --git a/content/docs/command-reference/apply.md b/content/docs/command-reference/apply.md index f697b59a..f53e71e7 100644 --- a/content/docs/command-reference/apply.md +++ b/content/docs/command-reference/apply.md @@ -6,11 +6,15 @@ provided. Otherwise, it will be printed to `stdout`. ## Synopsis ```usage -usage: mlem apply [options] model data +usage: mlem apply [-p ] [--rev ] [-o ] + [-m ] [--dr ] + [--data-rev ] [-i] [--it ] + [-b ] [--index] [-e] [--json] [-h] + model data arguments: -MODEL Path to model object [required] -DATA Path to dataset object [required] + model Path to model object + data Path to data object ``` ## Description @@ -29,20 +33,22 @@ datasets. ## Options -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `--rev TEXT`: Repo revision to use [default: (none)] -- `-o, --output TEXT`: Where to store the outputs. -- `-m, --method TEXT`: Which model method is to be applied [default: predict] -- `--data-project, --dr TEXT`: Project with data -- `--data-rev TEXT`: Revision of data -- `-i, --import`: Try to import data on-the-fly -- `--import-type, --it TEXT`: Specify how to read data file for import. - Available types: ['pandas', 'pickle'] -- `-b, --batch_size INTEGER`: Batch size for reading data in batches. -- `--index / --no-index`: Whether to index output in .mlem directory -- `-e, --external`: Save result not in .mlem, but directly in project -- `--json`: Output as json -- `-h, --help`: Show this message and exit. +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `--rev ` - Repo revision to use [default: (none)] +- `-o `, `--output ` - Where to save model outputs +- `-m `, `--method ` - Which model method is to be applied [default: + predict] +- `--dr `, `--data-project ` - Project with data +- `--data-rev ` - Revision of data +- `-i`, `--import` - Try to import data on-the-fly +- `--it `, `--import-type ` - Specify how to read data file for + import. Available types: ['pandas', 'pickle', 'torch'] +- `-b `, `--batch_size ` - Batch size for reading data in + batches +- `--index` / `--no-index` - Whether to index output in .mlem directory +- `-e`, `--external` - Save result not in .mlem, but directly in project +- `--json` - Output as json +- `-h`, `--help` - Show this message and exit. ## Examples diff --git a/content/docs/command-reference/build.md b/content/docs/command-reference/build.md index b5fd8aa1..1da9ce31 100644 --- a/content/docs/command-reference/build.md +++ b/content/docs/command-reference/build.md @@ -1,16 +1,21 @@ # build -Build models to create re-usable, ship-able entities such as a Docker image or -Python package. +Build models into re-usable assets you can distribute and use in production, +such as a Docker image or Python package. ## Synopsis ```usage -usage: mlem build [options] model [subtype] - -arguments: -MODEL Path to model [required] -[SUBTYPE] Type of build. Choices: ['whl', 'pip', 'docker_dir', 'docker'] +usage: mlem build [-m ] [-p ] [--rev ] + [-f ] [-h] + [ [builder options] | --load ] + +Builtin builders: +- docker +- docker_dir +- kubernetes +- pip +- whl ``` ## Description @@ -21,13 +26,12 @@ images. ## Options -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `--rev TEXT`: Repo revision to use [default: (none)] -- `-l, --load TEXT`: File to load builder config from -- `-c, --conf TEXT`: Options for builder in format `field.name=value` -- `-f, --file_conf TEXT`: File with options for builder in format +- `-m `, `--model ` - Path to MLEM model [required] +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `--rev ` - Repo revision to use [default: (none)] +- `-f `, `--file_conf ` - File with options for builder in format `field.name=path_to_config` -- `-h, --help`: Show this message and exit. +- `-h`, `--help` - Show this message and exit. ## Examples diff --git a/content/docs/command-reference/checkenv.md b/content/docs/command-reference/checkenv.md new file mode 100644 index 00000000..0045dc6a --- /dev/null +++ b/content/docs/command-reference/checkenv.md @@ -0,0 +1,21 @@ +# checkenv + +Check that current environment satisfies object requrements + +## Synopsis + +```usage +usage: mlem checkenv [-p ] [--rev ] [-h] + path + +arguments: + path Path to object +``` + +## Options + +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `--rev ` - Repo revision to use [default: (none)] +- `-h`, `--help` - Show this message and exit. + +## Examples diff --git a/content/docs/command-reference/clone.md b/content/docs/command-reference/clone.md index 070b8a09..be05f727 100644 --- a/content/docs/command-reference/clone.md +++ b/content/docs/command-reference/clone.md @@ -6,11 +6,13 @@ saves a copy of it to `target` path. ## Synopsis ```usage -usage: mlem clone [options] uri target +usage: mlem clone [-p ] [--rev ] + [--tp ] [-e] [--index] [-h] + uri target arguments: -URI URI to object you want to clone [required] -TARGET Path to store the downloaded object. [required] + uri URI to object you want to clone + target Path to store the downloaded object. ``` ## Description @@ -22,12 +24,13 @@ repository. ## Options -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `--rev TEXT`: Repo revision to use [default: (none)] -- `--target-project, --tp TEXT`: Project to save target to [default: (none)] -- `-e, --external`: Save result not in .mlem, but directly in project -- `--link / --no-link`: Whether to create link for output in .mlem directory -- `--help`: Show this message and exit. +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `--rev ` - Repo revision to use [default: (none)] +- `--tp `, `--target-project ` - Project to save target to [default: + (none)] +- `-e`, `--external` - Save result not in .mlem, but directly in project +- `--index` / `--no-index` - Whether to index output in .mlem directory +- `-h`, `--help` - Show this message and exit. ## Examples diff --git a/content/docs/command-reference/config.md b/content/docs/command-reference/config.md new file mode 100644 index 00000000..736fda9b --- /dev/null +++ b/content/docs/command-reference/config.md @@ -0,0 +1,15 @@ +# config + +Manipulate MLEM configuration. + +## Synopsys + +```usage +usage: mlem config [-h] +``` + +## Options + +- `-h`, `--help` - Show this message and exit. + +## Examples diff --git a/content/docs/command-reference/declare.md b/content/docs/command-reference/declare.md index fef69f57..3fd727f3 100644 --- a/content/docs/command-reference/declare.md +++ b/content/docs/command-reference/declare.md @@ -6,12 +6,16 @@ metafile from config args and config files. ## Synopsis ```usage -usage: mlem declare [options] object_type [subtype] path - -arguments: -OBJECT_TYPE Type of metafile to declare [required] -[SUBTYPE] Subtype of MLEM object [default: ] -PATH Where to save object [required] +usage: mlem declare [-h] + [ [subtype options] | --load ] + +Builtin subtypes: +- builder +- client +- deployment +- docker_registry +- env +- server ``` ## Description @@ -28,12 +32,7 @@ check out the last example [here](/doc/command-reference/types#examples) ## Options -- `-c, --conf TEXT`: Values for object fields in format - `field.nested.name=value` -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `-e, --external`: Save result not in .mlem, but directly in project -- `--index / --no-index`: Whether to index output in .mlem directory -- `-h, --help`: Show this message and exit. +- `-h`, `--help` - Show this message and exit. ## Examples @@ -45,7 +44,7 @@ $ mlem types env heroku [not required] api_key: str = None # Declare the heroku env -$ mlem declare env heroku production --conf api_key="mlem_heroku_staging" +$ mlem declare env heroku production --api_key mlem_heroku_staging 💾 Saving env to .mlem/env/staging.mlem # Print the contents of the new heroku env metafile diff --git a/content/docs/command-reference/deployment/apply.md b/content/docs/command-reference/deployment/apply.md index 0c6a897a..8221e9f7 100644 --- a/content/docs/command-reference/deployment/apply.md +++ b/content/docs/command-reference/deployment/apply.md @@ -5,11 +5,15 @@ Apply a deployed model to data. ## Synopsis ```usage -usage: mlem deployment apply [options] path data +usage: mlem deployment apply [-p ] [--rev ] + [--dr ] [--data-rev ] + [-o ] [--tp ] [-m ] + [--index] [--json] [-h] + path data arguments: -PATH Path to deployment meta [required] -DATA Path to data object [required] + path Path to deployment meta + data Path to data object ``` ## Description @@ -21,16 +25,18 @@ gathered and returned, also as a MLEM Object. ## Options -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `--rev TEXT`: Repo revision to use [default: (none)] -- `--data-project, --dr TEXT`: Project with data -- `--data-rev TEXT`: Revision of data -- `-o, --output TEXT`: Where to store the outputs. -- `--target-project, --tp TEXT`: Project to save target to [default: (none)] -- `-m, --method TEXT`: Which model method is to be applied [default: predict] -- `--index / --no-index`: Whether to index output in .mlem directory -- `--json`: Output as json -- `-h, --help`: Show this message and exit. +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `--rev ` - Repo revision to use [default: (none)] +- `--dr `, `--data-project ` - Project with data +- `--data-rev ` - Revision of data +- `-o `, `--output ` - Where to store the outputs. +- `--tp `, `--target-project ` - Project to save target to [default: + (none)] +- `-m `, `--method ` - Which model method is to be applied [default: + predict] +- `--index` / `--no-index` - Whether to index output in .mlem directory +- `--json` - Output as json +- `-h`, `--help` - Show this message and exit. ## Example: Apply a dataset on a deployed model diff --git a/content/docs/command-reference/deployment/index.md b/content/docs/command-reference/deployment/index.md index 08cc5ae1..e9296e3c 100644 --- a/content/docs/command-reference/deployment/index.md +++ b/content/docs/command-reference/deployment/index.md @@ -1,18 +1,19 @@ # deployment -A set of commands to set up and manage deployments. +A set of commands to set up and manage deployments ## Synopsis ```usage -usage: mlem deployment [options] COMMAND [ARGS]... - -arguments: -COMMAND - apply Apply method of deployed service - run Deploy a model to target environment - status Print status of deployed service - remove Stop and destroy deployed instance +usage: mlem deployment [-h] + command + +subcommands: + run Deploy a model to a target environment. + remove Stop and destroy deployed instance. + status Print status of deployed service. + wait Wait for status of deployed service + apply Apply a deployed model to data. ``` ## Description diff --git a/content/docs/command-reference/deployment/remove.md b/content/docs/command-reference/deployment/remove.md index c2ec20f6..8dcb65e8 100644 --- a/content/docs/command-reference/deployment/remove.md +++ b/content/docs/command-reference/deployment/remove.md @@ -5,10 +5,11 @@ Stop and destroy deployed instance. ## Synopsis ```usage -usage: mlem deployment remove [options] path +usage: mlem deployment remove [-p ] [-h] + path arguments: -PATH Path to deployment meta [required] + path Path to deployment meta ``` ## Description @@ -19,8 +20,8 @@ deleting its associated runtime resources. ## Options -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `-h, --help`: Show this message and exit. +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `-h`, `--help` - Show this message and exit. ## Example: Stop and destroy a deployment diff --git a/content/docs/command-reference/deployment/run.md b/content/docs/command-reference/deployment/run.md index c3ffc808..9dbb6f28 100644 --- a/content/docs/command-reference/deployment/run.md +++ b/content/docs/command-reference/deployment/run.md @@ -6,10 +6,13 @@ declaration or create a new one on-the-fly. ## Synopsis ```usage -usage: mlem deployment run [options] path +usage: mlem deployment run [-m ] [-t ] [-p ] [-e] + [--index] [-c ] [-h] + path arguments: -PATH Path to deployment meta (will be created if it does not exist) [required] + path Path to deployment meta (will be created if it does not + exist) ``` ## Description @@ -21,13 +24,13 @@ options (see below). ## Options -- `-m, --model TEXT`: Path to model -- `-t, --env TEXT`: Path to target environment -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `-e, --external`: Save result not in .mlem, but directly in project -- `--index / --no-index`: Whether to index output in .mlem directory -- `-c, --conf TEXT`: Configuration for new deployment meta if it does not exist -- `-h, --help`: Show this message and exit. +- `-m `, `--model ` - Path to model +- `-t `, `--env ` - Path to target environment +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `-e`, `--external` - Save result not in .mlem, but directly in project +- `--index` / `--no-index` - Whether to index output in .mlem directory +- `-c `, `--conf ` - Options in format `field.name=value` +- `-h`, `--help` - Show this message and exit. ## Example: Create a new deployment from scratch diff --git a/content/docs/command-reference/deployment/status.md b/content/docs/command-reference/deployment/status.md index cff6f553..b68bb594 100644 --- a/content/docs/command-reference/deployment/status.md +++ b/content/docs/command-reference/deployment/status.md @@ -5,10 +5,11 @@ Print status of deployed service. ## Synopsis ```usage -usage: mlem deployment status [options] path +usage: mlem deployment status [-p ] [-h] + path arguments: -PATH Path to deployment meta [required] + path Path to deployment meta ``` ## Description @@ -29,8 +30,8 @@ The possible statuses for deployments using the `heroku` target platform is: ## Options -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `-h, --help`: Show this message and exit. +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `-h`, `--help` - Show this message and exit. ## Example: Get the status of a deployment diff --git a/content/docs/command-reference/deployment/wait.md b/content/docs/command-reference/deployment/wait.md new file mode 100644 index 00000000..1f55cd1d --- /dev/null +++ b/content/docs/command-reference/deployment/wait.md @@ -0,0 +1,41 @@ +# deployment wait + +Wait for status of deployed service + +## Synopsis + +```usage +usage: mlem deployment wait [-p ] + [-s <[unknown|not_deployed|starting|crashed|stopped|running]>] + [-i <[unknown|not_deployed|starting|crashed|stopped|running]>] + [-p ] [-t ] [-h] + path + +arguments: + path Path to deployment meta +``` + +## Description + +TODO + +## Options + +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `-s <[unknown|not_deployed|starting|crashed|stopped|running]>`, + `--status <[unknown|not_deployed|starting|crashed|stopped|running]>` - + statuses to wait for [default: DeployStatus.RUNNING] +- `-i <[unknown|not_deployed|starting|crashed|stopped|running]>`, + `--intermediate <[unknown|not_deployed|starting|crashed|stopped|running]>` - + Possible intermediate statuses +- `-p `, `--poll-timeout ` - Timeout between attempts [default: + 1.0] +- `-t `, `--times ` - Number of attempts. 0 -> indefinite + [default: 0] +- `-h`, `--help` - Show this message and exit. + +## Example: Wait for deployment setup (after mlem deployment run command) + +```cli +$ mlem deployment wait service_name -i starting +``` diff --git a/content/docs/command-reference/import.md b/content/docs/command-reference/import.md index 98503f4e..d57e658b 100644 --- a/content/docs/command-reference/import.md +++ b/content/docs/command-reference/import.md @@ -5,11 +5,14 @@ Create a `.mlem` metafile for a model or data in any file or directory. ## Synopsis ```usage -usage: mlem import [options] uri target +usage: mlem import [-p ] [--rev ] + [--tp ] [--copy] [--type ] + [--index] [-e] [-h] + uri target arguments: -URI File to import [required] -TARGET Path to save MLEM object [required] + uri File to import + target Path to save MLEM object ``` ## Description @@ -24,16 +27,17 @@ load those models/datasets into object for subsequent usage in MLEM context. ## Options -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `--rev TEXT`: Repo revision to use [default: (none)] -- `--target-project, --tp TEXT`: Project to save target to [default: (none)] -- `--copy / --no-copy`: Whether to create a copy of file in target location or - just link existing file [default: copy] -- `--type TEXT`: Specify how to read file Available types: ['pandas', 'pickle'] - [default: (auto infer)] -- `--index / --no-index`: Whether to index output in .mlem directory -- `-e, --external`: Save result not in .mlem, but directly in project -- `-h, --help`: Show this message and exit. +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `--rev ` - Repo revision to use [default: (none)] +- `--tp `, `--target-project ` - Project to save target to [default: + (none)] +- `--copy` / `--no-copy` - Whether to create a copy of file in target location + or just link existing file [default: copy] +- `--type ` - Specify how to read file Available types: ['pandas', + 'pickle', 'torch'] [default: (auto infer)] +- `--index` / `--no-index` - Whether to index output in .mlem directory +- `-e`, `--external` - Save result not in .mlem, but directly in project +- `-h`, `--help` - Show this message and exit. ## Examples diff --git a/content/docs/command-reference/init.md b/content/docs/command-reference/init.md index dfaa6cec..36f6e485 100644 --- a/content/docs/command-reference/init.md +++ b/content/docs/command-reference/init.md @@ -1,14 +1,15 @@ # init -Initialize a MLEM project. +Initialize a [MLEM project](/doc/user-guide/project-structure). ## Synopsis ```usage -usage: mlem init [options] [path] +usage: mlem init [-h] + path arguments: -[path] Location (file path or URL) to initialize a MLEM project + path Where to init project ``` ## Description @@ -32,7 +33,7 @@ and manage them using standard Git workflows. ## Options -- `-h, --help`: Show this message and exit. +- `-h`, `--help` - Show this message and exit. ## Examples diff --git a/content/docs/command-reference/link.md b/content/docs/command-reference/link.md index 205c9ebd..2dfa5801 100644 --- a/content/docs/command-reference/link.md +++ b/content/docs/command-reference/link.md @@ -1,17 +1,19 @@ # link -Create a link (read alias) for an existing -[MLEM Object](/doc/user-guide/basic-concepts#mlem-objects), including from -remote MLEM projects. +Create a link (read alias) for an existing [MLEM Object](/doc/user-guide/basic- +concepts#mlem-objects), including from remote [MLEM project](/doc/user- +guide/project-structure)s. ## Synopsis ```usage -usage: mlem link [options] source target +usage: mlem link [--sp ] [--rev ] + [--tp ] [-e] [--f] [--abs] [-h] + source target arguments: -SOURCE URI of the object you are creating a link to [required] -TARGET Path to save link object [required] + source URI of the MLEM object you are creating a link to + target Path to save link object ``` ## Description @@ -26,15 +28,17 @@ to incorporate them in the local workspace. ## Options -- `--source-project, --sp TEXT`: Project for source object -- `--rev TEXT`: Repo revision to use [default: (none)] -- `--target-project, --tp TEXT`: Project to save target to [default: (none)] -- `-e, --external`: Save result not in .mlem, but directly in project -- `--follow-links, --f / --no-follow-links, --nf`: If True, first follow links - while reading {source} before creating this link. [default: follow-links] -- `--absolute, --abs / --relative, --rel`: Which path to linked object to +- `--sp `, `--source-project ` - Project for source object +- `--rev ` - Repo revision to use [default: (none)] +- `--tp `, `--target-project ` - Project to save target to [default: + (none)] +- `-e`, `--external` - Save result not in .mlem, but directly in project +- `--follow-links`, `--f` / `--no-follow-links`, `--nf` - If True, first follow + links while reading {source} before creating this link. [default: follow- + links] +- `--absolute`, `--abs` / `--relative`, `--rel` - Which path to linked object to specify: absolute or relative. [default: relative] -- `-h, --help`: Show this message and exit. +- `-h`, `--help` - Show this message and exit. ## Examples diff --git a/content/docs/command-reference/list.md b/content/docs/command-reference/list.md index 64396b99..49d6eccf 100644 --- a/content/docs/command-reference/list.md +++ b/content/docs/command-reference/list.md @@ -1,16 +1,17 @@ # list -List [MLEM objects](/doc/user-guide/basic-concepts#mlem-objects) inside a MLEM -project (location should be [initialized](/doc/command-reference/init)). - -> Aliased to `mlem ls` +List [MLEM objects](/doc/user-guide/basic-concepts#mlem-objects) inside a +[MLEM project](/doc/user-guide/project-structure). ## Synopsis ```usage -usage: mlem list [options] [project] +usage: mlem list [-t <[all|link|model|data|builder|env|deployment]>] + [--rev ] [+l] [--json] [-i] [-h] + project -arguments: [PROJECT] Project to list from [default: (current directory)] +arguments: + project Project to list from [default: (current directory)] ``` ## Description @@ -30,12 +31,14 @@ remote repositories. ## Options -- `-t, --type [all|link|model|dataset|env|deployment|packager]`: Type of objects - to list [default: all] -- `--rev TEXT`: Repo revision to use [default: (none)] -- `+l, --links / -l, --no-links`: Whether to include links [default: +l] -- `--json`: Output as json -- `--help`: Show this message and exit. +- `-t <[all|link|model|data|builder|env|deployment]>`, + `--type <[all|link|model|data|builder|env|deployment]>` - Type of objects to + list [default: all] +- `--rev ` - Repo revision to use [default: (none)] +- `--links`, `+l` / `-l`, `--no-links` - Whether to include links [default: l] +- `--json` - Output as json +- `-i`, `--ignore-errors` - Ignore corrupted objects +- `-h`, `--help` - Show this message and exit. ## Examples diff --git a/content/docs/command-reference/pprint.md b/content/docs/command-reference/pprint.md index ba2a775f..3116f809 100644 --- a/content/docs/command-reference/pprint.md +++ b/content/docs/command-reference/pprint.md @@ -1,15 +1,17 @@ # pprint -Display all details about a specific -[MLEM object](/doc/user-guide/basic-concepts#mlem-objects) from an existing MLEM -project. +Display all details about a specific [MLEM Object](/doc/user-guide/basic- +concepts#mlem-objects) from an existing MLEM project. ## Synopsis ```usage -usage: mlem pprint [options] path +usage: mlem pprint [-p ] [--rev ] [-f] + [--json] [-h] + path -arguments: PATH Path to object [required] +arguments: + path Path to object ``` ## Description @@ -29,11 +31,11 @@ You can use [`mlem list`](/doc/command-reference/list) to list MLEM objects. ## Options -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `--rev TEXT`: Repo revision to use [default: (none)] -- `-f, --follow-links`: If specified, follow the link to the actual object. -- `--json`: Output as json -- `--help`: Show this message and exit. +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `--rev ` - Repo revision to use [default: (none)] +- `-f`, `--follow-links` - If specified, follow the link to the actual object. +- `--json` - Output as json +- `-h`, `--help` - Show this message and exit. ## Example: Showing local model diff --git a/content/docs/command-reference/serve.md b/content/docs/command-reference/serve.md index 3d30a2ca..65fcce1d 100644 --- a/content/docs/command-reference/serve.md +++ b/content/docs/command-reference/serve.md @@ -1,16 +1,17 @@ # serve -Locally deploy the model using a server implementation and expose its methods as -endpoints. +Create an API from model methods using a server implementation. ## Synopsis ```usage -usage: mlem serve [options] model [subtype] +usage: mlem serve [-m ] [-p ] [--rev ] + [-f ] [-h] + [ [server options] | --load ] -arguments: -MODEL Model to create service from [required] -[SUBTYPE] Server type. Choices: ['fastapi', 'heroku', 'rmq'] [default: ] +Builtin servers: +- fastapi +- rmq ``` ## Description @@ -30,13 +31,12 @@ built-in client, or common HTTP clients, such as [`curl`](https://curl.se/) and ## Options -- `-p, --project TEXT`: Path to MLEM project [default: (none)] -- `--rev TEXT`: Repo revision to use [default: (none)] -- `-l, --load TEXT`: File to load server config from -- `-c, --conf TEXT`: Options for server in format `field.name=value` -- `-f, --file_conf TEXT`: File with options for server in format +- `-m `, `--model ` - Path to MLEM model [required] +- `-p `, `--project ` - Path to MLEM project [default: (none)] +- `--rev ` - Repo revision to use [default: (none)] +- `-f `, `--file_conf ` - File with options for server in format `field.name=path_to_config` -- `--help`: Show this message and exit. +- `-h`, `--help` - Show this message and exit. ## Example: FastAPI HTTP server diff --git a/content/docs/command-reference/types.md b/content/docs/command-reference/types.md index bca7192d..f588b985 100644 --- a/content/docs/command-reference/types.md +++ b/content/docs/command-reference/types.md @@ -6,11 +6,13 @@ subtype is not provided, list all available MLEM types. ## Synopsis ```usage -usage: mlem types [options] [abc] [sub_type] +usage: mlem types [-h] + [abc] [sub_type] arguments: -[ABC] Subtype to list implementations. List subtypes if not provided -[SUB_TYPE] Type of `meta` subtype + [abc] Subtype to list implementations. List subtypes if not + provided. + [sub_type] Type of `meta` subtype ``` ## Description @@ -28,7 +30,7 @@ capabilities. ## Options -- `-h, --help`: Show this message and exit. +- `-h`, `--help` - Show this message and exit. ## Examples diff --git a/scripts/docs/bootstrap_cli.py b/scripts/docs/bootstrap_cli.py new file mode 100644 index 00000000..87741fa3 --- /dev/null +++ b/scripts/docs/bootstrap_cli.py @@ -0,0 +1,162 @@ +import json +import os +import re +import subprocess +import textwrap +from typing import Dict, List + +from pydantic import BaseModel, parse_obj_as + +from scripts.docs.cli_generate_spec import Opt, Spec + +CLI_DOCS_PATH = "../../content/docs/command-reference" +DOC_AUTO_REPLACE = { + "MLEM Object": "[MLEM Object](/doc/user-guide/basic-concepts#mlem-objects)", + "MLEM objects": "[MLEM objects](/doc/user-guide/basic-concepts#mlem-objects)", + "MLEM project": "[MLEM project](/doc/user-guide/project-structure)" +} +LINE_WIDTH = 80 + + +def replace_section(data: str, section_name: str, new_value: str, + section_prefix: str = "## ") -> str: + return re.sub(f"{section_prefix}{section_name}(.*?)^{section_prefix}", + f"{section_prefix}{section_name}{new_value}{section_prefix}", + data, flags=re.MULTILINE | re.DOTALL) + + +def repr_option(option: Opt): + decls = ", ".join(f"`{d} <{option.metavar.lower()}>`" for d in option.decls) + if option.is_flag: + decls = ", ".join(f"`{d}`" for d in option.decls) + if option.secondary: + decls += " / " + ", ".join(f"`{d}`" for d in option.secondary) + return textwrap.fill(f"- {decls} - {option.help}", width=LINE_WIDTH, + subsequent_indent=" ") + + +def repr_arg(option: Opt): + margin = 17 + metavar = option.metavar.lower() + option_help = option.help + if option_help.endswith(" [required]"): + option_help = option_help[:-len(" [required]")] + return textwrap.fill(f" {metavar:{margin}}{option_help}", width=LINE_WIDTH, + subsequent_indent=" " * (margin + 2)) + + +def generate_options(options: List[Opt]): + res = ["", ""] + for option in options: + res.append(repr_option(option)) + return "\n".join(res + ["", ""]) + + +def _gen_usage_string(spec: Spec): + usage = f"usage: mlem {spec.name} " + indent = len(usage) + options = [] + for opt in spec.options: + decl = min(opt.decls, key=len) + metavar = opt.metavar.lower() + if metavar == "boolean": + options.append(f"[{decl}]") + else: + options.append(f"[{decl} <{metavar}>]") + max_opts_len = min(45, LINE_WIDTH - indent) + option_lines = [""] + for o in options: + line = f"{option_lines[-1]}{o} " + if len(line) > max_opts_len and option_lines[-1] != "": + option_lines[-1] = option_lines[-1].strip() + option_lines.append(o + " ") + else: + option_lines[-1] = line + option_lines[-1] = option_lines[-1].strip() + options = ("\n" + " " * indent).join(option_lines) + impl = "" + if spec.args.impl_metavar: + impl = f"[<{spec.args.impl_metavar}> [{spec.args.impl_metavar} options] | --load ]" + args = impl + " ".join(a.metavar for a in spec.args.args).lower() + if spec.args.subcommands: + args += "command" + res = f"{usage}{options}" + if args: + res += "\n" + " " * indent + f"{args}" + return res + + +def generate_usage(spec: Spec): + usage = _gen_usage_string(spec) + argspec = spec.args + if argspec.args: + args = '\n'.join(repr_arg(a) for a in argspec.args) + args = f"\n\narguments:\n{args}" + else: + args = "" + if argspec.impls: + impls = "\n".join(f"- {c}" for c in argspec.impls) + impls = f"\n\nBuiltin {argspec.impl_metavar}s:\n{impls}" + else: + impls = "" + if argspec.subcommands: + margin = 17 + subcommands = "\n".join( + f" {k:{margin}}{v}" for k, v in argspec.subcommands.items()) + subcommands = f"\n\nsubcommands:\n{subcommands}" + else: + subcommands = "" + usage = usage[0].lower() + usage[1:] + return f"\n{usage}{subcommands}{impls}{args}\n" + + +def generate_doc(doc): + for k, v in DOC_AUTO_REPLACE.items(): + doc = doc.replace(k, v) + return f"\n\n{textwrap.fill(doc, width=LINE_WIDTH)}\n\n" + + +def generate_cli_command(name: str, spec: Spec): + path = os.path.join(CLI_DOCS_PATH, f"{name}.md") + with open(path, "r", encoding="utf8") as f: + data = f.read() + + data = replace_section(data, "usage", generate_usage(spec), + section_prefix="```") + data = replace_section(data, "Options", generate_options(spec.options)) + + cmd_name = name.replace("/", " ") + if cmd_name.endswith(" index"): + cmd_name = cmd_name[:-len(" index")] + data = replace_section(data, " " + cmd_name, generate_doc(spec.doc), + section_prefix="#") + with open(path, "w", encoding="utf8") as f: + f.write(data) + + +class AllSpec(BaseModel): + __root__: Dict[str, Spec] + + +def main(): + with open("spec.json", "r", encoding="utf8") as f: + spec = parse_obj_as(AllSpec, json.load(f)) + + # spec.__root__ = {"apply": spec.__root__["apply"]} + for k, s in spec.__root__.items(): + generate_cli_command(k, s) + + os.unlink("spec.json") + + +def run_lint(): + print("Running liniter") + subprocess.check_output("yarn run format", shell=True, cwd="../../") + + +if __name__ == '__main__': + from scripts.docs.cli_generate_spec import main as spec_main + + spec_main() + main() + run_lint() diff --git a/scripts/docs/cli_generate_spec.py b/scripts/docs/cli_generate_spec.py new file mode 100644 index 00000000..b08b7837 --- /dev/null +++ b/scripts/docs/cli_generate_spec.py @@ -0,0 +1,134 @@ +import json +from typing import Dict, List, Optional + +from click import Command, Context, Group, Option +from pydantic import BaseModel +from typer.main import get_group + +from mlem import cli +from mlem.cli.main import get_cmd_name + +use_group = ["deployment"] +skip = ["dev"] + +abc_group = ["apply-remote", "build", "declare", "serve"] + + +class Opt(BaseModel): + decls: List[str] + secondary: List[str] + metavar: str + help: str + is_flag: bool + + +class Args(BaseModel): + args: List[Opt] + impls: Optional[List[str]] + impl_metavar: Optional[str] + subcommands: Optional[Dict[str, str]] + + +class Spec(BaseModel): + args: Args + options: List[Opt] + doc: str + name: str + + +def get_options(command: Command, ctx): + if command.name not in abc_group: + yield from command.get_params(ctx) + return + + options = None + for subcommand in command.commands.values(): + if options is None: + options = list(get_options(subcommand, ctx)) + continue + new_options = {o.help for o in + get_options(subcommand, ctx)} + options = [o for o in options if o.help in new_options] + yield from options + + +def repr_option(option, ctx) -> Opt: + _, help_ = option.get_help_record(ctx) + help_ = help_.replace(" ", " ") # TODO: maybe fix in typer code? + return Opt(decls=sorted(option.opts, reverse=True), + secondary=sorted(option.secondary_opts, reverse=True), + metavar=option.make_metavar(), + help=help_, + is_flag=option.is_flag if isinstance(option, Option) else False) + + +def generate_options(command: Command, ctx): + res = [] + for option in get_options(command, ctx): + if not isinstance(option, Option): + continue + res.append(repr_option(option, ctx)) + return res + + +def generate_args(command, ctx): + args = [] + for arg in command.get_params(ctx): + if isinstance(arg, Option): + continue + args.append(repr_option(arg, ctx)) + impls = None + metavar = None + subcommands = None + if command.name in abc_group: + impls = list( + sorted([c for c in command.commands if not c.startswith("_")])) + metavar = command.subcommand_metavar + args.extend(generate_args(list(command.commands.values())[0], ctx).args) + if command.name in use_group: + subcommands = {c.name: c.get_short_help_str() for c in + command.commands.values()} + return Args(args=args, impls=impls, impl_metavar=metavar, + subcommands=subcommands) + + +def generate_usage(command: Command, ctx): + if command.name not in abc_group: + return command.get_usage(ctx) + subcommand = list(command.commands.values())[0] + subctx = Context(subcommand, parent=ctx, info_name=subcommand.name) + sub_usage = generate_usage(subcommand, subctx) + return sub_usage.replace(subcommand.name, command.subcommand_metavar) + + +def generate_cli_command(command: Command, ctx): + return Spec(args=generate_args(command, ctx), + options=generate_options(command, ctx), + doc=command.help.strip(), + name=get_cmd_name(ctx)) + + +def main(): + group = get_group(cli.app) + ctx = Context(group, info_name="mlem", help_option_names=["-h", "--help"]) + spec = {} + for name, command in group.commands.items(): + if name in skip: + continue + subctx = Context(command, ctx, info_name=name) + if isinstance(command, Group) and name in use_group: + + spec[f"{name}/index"] = generate_cli_command(command, subctx) + for subname, subcommand in command.commands.items(): + subsubctx = Context(subcommand, subctx, info_name=subname) + spec[f"{name}/{subname}"] = generate_cli_command(subcommand, + subsubctx) + continue + spec[name] = generate_cli_command(command, subctx) + + with open("spec.json", "w", encoding="utf8") as f: + json.dump({k: v.dict() for k, v in spec.items()}, f, indent=2) + + +if __name__ == '__main__': + main()