From 98f7797d58d86986cb5f41c71cd0ce64334b88c0 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Thu, 31 Mar 2022 19:15:11 +0200 Subject: [PATCH 01/17] DVCLive: Deprecate `live` section of `dvc.yaml`. In favor of reusing `metrics` and `plots`. Closes https://github.com/iterative/dvclive/issues/229 --- content/docs/command-reference/exp/init.md | 8 +- content/docs/command-reference/run.md | 14 +--- content/docs/command-reference/stage/add.md | 14 +--- .../docs/dvclive/api-reference/live/index.md | 21 +++-- .../dvclive/api-reference/live/make_report.md | 31 ++++++++ .../dvclive/api-reference/live/next_step.md | 16 ++-- .../dvclive/api-reference/live/set_step.md | 16 ++-- content/docs/dvclive/dvclive-with-dvc.md | 78 +++++++++---------- content/docs/dvclive/get-started.md | 16 ++++ content/docs/sidebar.json | 4 + .../experiment-management/checkpoints.md | 15 ++-- .../project-structure/pipelines-files.md | 1 - 12 files changed, 129 insertions(+), 105 deletions(-) create mode 100644 content/docs/dvclive/api-reference/live/make_report.md diff --git a/content/docs/command-reference/exp/init.md b/content/docs/command-reference/exp/init.md index f28bfeb665..bbd90ee5c8 100644 --- a/content/docs/command-reference/exp/init.md +++ b/content/docs/command-reference/exp/init.md @@ -11,7 +11,7 @@ Quickly setup any project to use [DVC Experiments]. usage: dvc exp init [-h] [-q | -v] [--run] [--interactive] [-f] [--explicit] [--name NAME] [--code CODE] [--data DATA] [--models MODELS] [--params PARAMS] - [--metrics METRICS] [--plots PLOTS] [--live LIVE] + [--metrics METRICS] [--plots PLOTS] [--type {default,dl}] [command] ``` @@ -128,12 +128,6 @@ $ dvc exp init './another_script.sh $MYENVVAR' your experiment can be found (if any). Overrides other configuration and default value (`plots/`). -- `--live` - configure the `path` directory for [DVCLive](/doc/dvclive). This is - where experiment logs will be written. Overrides other configuration and - default value (`dvclive/`). - - > This only has an effect when used with `--type=dl`. - - `--explicit` - do not assume default locations of project dependencies and outputs. You'll have to provide specific locations via other options or `dvc config exp`. In `--interactive` this removes default values from prompts. diff --git a/content/docs/command-reference/run.md b/content/docs/command-reference/run.md index 5f534eaee5..c5850bb3db 100644 --- a/content/docs/command-reference/run.md +++ b/content/docs/command-reference/run.md @@ -13,9 +13,7 @@ usage: dvc run [-h] [-q | -v] [-n ] [-f] [--outs-persist-no-cache ] [-m ] [-M ] [--plots ] [--plots-no-cache ] - [--live ] [--live-no-cache ] - [--live-no-html] [-w ] - [--always-changed] [--desc ] + [-w ] [--always-changed] [--desc ] [--no-exec] [--no-commit] [--no-run-cache] command @@ -269,16 +267,6 @@ $ dvc run -n second_stage './another_script.sh $MYENVVAR' - `--desc ` - user description of the stage (optional). This doesn't affect any DVC operations. -- `--live ` - specify the directory `path` for - [DVCLive](/doc/dvclive/dvclive-with-dvc) to write logs in. `path` will be - tracked (cached) by DVC. Saved in the `live` field of `dvc.yaml`. - -- `--live-no-cache ` - the same as `--live` except that the `path` is not - tracked by DVC. Useful if you prefer to track it with Git. - -- `--live-no-html` - deactivates DVCLive - [HTML report](/doc/dvclive/dvclive-with-dvc#html-report) generation. - - `-h`, `--help` - prints the usage/help message, and exit. - `-q`, `--quiet` - do not write anything to standard output. Exit with 0 if no diff --git a/content/docs/command-reference/stage/add.md b/content/docs/command-reference/stage/add.md index 218ead53d2..8fad58d475 100644 --- a/content/docs/command-reference/stage/add.md +++ b/content/docs/command-reference/stage/add.md @@ -12,9 +12,7 @@ usage: dvc stage add [-h] [-q | -v] -n [-f] [--outs-persist-no-cache ] [-m ] [-M ] [--plots ] [--plots-no-cache ] - [--live ] [--live-no-cache ] - [--live-no-html] [-w ] - [--always-changed] [--desc ] + [-w ] [--always-changed] [--desc ] command positional arguments: @@ -246,16 +244,6 @@ data science experiments. - `--desc ` - user description of the stage (optional). This doesn't affect any DVC operations. -- `--live ` - specify the directory `path` for - [DVCLive](/doc/dvclive/dvclive-with-dvc) to write logs in. `path` will be - tracked (cached) by DVC. Saved in the `live` field of `dvc.yaml`. - -- `--live-no-cache ` - the same as `--live` except that the `path` is not - tracked by DVC. Useful if you prefer to track it with Git. - -- `--live-no-html` - deactivates DVCLive - [HTML report](/doc/dvclive/dvclive-with-dvc#html-report) generation. - - `-h`, `--help` - prints the usage/help message, and exit. - `-q`, `--quiet` - do not write anything to standard output. Exit with 0 if no diff --git a/content/docs/dvclive/api-reference/live/index.md b/content/docs/dvclive/api-reference/live/index.md index 2a42c9a48c..97635cb818 100644 --- a/content/docs/dvclive/api-reference/live/index.md +++ b/content/docs/dvclive/api-reference/live/index.md @@ -9,6 +9,8 @@ class Live: self, path: Optional[str] = None, resume: bool = False, + report: Optional[str] = "html", + auto_open: bool = True, ): ``` @@ -36,10 +38,12 @@ other metadata. - `dir` - Location of the directory to store [outputs](/doc/dvclive/get-started#outputs). -- `summary_path` - Location of the - [summary](/doc/dvclive/api-reference/live/#parameters). -- `html_path` - Location of the - [html report](/doc/dvclive/dvclive-with-dvc#html-report). + +- `summary_path` - `{dir}.json`. Location of the + [summary](/doc/dvclive/api-reference/log##description). + +- `html_path` - `{dir}/report.html`. Location of the + [html report](/doc/dvclive/api-reference/make_report##description). ## Parameters @@ -59,17 +63,18 @@ other metadata. -## Exceptions +- `report` - If `html`, DVCLive will call `Live.make_report()` on each step + update . _Default_: `html`. -- `dvclive.error.ConfigMismatchError` - thrown if the provided `path` does not - match with the one set in DVC (see - [DVCLive with DVC](/docs/dvclive/dvclive-with-dvc)) +- `auto_open` - If `True`, on the first `Live.make_report()` call, DVCLive will + automatically open `html_path` in a browser. _Default_: `True`. ## Methods - `Live.log()` - `Live.log_image()` - `Live.log_plot()` +- `Live.make_report()` - `Live.get_step()` - `Live.next_step()` - `Live.set_step()` diff --git a/content/docs/dvclive/api-reference/live/make_report.md b/content/docs/dvclive/api-reference/live/make_report.md new file mode 100644 index 0000000000..944840b763 --- /dev/null +++ b/content/docs/dvclive/api-reference/live/make_report.md @@ -0,0 +1,31 @@ +# Live.make_report() + +Generates an HTML Report from the logged data. + +```py +def make_report() +``` + +#### Usage: + +```py +from dvclive import Live + +live = Live() +live.log_plot("confusion_matrix", [0, 0, 1, 1], [1, 0, 0, 1]) +live.make_report() +``` + +## Description + +On each call, DVCLive will collect all the data logged in `{dir}`, generate an +HTML report and save it in `{dir}/report.html`. + +![](/img/dvclive-html.gif) + + + +This function gets automatically called on each `step` update, if +`report="html"` was passed to `Live()` + + diff --git a/content/docs/dvclive/api-reference/live/next_step.md b/content/docs/dvclive/api-reference/live/next_step.md index 7b4a30cbfe..48eceb9fb3 100644 --- a/content/docs/dvclive/api-reference/live/next_step.md +++ b/content/docs/dvclive/api-reference/live/next_step.md @@ -28,17 +28,19 @@ You can use `Live.next_step()` to increase the `step` by 1 (one). Each metric logged in between `Live.next_step()` (or `Live.set_step()`) calls will be associated to the updated `step` value. + + +If `report="html"` was passed to `Live()`, each `Live.next_step()` will also +call `Live.make_report()`. + + + ### DVC integration When `dvclive` is used alongside `DVC`, each `Live.next_step()` call will have additional effects. -By default, on each `Live.next_step()` call, `DVC` will prepare an -[HTML report](/doc/dvclive/dvclive-with-dvc#html-report) with the -[metrics history](/doc/dvclive/get-started#history). - -In addition, when -[checkpoints](/doc/user-guide/experiment-management/checkpoints) are enabled in -the pipeline, `DVC` will +When [checkpoints](/doc/user-guide/experiment-management/checkpoints) are +enabled in the pipeline, `DVC` will [create a new checkpoint](/doc/dvclive/dvclive-with-dvc#checkpoints) on each `Live.next_step()` call. diff --git a/content/docs/dvclive/api-reference/live/set_step.md b/content/docs/dvclive/api-reference/live/set_step.md index be212d2cef..3e8ba884ac 100644 --- a/content/docs/dvclive/api-reference/live/set_step.md +++ b/content/docs/dvclive/api-reference/live/set_step.md @@ -29,18 +29,20 @@ You can use `Live.set_step()` to set `step` to any value. Each metric logged in between `Live.set_step()` (or `Live.next_step()`) calls will be associated to the provided `step` value. + + +If `report="html"` was passed to `Live()`, each `Live.next_step()` will also +call `Live.make_report()`. + + + ### DVC integration When `dvclive` is used alongside `DVC`, each `Live.set_step()` call will have additional effects. -By default, on each `Live.set_step()` call, `DVC` will prepare an -[HTML report](/doc/dvclive/dvclive-with-dvc#html-report) with the -[metrics history](/doc/dvclive/get-started#lhistory). - -In addition, when -[checkpoints](/doc/user-guide/experiment-management/checkpoints) are enabled in -the pipeline, `DVC` will +When [checkpoints](/doc/user-guide/experiment-management/checkpoints) are +enabled in the pipeline, `DVC` will [create a new checkpoint](/doc/dvclive/dvclive-with-dvc#checkpoints) on each `Live.set_step()` call. diff --git a/content/docs/dvclive/dvclive-with-dvc.md b/content/docs/dvclive/dvclive-with-dvc.md index 6837e16042..c7881c15eb 100644 --- a/content/docs/dvclive/dvclive-with-dvc.md +++ b/content/docs/dvclive/dvclive-with-dvc.md @@ -7,9 +7,6 @@ ways: `dvc metrics` and `dvc plots`. Those same outputs can be visualized in [Iterative Studio](#iterative-studio). -- You can monitor model performance in realtime with the - [HTML report](#html-report) that DVCLive generates when used alongside DVC. - - DVCLive is also capable of generating [checkpoint](#checkpoints) signal files used by DVC experiments. @@ -22,7 +19,7 @@ We will refer to a training script (`train.py`) already using `dvclive`: from dvclive import Live -live = Live() +live = Live("training_metrics") for epoch in range(NUM_EPOCHS): train_model(...) @@ -38,12 +35,23 @@ Let's use `dvc stage add` to create a stage to wrap this code (don't forget to `dvc init` first): ```dvc -$ dvc stage add -n train --live training_metrics - -d train.py python train.py +$ dvc stage add \ +--name train \ +--deps train.py \ +--metrics-no-cache training_metrics.json \ +--plots training_metrics/scalars \ +python train.py ``` -`dvc.yaml` will contain a new `train` stage with the DVCLive configuration (in -the `live` field): + + +Note that the paths indicated in the `metrics` and `plots` options match the +`path` passed to `Live()` in the Python code. + + + +`dvc.yaml` will contain a new `train` stage using the DVCLive outputs as +`dvc metrics` and `dvc plots`: ```yaml stages: @@ -51,23 +59,13 @@ stages: cmd: python train.py deps: - train.py - live: - training_metrics: - summary: true - html: true + metrics: + - training_metrics.json: + cache: false + plots: + - training_metrics/scalars ``` -The value passed to `--live` (`training_metrics`) became the directory `path` -for DVCLive to write logs in, and DVC will now -[track](/doc/use-cases/versioning-data-and-model-files) it. Other supported -command options for the DVC integration: - -- `--live-no-cache ` - specify a DVCLive log directory `path` but don't - track it with DVC. Useful if you prefer to track it with Git. -- `--live-no-html` - deactivates [HTML report](#html-report) generation. - -> Note that DVC will not track summary files or the HTML report. - Run the training with `dvc repro` or `dvc exp run`: ```dvc @@ -83,6 +81,7 @@ $ tree ├── dvc.lock ├── dvc.yaml ├── training_metrics +│ ├── report.html │ └── scalars │ ├── acc.tsv │ └── loss.tsv @@ -94,9 +93,21 @@ The [metrics summary](/doc/dvclive/api-reference/live/log#description) `training_metrics.json` can be used by `dvc metrics` and visualized with `dvc exp show`/`dvc exp diff`. -In addition, the -[metrics history](/doc/dvclive/api-reference/live/log#step-updates) generated -under `training_metrics/scalars` can be visualized with `dvc plots`. +The [metrics history](/doc/dvclive/api-reference/live/log#step-updates) +`training_metrics/scalars` can be visualized with `dvc plots`. + +The [HTML report](/doc/dvclive/api-reference/live/make_report##description) +`training_metrics/report.html` will contain all the logged data and will be +automatically updated during training on each `step` update! + +![](/img/dvclive-html.gif) + + + +If you don't update the step number, the HTML report won't be generated unless +you call `Live.make_report()` directly. + + ### Iterative Studio @@ -107,21 +118,6 @@ experiments using DVCLive in Iterative Studio. ![](/img/dvclive-studio-plots.png) -### HTML report - -When `html: true`, DVC generates an _HTML report_. - -If you open `training_metrics_dvc_plots/index.html` in a browser, you'll see a -plot for the logged data automatically updated during the model training! - -![](/img/dvclive-html.gif) - - - -If you don't update the step number, the HTML report won't be generated. - - - ### Checkpoints When used alongside DVC, DVCLive can create _checkpoint_ signal files used by diff --git a/content/docs/dvclive/get-started.md b/content/docs/dvclive/get-started.md index 5952cc59c1..37722f639b 100644 --- a/content/docs/dvclive/get-started.md +++ b/content/docs/dvclive/get-started.md @@ -109,6 +109,22 @@ not. See `Live.log()`, `Live.log_image()` and `Live.log_plot()` for more details. +### HTML report + +By default, DVCLive generates an HTML report in `dvclive/report.html`. + +This report will contain all the logged data and will be automatically updated +during training on each `step` update! + +![](/img/dvclive-html.gif) + + + +If you don't update the step number, the HTML report won't be generated unless +you call `Live.make_report()` directly. + + + ## What next? Learn how to use DVCLive alongside other tools: diff --git a/content/docs/sidebar.json b/content/docs/sidebar.json index 3b873c0ed1..13bc84d703 100644 --- a/content/docs/sidebar.json +++ b/content/docs/sidebar.json @@ -670,6 +670,10 @@ "slug": "log_plot", "label": "log_plot()" }, + { + "slug": "make_report", + "label": "make_report()" + }, { "slug": "get_step", "label": "get_step()" diff --git a/content/docs/user-guide/experiment-management/checkpoints.md b/content/docs/user-guide/experiment-management/checkpoints.md index 82b975b762..65ccdcf314 100644 --- a/content/docs/user-guide/experiment-management/checkpoints.md +++ b/content/docs/user-guide/experiment-management/checkpoints.md @@ -97,13 +97,11 @@ $ dvc stage add --name train \ --params seed,lr,weight_decay \ --checkpoints model.pt \ --plots-no-cache predictions.json \ - --live dvclive \ + --plots-no-cache dvclive/scalars \ + --metrics-no-cache dvclive.json python train.py ``` -💡 The `--live dvclive` option enables our special logger -[DVCLive](/doc/dvclive), which helps you register checkpoints from code. - The checkpoints need to be enabled in DVC at the pipeline level. The `-c / --checkpoint` option of the `dvc stage add` command defines the checkpoint file or directory. The checkpoint file, _model.pt_, is an output from one @@ -132,13 +130,14 @@ stages: outs: - model.pt: checkpoint: true + metrics: + - dvclive.json: + cache: false plots: + - dvclive/scalars: + cache: false - predictions.json: cache: false - live: - dvclive: - summary: true - html: true ``` ⚠️ Note that enabling checkpoints in a `dvc.yaml` file makes it incompatible diff --git a/content/docs/user-guide/project-structure/pipelines-files.md b/content/docs/user-guide/project-structure/pipelines-files.md index 674b6c906f..54fd433ed2 100644 --- a/content/docs/user-guide/project-structure/pipelines-files.md +++ b/content/docs/user-guide/project-structure/pipelines-files.md @@ -362,7 +362,6 @@ These are the fields that are accepted in each stage: | `always_changed` | Causes this stage to be always considered as [changed] by commands such as `dvc status` and `dvc repro`. `false` by default | | `meta` | (Optional) arbitrary metadata can be added manually with this field. Any YAML content is supported. `meta` contents are ignored by DVC, but they can be meaningful for user processes that read or write `.dvc` files directly. | | `desc` | (Optional) user description for this stage. This doesn't affect any DVC operations. | -| `live` | (Optional) [Dvclive](/doc/dvclive/dvclive-with-dvc) configuration field | [changed]: /doc/command-reference/status#local-workspace-status From 3fdfce9d9d55937e20685b6ee44627954e820b9b Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Mon, 18 Apr 2022 16:59:50 +0200 Subject: [PATCH 02/17] Fix links --- content/docs/dvclive/api-reference/live/index.md | 4 ++-- content/docs/dvclive/dvclive-with-dvc.md | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/content/docs/dvclive/api-reference/live/index.md b/content/docs/dvclive/api-reference/live/index.md index 97635cb818..64907a2510 100644 --- a/content/docs/dvclive/api-reference/live/index.md +++ b/content/docs/dvclive/api-reference/live/index.md @@ -40,10 +40,10 @@ other metadata. [outputs](/doc/dvclive/get-started#outputs). - `summary_path` - `{dir}.json`. Location of the - [summary](/doc/dvclive/api-reference/log##description). + [summary](/doc/dvclive/api-reference/live/log#description). - `html_path` - `{dir}/report.html`. Location of the - [html report](/doc/dvclive/api-reference/make_report##description). + [html report](/doc/dvclive/api-reference/live/make_report#description). ## Parameters diff --git a/content/docs/dvclive/dvclive-with-dvc.md b/content/docs/dvclive/dvclive-with-dvc.md index c7881c15eb..8844f98df5 100644 --- a/content/docs/dvclive/dvclive-with-dvc.md +++ b/content/docs/dvclive/dvclive-with-dvc.md @@ -14,6 +14,13 @@ ways: We will refer to a training script (`train.py`) already using `dvclive`: + + +If you use one of the supported [ML Frameworks](/doc/dvclive/ml-frameworks), you +can jump to its corresponding page to find an example usage. + + + ```python # train.py @@ -46,7 +53,7 @@ python train.py Note that the paths indicated in the `metrics` and `plots` options match the -`path` passed to `Live()` in the Python code. +`path` passed to `Live()` in the Python code (`"training_metrics"`). From caf1e18c6e49d0c7b08b707fa93678a5c78c0f41 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 19 Apr 2022 12:04:02 +0200 Subject: [PATCH 03/17] Apply suggestions from code review Co-authored-by: Dave Berenbaum --- content/docs/dvclive/api-reference/live/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/docs/dvclive/api-reference/live/index.md b/content/docs/dvclive/api-reference/live/index.md index 64907a2510..dbb0458ab5 100644 --- a/content/docs/dvclive/api-reference/live/index.md +++ b/content/docs/dvclive/api-reference/live/index.md @@ -64,10 +64,10 @@ other metadata. - `report` - If `html`, DVCLive will call `Live.make_report()` on each step - update . _Default_: `html`. + update. _Default_: `html`. - `auto_open` - If `True`, on the first `Live.make_report()` call, DVCLive will - automatically open `html_path` in a browser. _Default_: `True`. + automatically open `html_path` in a browser. _Default_: `False`. ## Methods From 4e7a3b1ed9f7269198f49ca1edb8187827457d0c Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 19 Apr 2022 12:07:50 +0200 Subject: [PATCH 04/17] auto_open False --- content/docs/dvclive/api-reference/live/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/dvclive/api-reference/live/index.md b/content/docs/dvclive/api-reference/live/index.md index dbb0458ab5..1263d5708c 100644 --- a/content/docs/dvclive/api-reference/live/index.md +++ b/content/docs/dvclive/api-reference/live/index.md @@ -10,7 +10,7 @@ class Live: path: Optional[str] = None, resume: bool = False, report: Optional[str] = "html", - auto_open: bool = True, + auto_open: bool = False, ): ``` From 125dfa633c256d5bf1f3a1d081d7cfa726c37e28 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 19 Apr 2022 12:09:41 +0200 Subject: [PATCH 05/17] Use plots-no-cache --- content/docs/dvclive/dvclive-with-dvc.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/content/docs/dvclive/dvclive-with-dvc.md b/content/docs/dvclive/dvclive-with-dvc.md index 8844f98df5..3dbd7d0b86 100644 --- a/content/docs/dvclive/dvclive-with-dvc.md +++ b/content/docs/dvclive/dvclive-with-dvc.md @@ -46,7 +46,7 @@ $ dvc stage add \ --name train \ --deps train.py \ --metrics-no-cache training_metrics.json \ ---plots training_metrics/scalars \ +--plots-no-cache training_metrics/scalars \ python train.py ``` @@ -70,7 +70,8 @@ stages: - training_metrics.json: cache: false plots: - - training_metrics/scalars + - training_metrics/scalars: + cache: false ``` Run the training with `dvc repro` or `dvc exp run`: From f82ab07269e123e1e27ee4a8d2e66877a3641180 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Wed, 20 Apr 2022 10:06:02 +0200 Subject: [PATCH 06/17] Update exp init --- content/docs/command-reference/exp/init.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/content/docs/command-reference/exp/init.md b/content/docs/command-reference/exp/init.md index bbd90ee5c8..1c17c11865 100644 --- a/content/docs/command-reference/exp/init.md +++ b/content/docs/command-reference/exp/init.md @@ -11,7 +11,7 @@ Quickly setup any project to use [DVC Experiments]. usage: dvc exp init [-h] [-q | -v] [--run] [--interactive] [-f] [--explicit] [--name NAME] [--code CODE] [--data DATA] [--models MODELS] [--params PARAMS] - [--metrics METRICS] [--plots PLOTS] + [--metrics METRICS] [--plots PLOTS] [--live LIVE] [--type {default,dl}] [command] ``` @@ -128,6 +128,10 @@ $ dvc exp init './another_script.sh $MYENVVAR' your experiment can be found (if any). Overrides other configuration and default value (`plots/`). +- `--live` - use the given path to set up `metrics` and `plots` as in + [DVCLive with DVC](/doc/dvclive/dvclive-with-dvc). Overrides other + configuration and default value (`dvclive/`). + - `--explicit` - do not assume default locations of project dependencies and outputs. You'll have to provide specific locations via other options or `dvc config exp`. In `--interactive` this removes default values from prompts. From beb35b7d36dc479304fb688d242a37a8b8e24524 Mon Sep 17 00:00:00 2001 From: Jorge Orpinel Date: Sat, 30 Apr 2022 13:00:42 -0500 Subject: [PATCH 07/17] Update content/docs/dvclive/get-started.md --- content/docs/dvclive/get-started.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/content/docs/dvclive/get-started.md b/content/docs/dvclive/get-started.md index 37722f639b..00f3ec38ed 100644 --- a/content/docs/dvclive/get-started.md +++ b/content/docs/dvclive/get-started.md @@ -111,10 +111,8 @@ See `Live.log()`, `Live.log_image()` and `Live.log_plot()` for more details. ### HTML report -By default, DVCLive generates an HTML report in `dvclive/report.html`. - -This report will contain all the logged data and will be automatically updated -during training on each `step` update! +If and when `step` is updated, DVCLive generates or updates an HTML report in +`dvclive/report.html` which will contain all the logged data. ![](/img/dvclive-html.gif) From 77c910c8865f2f5b26b92a97cc29832302a4799b Mon Sep 17 00:00:00 2001 From: Jorge Orpinel Date: Sat, 30 Apr 2022 13:15:51 -0500 Subject: [PATCH 08/17] Apply suggestions from code review --- content/docs/dvclive/api-reference/live/next_step.md | 2 +- content/docs/dvclive/api-reference/live/set_step.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/content/docs/dvclive/api-reference/live/next_step.md b/content/docs/dvclive/api-reference/live/next_step.md index 48eceb9fb3..a8c45f52ad 100644 --- a/content/docs/dvclive/api-reference/live/next_step.md +++ b/content/docs/dvclive/api-reference/live/next_step.md @@ -41,6 +41,6 @@ When `dvclive` is used alongside `DVC`, each `Live.next_step()` call will have additional effects. When [checkpoints](/doc/user-guide/experiment-management/checkpoints) are -enabled in the pipeline, `DVC` will +enabled in the pipeline, DVC will [create a new checkpoint](/doc/dvclive/dvclive-with-dvc#checkpoints) on each `Live.next_step()` call. diff --git a/content/docs/dvclive/api-reference/live/set_step.md b/content/docs/dvclive/api-reference/live/set_step.md index 3e8ba884ac..100cdf6a58 100644 --- a/content/docs/dvclive/api-reference/live/set_step.md +++ b/content/docs/dvclive/api-reference/live/set_step.md @@ -42,7 +42,7 @@ When `dvclive` is used alongside `DVC`, each `Live.set_step()` call will have additional effects. When [checkpoints](/doc/user-guide/experiment-management/checkpoints) are -enabled in the pipeline, `DVC` will +enabled in the pipeline, DVC will [create a new checkpoint](/doc/dvclive/dvclive-with-dvc#checkpoints) on each `Live.set_step()` call. From b51b948b2c2543755e76b433d2d3c5fdb2bba50d Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Mon, 2 May 2022 11:47:06 +0200 Subject: [PATCH 09/17] Apply suggestions from code review Co-authored-by: Jorge Orpinel --- content/docs/command-reference/exp/init.md | 6 +++--- content/docs/dvclive/api-reference/live/next_step.md | 4 ++-- content/docs/dvclive/api-reference/live/set_step.md | 4 ++-- content/docs/dvclive/dvclive-with-dvc.md | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/content/docs/command-reference/exp/init.md b/content/docs/command-reference/exp/init.md index 1c17c11865..26c19d2442 100644 --- a/content/docs/command-reference/exp/init.md +++ b/content/docs/command-reference/exp/init.md @@ -128,9 +128,9 @@ $ dvc exp init './another_script.sh $MYENVVAR' your experiment can be found (if any). Overrides other configuration and default value (`plots/`). -- `--live` - use the given path to set up `metrics` and `plots` as in - [DVCLive with DVC](/doc/dvclive/dvclive-with-dvc). Overrides other - configuration and default value (`dvclive/`). +- `--live` - set the path to the directory where the metrics and plots + [produced by DVCLive](https://dvc.org/doc/dvclive/dvclive-with-dvc#outputs) + will be found. Overrides other configuration and default value (`dvclive/`). - `--explicit` - do not assume default locations of project dependencies and outputs. You'll have to provide specific locations via other options or diff --git a/content/docs/dvclive/api-reference/live/next_step.md b/content/docs/dvclive/api-reference/live/next_step.md index a8c45f52ad..2d05acafdf 100644 --- a/content/docs/dvclive/api-reference/live/next_step.md +++ b/content/docs/dvclive/api-reference/live/next_step.md @@ -30,8 +30,8 @@ will be associated to the updated `step` value. -If `report="html"` was passed to `Live()`, each `Live.next_step()` will also -call `Live.make_report()`. +Each `Live.next_step()` will call `Live.make_report()` internally by default +(unless `report` is passed to `Live()` with a value other than `"html"`). diff --git a/content/docs/dvclive/api-reference/live/set_step.md b/content/docs/dvclive/api-reference/live/set_step.md index 100cdf6a58..a5e3432fbb 100644 --- a/content/docs/dvclive/api-reference/live/set_step.md +++ b/content/docs/dvclive/api-reference/live/set_step.md @@ -31,8 +31,8 @@ will be associated to the provided `step` value. -If `report="html"` was passed to `Live()`, each `Live.next_step()` will also -call `Live.make_report()`. +Each `Live.set_step()` will call `Live.make_report()` internally by default +(unless `report` is passed to `Live()` with a value other than `"html"`). diff --git a/content/docs/dvclive/dvclive-with-dvc.md b/content/docs/dvclive/dvclive-with-dvc.md index 3dbd7d0b86..da3a3b2938 100644 --- a/content/docs/dvclive/dvclive-with-dvc.md +++ b/content/docs/dvclive/dvclive-with-dvc.md @@ -98,14 +98,14 @@ $ tree ``` The [metrics summary](/doc/dvclive/api-reference/live/log#description) -`training_metrics.json` can be used by `dvc metrics` and visualized with +in `training_metrics.json` can be used by `dvc metrics` and visualized with `dvc exp show`/`dvc exp diff`. The [metrics history](/doc/dvclive/api-reference/live/log#step-updates) `training_metrics/scalars` can be visualized with `dvc plots`. -The [HTML report](/doc/dvclive/api-reference/live/make_report##description) -`training_metrics/report.html` will contain all the logged data and will be +The [HTML report](/doc/dvclive/api-reference/live/make_report#description) +in `training_metrics/report.html` will contain all the logged data and will be automatically updated during training on each `step` update! ![](/img/dvclive-html.gif) From 771f899de64b3f986c945e99317450ecc8ad2b21 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Mon, 2 May 2022 11:47:41 +0200 Subject: [PATCH 10/17] Update dvclive-with-dvc.md --- content/docs/dvclive/dvclive-with-dvc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/dvclive/dvclive-with-dvc.md b/content/docs/dvclive/dvclive-with-dvc.md index da3a3b2938..199f761145 100644 --- a/content/docs/dvclive/dvclive-with-dvc.md +++ b/content/docs/dvclive/dvclive-with-dvc.md @@ -1,6 +1,6 @@ # DVCLive with DVC -Even though DVCLive does not require DVC, they can integrate in several useful +Even though DVCLive does not require DVC, they can integrate in a couple useful ways: - The [outputs](#outputs) DVCLive produces are recognized by `dvc exp`, From 2050ce893643e5cc87a3b119e99de0fa9143c99f Mon Sep 17 00:00:00 2001 From: "restyled-io[bot]" <32688539+restyled-io[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 11:49:51 +0200 Subject: [PATCH 11/17] Restyled by prettier (#3499) Co-authored-by: Restyled.io --- content/docs/dvclive/dvclive-with-dvc.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/docs/dvclive/dvclive-with-dvc.md b/content/docs/dvclive/dvclive-with-dvc.md index 199f761145..d1924a16fc 100644 --- a/content/docs/dvclive/dvclive-with-dvc.md +++ b/content/docs/dvclive/dvclive-with-dvc.md @@ -97,15 +97,15 @@ $ tree └── train.py ``` -The [metrics summary](/doc/dvclive/api-reference/live/log#description) -in `training_metrics.json` can be used by `dvc metrics` and visualized with +The [metrics summary](/doc/dvclive/api-reference/live/log#description) in +`training_metrics.json` can be used by `dvc metrics` and visualized with `dvc exp show`/`dvc exp diff`. The [metrics history](/doc/dvclive/api-reference/live/log#step-updates) `training_metrics/scalars` can be visualized with `dvc plots`. -The [HTML report](/doc/dvclive/api-reference/live/make_report#description) -in `training_metrics/report.html` will contain all the logged data and will be +The [HTML report](/doc/dvclive/api-reference/live/make_report#description) in +`training_metrics/report.html` will contain all the logged data and will be automatically updated during training on each `step` update! ![](/img/dvclive-html.gif) From e16598072dc6aa395b913022feaecb0cc0839205 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 10 May 2022 11:29:06 +0200 Subject: [PATCH 12/17] Remove outdated auto_bool --- content/docs/dvclive/api-reference/live/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/content/docs/dvclive/api-reference/live/index.md b/content/docs/dvclive/api-reference/live/index.md index 1263d5708c..8091103f47 100644 --- a/content/docs/dvclive/api-reference/live/index.md +++ b/content/docs/dvclive/api-reference/live/index.md @@ -10,7 +10,6 @@ class Live: path: Optional[str] = None, resume: bool = False, report: Optional[str] = "html", - auto_open: bool = False, ): ``` From f85e05d28e5e2eefe43c47b6998651fc3adfec31 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 10 May 2022 11:29:18 +0200 Subject: [PATCH 13/17] Replace exclamation with warn --- content/docs/dvclive/dvclive-with-dvc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/dvclive/dvclive-with-dvc.md b/content/docs/dvclive/dvclive-with-dvc.md index d1924a16fc..9759a1bb9a 100644 --- a/content/docs/dvclive/dvclive-with-dvc.md +++ b/content/docs/dvclive/dvclive-with-dvc.md @@ -50,7 +50,7 @@ $ dvc stage add \ python train.py ``` - + Note that the paths indicated in the `metrics` and `plots` options match the `path` passed to `Live()` in the Python code (`"training_metrics"`). From 0345cbf4549967e0b512d956065dda66c69f426b Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 10 May 2022 11:39:19 +0200 Subject: [PATCH 14/17] Cleanup --- content/docs/command-reference/exp/init.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/command-reference/exp/init.md b/content/docs/command-reference/exp/init.md index 26c19d2442..6e78a2a353 100644 --- a/content/docs/command-reference/exp/init.md +++ b/content/docs/command-reference/exp/init.md @@ -130,7 +130,7 @@ $ dvc exp init './another_script.sh $MYENVVAR' - `--live` - set the path to the directory where the metrics and plots [produced by DVCLive](https://dvc.org/doc/dvclive/dvclive-with-dvc#outputs) - will be found. Overrides other configuration and default value (`dvclive/`). + will be found. - `--explicit` - do not assume default locations of project dependencies and outputs. You'll have to provide specific locations via other options or From 1a99e3703cb3f0df22e81fc75899323759a31b6c Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Tue, 10 May 2022 11:40:55 +0200 Subject: [PATCH 15/17] Update description --- content/docs/dvclive/api-reference/live/make_report.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/docs/dvclive/api-reference/live/make_report.md b/content/docs/dvclive/api-reference/live/make_report.md index 944840b763..183bf4e7e8 100644 --- a/content/docs/dvclive/api-reference/live/make_report.md +++ b/content/docs/dvclive/api-reference/live/make_report.md @@ -25,7 +25,7 @@ HTML report and save it in `{dir}/report.html`. -This function gets automatically called on each `step` update, if -`report="html"` was passed to `Live()` +This function gets automatically called on each `step` update, unless +`report=None` was passed to `Live()` From f06daac6881b8e5ee8c5c8375fa8d2413fec96d8 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Wed, 11 May 2022 12:38:15 +0200 Subject: [PATCH 16/17] Update content/docs/dvclive/api-reference/live/make_report.md Co-authored-by: Jorge Orpinel --- content/docs/dvclive/api-reference/live/make_report.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/docs/dvclive/api-reference/live/make_report.md b/content/docs/dvclive/api-reference/live/make_report.md index 183bf4e7e8..eecbe2288a 100644 --- a/content/docs/dvclive/api-reference/live/make_report.md +++ b/content/docs/dvclive/api-reference/live/make_report.md @@ -25,7 +25,7 @@ HTML report and save it in `{dir}/report.html`. -This function gets automatically called on each `step` update, unless -`report=None` was passed to `Live()` +This function gets called internally on each `step` update by default +(unless `report` is passed to `Live()` with a value other than `"html"`). From 074a128a0f7747351a0c2b697ab1c5545ea7a4ce Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Wed, 11 May 2022 12:57:28 +0200 Subject: [PATCH 17/17] fix --- content/docs/dvclive/api-reference/live/make_report.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/docs/dvclive/api-reference/live/make_report.md b/content/docs/dvclive/api-reference/live/make_report.md index eecbe2288a..2f66156e7a 100644 --- a/content/docs/dvclive/api-reference/live/make_report.md +++ b/content/docs/dvclive/api-reference/live/make_report.md @@ -25,7 +25,7 @@ HTML report and save it in `{dir}/report.html`. -This function gets called internally on each `step` update by default -(unless `report` is passed to `Live()` with a value other than `"html"`). +This function gets called internally on each `step` update by default (unless +`report` is passed to `Live()` with a value other than `"html"`).