From b2a7b9e3b0dc67388e6ef751c55d9e2d99e450cb Mon Sep 17 00:00:00 2001 From: Carlo Siebenschuh Date: Wed, 1 Feb 2023 18:47:03 -0600 Subject: [PATCH 1/8] Update exact_match.py Typo in line 235. Changed `multilalbe` to `multilalbe`. --- src/torchmetrics/functional/classification/exact_match.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/torchmetrics/functional/classification/exact_match.py b/src/torchmetrics/functional/classification/exact_match.py index b4ca3ce7dd1..330eea49e26 100644 --- a/src/torchmetrics/functional/classification/exact_match.py +++ b/src/torchmetrics/functional/classification/exact_match.py @@ -232,7 +232,7 @@ def exact_match( if task == "multiclass": assert num_classes is not None return multiclass_exact_match(preds, target, num_classes, multidim_average, ignore_index, validate_args) - if task == "multilalbe": + if task == "multilabel": assert num_labels is not None return multilabel_exact_match( preds, target, num_labels, threshold, multidim_average, ignore_index, validate_args From 97997d910b8039df03ee075ac05bfb097156dfbc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 2 Feb 2023 00:50:52 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index c45228afe01..5165b75ca23 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,6 @@ def metric_ddp(rank, world_size): n_epochs = 5 # this shows iteration over multiple training epochs for n in range(n_epochs): - # this will be replaced by a DataLoader with a DistributedSampler n_batches = 10 for i in range(n_batches): From 647746a9c9a5d9ba3f242252f5c47ae12997c242 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 2 Feb 2023 12:22:41 +0100 Subject: [PATCH 3/8] chlog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cf7115e8b8..a52c114b4ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed type hints in methods belonging to `MetricTracker` wrapper ([#1472](https://github.com/Lightning-AI/metrics/pull/1472)) +- Fixed `multilabel` in `ExactMatch` ([#1474](https://github.com/Lightning-AI/metrics/pull/1474)) + + + ## [0.11.1] - 2023-01-30 ### Fixed From 3bec0238c02aa75cfde9c159829af3f22c7e04b0 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Fri, 3 Feb 2023 22:24:34 +0900 Subject: [PATCH 4/8] gh: update templates (#1477)Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Nicki Skafte Detlefsen * gh: update templates --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.md | 5 ++++- .github/ISSUE_TEMPLATE/documentation.md | 5 ++--- .github/PULL_REQUEST_TEMPLATE.md | 12 +++++++++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 487b6e8bdc2..150f7849963 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -16,11 +16,14 @@ Steps to reproduce the behavior... -#### Code sample +
+ Code sample +
+ ### Expected behavior diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md index c74b3408000..456d49be454 100644 --- a/.github/ISSUE_TEMPLATE/documentation.md +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -10,8 +10,7 @@ assignees: '' For typos and doc fixes, please go ahead and: -1. Create an issue. -1. Fix the typo. -1. Submit a PR. +- For a simple typo or fix, please send directly a PR (no need to create an issue) +- If you are not sure about the proper solution, please describe here your finding... Thanks! diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0c6881d0228..318ee5483eb 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,18 +2,24 @@ Fixes #\ -## Before submitting +
+ Before submitting -- [ ] Was this **discussed/approved** via a Github issue? (no need for typos and docs improvements) +- [ ] Was this **discussed/agreed** via a Github issue? (no need for typos and docs improvements) - [ ] Did you read the [contributor guideline](https://github.com/Lightning-AI/metrics/blob/master/.github/CONTRIBUTING.md), Pull Request section? - [ ] Did you make sure to **update the docs**? - [ ] Did you write any new **necessary tests**? -## PR review +
+ +
+ PR review Anyone in the community is free to review the PR once the tests have passed. If we didn't discuss your PR in Github issues there's a high chance it will not be merged. +
+ ## Did you have fun? Make sure you had fun coding 🙃 From 881db8f27e20b1a3605b313505bc0890b8d60d3b Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Sun, 5 Feb 2023 19:57:40 +0100 Subject: [PATCH 5/8] try increasing timeout --- .azure/gpu-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index f2feb2d25dd..2314e93591f 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -122,7 +122,7 @@ jobs: displayName: 'Pull testing data from S3' - bash: | - python -m pytest unittests -v --cov=torchmetrics --junitxml=$(Build.StagingDirectory)/test-results.xml --timeout=180 --durations=50 + python -m pytest unittests -v --cov=torchmetrics --junitxml=$(Build.StagingDirectory)/test-results.xml --timeout=240 --durations=50 env: CUDA_LAUNCH_BLOCKING: 1 workingDirectory: tests From 6fd756273f368adae34448910a0633b5b963c40c Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Sun, 5 Feb 2023 20:18:05 +0100 Subject: [PATCH 6/8] further increase timeout --- .azure/gpu-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index 2314e93591f..e86547e1a53 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -122,7 +122,7 @@ jobs: displayName: 'Pull testing data from S3' - bash: | - python -m pytest unittests -v --cov=torchmetrics --junitxml=$(Build.StagingDirectory)/test-results.xml --timeout=240 --durations=50 + python -m pytest unittests -v --cov=torchmetrics --junitxml=$(Build.StagingDirectory)/test-results.xml --timeout=300 --durations=50 env: CUDA_LAUNCH_BLOCKING: 1 workingDirectory: tests From a6cd44f60a65ab131dac402da03fbdbf1e09bf0b Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 6 Feb 2023 06:17:03 +0100 Subject: [PATCH 7/8] links --- docs/source/links.rst | 4 +- src/torchmetrics/classification/accuracy.py | 40 +++++++++---------- src/torchmetrics/functional/image/psnr.py | 8 ++-- src/torchmetrics/functional/regression/mae.py | 4 +- src/torchmetrics/image/psnr.py | 2 +- src/torchmetrics/regression/mae.py | 4 +- 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/docs/source/links.rst b/docs/source/links.rst index f5b847fe601..ed3f7732cff 100644 --- a/docs/source/links.rst +++ b/docs/source/links.rst @@ -64,10 +64,10 @@ .. _Improved Techniques for Training GANs: https://arxiv.org/abs/1606.03498 .. _KID Score: https://github.com/toshas/torch-fidelity/blob/v0.3.0/torch_fidelity/metric_kid.py .. _Demystifying MMD GANs: https://arxiv.org/abs/1801.01401 -.. _Computes Peak Signal-to-Noise Ratio: https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio +.. _Compute Peak Signal-to-Noise Ratio: https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio .. _Turn a Metric into a Bootstrapped: https://en.wikipedia.org/wiki/Bootstrapping_(statistics) .. _Metric Test for Reset: https://github.com/Lightning-AI/pytorch-lightning/pull/7055 -.. _Computes Mean Absolute Error: https://en.wikipedia.org/wiki/Mean_absolute_error +.. _Compute Mean Absolute Error: https://en.wikipedia.org/wiki/Mean_absolute_error .. _Mean Absolute Percentage Error: https://en.wikipedia.org/wiki/Mean_absolute_percentage_error .. _mean squared error: https://en.wikipedia.org/wiki/Mean_squared_error .. _Aggregate the statistics from multiple devices: https://stackoverflow.com/questions/68395368/estimate-running-correlation-on-multiple-nodes diff --git a/src/torchmetrics/classification/accuracy.py b/src/torchmetrics/classification/accuracy.py index 6cb1704ab04..bc52756ba74 100644 --- a/src/torchmetrics/classification/accuracy.py +++ b/src/torchmetrics/classification/accuracy.py @@ -32,7 +32,7 @@ class BinaryAccuracy(BinaryStatScores): - r"""Computes `Accuracy`_ for binary tasks: + r"""Compute `Accuracy`_ for binary tasks. .. math:: \text{Accuracy} = \frac{1}{N}\sum_i^N 1(y_i = \hat{y}_i) @@ -41,16 +41,16 @@ class BinaryAccuracy(BinaryStatScores): As input to ``forward`` and ``update`` the metric accepts the following input: - - ``preds`` (:class:`~torch.Tensor`): An int or float tensor of shape ``(N, ...)``. If preds is a floating - point tensor with values outside [0,1] range we consider the input to be logits and will auto apply sigmoid - per element. Addtionally, we convert to int tensor with thresholding using the value in ``threshold``. - - ``target`` (:class:`~torch.Tensor`): An int tensor of shape ``(N, ...)`` + - ``preds`` (:class:`~torch.Tensor`): An int or float tensor of shape ``(N, ...)``. If preds is a floating + point tensor with values outside [0,1] range we consider the input to be logits and will auto apply sigmoid + per element. Addtionally, we convert to int tensor with thresholding using the value in ``threshold``. + - ``target`` (:class:`~torch.Tensor`): An int tensor of shape ``(N, ...)`` As output to ``forward`` and ``compute`` the metric returns the following output: - - ``ba`` (:class:`~torch.Tensor`): If ``multidim_average`` is set to ``global``, the metric returns a scalar value. - If ``multidim_average`` is set to ``samplewise``, the metric returns ``(N,)`` vector consisting of a scalar - value per sample. + - ``ba`` (:class:`~torch.Tensor`): If ``multidim_average`` is set to ``global``, metric returns a scalar value. + If ``multidim_average`` is set to ``samplewise``, the metric returns ``(N,)`` vector consisting of a scalar + value per sample. Args: threshold: Threshold for transforming probability to binary {0,1} predictions @@ -161,25 +161,25 @@ class MulticlassAccuracy(MulticlassStatScores): As input to ``forward`` and ``update`` the metric accepts the following input: - - ``preds`` (:class:`~torch.Tensor`): An int tensor of shape ``(N, ...)`` or float tensor of shape ``(N, C, ..)``. - If preds is a floating point we apply ``torch.argmax`` along the ``C`` dimension to automatically convert - probabilities/logits into an int tensor. - - ``target`` (:class:`~torch.Tensor`): An int tensor of shape ``(N, ...)`` + - ``preds`` (:class:`~torch.Tensor`): An int tensor of shape ``(N, ...)`` or float tensor + of shape ``(N, C, ..)``. If preds is a floating point we apply ``torch.argmax`` along the ``C`` dimension + to automatically convert probabilities/logits into an int tensor. + - ``target`` (:class:`~torch.Tensor`): An int tensor of shape ``(N, ...)`` As output to ``forward`` and ``compute`` the metric returns the following output: - - ``mca`` (:class:`~torch.Tensor`): A tensor with the accuracy score whose returned shape depends on the - ``average`` and ``multidim_average`` arguments: + - ``mca`` (:class:`~torch.Tensor`): A tensor with the accuracy score whose returned shape depends on the + ``average`` and ``multidim_average`` arguments: - - If ``multidim_average`` is set to ``global``: + - If ``multidim_average`` is set to ``global``: - - If ``average='micro'/'macro'/'weighted'``, the output will be a scalar tensor - - If ``average=None/'none'``, the shape will be ``(C,)`` + - If ``average='micro'/'macro'/'weighted'``, the output will be a scalar tensor + - If ``average=None/'none'``, the shape will be ``(C,)`` - - If ``multidim_average`` is set to ``samplewise``: + - If ``multidim_average`` is set to ``samplewise``: - - If ``average='micro'/'macro'/'weighted'``, the shape will be ``(N,)`` - - If ``average=None/'none'``, the shape will be ``(N, C)`` + - If ``average='micro'/'macro'/'weighted'``, the shape will be ``(N,)`` + - If ``average=None/'none'``, the shape will be ``(N, C)`` Args: num_classes: Integer specifing the number of classes diff --git a/src/torchmetrics/functional/image/psnr.py b/src/torchmetrics/functional/image/psnr.py index 77db51915d0..1fb7024f1e1 100644 --- a/src/torchmetrics/functional/image/psnr.py +++ b/src/torchmetrics/functional/image/psnr.py @@ -27,7 +27,7 @@ def _psnr_compute( base: float = 10.0, reduction: Literal["elementwise_mean", "sum", "none", None] = "elementwise_mean", ) -> Tensor: - """Computes peak signal-to-noise ratio. + """Compute peak signal-to-noise ratio. Args: sum_squared_error: Sum of square of errors over all observations @@ -60,13 +60,13 @@ def _psnr_update( target: Tensor, dim: Optional[Union[int, Tuple[int, ...]]] = None, ) -> Tuple[Tensor, Tensor]: - """Updates and returns variables required to compute peak signal-to-noise ratio. + """Update and return variables required to compute peak signal-to-noise ratio. Args: preds: Predicted tensor target: Ground truth tensor - dim: Dimensions to reduce PSNR scores over provided as either an integer or a list of integers. Default is - None meaning scores will be reduced across all dimensions. + dim: Dimensions to reduce PSNR scores over provided as either an integer or a list of integers. + Default is None meaning scores will be reduced across all dimensions. """ if dim is None: diff --git a/src/torchmetrics/functional/regression/mae.py b/src/torchmetrics/functional/regression/mae.py index f3508c02cea..4f38f8f4397 100644 --- a/src/torchmetrics/functional/regression/mae.py +++ b/src/torchmetrics/functional/regression/mae.py @@ -37,7 +37,7 @@ def _mean_absolute_error_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, def _mean_absolute_error_compute(sum_abs_error: Tensor, n_obs: int) -> Tensor: - """Computes Mean Absolute Error. + """Compute Mean Absolute Error. Args: sum_abs_error: Sum of absolute value of errors over all observations @@ -55,7 +55,7 @@ def _mean_absolute_error_compute(sum_abs_error: Tensor, n_obs: int) -> Tensor: def mean_absolute_error(preds: Tensor, target: Tensor) -> Tensor: - """Computes mean absolute error. + """Compute mean absolute error. Args: preds: estimated labels diff --git a/src/torchmetrics/image/psnr.py b/src/torchmetrics/image/psnr.py index ca2e2ad6f5b..8b4eb0c169d 100644 --- a/src/torchmetrics/image/psnr.py +++ b/src/torchmetrics/image/psnr.py @@ -23,7 +23,7 @@ class PeakSignalNoiseRatio(Metric): - r"""Computes `Computes Peak Signal-to-Noise Ratio`_ (PSNR): + r"""`Compute Peak Signal-to-Noise Ratio`_ (PSNR): .. math:: \text{PSNR}(I, J) = 10 * \log_{10} \left(\frac{\max(I)^2}{\text{MSE}(I, J)}\right) diff --git a/src/torchmetrics/regression/mae.py b/src/torchmetrics/regression/mae.py index 33f1b5a1e67..79c5b243f54 100644 --- a/src/torchmetrics/regression/mae.py +++ b/src/torchmetrics/regression/mae.py @@ -20,7 +20,7 @@ class MeanAbsoluteError(Metric): - r"""`Computes Mean Absolute Error`_ (MAE): + r"""`Compute Mean Absolute Error`_ (MAE): .. math:: \text{MAE} = \frac{1}{N}\sum_i^N | y_i - \hat{y_i} | @@ -70,5 +70,5 @@ def update(self, preds: Tensor, target: Tensor) -> None: # type: ignore self.total += n_obs def compute(self) -> Tensor: - """Computes mean absolute error over state.""" + """Compute mean absolute error over state.""" return _mean_absolute_error_compute(self.sum_abs_error, self.total) From a9b655394901a819b343f72ac1fcee1834151287 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Mon, 6 Feb 2023 13:18:11 +0100 Subject: [PATCH 8/8] Apply suggestions from code review --- .azure/gpu-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index e86547e1a53..f2feb2d25dd 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -122,7 +122,7 @@ jobs: displayName: 'Pull testing data from S3' - bash: | - python -m pytest unittests -v --cov=torchmetrics --junitxml=$(Build.StagingDirectory)/test-results.xml --timeout=300 --durations=50 + python -m pytest unittests -v --cov=torchmetrics --junitxml=$(Build.StagingDirectory)/test-results.xml --timeout=180 --durations=50 env: CUDA_LAUNCH_BLOCKING: 1 workingDirectory: tests