Remove error in ROC and AUROC (#583)

Co-authored-by: Jirka Borovec <[email protected]>

CHANGELOG.md

@@ -39,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `half`, `double`, `float` will no longer change the dtype of the metric states. Use `metric.set_dtype` instead ([#493](https://github.com/PyTorchLightning/metrics/pull/493))
 - Renamed `AverageMeter` to `MeanMetric` ([#506](https://github.com/PyTorchLightning/metrics/pull/506))
 - Changed `is_differentiable` from property to a constant attribute ([#551](https://github.com/PyTorchLightning/metrics/pull/551))
+- `ROC` and `AUROC` will no longer throw an error when either the positive or negative class is missing. Instead return 0 score and give a warning
 
 ### Deprecated
 

tests/classification/test_auroc.py

@@ -216,3 +216,24 @@ def test_weighted_with_empty_classes():
     target = torch.zeros_like(target)
     with pytest.raises(ValueError, match="Found 1 non-empty class in `multiclass` AUROC calculation"):
         _ = auroc(preds, target, average="weighted", num_classes=num_classes + 1)
+
+
+def test_warnings_on_missing_class():
+    """Test that a warning is given if either the positive or negative class is missing."""
+    metric = AUROC()
+    # no positive samples
+    warning = (
+        "No positive samples in targets, true positive value should be meaningless."
+        " Returning zero tensor in true positive score"
+    )
+    with pytest.warns(UserWarning, match=warning):
+        score = metric(torch.randn(10).sigmoid(), torch.zeros(10).int())
+    assert score == 0
+
+    warning = (
+        "No negative samples in targets, false positive value should be meaningless."
+        " Returning zero tensor in false positive score"
+    )
+    with pytest.warns(UserWarning, match=warning):
+        score = metric(torch.randn(10).sigmoid(), torch.ones(10).int())
+    assert score == 0

tests/classification/test_roc.py

@@ -144,3 +144,24 @@ def test_roc_curve(pred, target, expected_tpr, expected_fpr):
     assert fpr.size(0) == thresh.size(0)
     assert torch.allclose(fpr, tensor(expected_fpr).to(fpr))
     assert torch.allclose(tpr, tensor(expected_tpr).to(tpr))
+
+
+def test_warnings_on_missing_class():
+    """Test that a warning is given if either the positive or negative class is missing."""
+    metric = ROC()
+    # no positive samples
+    warning = (
+        "No positive samples in targets, true positive value should be meaningless."
+        " Returning zero tensor in true positive score"
+    )
+    with pytest.warns(UserWarning, match=warning):
+        _, tpr, _ = metric(torch.randn(10).sigmoid(), torch.zeros(10))
+    assert all(tpr == 0)
+
+    warning = (
+        "No negative samples in targets, false positive value should be meaningless."
+        " Returning zero tensor in false positive score"
+    )
+    with pytest.warns(UserWarning, match=warning):
+        fpr, _, _ = metric(torch.randn(10).sigmoid(), torch.ones(10))
+    assert all(fpr == 0)

torchmetrics/classification/auroc.py

@@ -41,6 +41,11 @@ class AUROC(Metric):
     dimension more than the ``target`` tensor the input will be interpretated as
     multiclass.
 
+    .. note::
+        If either the positive class or negative class is completly missing in the target tensor,
+        the auroc score is meaningless in this case and a score of 0 will be returned together
+        with an warning.
+
     Args:
        num_classes: integer with number of classes for multi-label and multiclass problems.
            Should be set to ``None`` for binary problems

torchmetrics/classification/roc.py

@@ -32,6 +32,11 @@ class ROC(Metric):
 
     - ``target`` (long tensor): ``(N, ...)`` or ``(N, C, ...)`` with integer labels
 
+    .. note::
+        If either the positive class or negative class is completly missing in the target tensor,
+        the roc values are not well defined in this case and a tensor of zeros will be returned (either fpr
+        or tpr depending on what class is missing) together with an warning.
+
     Args:
         num_classes: integer with number of classes for multi-label and multiclass problems.
             Should be set to ``None`` for binary problems

torchmetrics/functional/classification/auroc.py

@@ -204,6 +204,16 @@ def auroc(
 ) -> Tensor:
     """Compute Area Under the Receiver Operating Characteristic Curve (`ROC AUC`_)
 
+    For non-binary input, if the ``preds`` and ``target`` tensor have the same
+    size the input will be interpretated as multilabel and if ``preds`` have one
+    dimension more than the ``target`` tensor the input will be interpretated as
+    multiclass.
+
+    .. note::
+        If either the positive class or negative class is completly missing in the target tensor,
+        the auroc score is meaningless in this case and a score of 0 will be returned together
+        with an warning.
+
     Args:
         preds: predictions from model (logits or probabilities)
         target: Ground truth labels

torchmetrics/functional/classification/roc.py

@@ -20,6 +20,7 @@
     _binary_clf_curve,
     _precision_recall_curve_update,
 )
+from torchmetrics.utilities import rank_zero_warn
 
 
 def _roc_update(
@@ -73,12 +74,24 @@ def _roc_compute_single_class(
     thresholds = torch.cat([thresholds[0][None] + 1, thresholds])
 
     if fps[-1] <= 0:
-        raise ValueError("No negative samples in targets, false positive value should be meaningless")
-    fpr = fps / fps[-1]
+        rank_zero_warn(
+            "No negative samples in targets, false positive value should be meaningless."
+            " Returning zero tensor in false positive score",
+            UserWarning,
+        )
+        fpr = torch.zeros_like(thresholds)
+    else:
+        fpr = fps / fps[-1]
 
     if tps[-1] <= 0:
-        raise ValueError("No positive samples in targets, true positive value should be meaningless")
-    tpr = tps / tps[-1]
+        rank_zero_warn(
+            "No positive samples in targets, true positive value should be meaningless."
+            " Returning zero tensor in true positive score",
+            UserWarning,
+        )
+        tpr = torch.zeros_like(thresholds)
+    else:
+        tpr = tps / tps[-1]
 
     return fpr, tpr, thresholds
 
@@ -196,6 +209,11 @@ def roc(
     """Computes the Receiver Operating Characteristic (ROC). Works with both binary, multiclass and multilabel
     input.
 
+    .. note::
+        If either the positive class or negative class is completly missing in the target tensor,
+        the roc values are not well defined in this case and a tensor of zeros will be returned (either fpr
+        or tpr depending on what class is missing) together with an warning.
+
     Args:
         preds: predictions from model (logits or probabilities)
         target: ground truth values