New Metric: Dunn Index (Lightning-AI#2049)

CHANGELOG.md

@@ -12,17 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added `MutualInformationScore` metric to cluster package ([#2008](https://github.com/Lightning-AI/torchmetrics/pull/2008))
-
-
 - Added `RandScore` metric to cluster package ([#2025](https://github.com/Lightning-AI/torchmetrics/pull/2025))
-
-
-- Added `CalinskiHarabaszScore` metric to cluster package ([#2036](https://github.com/Lightning-AI/torchmetrics/pull/2036))
-
-
 - Added `NormalizedMutualInfoScore` metric to cluster package ([#2029](https://github.com/Lightning-AI/torchmetrics/pull/2029))
-
-
+- Added `CalinskiHarabaszScore` metric to cluster package ([#2036](https://github.com/Lightning-AI/torchmetrics/pull/2036))
+- Added `DunnIndex` metric to cluster package ([#2049](https://github.com/Lightning-AI/torchmetrics/pull/2049))
 
 ### Changed
 

docs/source/clustering/dunn_index.rst

@@ -0,0 +1,21 @@
+.. customcarditem::
+   :header: Dunn Index
+   :image: https://pl-flash-data.s3.amazonaws.com/assets/thumbnails/clustering.svg
+   :tags: Clustering
+
+.. include:: ../links.rst
+
+##########
+Dunn Index
+##########
+
+Module Interface
+________________
+
+.. autoclass:: torchmetrics.clustering.DunnIndex
+    :exclude-members: update, compute
+
+Functional Interface
+____________________
+
+.. autofunction:: torchmetrics.functional.clustering.dunn_index

docs/source/links.rst

@@ -154,3 +154,4 @@
 .. _Normalized Mutual Information Score: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.normalized_mutual_info_score.html
 .. _pycocotools: https://github.com/cocodataset/cocoapi/tree/master/PythonAPI/pycocotools
 .. _Rand Score: https://link.springer.com/article/10.1007/BF01908075
+.. _Dunn Index: https://en.wikipedia.org/wiki/Dunn_index

src/torchmetrics/clustering/__init__.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from torchmetrics.clustering.calinski_harabasz_score import CalinskiHarabaszScore
+from torchmetrics.clustering.dunn_index import DunnIndex
 from torchmetrics.clustering.mutual_info_score import MutualInfoScore
 from torchmetrics.clustering.normalized_mutual_info_score import NormalizedMutualInfoScore
 from torchmetrics.clustering.rand_score import RandScore
 
 __all__ = [
     "CalinskiHarabaszScore",
+    "DunnIndex",
     "MutualInfoScore",
     "NormalizedMutualInfoScore",
     "RandScore",

src/torchmetrics/clustering/dunn_index.py

@@ -0,0 +1,116 @@
+# Copyright The Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, List, Optional, Sequence, Union
+
+from torch import Tensor
+
+from torchmetrics.functional.clustering.dunn_index import dunn_index
+from torchmetrics.metric import Metric
+from torchmetrics.utilities.data import dim_zero_cat
+from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
+
+if not _MATPLOTLIB_AVAILABLE:
+    __doctest_skip__ = ["DunnIndex.plot"]
+
+
+class DunnIndex(Metric):
+    r"""Compute `Dunn Index`_.
+
+    .. math::
+        DI_m = \frac{\min_{1\leq i<j\leq m} \delta(C_i,C_j)}{\max_{1\leq k\leq m} \Delta_k}
+
+    Where :math:`C_i` is a cluster of tensors, :math:`C_j` is a cluster of tensors,
+    and :math:`\delta(C_i,C_j)` is the intercluster distance metric for :math:`m` clusters.
+
+    This clustering metric is an intrinsic measure, because it does not rely on ground truth labels for the evaluation.
+    Instead it examines how well the clusters are separated from each other. The score is higher when clusters are dense
+    and well separated, which relates to a standard concept of a cluster.
+
+    As input to ``forward`` and ``update`` the metric accepts the following input:
+
+    - ``data`` (:class:`~torch.Tensor`): float tensor with shape ``(N,d)`` with the embedded data.
+        ``d`` is the dimensionality of the embedding space.
+    - ``labels`` (:class:`~torch.Tensor`): single integer tensor with shape ``(N,)`` with cluster labels
+
+    As output of ``forward`` and ``compute`` the metric returns the following output:
+
+    - ``dunn_index`` (:class:`~torch.Tensor`): A tensor with the Dunn Index
+
+    Args:
+        kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
+
+    Example:
+        >>> import torch
+        >>> from torchmetrics.clustering import DunnIndex
+        >>> data = torch.tensor([[0, 0], [0.5, 0], [1, 0], [0.5, 1]])
+        >>> labels = torch.tensor([0, 0, 0, 1])
+        >>> dunn_index = DunnIndex(p=2)
+        >>> dunn_index(data, labels)
+        tensor(2.)
+
+    """
+
+    is_differentiable: bool = True
+    higher_is_better: bool = True
+    full_state_update: bool = True
+    plot_lower_bound: float = 0.0
+    data: List[Tensor]
+    labels: List[Tensor]
+
+    def __init__(self, p: float = 2, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+        self.p = p
+
+        self.add_state("data", default=[], dist_reduce_fx="cat")
+        self.add_state("labels", default=[], dist_reduce_fx="cat")
+
+    def update(self, data: Tensor, labels: Tensor) -> None:
+        """Update state with predictions and targets."""
+        self.data.append(data)
+        self.labels.append(labels)
+
+    def compute(self) -> Tensor:
+        """Compute mutual information over state."""
+        return dunn_index(dim_zero_cat(self.data), dim_zero_cat(self.labels), self.p)
+
+    def plot(self, val: Union[Tensor, Sequence[Tensor], None] = None, ax: Optional[_AX_TYPE] = None) -> _PLOT_OUT_TYPE:
+        """Plot a single or multiple values from the metric.
+
+        Args:
+            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
+                If no value is provided, will automatically call `metric.compute` and plot that result.
+            ax: An matplotlib axis object. If provided will add plot to that axis
+
+        Returns:
+            Figure and Axes object
+
+        Raises:
+            ModuleNotFoundError:
+                If `matplotlib` is not installed
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting a single value
+            >>> import torch
+            >>> from torchmetrics.clustering import DunnIndex
+            >>> data = torch.tensor([[0, 0], [0.5, 0], [1, 0], [0.5, 1]])
+            >>> labels = torch.tensor([0, 0, 0, 1])
+            >>> metric = DunnIndex(p=2)
+            >>> metric.update(data, labels)
+            >>> fig_, ax_ = metric.plot(metric.compute())
+
+        """
+        return self._plot(val, ax)

src/torchmetrics/functional/clustering/__init__.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from torchmetrics.functional.clustering.calinski_harabasz_score import calinski_harabasz_score
+from torchmetrics.functional.clustering.dunn_index import dunn_index
 from torchmetrics.functional.clustering.mutual_info_score import mutual_info_score
 from torchmetrics.functional.clustering.normalized_mutual_info_score import normalized_mutual_info_score
 from torchmetrics.functional.clustering.rand_score import rand_score
 
 __all__ = [
     "calinski_harabasz_score",
+    "dunn_index",
     "mutual_info_score",
     "normalized_mutual_info_score",
     "rand_score",

src/torchmetrics/functional/clustering/dunn_index.py

@@ -0,0 +1,83 @@
+# Copyright The Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from itertools import combinations
+from typing import Tuple
+
+import torch
+from torch import Tensor
+
+
+def _dunn_index_update(data: Tensor, labels: Tensor, p: float) -> Tuple[Tensor, Tensor]:
+    """Update and return variables required to compute the Dunn index.
+
+    Args:
+        data: feature vectors of shape (n_samples, n_features)
+        labels: cluster labels
+        p: p-norm (distance metric)
+
+    Returns:
+        intercluster_distance: intercluster distances
+        max_intracluster_distance: max intracluster distances
+
+    """
+    unique_labels, inverse_indices = labels.unique(return_inverse=True)
+    clusters = [data[inverse_indices == label_idx] for label_idx in range(len(unique_labels))]
+    centroids = [c.mean(dim=0) for c in clusters]
+
+    intercluster_distance = torch.linalg.norm(
+        torch.stack([a - b for a, b in combinations(centroids, 2)], dim=0), ord=p, dim=1
+    )
+
+    max_intracluster_distance = torch.stack(
+        [torch.linalg.norm(ci - mu, ord=p, dim=1).max() for ci, mu in zip(clusters, centroids)]
+    )
+
+    return intercluster_distance, max_intracluster_distance
+
+
+def _dunn_index_compute(intercluster_distance: Tensor, max_intracluster_distance: Tensor) -> Tensor:
+    """Compute the Dunn index based on updated state.
+
+    Args:
+        intercluster_distance: intercluster distances
+        max_intracluster_distance: max intracluster distances
+
+    Returns:
+        scalar tensor with the dunn index
+
+    """
+    return intercluster_distance.min() / max_intracluster_distance.max()
+
+
+def dunn_index(data: Tensor, labels: Tensor, p: float = 2) -> Tensor:
+    """Compute the Dunn index.
+
+    Args:
+        data: feature vectors
+        labels: cluster labels
+        p: p-norm used for distance metric
+
+    Returns:
+        scalar tensor with the dunn index
+
+    Example:
+        >>> from torchmetrics.functional.clustering import dunn_index
+        >>> data = torch.tensor([[0, 0], [0.5, 0], [1, 0], [0.5, 1]])
+        >>> labels = torch.tensor([0, 0, 0, 1])
+        >>> dunn_index(data, labels)
+        tensor(2.)
+
+    """
+    pairwise_distance, max_distance = _dunn_index_update(data, labels, p)
+    return _dunn_index_compute(pairwise_distance, max_distance)

src/torchmetrics/functional/clustering/utils.py

@@ -151,6 +151,13 @@ def calculate_contingency_matrix(
     return contingency
 
 
+def _is_real_discrete_label(x: Tensor) -> bool:
+    """Check if tensor of labels is real and discrete."""
+    if x.ndim != 1:
+        raise ValueError(f"Expected arguments to be 1-d tensors but got {x.ndim}-d tensors.")
+    return not (torch.is_floating_point(x) or torch.is_complex(x))
+
+
 def check_cluster_labels(preds: Tensor, target: Tensor) -> None:
     """Check shape of input tensors and if they are real, discrete tensors.
 
@@ -160,18 +167,8 @@ def check_cluster_labels(preds: Tensor, target: Tensor) -> None:
 
     """
     _check_same_shape(preds, target)
-    if preds.ndim != 1:
-        raise ValueError(f"Expected arguments to be 1d tensors but got {preds.ndim} and {target.ndim}")
-    if (
-        torch.is_floating_point(preds)
-        or torch.is_complex(preds)
-        or torch.is_floating_point(target)
-        or torch.is_complex(target)
-    ):
-        raise ValueError(
-            f"Expected real, discrete values but received {preds.dtype} for"
-            f"predictions and {target.dtype} for target labels instead."
-        )
+    if not (_is_real_discrete_label(preds) and _is_real_discrete_label(target)):
+        raise ValueError(f"Expected real, discrete values for x but received {preds.dtype} and {target.dtype}.")
 
 
 def calcualte_pair_cluster_confusion_matrix(

tests/unittests/clustering/inputs.py

@@ -14,38 +14,44 @@
 from collections import namedtuple
 
 import torch
+from sklearn.datasets import make_blobs
 
-from unittests import BATCH_SIZE, EXTRA_DIM, NUM_BATCHES
+from unittests import BATCH_SIZE, EXTRA_DIM, NUM_BATCHES, NUM_CLASSES
 from unittests.helpers import seed_all
 
 seed_all(42)
 
 
-Input = namedtuple("Input", ["preds", "target"])
-NUM_CLASSES = 10
-
 # extrinsic input for clustering metrics that requires predicted clustering labels and target clustering labels
-_single_target_extrinsic1 = Input(
+ExtrinsicInput = namedtuple("ExtrinsicInput", ["preds", "target"])
+
+# intrinsic input for clustering metrics that requires only predicted clustering labels and the cluster embeddings
+IntrinsicInput = namedtuple("IntrinsicInput", ["data", "labels"])
+
+
+def _batch_blobs(num_batches, num_samples, num_features, num_classes):
+    data, labels = [], []
+    for _ in range(num_batches):
+        _data, _labels = make_blobs(num_samples, num_features, centers=num_classes)
+        data.append(torch.tensor(_data))
+        labels.append(torch.tensor(_labels))
+
+    return IntrinsicInput(data=torch.stack(data), labels=torch.stack(labels))
+
+
+_single_target_extrinsic1 = ExtrinsicInput(
     preds=torch.randint(high=NUM_CLASSES, size=(NUM_BATCHES, BATCH_SIZE)),
     target=torch.randint(high=NUM_CLASSES, size=(NUM_BATCHES, BATCH_SIZE)),
 )
 
-_single_target_extrinsic2 = Input(
+_single_target_extrinsic2 = ExtrinsicInput(
     preds=torch.randint(high=NUM_CLASSES, size=(NUM_BATCHES, BATCH_SIZE)),
     target=torch.randint(high=NUM_CLASSES, size=(NUM_BATCHES, BATCH_SIZE)),
 )
 
-_float_inputs_extrinsic = Input(
+_float_inputs_extrinsic = ExtrinsicInput(
     preds=torch.rand((NUM_BATCHES, BATCH_SIZE)), target=torch.rand((NUM_BATCHES, BATCH_SIZE))
 )
 
-# intrinsic input for clustering metrics that requires only predicted clustering labels and the cluster embeddings
-_single_target_intrinsic1 = Input(
-    preds=torch.randn(NUM_BATCHES, BATCH_SIZE, EXTRA_DIM),
-    target=torch.randint(high=NUM_CLASSES, size=(NUM_BATCHES, BATCH_SIZE)),
-)
-
-_single_target_intrinsic2 = Input(
-    preds=torch.randn(NUM_BATCHES, BATCH_SIZE, EXTRA_DIM),
-    target=torch.randint(high=NUM_CLASSES, size=(NUM_BATCHES, BATCH_SIZE)),
-)
+_single_target_intrinsic1 = _batch_blobs(NUM_BATCHES, BATCH_SIZE, EXTRA_DIM, NUM_CLASSES)
+_single_target_intrinsic2 = _batch_blobs(NUM_BATCHES, BATCH_SIZE, EXTRA_DIM, NUM_CLASSES)