From d1e2153c595f9d18f991e807d6b05281001cbaf0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 20:20:59 +0000 Subject: [PATCH 1/8] Bump torchmetrics from 0.11.4 to 1.0.0 in /requirements Bumps [torchmetrics](https://github.com/Lightning-AI/torchmetrics) from 0.11.4 to 1.0.0. - [Release notes](https://github.com/Lightning-AI/torchmetrics/releases) - [Changelog](https://github.com/Lightning-AI/torchmetrics/blob/master/CHANGELOG.md) - [Commits](https://github.com/Lightning-AI/torchmetrics/compare/v0.11.4...v1.0.0) --- updated-dependencies: - dependency-name: torchmetrics dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements/required.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/required.txt b/requirements/required.txt index fd8788f1d67..94a1a016552 100644 --- a/requirements/required.txt +++ b/requirements/required.txt @@ -17,5 +17,5 @@ segmentation-models-pytorch==0.3.3 shapely==2.0.1 timm==0.9.2 torch==2.0.1 -torchmetrics==0.11.4 +torchmetrics==1.0.0 torchvision==0.15.2 From fc689ca2d4fc1f49c2a51f809f3a0cb641c7b27f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 5 Jul 2023 15:25:48 -0500 Subject: [PATCH 2/8] Bump max version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5107822c1c4..c167af3458b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ dependencies = [ # torch 1.12+ required by torchvision "torch>=1.12,<3", # torchmetrics 0.10+ required for binary/multiclass/multilabel classification metrics - "torchmetrics>=0.10,<0.12", + "torchmetrics>=0.10,<2", # torchvision 0.13+ required for torchvision.models._api.WeightsEnum "torchvision>=0.13,<0.16", ] From e42efc685e027313d37d5dd8b79fafd3b8addc35 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 6 Jul 2023 09:42:29 -0500 Subject: [PATCH 3/8] mdmc_average -> multidim_average --- torchgeo/trainers/segmentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchgeo/trainers/segmentation.py b/torchgeo/trainers/segmentation.py index af20fb8687a..2107371cf53 100644 --- a/torchgeo/trainers/segmentation.py +++ b/torchgeo/trainers/segmentation.py @@ -175,7 +175,7 @@ class and used with 'ce' loss MulticlassAccuracy( num_classes=self.hyperparams["num_classes"], ignore_index=self.ignore_index, - mdmc_average="global", + multidim_average="global", average="micro", ), MulticlassJaccardIndex( From 1fe1b2df141b263bcc8ec4f84644cb8e066629fa Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 6 Jul 2023 10:10:55 -0500 Subject: [PATCH 4/8] debugging --- torchgeo/trainers/detection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/torchgeo/trainers/detection.py b/torchgeo/trainers/detection.py index dfa2bd924a8..91f1e1702dc 100644 --- a/torchgeo/trainers/detection.py +++ b/torchgeo/trainers/detection.py @@ -273,8 +273,10 @@ def validation_step(self, *args: Any, **kwargs: Any) -> None: def on_validation_epoch_end(self) -> None: """Logs epoch level validation metrics.""" metrics = self.val_metrics.compute() + print(metrics) renamed_metrics = {f"val_{i}": metrics[i] for i in metrics.keys()} - self.log_dict(renamed_metrics) + print(renamed_metrics) + self.log_dict(metrics) self.val_metrics.reset() def test_step(self, *args: Any, **kwargs: Any) -> None: From 24f2c5db0c58845cc3d6d8dd1b0bcafcf2c081a2 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 6 Jul 2023 10:38:58 -0500 Subject: [PATCH 5/8] Workaround for bug in MAP dict --- torchgeo/trainers/detection.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/torchgeo/trainers/detection.py b/torchgeo/trainers/detection.py index 91f1e1702dc..74a0b38715b 100644 --- a/torchgeo/trainers/detection.py +++ b/torchgeo/trainers/detection.py @@ -273,10 +273,12 @@ def validation_step(self, *args: Any, **kwargs: Any) -> None: def on_validation_epoch_end(self) -> None: """Logs epoch level validation metrics.""" metrics = self.val_metrics.compute() - print(metrics) renamed_metrics = {f"val_{i}": metrics[i] for i in metrics.keys()} - print(renamed_metrics) - self.log_dict(metrics) + + # https://github.com/Lightning-AI/torchmetrics/pull/1832#issuecomment-1623890714 + renamed_metrics.pop("classes", None) + + self.log_dict(renamed_metrics) self.val_metrics.reset() def test_step(self, *args: Any, **kwargs: Any) -> None: @@ -300,6 +302,10 @@ def on_test_epoch_end(self) -> None: """Logs epoch level test metrics.""" metrics = self.test_metrics.compute() renamed_metrics = {f"test_{i}": metrics[i] for i in metrics.keys()} + + # https://github.com/Lightning-AI/torchmetrics/pull/1832#issuecomment-1623890714 + renamed_metrics.pop("classes", None) + self.log_dict(renamed_metrics) self.test_metrics.reset() From 5972d023dc352155b2f58a887a9050ee5fd0fe16 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 6 Jul 2023 10:54:32 -0500 Subject: [PATCH 6/8] Workaround for bug in MAP dict --- torchgeo/trainers/detection.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/torchgeo/trainers/detection.py b/torchgeo/trainers/detection.py index 74a0b38715b..4d1f72e8041 100644 --- a/torchgeo/trainers/detection.py +++ b/torchgeo/trainers/detection.py @@ -187,8 +187,9 @@ def __init__(self, **kwargs: Any) -> None: self.config_task() - self.val_metrics = MeanAveragePrecision() - self.test_metrics = MeanAveragePrecision() + metrics = MeanAveragePrecision() + self.val_metrics = metrics.clone(prefix="val_") + self.test_metrics = metrics.clone(prefix="test_") def forward(self, *args: Any, **kwargs: Any) -> Any: """Forward pass of the model. @@ -273,12 +274,11 @@ def validation_step(self, *args: Any, **kwargs: Any) -> None: def on_validation_epoch_end(self) -> None: """Logs epoch level validation metrics.""" metrics = self.val_metrics.compute() - renamed_metrics = {f"val_{i}": metrics[i] for i in metrics.keys()} # https://github.com/Lightning-AI/torchmetrics/pull/1832#issuecomment-1623890714 - renamed_metrics.pop("classes", None) + metrics.pop("classes", None) - self.log_dict(renamed_metrics) + self.log_dict(metrics) self.val_metrics.reset() def test_step(self, *args: Any, **kwargs: Any) -> None: @@ -301,12 +301,11 @@ def test_step(self, *args: Any, **kwargs: Any) -> None: def on_test_epoch_end(self) -> None: """Logs epoch level test metrics.""" metrics = self.test_metrics.compute() - renamed_metrics = {f"test_{i}": metrics[i] for i in metrics.keys()} # https://github.com/Lightning-AI/torchmetrics/pull/1832#issuecomment-1623890714 - renamed_metrics.pop("classes", None) + metrics.pop("classes", None) - self.log_dict(renamed_metrics) + self.log_dict(metrics) self.test_metrics.reset() def predict_step(self, *args: Any, **kwargs: Any) -> list[dict[str, Tensor]]: From d1000c57e552e363dee157749ff6f2247a4ced45 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 6 Jul 2023 11:01:05 -0500 Subject: [PATCH 7/8] Workaround for bug in MAP dict --- torchgeo/trainers/detection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/torchgeo/trainers/detection.py b/torchgeo/trainers/detection.py index 4d1f72e8041..668b23e18ac 100644 --- a/torchgeo/trainers/detection.py +++ b/torchgeo/trainers/detection.py @@ -12,6 +12,7 @@ from lightning.pytorch import LightningModule from torch import Tensor from torch.optim.lr_scheduler import ReduceLROnPlateau +from torchmetrics import MetricCollection from torchmetrics.detection.mean_ap import MeanAveragePrecision from torchvision.models import resnet as R from torchvision.models.detection.backbone_utils import resnet_fpn_backbone @@ -187,7 +188,7 @@ def __init__(self, **kwargs: Any) -> None: self.config_task() - metrics = MeanAveragePrecision() + metrics = MetricCollection([MeanAveragePrecision()]) self.val_metrics = metrics.clone(prefix="val_") self.test_metrics = metrics.clone(prefix="test_") From 5841224795bcae4ffb60c3baf08bc43f5b1cf762 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 6 Jul 2023 11:36:08 -0500 Subject: [PATCH 8/8] Workaround for bug in MAP dict --- torchgeo/trainers/detection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchgeo/trainers/detection.py b/torchgeo/trainers/detection.py index 668b23e18ac..ed58bf945a2 100644 --- a/torchgeo/trainers/detection.py +++ b/torchgeo/trainers/detection.py @@ -277,7 +277,7 @@ def on_validation_epoch_end(self) -> None: metrics = self.val_metrics.compute() # https://github.com/Lightning-AI/torchmetrics/pull/1832#issuecomment-1623890714 - metrics.pop("classes", None) + metrics.pop("val_classes", None) self.log_dict(metrics) self.val_metrics.reset() @@ -304,7 +304,7 @@ def on_test_epoch_end(self) -> None: metrics = self.test_metrics.compute() # https://github.com/Lightning-AI/torchmetrics/pull/1832#issuecomment-1623890714 - metrics.pop("classes", None) + metrics.pop("test_classes", None) self.log_dict(metrics) self.test_metrics.reset()