Feature/sg 404 ssd reproduce

src/super_gradients/recipes/coco2017_ssd_lite_mobilenet_v2.yaml

@@ -1,9 +1,9 @@
 # SSD MobileNetV2 Detection training on CoCo2017 Dataset:
-# Trained in 320x320 [email protected]@0.95 (COCO API, confidence 0.001, IoU threshold 0.6, test on 320x320 images) ~20.52
-# Checkpoint path: https://deci-pretrained-models.s3.amazonaws.com/ssd_lite_mobilenet_v2_coco_res320_new_coco_filtered_affine_scale_5_15_no_mosaic/ckpt_best.pth
+# Trained in 320x320 [email protected]@0.95 (COCO API, confidence 0.001, IoU threshold 0.6, test on 320x320 images) ~20.41
+# Checkpoint path: https://deci-pretrained-models.s3.amazonaws.com/ssd_lite_mobilenet_v2/coco2017/2022-11-28/average_model.pth
 # (trained with stride_16_plus_big)
-# Hardware: 8 NVIDIA RTX 3090
-# Training time: ±17 hours
+# Hardware: 4 NVIDIA RTX 2080Ti
+# Training time: ±35 hours
 #
 
 
@@ -53,7 +53,7 @@ training_hyperparams:
     dboxes: ${dboxes}
 
 multi_gpu: DDP
-num_gpus: 8
+num_gpus: 4
 
 ckpt_root_dir:
 

src/super_gradients/recipes/training_hyperparams/coco2017_ssd_lite_mobilenet_v2_train_params.yaml

@@ -3,7 +3,7 @@ max_epochs: 400
 lr_mode: cosine
 cosine_final_lr_ratio: 0.01
 batch_accumulate: 1
-initial_lr: 0.02
+initial_lr: 0.01
 loss: ssd_loss
 
 criterion_params:
@@ -29,4 +29,4 @@ valid_metrics_list:
       num_cls: 80
 
 metric_to_watch: '[email protected]:0.95'
-greater_metric_to_watch_is_better: True
+greater_metric_to_watch_is_better: True

src/super_gradients/training/pretrained_models.py

@@ -1,55 +1,41 @@
-MODEL_URLS = {"regnetY800_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/RegnetY800/average_model.pth",
-              "regnetY600_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/RegnetY600/average_model_regnety600.pth",
-              "regnetY400_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/RegnetY400/average_model_regnety400.pth",
-              "regnetY200_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/RegnetY200/average_model_regnety200.pth",
-
-              "resnet50_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/KD_ResNet50_Beit_Base_ImageNet/resnet.pth",
-              "resnet34_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/resent_34/average_model.pth",
-              "resnet18_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/resnet18/average_model.pth",
-
-              "repvgg_a0_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/repvgg_a0_imagenet.pth",
-
-              "shelfnet34_lw_coco_segmentation_subclass": "https://deci-pretrained-models.s3.amazonaws.com"
-                                                          "/shelfnet34_coco_segmentation_subclass.pth",
-
-              "ddrnet_23_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ddrnet/cityscapes/ddrnet23/average_model.pth",
-              "ddrnet_23_slim_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ddrnet/cityscapes/ddrnet23_slim/average_model.pth",
-              "stdc1_seg50_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/cityscapes_stdc1_seg50_dice_edge/ckpt_best.pth",
-              "stdc1_seg75_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/stdc1_seg75_cityscapes/ckpt_best.pth",
-              "stdc2_seg50_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/cityscapes_stdc2_seg50_dice_edge/ckpt_best.pth",
-              "stdc2_seg75_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/stdc2_seg75_cityscapes/ckpt_best.pth",
-              "efficientnet_b0_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/efficientnet_b0/average_model-3.pth",
-              "ssd_lite_mobilenet_v2_coco":
-                  "https://deci-pretrained-models.s3.amazonaws.com/"
-                  "ssd_lite_mobilenet_v2_coco_res320_new_coco_filtered_affine_scale_5_15_no_mosaic/ckpt_best.pth",
-              "ssd_mobilenet_v1_coco": "https://deci-pretrained-models.s3.amazonaws.com/ssd_mobilenet_v1_coco_res320/ckpt_best.pth",
-
-              "mobilenet_v3_large_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/mobilenetv3+large+300epoch/average_model.pth",
-              "mobilenet_v3_small_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/mobilenetv3+small/ckpt_best.pth",
-              "mobilenet_v2_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/mobilenetv2+w1/ckpt_best.pth",
-
-              "regseg48_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/regseg48_cityscapes/ckpt_best.pth",
-              "vit_base_imagenet21k": "https://deci-pretrained-models.s3.amazonaws.com/vit_pretrained_imagenet21k/vit_base_16_imagenet21K.pth",
-              "vit_large_imagenet21k": "https://deci-pretrained-models.s3.amazonaws.com/vit_pretrained_imagenet21k/vit_large_16_imagenet21K.pth",
-              "vit_base_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/vit_base_imagenet1k/ckpt_best.pth",
-              "vit_large_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/vit_large_cutmix_randaug_v2_lr%3D0.03/average_model.pth",
-              "beit_base_patch16_224_imagenet": 'https://deci-pretrained-models.s3.amazonaws.com/beit_base_patch16_224_imagenet.pth',
-              "beit_base_patch16_224_cifar10": 'https://deci-pretrained-models.s3.amazonaws.com/beit_cifar10.pth',
-              "yolox_s_coco": "https://deci-pretrained-models.s3.amazonaws.com/yolox_coco/yolox_s_coco/average_model.pth",
-              "yolox_m_coco": "https://deci-pretrained-models.s3.amazonaws.com/yolox_coco/yolox_m_coco/average_model.pth",
-              "yolox_l_coco": "https://deci-pretrained-models.s3.amazonaws.com/yolox_coco/yolox_l_coco/average_model.pth",
-              "yolox_t_coco": "https://deci-pretrained-models.s3.amazonaws.com/yolox_coco/yolox_tiny_coco/ckpt_best.pth",
-              "yolox_n_coco": "https://deci-pretrained-models.s3.amazonaws.com/yolox_coco/yolox_n_coco/ckpt_best.pth",
-
-              "pp_lite_t_seg50_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ppliteseg/cityscapes/pplite_t_seg50/average_model.pth",
-              "pp_lite_t_seg75_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ppliteseg/cityscapes/pplite_t_seg75/average_model.pth",
-              "pp_lite_b_seg50_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ppliteseg/cityscapes/pplite_b_seg50/average_model.pth",
-              "pp_lite_b_seg75_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ppliteseg/cityscapes/pplite_b_seg75/average_model.pth",
-              }
-
-PRETRAINED_NUM_CLASSES = {"imagenet": 1000,
-                          "imagenet21k": 21843,
-                          "coco_segmentation_subclass": 21,
-                          "cityscapes": 19,
-                          "coco": 80,
-                          "cifar10": 10}
+MODEL_URLS = {
+    "regnetY800_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/RegnetY800/average_model.pth",
+    "regnetY600_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/RegnetY600/average_model_regnety600.pth",
+    "regnetY400_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/RegnetY400/average_model_regnety400.pth",
+    "regnetY200_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/RegnetY200/average_model_regnety200.pth",
+    "resnet50_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/KD_ResNet50_Beit_Base_ImageNet/resnet.pth",
+    "resnet34_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/resent_34/average_model.pth",
+    "resnet18_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/resnet18/average_model.pth",
+    "repvgg_a0_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/repvgg_a0_imagenet.pth",
+    "shelfnet34_lw_coco_segmentation_subclass": "https://deci-pretrained-models.s3.amazonaws.com" "/shelfnet34_coco_segmentation_subclass.pth",
+    "ddrnet_23_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ddrnet/cityscapes/ddrnet23/average_model.pth",
+    "ddrnet_23_slim_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ddrnet/cityscapes/ddrnet23_slim/average_model.pth",
+    "stdc1_seg50_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/cityscapes_stdc1_seg50_dice_edge/ckpt_best.pth",
+    "stdc1_seg75_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/stdc1_seg75_cityscapes/ckpt_best.pth",
+    "stdc2_seg50_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/cityscapes_stdc2_seg50_dice_edge/ckpt_best.pth",
+    "stdc2_seg75_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/stdc2_seg75_cityscapes/ckpt_best.pth",
+    "efficientnet_b0_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/efficientnet_b0/average_model-3.pth",
+    "ssd_lite_mobilenet_v2_coco": "https://deci-pretrained-models.s3.amazonaws.com/ssd_lite_mobilenet_v2/coco2017/2022-11-28/average_model.pth",
+    "ssd_mobilenet_v1_coco": "https://deci-pretrained-models.s3.amazonaws.com/ssd_mobilenet_v1_coco_res320/ckpt_best.pth",
+    "mobilenet_v3_large_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/mobilenetv3+large+300epoch/average_model.pth",
+    "mobilenet_v3_small_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/mobilenetv3+small/ckpt_best.pth",
+    "mobilenet_v2_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/mobilenetv2+w1/ckpt_best.pth",
+    "regseg48_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/regseg48_cityscapes/ckpt_best.pth",
+    "vit_base_imagenet21k": "https://deci-pretrained-models.s3.amazonaws.com/vit_pretrained_imagenet21k/vit_base_16_imagenet21K.pth",
+    "vit_large_imagenet21k": "https://deci-pretrained-models.s3.amazonaws.com/vit_pretrained_imagenet21k/vit_large_16_imagenet21K.pth",
+    "vit_base_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/vit_base_imagenet1k/ckpt_best.pth",
+    "vit_large_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/vit_large_cutmix_randaug_v2_lr%3D0.03/average_model.pth",
+    "beit_base_patch16_224_imagenet": "https://deci-pretrained-models.s3.amazonaws.com/beit_base_patch16_224_imagenet.pth",
+    "beit_base_patch16_224_cifar10": "https://deci-pretrained-models.s3.amazonaws.com/beit_cifar10.pth",
+    "yolox_s_coco": "https://deci-pretrained-models.s3.amazonaws.com/yolox_coco/yolox_s_coco/average_model.pth",
+    "yolox_m_coco": "https://deci-pretrained-models.s3.amazonaws.com/yolox_coco/yolox_m_coco/average_model.pth",
+    "yolox_l_coco": "https://deci-pretrained-models.s3.amazonaws.com/yolox_coco/yolox_l_coco/average_model.pth",
+    "yolox_t_coco": "https://deci-pretrained-models.s3.amazonaws.com/yolox_coco/yolox_tiny_coco/ckpt_best.pth",
+    "yolox_n_coco": "https://deci-pretrained-models.s3.amazonaws.com/yolox_coco/yolox_n_coco/ckpt_best.pth",
+    "pp_lite_t_seg50_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ppliteseg/cityscapes/pplite_t_seg50/average_model.pth",
+    "pp_lite_t_seg75_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ppliteseg/cityscapes/pplite_t_seg75/average_model.pth",
+    "pp_lite_b_seg50_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ppliteseg/cityscapes/pplite_b_seg50/average_model.pth",
+    "pp_lite_b_seg75_cityscapes": "https://deci-pretrained-models.s3.amazonaws.com/ppliteseg/cityscapes/pplite_b_seg75/average_model.pth",
+}
+
+PRETRAINED_NUM_CLASSES = {"imagenet": 1000, "imagenet21k": 21843, "coco_segmentation_subclass": 21, "cityscapes": 19, "coco": 80, "cifar10": 10}

src/super_gradients/training/utils/ema.py

@@ -14,14 +14,14 @@
 def copy_attr(a: nn.Module, b: nn.Module, include: Union[list, tuple] = (), exclude: Union[list, tuple] = ()):
     # Copy attributes from b to a, options to only include [...] and to exclude [...]
     for k, v in b.__dict__.items():
-        if (len(include) and k not in include) or k.startswith('_') or k in exclude:
+        if (len(include) and k not in include) or k.startswith("_") or k in exclude:
             continue
         else:
             setattr(a, k, v)
 
 
 class ModelEMA:
-    """ Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models
+    """Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models
     Keep a moving average of everything in the model state_dict (parameters and buffers).
     This is intended to allow functionality like
     https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage
@@ -59,8 +59,7 @@ def __init__(self, model, decay: float = 0.9999, beta: float = 15, exp_activatio
             self.include_attributes = model.module.get_include_attributes()
             self.exclude_attributes = model.module.get_exclude_attributes()
         else:
-            warnings.warn("Warning: EMA should be used with SgModule instance. All attributes of the model will be "
-                          "included in EMA")
+            warnings.warn("Warning: EMA should be used with SgModule instance. All attributes of the model will be " "included in EMA")
             self.include_attributes = []
             self.exclude_attributes = []
         for p in self.ema.module.parameters():
@@ -78,7 +77,7 @@ def update(self, model, training_percent: float):
 
             for ema_v, model_v in zip(self.ema.module.state_dict().values(), model.state_dict().values()):
                 if ema_v.dtype.is_floating_point:
-                    ema_v.copy_(ema_v * decay + (1. - decay) * model_v.detach())
+                    ema_v.copy_(ema_v * decay + (1.0 - decay) * model_v.detach())
 
     def update_attr(self, model):
         """
@@ -93,7 +92,7 @@ def update_attr(self, model):
 
 
 class KDModelEMA(ModelEMA):
-    """ Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models
+    """Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models
     Keep a moving average of everything in the model state_dict (parameters and buffers).
     This is intended to allow functionality like
     https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage
@@ -114,14 +113,15 @@ def __init__(self, kd_model: KDModule, decay: float = 0.9999, beta: float = 15,
                      its final value. beta=15 is ~40% of the training process.
         """
         # Only work on the student (we don't want to update and to have a duplicate of the teacher)
-        super().__init__(model=core_utils.WrappedModel(kd_model.module.student),
-                         decay=decay,
-                         beta=beta,
-                         exp_activation=exp_activation)
+        super().__init__(model=core_utils.WrappedModel(kd_model.module.student), decay=decay, beta=beta, exp_activation=exp_activation)
 
         # Overwrite current ema attribute with combination of the student model EMA (current self.ema)
         # with already the instantiated teacher, to have the final KD EMA
-        self.ema = core_utils.WrappedModel(KDModule(arch_params=kd_model.module.arch_params,
-                                                    student=self.ema.module,
-                                                    teacher=kd_model.module.teacher,
-                                                    run_teacher_on_eval=kd_model.module.run_teacher_on_eval))
+        self.ema = core_utils.WrappedModel(
+            KDModule(
+                arch_params=kd_model.module.arch_params,
+                student=self.ema.module,
+                teacher=kd_model.module.teacher,
+                run_teacher_on_eval=kd_model.module.run_teacher_on_eval,
+            )
+        )

tests/integration_tests/pretrained_models_test.py

@@ -102,7 +102,7 @@ def setUp(self) -> None:
         }
 
         self.coco_pretrained_maps = {
-            "ssd_lite_mobilenet_v2": 0.2052,
+            "ssd_lite_mobilenet_v2": 0.2041,
             "coco_ssd_mobilenet_v1": 0.243,
             "yolox_s": 0.4047,
             "yolox_m": 0.4640,