Hide parameters of Adversary's Perturber from DDP (#166)

* Hide Perturber in a list in Adverary.

* Add both single-GPU and multi-GPU adversarial training as example tests.

* Add estimated time for test experiments.

.github/PULL_REQUEST_TEMPLATE.md

@@ -23,8 +23,9 @@ Please check all relevant options.
 
 Please describe the tests that you ran to verify your changes. Consider listing any relevant details of your test configuration.
 
-- [ ] Test A
-- [ ] Test B
+- [ ] `pytest`
+- [ ] `CUDA_VISIBLE_DEVICES=0 python -m mart experiment=CIFAR10_CNN_Adv trainer=gpu trainer.precision=16` reports 70% (21 sec/epoch).
+- [ ] `CUDA_VISIBLE_DEVICES=0,1 python -m mart experiment=CIFAR10_CNN_Adv trainer=ddp trainer.precision=16 trainer.devices=2 model.optimizer.lr=0.2 trainer.max_steps=2925 datamodule.ims_per_batch=256 datamodule.world_size=2` reports 70% (14 sec/epoch).
 
 ## Before submitting
 

mart/attack/adversary.py

@@ -64,7 +64,9 @@ def __init__(
             lambda state_dict, *args, **kwargs: state_dict.clear()
         )
 
-        self.perturber = perturber
+        # Hide the perturber module in a list, so that perturbation is not exported as a parameter in the model checkpoint.
+        # and DDP won't try to get the uninitialized parameters of perturbation.
+        self._perturber = [perturber]
         self.composer = composer
         self.optimizer = optimizer
         if not isinstance(self.optimizer, OptimizerFactory):
@@ -99,6 +101,12 @@ def __init__(
             assert self._attacker.max_epochs == 0
             assert self._attacker.limit_train_batches > 0
 
+    @property
+    def perturber(self) -> Perturber:
+        # Hide the perturber module in a list, so that perturbation is not exported as a parameter in the model checkpoint,
+        # and DDP won't try to get the uninitialized parameters of perturbation.
+        return self._perturber[0]
+
     def configure_optimizers(self):
         return self.optimizer(self.perturber)
 

tests/test_adversary.py

@@ -135,13 +135,13 @@ def test_hidden_params_after_forward(input_data, target_data, perturbation):
 
     output_data = adversary(input=input_data, target=target_data, model=model, sequence=sequence)
 
-    # Adversarial perturbation will have one parameter after forward is called.
+    # Adversary will have no parameter even after forward is called, because we hide Perturber in a list.
     params = [p for p in adversary.parameters()]
-    assert len(params) == 1
+    assert len(params) == 0
 
-    # Adversarial perturbation should have one state dict item being exported to the model checkpoint.
+    # Adversary should have no state dict item being exported to the model checkpoint, because we hide Perturber in a list.
     state_dict = adversary.state_dict()
-    assert len(state_dict) == 1
+    assert len(state_dict) == 0
 
 
 def test_loading_perturbation_from_state_dict():