Move RobustBench and Carla stuff from mart to examples

Makefile

@@ -46,65 +46,8 @@ debug: ## Enter debugging mode with pdb, an example.
 	#
 	python -m pdb -m mart experiment=CIFAR10_CNN debug=default
 
-.PHONY: cifar_attack
-cifar_attack: ## Evaluate adversarial robustness of a CIFAR-10 model from robustbench, expect 0.6171875
-	python -m mart experiment=CIFAR10_RobustBench \
-	trainer=gpu \
-	fit=false \
-	+trainer.limit_test_batches=1 \
-	[email protected]_adv_test=classification_autoattack \
-	+model.test_sequence.seq005=input_adv_test \
-	model.test_sequence.seq010.preprocessor=["input_adv_test"]
-
 .PHONY: cifar_train
 cifar_train: ## Adversarial training for a CIFAR-10 model.
 	python -m mart experiment=CIFAR10_CNN_Adv \
 	fit=true \
 	trainer=gpu
-
-
-# Download and extract dataset of carla_over_obj_det
-CARLA_OVERHEAD_DATASET_TRAIN ?= data/carla_over_obj_det/train/kwcoco_annotations.json
-CARLA_OVERHEAD_DATASET_DEV ?= data/carla_over_obj_det/dev/kwcoco_annotations.json
-
-data/carla_over_obj_det/carla_over_od_dev_1.0.0.tar.gz:
-	mkdir -p $(@D)
-	wget -O $@ https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_1.0.0.tar.gz
-
-$(CARLA_OVERHEAD_DATASET_DEV): data/carla_over_obj_det/carla_over_od_dev_1.0.0.tar.gz
-	tar -zxf $<  -C data/carla_over_obj_det
-
-data/carla_over_obj_det/carla_over_od_train_val_1.0.0.tar.gz:
-	mkdir -p $(@D)
-	wget -O $@ https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_train_val_1.0.0.tar.gz
-
-$(CARLA_OVERHEAD_DATASET_TRAIN): data/carla_over_obj_det/carla_over_od_train_val_1.0.0.tar.gz
-	tar -zxf $<  -C data/carla_over_obj_det
-
-
-.PHONY: carla_train
-carla_train: $(CARLA_OVERHEAD_DATASET_TRAIN) $(CARLA_OVERHEAD_DATASET_DEV) ## Train Faster R-CNN with the CarlaOverObjDet dataset from Armory.
-	python -m mart \
-	experiment=ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
-	trainer=gpu \
-	trainer.precision=16 \
-	fit=true \
-	tags=["regular_training","backbone_ImageNetPretrained"] \
-
-
-# You need to specify weights of target model in [model.modules.losses_and_detections.model.weights_fpath].
-.PHONY: carla_attack
-carla_attack: $(CARLA_OVERHEAD_DATASET_TRAIN) $(CARLA_OVERHEAD_DATASET_DEV) ## Evaluate adversarial robustness of a pretrained model.
-	python -m mart \
-	experiment=ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
-	trainer=gpu \
-	fit=false \
-	model.modules.losses_and_detections.model.weights_fpath=null \
-	[email protected]_adv_test=object_detection_mask_adversary \
-	model.modules.input_adv_test.optimizer.lr=5 \
-	model.modules.input_adv_test.max_iters=50 \
-	+model.test_sequence.seq001.input_adv_test._call_with_args_=[input,target] \
-	+model.test_sequence.seq001.input_adv_test.model=model \
-	+model.test_sequence.seq001.input_adv_test.step=step \
-	model.test_sequence.seq010.preprocessor=[input_adv_test] \
-	tags=["MaskPGD50_LR5"]

README.md

@@ -68,41 +68,12 @@ pre-commit install
 
 ## How to run
 
-The [configs](/configs) folder is required to run the toolkit. You can evaluate adversarial robustness of pretrained models with chosen experiment configuration from [configs/experiment/](configs/experiment/). Feel free to override any parameter from command line. Run `make` to learn more tasks pre-defined in [Makefile](Makefile).
+The toolkit comes with built-in experiment configurations in [mart/configs](mart/configs).
 
-```bash
-# run on CPU
-python -m mart experiment=CIFAR10_RobustBench \
-	trainer=default \
-	fit=false \
-	+trainer.limit_test_batches=1 \
-	[email protected]_adv_test=classification_eps8_pgd10_step1
-
-# run on GPU
-python -m mart experiment=CIFAR10_RobustBench \
-	trainer=gpu \
-	fit=false \
-	+trainer.limit_test_batches=1 \
-	[email protected]_adv_test=classification_eps8_pgd10_step1
-
-# train on 1 GPU
-python -m mart experiment=ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
-	task_name=1GPU_ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
-	trainer=gpu \
-	fit=true
-
-# train on multiple GPUs using Distributed Data Parallel
-python -m mart experiment=ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
-	task_name=2GPUs_ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
-	fit=true \
-	trainer=ddp \
-	trainer.devices=2 \
-	datamodule.ims_per_batch=4 \
-	model.optimizer.lr=0.025 \
-	trainer.max_steps=5244
-```
+For example, you can run a fast adversarial training experiment on CIFAR-10 with `python -m mart experiment=CIFAR10_CNN_Adv`.
+Running on GPU will make it even faster `CUDA_VISIBLE_DEVICES=0 python -m mart experiment=CIFAR10_CNN_Adv trainer=gpu trainer.precision=16`.
 
-You can also install the repository as a package, then run `python -m mart` from anywhere with your own `configs` folder.
+You can see other examples in [examples](/examples).
 
 ## Acknowledgements
 

examples/README.md

@@ -0,0 +1,15 @@
+# How to use Modular Adversarial Robustness Toolkit
+
+We provide examples on how to use the toolkit in your project.
+
+A typical procedure is
+
+1. Install the toolkit as a Python package `pip install https://github.com/IntelLabs/MART/archive/refs/heads/main.zip`;
+2. Create a `configs` folder;
+3. Add your configurations in `configs`;
+4. Run experiments at the folder that contains `configs`.
+
+The toolkit searches configurations in the order of `./configs` and `mart.configs`.
+Local configurations in `./configs` precede those built-in configurations in `mart/configs`.
+
+You can find specific examples in sub-folders.

examples/carla_overhead_object_detection/Makefile

@@ -0,0 +1,45 @@
+# Download and extract dataset of carla_over_obj_det
+CARLA_OVERHEAD_DATASET_TRAIN ?= data/carla_over_obj_det/train/kwcoco_annotations.json
+CARLA_OVERHEAD_DATASET_DEV ?= data/carla_over_obj_det/dev/kwcoco_annotations.json
+
+data/carla_over_obj_det/carla_over_od_dev_2.0.0.tar.gz:
+	mkdir -p $(@D)
+	wget -O $@ https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_dev_2.0.0.tar.gz
+
+$(CARLA_OVERHEAD_DATASET_DEV): data/carla_over_obj_det/carla_over_od_dev_2.0.0.tar.gz
+	tar -zxf $<  -C data/carla_over_obj_det
+
+data/carla_over_obj_det/carla_over_od_train_val_1.0.0.tar.gz:
+	mkdir -p $(@D)
+	wget -O $@ https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_over_od_train_val_1.0.0.tar.gz
+
+$(CARLA_OVERHEAD_DATASET_TRAIN): data/carla_over_obj_det/carla_over_od_train_val_1.0.0.tar.gz
+	tar -zxf $<  -C data/carla_over_obj_det
+
+
+.PHONY: carla_train
+carla_train: $(CARLA_OVERHEAD_DATASET_TRAIN) $(CARLA_OVERHEAD_DATASET_DEV) ## Train Faster R-CNN with the CarlaOverObjDet dataset from Armory.
+	python -m mart \
+	experiment=ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
+	trainer=gpu \
+	trainer.precision=16 \
+	fit=true \
+	tags=["regular_training","backbone_ImageNetPretrained"] \
+
+
+# You need to specify weights of target model in [model.modules.losses_and_detections.model.weights_fpath].
+.PHONY: carla_attack
+carla_attack: $(CARLA_OVERHEAD_DATASET_TRAIN) $(CARLA_OVERHEAD_DATASET_DEV) ## Evaluate adversarial robustness of a pretrained model.
+	python -m mart \
+	experiment=ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
+	trainer=gpu \
+	fit=false \
+	model.modules.losses_and_detections.model.weights_fpath=null \
+	[email protected]_adv_test=object_detection_mask_adversary \
+	model.modules.input_adv_test.optimizer.lr=5 \
+	model.modules.input_adv_test.max_iters=50 \
+	+model.test_sequence.seq001.input_adv_test._call_with_args_=[input,target] \
+	+model.test_sequence.seq001.input_adv_test.model=model \
+	+model.test_sequence.seq001.input_adv_test.step=step \
+	model.test_sequence.seq010.preprocessor=[input_adv_test] \
+	# tags=["MaskPGD50_LR5"]

examples/carla_overhead_object_detection/README.md

@@ -0,0 +1,29 @@
+# Introduction
+
+This example shows how to use MART to train an object detection model on the Carla overhead dataset.
+
+## Installation
+
+```bash
+pip install -r requirements.txt
+```
+
+```bash
+# train on 1 GPU
+CUDA_VISIBLE_DEVICES=0 \
+python -m mart experiment=ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
+	task_name=1GPU_ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
+	trainer=gpu \
+	fit=true
+
+# train on multiple GPUs using Distributed Data Parallel
+CUDA_VISIBLE_DEVICES=0,1 \
+python -m mart experiment=ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
+	task_name=2GPUs_ArmoryCarlaOverObjDet_TorchvisionFasterRCNN \
+	fit=true \
+	trainer=ddp \
+	trainer.devices=2 \
+	datamodule.ims_per_batch=4 \
+	model.optimizer.lr=0.025 \
+	trainer.max_steps=5244
+```

examples/carla_overhead_object_detection/requirements.txt

@@ -0,0 +1 @@
+mart @ git+https://github.com/IntelLabs/MART.git

examples/carla_overhead_object_detection/tests/conftest.py

@@ -0,0 +1,38 @@
+#
+# Copyright (C) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+import os
+from pathlib import Path
+
+import pyrootutils
+import pytest
+from hydra import compose, initialize
+from hydra.core.global_hydra import GlobalHydra
+from omegaconf import DictConfig
+
+root = Path(os.getcwd())
+pyrootutils.set_root(path=root, dotenv=True, pythonpath=True)
+
+experiments_names = [
+    "ArmoryCarlaOverObjDet_TorchvisionFasterRCNN",
+]
+
+
+# Loads the configuration file from a given experiment
+def get_cfg(experiment):
+    with initialize(version_base="1.2", config_path="../configs"):
+        params = "experiment=" + experiment
+        cfg = compose(config_name="lightning.yaml", return_hydra_config=True, overrides=[params])
+    return cfg
+
+
+@pytest.fixture(scope="function", params=experiments_names)
+def cfg_experiment(request) -> DictConfig:
+    cfg = get_cfg(request.param)
+
+    yield cfg
+
+    GlobalHydra.instance().clear()

examples/carla_overhead_object_detection/tests/helpers

@@ -0,0 +1 @@
+../../../tests/helpers/

examples/carla_overhead_object_detection/tests/test_configs.py

@@ -0,0 +1 @@
+../../../tests/test_configs.py

examples/carla_overhead_object_detection/tests/test_experiments.py

@@ -0,0 +1,69 @@
+import os
+from typing import Dict
+
+import pytest
+from hydra.core.global_hydra import GlobalHydra
+
+from tests.helpers.dataset_generator import FakeCOCODataset
+from tests.helpers.run_if import RunIf
+from tests.helpers.run_sh_command import run_sh_command
+
+module = "mart"
+
+
+@pytest.fixture(scope="function")
+def carla_cfg(tmp_path) -> Dict:
+    # Generate fake CARLA dataset on disk at tmp_path
+    dataset = FakeCOCODataset(tmp_path, config=carla_ds, name="carla_over_obj_det")
+    dataset.generate(num_images=2, num_annotations_per_image=2)
+
+    cfg = {
+        "trainer": [
+            "++trainer.fast_dev_run=3",
+        ],
+        "datamodel": [
+            "++paths.data_dir=" + str(tmp_path),
+            "datamodule.num_workers=0",
+        ],
+    }
+    yield cfg
+
+    GlobalHydra.instance().clear()
+
+
+carla_ds = {
+    "train": {
+        "folder": "train",
+        "modalities": ["rgb"],
+        "ann_folder": "train",
+        "ann_file": "kwcoco_annotations.json",
+    },
+    "val": {
+        "folder": "val",
+        "modalities": ["rgb"],
+        "ann_folder": "val",
+        "ann_file": "kwcoco_annotations.json",
+    },
+    "test": {
+        "folder": "dev",
+        "modalities": ["foreground_mask", "rgb"],
+        "ann_folder": "dev",
+        "ann_file": "kwcoco_annotations.json",
+    },
+}
+
+
+@RunIf(sh=True)
+@pytest.mark.slow
+def test_armory_carla_fasterrcnn_experiment(carla_cfg, tmp_path):
+    """Test Armory CARLA TorchVision FasterRCNN experiment."""
+    overrides = carla_cfg["trainer"] + carla_cfg["datamodel"]
+    command = [
+        "-m",
+        module,
+        "experiment=ArmoryCarlaOverObjDet_TorchvisionFasterRCNN",
+        "[email protected]_adv_test=object_detection_mask_adversary",
+        "hydra.sweep.dir=" + str(tmp_path),
+        "optimized_metric=training/loss_objectness",
+    ] + overrides
+    run_sh_command(command)

examples/robust_bench/README.md

@@ -0,0 +1,48 @@
+# Introduction
+
+This example shows how to use MART to evaluate adversarial robustness of models from RobustBench.
+
+Note that the attack algorithm here is not optimal, just for the demonstration purpose.
+
+The `requirements.txt` contains dependency of MART and RobustBench.
+
+The `./configs` folder contains configurations of the target model `classifier_robustbench` and the MART experiment `CIFAR10_RobustBench`.
+
+The configuration files in `./configs` precedes those in `mart.configs` (MART's built-in configs).
+
+## Installation
+
+```bash
+pip install -r requirements.txt
+```
+
+## How to run
+
+```bash
+# run on CPU
+python -m mart experiment=CIFAR10_RobustBench \
+	trainer=default \
+	fit=false \
+	+trainer.limit_test_batches=1 \
+	[email protected]_adv_test=classification_eps8_pgd10_step1
+
+# run on GPU
+CUDA_VISIBLE_DEVICES=0 \
+python -m mart experiment=CIFAR10_RobustBench \
+	trainer=gpu \
+	fit=false \
+	+trainer.limit_test_batches=1 \
+	[email protected]_adv_test=classification_eps8_pgd10_step1 \
+	+model.test_sequence.seq005=input_adv_test \
+	model.test_sequence.seq010.preprocessor=["input_adv_test"]
+
+# Evaluate with AutoAttack, expect 0.6171875
+CUDA_VISIBLE_DEVICES=0 \
+python -m mart experiment=CIFAR10_RobustBench \
+	trainer=gpu \
+	fit=false \
+	+trainer.limit_test_batches=1 \
+	[email protected]_adv_test=classification_autoattack \
+	+model.test_sequence.seq005=input_adv_test \
+	model.test_sequence.seq010.preprocessor=["input_adv_test"]
+```

examples/robust_bench/requirements.txt

@@ -0,0 +1,2 @@
+mart @ git+https://github.com/IntelLabs/MART.git
+robustbench @ git+https://github.com/RobustBench/robustbench.git@9a590683b7daecf963244dea402529f0d728c727

examples/robust_bench/tests/helpers

@@ -0,0 +1 @@
+../../../tests/helpers/