init the pFL-bench scripts

benchmark/pFL_Bench/FEMNIST-s02/fedavg_convnet2_on_femnist.yaml

@@ -0,0 +1,36 @@
+use_gpu: True
+device: -1
+early_stop:
+  patience: 5
+seed: 1
+federate:
+  mode: standalone
+  local_update_steps: 1
+  batch_or_epoch: epoch
+  total_round_num: 1000
+  sample_client_rate: 0.2
+  unseen_clients_rate: 0.2
+data:
+  root: data/
+  type: femnist
+  splits: [0.6,0.2,0.2]
+  batch_size: 32
+  subsample: 0.05
+  num_workers: 0
+  transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]]
+model:
+  type: convnet2
+  hidden: 2048
+  out_channels: 62
+  dropout: 0.0
+optimizer:
+  lr: 0.01
+  weight_decay: 0.0
+  grad_clip: 5.0
+criterion:
+  type: CrossEntropyLoss
+trainer:
+  type: cvtrainer
+eval:
+  freq: 10
+  metrics: ['acc', 'correct']

benchmark/pFL_Bench/FEMNIST-s02/run_fedopt_bn_plus_sweep.sh

@@ -0,0 +1,17 @@
+set -e
+
+wandb sweep sweep_fedOpt.yaml
+wandb sweep sweep_fedOpt_FT.yaml
+
+wandb sweep sweep_ditto_fedBN.yaml
+wandb sweep sweep_ditto_fedBN_fedOpt.yaml
+wandb sweep sweep_ditto_FT_fedBN.yaml
+wandb sweep sweep_ditto_FT_fedBN_fedOpt.yaml
+
+wandb sweep sweep_fedBN_fedOpt.yaml
+wandb sweep sweep_fedBN_FT_fedOpt.yaml
+
+wandb sweep sweep_fedEM_fedBN.yaml
+wandb sweep sweep_fedEM_fedBN_fedOpt.yaml
+wandb sweep sweep_fedEM_FT_fedBN.yaml
+wandb sweep sweep_fedEM_FT_fedBN_fedOpt.yaml

benchmark/pFL_Bench/FEMNIST-s02/sweep_ditto.yaml

@@ -0,0 +1,38 @@
+
+program: federatedscope/main.py
+project: pFL-bench
+name: ditto,FEMNIST-s02
+method: grid
+#method: bayes
+metric:
+  goal: maximize
+  name: best_client_summarized_weighted_avg/val_acc
+command:
+  - ${env}
+  - ${interpreter}
+  - ${program}
+  - "--cfg"
+  - "scripts/personalization_exp_scripts/pfl_bench/FEMNIST-s02/fedavg_convnet2_on_femnist.yaml"
+  - "outdir"
+  - "exp_pfl_bench"
+  - "wandb.use"
+  - "True"
+  - "wandb.name_project"
+  - "pFL-bench"
+  - "wandb.name_user"
+  - "daoyuan"
+  - "federate.method"
+  - "Ditto"
+  - ${args_no_hyphens}
+parameters:
+  optimizer.lr:
+    values: [0.05, 0.005, 0.5, 0.01, 0.1]
+  federate.local_update_steps:
+    values: [1, 3]
+  personalization.regular_weight:
+    values: [0.05, 0.1, 0.5, 0.8]
+
+early_terminate:
+  type: hyperband
+  min_iter: 5  # the first bucket indicates we called wandb at least 5 times
+

benchmark/pFL_Bench/FEMNIST-s02/sweep_fedAvg_FT.yaml

@@ -0,0 +1,40 @@
+
+program: federatedscope/main.py
+project: pFL-bench
+name: fedAvg,FEMNIST-s02,FT
+method: grid
+#method: bayes
+metric:
+  goal: maximize
+  name: best_client_summarized_weighted_avg/val_acc
+command:
+  - ${env}
+  - ${interpreter}
+  - ${program}
+  - "--cfg"
+  - "scripts/personalization_exp_scripts/pfl_bench/FEMNIST-s02/fedavg_convnet2_on_femnist.yaml"
+  - "outdir"
+  - "exp_pfl_bench"
+  - "wandb.use"
+  - "True"
+  - "wandb.name_project"
+  - "pFL-bench"
+  - "wandb.name_user"
+  - "daoyuan"
+  - "expname_tag"
+  - "finetune"
+  - "trainer.finetune.before_eval"
+  - "True"
+  - "trainer.finetune.steps"
+  - "5"
+  - ${args_no_hyphens}
+parameters:
+  optimizer.lr:
+    values: [0.05, 0.005, 0.5, 0.01, 0.1]
+  federate.local_update_steps:
+    values: [1, 3]
+
+early_terminate:
+  type: hyperband
+  min_iter: 5  # the first bucket indicates we called wandb at least 5 times
+

benchmark/pFL_Bench/FEMNIST-s02/sweep_fedEM_FT_fedBN_fedOpt.yaml

@@ -0,0 +1,50 @@
+
+program: federatedscope/main.py
+project: pFL-bench
+name: FedEM-FT-FedBN+fedOpt,FEMNIST-s02
+method: grid
+#method: bayes
+metric:
+  goal: maximize
+  name: best_client_summarized_weighted_avg/val_acc
+command:
+  - ${env}
+  - ${interpreter}
+  - ${program}
+  - "--cfg"
+  - "scripts/personalization_exp_scripts/pfl_bench/FEMNIST-s02/fedavg_convnet2_on_femnist.yaml"
+  - "outdir"
+  - "exp_pfl_bench"
+  - "wandb.use"
+  - "True"
+  - "wandb.name_project"
+  - "pFL-bench"
+  - "wandb.name_user"
+  - "daoyuan"
+  - "federate.method"
+  - "FedEM"
+  - "model.model_num_per_trainer"
+  - "3"
+  - "expname_tag"
+  - "+fedBN+finetune+fedOpt"
+  - "trainer.finetune.before_eval"
+  - "True"
+  - "trainer.finetune.steps"
+  - "5"
+  - "personalization.local_param"
+  - "['bn', 'norms']"  # FedBN
+  - "fedopt.use"
+  - "True"
+  - ${args_no_hyphens}
+parameters:
+  optimizer.lr:
+    values: [0.05, 0.005, 0.5, 0.01, 0.1]
+  federate.local_update_steps:
+    values: [1, 3]
+  fedopt.lr_server:
+    values: [ 0.5, 0.05, 1.5, 0.1, 1.0]
+
+early_terminate:
+  type: hyperband
+  min_iter: 5  # the first bucket indicates we called wandb at least 5 times
+

benchmark/pFL_Bench/README.md

@@ -0,0 +1,106 @@
+# pFL-Bench
+The **pFL-Bench** is a comprehensive benchmark for personalized Federated Learning (pFL), which contains more than 10 diverse datasets, 20 competitive pFL baselines, and systematic evaluation with highlighted benefits and potential of pFL. See more details in our [paper](https://arxiv.org/abs/2206.03655).
+
+
+This repository includes the experimental data, environments, scripts and codes of **pFL-Bench**. We welcome contributions of new pFL methods and datasets to keep pFL-Bench up-to-date and to evolve it! See more details about contribution [here](https://github.com/alibaba/FederatedScope#contributing).
+
+**NOTICE:** We are working on seamlessly and consistently fusing the new features in pFL-Bench into the *FederatedScope*. However, since the underling package *FederatedScope* is still being continuously and actively updated, the results can be a little different to the ones in our paper.
+To fully reproduce the experimental results reported in the paper, please use the code versioned by this [branch](https://github.com/alibaba/FederatedScope/tree/Feature/pfl_bench) on which the experiments were conducted at the time.
+
+
+# 1. Data
+All the experimental data can be automatically downloaded and processed via *FederatedScope* from the original public data links.
+
+In case the slow or blocked internet connection prevents your downloading, we also provide a public [mirror](https://federatedscope.oss-cn-beijing.aliyuncs.com/pFL-Bench-data.zip) with *aliyun*.
+You can download the pre-processed datasets from our public mirror and unzip them in `data` directory under your project root.
+
+If you use other customized data directory, please replace the value of `data.root` in the scripts accordingly.
+
+# 2. Docker Environment
+The experiments are conducted on the *federatedscope-torch1.8-application* docker image, you can build it using the [Dockfile](https://github.com/alibaba/FederatedScope/blob/master/enviroment/docker_files/federatedscope-torch1.8-application.Dockerfile). 
+
+We also provide a built docker [image](https://federatedscope.oss-cn-beijing.aliyuncs.com/federatedscope_cuda10_torch18_app.tar), you can download it and creat your image as 
+```
+docker load < federatedscope_cuda10_torch18_app.tar & docker tag 188b4 alibaba/federatedscope:app-env-torch1.8
+```
+
+# 3. Run the experiments
+We first use wandb sweep to find the best hyper-parameters, then repeat the results three times.
+Here we provide some examples for FEMNIST-s02 at `benchmark/pfl_bench/FEMNIST-s02` for the hyper-parameter search space and hyper-parameter optimization (HPO) scripts, and the searched best configurations as yaml files for FEMNIST-s02 at `benchmark/pfl_bench/yaml_best_runs_example`.
+
+Since the searching scripts and best config yaml files for all experiments involve about 600 files and 6w+ code lines, we omit them here.
+
+You can find the full scripts from the another [branch](https://github.com/alibaba/FederatedScope/tree/Feature/pfl_bench/scripts/personalization_exp_scripts/pfl_bench) or the packed small zip [file](https://federatedscope.oss-cn-beijing.aliyuncs.com/pfl_bench_scripts.zip), in which we organize the scripts for all the methods and all the datasets as multiple directories named by dataset name.
+
+## 3.1 The searched best configs
+We put all the config yaml file in the directory `benchmark/pfl_bench/yaml_best_runs`.
+To reproduce the experiments with searched best configurations, you can run the experiment as the following example:
+```
+python federatedscope/main.py --cfg benchmark/pfl_bench/yaml_best_runs/FedBN_FEMNIST-s02.yaml
+```
+Then all the metrics will be tracked in your logfile and send to wandb monitor.
+
+You can customize the yaml file such as your wandb project name, or directly add new config in the command such as 
+```
+python federatedscope/main.py --cfg benchmark/pfl_bench/yaml_best_runs/FedBN_FEMNIST-s02.yaml federate.local_update_steps 1
+```
+More examples for other methods including the combined pFL method variants (e.g., `FedEM-FedBN-FedOPT-FT_cifar10-alpha01.yaml`) are in the directory `benchmark/pfl_bench/yaml_best_runs`.
+
+## 3.2 Scripts for HPO
+We use wandb sweep to find the best hyper-parameters, here are some scripts to do the sweep,
+
+### 3.2.1 For sweep machine
+0. login to the wandb host, if you need private hosting, try wandb login [here](https://docs.wandb.ai/guides/self-hosted/local).
+1. write your sweep HPO scripts, we provide the full HPO yamls in the `benchmark/pfl_bench` directory and organized by dataset name. See more details about sweep [here](https://docs.wandb.ai/guides/sweeps).
+
+2. start your sweep by `wandb sweep my_hpo.yaml`, it will print the sweep id such as 
+```
+wandb: Creating sweep from: sweep_fedAvg_FT.yaml
+wandb: Created sweep with ID: mo45xa3d
+wandb: View sweep at: http://xx.xx.xxx.xxx:8080/your_sweep_name/pFL-bench/sweeps/mo45xa3d
+```
+
+
+### 3.2.2 For agent machine 
+0. - sync your FederatedScope codes to the agent machine
+    - sync your data to the agent machine, and make sure you put them in the right path, e.g., `/mnt1/user_name/FederatedScope`
+
+
+1. enter the container
+```
+docker run -u root --gpus all -it --rm -v "/mnt1:/mnt" --name your_name-pfl-bench -w /mnt/user_name/FederatedScope alibaba/federatedscope:app-env-torch1.8 /bin/bash
+```
+
+2. setup wandb and FederatedScope
+```bash
+wandb login --host=http://xx.xx.xx.xx:8080/
+python setup.py install
+```
+
+If necessary, install several missing packages in case of the docker image misses these package
+`conda install fvcore pympler iopath`
+
+
+3. run sweep agent, e.g.,
+```bash
+nohup wandb agent your_name/pFL-bench/sweep_id &
+```
+
+### 3.2.3 For develop/debug machine
+For the machine used for remote development and debug
+1. enter the container:
+```
+docker run -u root -p 8023:22 --gpus all -it --rm -v "/mnt1:/mnt"  --name your_name-pfl-bench-debug -w /mnt/user_name/FederatedScope alibaba/federatedscope:app-env-torch1.8 /bin/bash
+```
+
+2. prepare the ssh and wandb
+```bash
+apt-get update && apt-get install -y openssh-server
+mkdir /var/run/sshd
+echo 'root:fsdebug' | chpasswd
+sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+service ssh start
+wandb login --host=http://xx.xx.xx.xx:8080/
+```
+
+3. connect the machine and develop your pFL algorithm

benchmark/pFL_Bench/res_analysis_plot/get_total_run_time.py

@@ -0,0 +1,52 @@
+
+import wandb
+
+api = wandb.Api()
+
+name_project = ["daoyuan/pFL-bench",
+                "daoyuan/pfl-bench-best-repeat"]
+
+total_run_time = 0
+run_cnt = 0
+run_finish_cnt = 0
+total_run_time_finished = 0
+
+
+def convert(seconds):
+    seconds_in_day = 60 * 60 * 24
+    seconds_in_hour = 60 * 60
+    seconds_in_minute = 60
+
+    days = seconds // seconds_in_day
+    hours = (seconds - (days * seconds_in_day)) // seconds_in_hour
+    minutes = (seconds - (days * seconds_in_day) - (hours * seconds_in_hour)) // seconds_in_minute
+
+    return "%d:%02d:%02d" % (days, hours, minutes)
+
+
+def print_run_time():
+    print(f"Total_run_t: {convert(total_run_time)}, run_cnt={run_cnt}")
+    print(f"Total_run_t_finished: {convert(total_run_time_finished)}, run_cnt_finish={run_finish_cnt}")
+
+
+for p in name_project:
+    runs = api.runs(p)
+    for run in runs:
+        try:
+            if '_runtime' in run.summary:
+                time_run = run.summary["_runtime"]
+            else:
+                time_run = run.summary['_wandb']["runtime"]
+
+            if run.state == "finished":
+                total_run_time_finished += time_run
+                run_finish_cnt += 1
+            total_run_time += time_run
+            run_cnt += 1
+            if run_cnt % 200 == 0:
+                print_run_time()
+        except:
+            #print("something wrong")
+            continue
+
+print_run_time()