Add condition param to autotune; Add feature engineering module

.github/workflows/test_autotune.yml

@@ -0,0 +1,35 @@
+name: UnitTests for Autotune Module
+
+on: [push, pull_request]
+
+jobs:
+  run:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9']
+        torch-version: ['1.10.1']
+        torchvision-version: ['0.11.2']
+        torchaudio-version: ['0.10.1']
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: '3.9'
+    steps:
+    - uses: actions/checkout@master
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@master
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install PyTorch ${{ matrix.torch-version }}+cpu
+      run: |
+        pip install numpy typing-extensions dataclasses
+        pip install torch==${{ matrix.torch-version}}+cpu torchvision==${{matrix.torchvision-version}}+cpu torchaudio==${{matrix.torchaudio-version}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+    - name: Install FS
+      run: |
+        pip install -e .[test,hpo]
+    - name: Test Autotune
+      run: |
+        python federatedscope/hpo.py --cfg federatedscope/autotune/baseline/fedhpo_vfl.yaml
+        [ $? -eq 1 ] && exit 1 || echo "Passed"

federatedscope/autotune/__init__.py

@@ -2,8 +2,9 @@
 from federatedscope.autotune.utils import parse_search_space, \
     config2cmdargs, config2str
 from federatedscope.autotune.algos import get_scheduler
+from federatedscope.autotune.run import run_scheduler
 
 __all__ = [
     'Continuous', 'Discrete', 'parse_search_space', 'config2cmdargs',
-    'config2str', 'get_scheduler'
+    'config2str', 'get_scheduler', 'run_scheduler'
 ]

federatedscope/autotune/algos.py

@@ -1,7 +1,6 @@
 import os
 import logging
 from copy import deepcopy
-from contextlib import redirect_stdout
 import threading
 import math
 
@@ -15,7 +14,7 @@
     get_server_cls
 from federatedscope.core.auxiliaries.runner_builder import get_runner
 from federatedscope.autotune.utils import parse_search_space, \
-    config2cmdargs, config2str, summarize_hpo_results
+    config2cmdargs, config2str, summarize_hpo_results, log2wandb
 
 logger = logging.getLogger(__name__)
 
@@ -25,14 +24,13 @@ def make_trial(trial_cfg, client_cfgs=None):
     data, modified_config = get_data(config=trial_cfg.clone())
     trial_cfg.merge_from_other_cfg(modified_config)
     trial_cfg.freeze()
-    Fed_runner = get_runner(data=data,
+    fed_runner = get_runner(data=data,
                             server_class=get_server_cls(trial_cfg),
                             client_class=get_client_cls(trial_cfg),
                             config=trial_cfg.clone(),
-                            client_config=client_cfgs)
-    results = Fed_runner.run()
-    key1, key2 = trial_cfg.hpo.metric.split('.')
-    return results[key1][key2]
+                            client_configs=client_cfgs)
+    results = fed_runner.run()
+    return results
 
 
 class TrialExecutor(threading.Thread):
@@ -57,7 +55,7 @@ def run(self):
                                 server_class=get_server_cls(self._trial_cfg),
                                 client_class=get_client_cls(self._trial_cfg),
                                 config=self._trial_cfg.clone(),
-                                client_config=self._client_cfgs)
+                                client_configs=self._client_cfgs)
         results = Fed_runner.run()
         key1, key2 = self._trial_cfg.hpo.metric.split('.')
         self._returns['perf'] = results[key1][key2]
@@ -177,6 +175,7 @@ def _evaluate(self, configs):
                     completed_trial_results = thread_results[i]
                     cfg_idx = completed_trial_results['cfg_idx']
                     perfs[cfg_idx] = completed_trial_results['perf']
+                    # TODO: Support num_worker in WandB
                     logger.info(
                         "Evaluate the {}-th config {} and get performance {}".
                         format(cfg_idx, configs[cfg_idx], perfs[cfg_idx]))
@@ -187,21 +186,24 @@ def _evaluate(self, configs):
             for i, config in enumerate(configs):
                 trial_cfg = self._cfg.clone()
                 trial_cfg.merge_from_list(config2cmdargs(config))
-                perfs[i] = make_trial(trial_cfg, self._client_cfgs)
+                results = make_trial(trial_cfg, self._client_cfgs)
+                key1, key2 = trial_cfg.hpo.metric.split('.')
+                perfs[i] = results[key1][key2]
                 logger.info(
                     "Evaluate the {}-th config {} and get performance {}".
                     format(i, config, perfs[i]))
-
+                if self._cfg.wandb.use:
+                    log2wandb(i, config, results, trial_cfg)
         return perfs
 
     def optimize(self):
         perfs = self._evaluate(self._init_configs)
-
         results = summarize_hpo_results(self._init_configs,
                                         perfs,
                                         white_list=set(
                                             self._search_space.keys()),
-                                        desc=self._cfg.hpo.larger_better)
+                                        desc=self._cfg.hpo.larger_better,
+                                        use_wandb=self._cfg.wandb.use)
         logger.info(
             "========================== HPO Final ==========================")
         logger.info("\n{}".format(results))
@@ -263,7 +265,8 @@ def optimize(self):
                 current_configs,
                 current_perfs,
                 white_list=set(self._search_space.keys()),
-                desc=self._cfg.hpo.larger_better)
+                desc=self._cfg.hpo.larger_better,
+                use_wandb=self._cfg.wandb.use)
             self._stage += 1
             logger.info(
                 "========================== Stage{} =========================="

federatedscope/autotune/baseline/fedhpo_vfl.yaml

@@ -0,0 +1,54 @@
+use_gpu: False
+device: 0
+backend: torch
+outdir: vFL_adult
+federate:
+  mode: standalone
+  client_num: 2
+  total_round_num: 30
+model:
+  type: lr
+  use_bias: False
+train:
+  optimizer:
+    lr: 0.5
+    bin_num: 100
+    lambda_: 0.1
+    gamma: 0
+    num_of_trees: 5
+    max_tree_depth: 3
+xgb_base:
+  use: True
+  use_bin: False
+data:
+  root: data/
+  type: adult
+  splits: [1.0, 0.0]
+  args: [{normalization: False, standardization: True}]
+feat_engr:
+  scenario: vfl
+dataloader:
+  type: raw
+  batch_size: 50
+criterion:
+  type: CrossEntropyLoss
+trainer:
+  type: none
+vertical_dims: [7, 14]
+vertical:
+  use: False
+  key_size: 256
+eval:
+  freq: 5
+  best_res_update_round_wise_key: test_loss
+hpo:
+  scheduler: sha
+  num_workers: 0
+  init_cand_num: 9
+  ss: 'federatedscope/autotune/baseline/vfl_ss.yaml'
+  sha:
+    budgets: [ 3, 9 ]
+    elim_rate: 3
+    iter: 1
+  metric: 'server_global_eval.test_loss'
+  working_folder: sha

federatedscope/autotune/baseline/vfl_ss.yaml

@@ -0,0 +1,25 @@
+train.optimizer.lr:
+  type: float
+  lower: 0.01
+  upper: 1.0
+  log: True
+train.optimizer.num_of_trees:
+  type: int
+  lower: 3
+  upper: 5
+vertical.use:
+  type: cate
+  choices: [True, False]
+feat_engr.type:
+  type: cate
+  choices: ['', 'min_max_norm', 'instance_norm', 'standardization', 'log_transform', 'uniform_binning', 'quantile_binning', 'correlation_filter', 'variance_filter', 'iv_filter']
+condition1:
+  type: equal
+  child: train.optimizer.num_of_trees
+  parent: vertical.use
+  value: False
+condition2:
+  type: equal
+  child: train.optimizer.lr
+  parent: vertical.use
+  value: True

federatedscope/autotune/hpbandster.py

@@ -9,7 +9,7 @@
 from hpbandster.core.worker import Worker
 from hpbandster.optimizers import BOHB, HyperBand, RandomSearch
 
-from federatedscope.autotune.utils import eval_in_fs
+from federatedscope.autotune.utils import eval_in_fs, log2wandb
 
 logging.basicConfig(level=logging.WARNING)
 logger = logging.getLogger(__name__)
@@ -63,27 +63,33 @@ def __init__(self,
         super(MyWorker, self).__init__(**kwargs)
         self.sleep_interval = sleep_interval
         self.cfg = cfg
+        self.client_cfgs = client_cfgs
         self._ss = ss
         self._init_configs = []
         self._perfs = []
 
     def compute(self, config, budget, **kwargs):
-        res = eval_in_fs(self.cfg, config, int(budget), self.client_cfgs)
+        results = eval_in_fs(self.cfg, config, int(budget), self.client_cfgs)
+        key1, key2 = self.cfg.hpo.metric.split('.')
+        res = results[key1][key2]
         config = dict(config)
         config['federate.total_round_num'] = budget
         self._init_configs.append(config)
         self._perfs.append(float(res))
         time.sleep(self.sleep_interval)
         logger.info(f'Evaluate the {len(self._perfs)-1}-th config '
                     f'{config}, and get performance {res}')
+        if self.cfg.wandb.use:
+            log2wandb(len(self._perfs) - 1, config, results, self.cfg)
         return {'loss': float(res), 'info': res}
 
     def summarize(self):
         from federatedscope.autotune.utils import summarize_hpo_results
         results = summarize_hpo_results(self._init_configs,
                                         self._perfs,
                                         white_list=set(self._ss.keys()),
-                                        desc=self.cfg.hpo.larger_better)
+                                        desc=self.cfg.hpo.larger_better,
+                                        use_wandb=self.cfg.wandb.use)
         logger.info(
             "========================== HPO Final ==========================")
         logger.info("\n{}".format(results))

federatedscope/autotune/run.py

@@ -0,0 +1,14 @@
+def run_scheduler(scheduler, cfg, client_cfgs=None):
+    if cfg.hpo.scheduler in ['sha', 'wrap_sha']:
+        _ = scheduler.optimize()
+    elif cfg.hpo.scheduler in [
+            'rs', 'bo_kde', 'hb', 'bohb', 'wrap_rs', 'wrap_bo_kde', 'wrap_hb',
+            'wrap_bohb'
+    ]:
+        from federatedscope.autotune.hpbandster import run_hpbandster
+        run_hpbandster(cfg, scheduler, client_cfgs)
+    elif cfg.hpo.scheduler in ['bo_gp', 'bo_rf', 'wrap_bo_gp', 'wrap_bo_rf']:
+        from federatedscope.autotune.smac import run_smac
+        run_smac(cfg, scheduler, client_cfgs)
+    else:
+        raise ValueError(f'No scheduler named {cfg.hpo.scheduler}')

federatedscope/autotune/smac.py

@@ -1,7 +1,7 @@
 import logging
 import numpy as np
 import ConfigSpace as CS
-from federatedscope.autotune.utils import eval_in_fs
+from federatedscope.autotune.utils import eval_in_fs, log2wandb
 from smac.facade.smac_bb_facade import SMAC4BB
 from smac.facade.smac_hpo_facade import SMAC4HPO
 from smac.scenario.scenario import Scenario
@@ -16,21 +16,26 @@ def run_smac(cfg, scheduler, client_cfgs=None):
 
     def optimization_function_wrapper(config):
         budget = cfg.hpo.sha.budgets[-1]
-        res = eval_in_fs(cfg, config, budget, client_cfgs)
+        results = eval_in_fs(cfg, config, budget, client_cfgs)
+        key1, key2 = cfg.hpo.metric.split('.')
+        res = results[key1][key2]
         config = dict(config)
         config['federate.total_round_num'] = budget
         init_configs.append(config)
         perfs.append(res)
         logger.info(f'Evaluate the {len(perfs)-1}-th config '
                     f'{config}, and get performance {res}')
+        if cfg.wandb.use:
+            log2wandb(len(perfs) - 1, config, results, cfg)
         return res
 
     def summarize():
         from federatedscope.autotune.utils import summarize_hpo_results
         results = summarize_hpo_results(init_configs,
                                         perfs,
                                         white_list=set(config_space.keys()),
-                                        desc=cfg.hpo.larger_better)
+                                        desc=cfg.hpo.larger_better,
+                                        use_wandb=cfg.wandb.use)
         logger.info(
             "========================== HPO Final ==========================")
         logger.info("\n{}".format(results))