updated hpo args (#297)

federatedscope/core/configs/README.md

@@ -154,7 +154,7 @@ The configurations related to NbAFL method.
 | `nbafl.w_clip` |      (float) 1.      | The threshold used for weight clipping.    | - |
 | `nbafl.constant` |     (float) 30. | The constant used in NbAFL.                | - |
 
-# SGDMF
+#### SGDMF
 The configurations related to SGDMF method (only used in matrix factorization tasks).
 
 |      Name       | (Type) Default Value | Description                        | Note                                                    |
@@ -165,3 +165,48 @@ The configurations related to SGDMF method (only used in matrix factorization ta
 | `sgdmf.delta` |     (float) 0.5      | The $\delta$ used in DP.           | -                                                       |
 | `sgdmf.constant` |      (float) 1. | The constant in SGDMF | -                                                       |
 | `sgdmf.theta` | (int) -1 | - | -1 means per-rating privacy, otherwise per-user privacy |
+
+### Auto-tuning Components
+
+These arguments are exposed for customizing our provided auto-tuning components.
+
+#### General
+
+| Name | (Type) Default Value | Description                                | Note |
+|:----:|:--------------------:|:-------------------------------------------|:-----|
+| `hpo.working_folder` |     (string) 'hpo'     | Save model checkpoints and search space configurations to this folder.         | Trials in the next stage of an iterative HPO algorithm can restore from the checkpoints of their corresponding last trials. |
+| `hpo.ss` |     (string) 'hpo'     | File path of the .yaml that specifying the search space.         | - |
+| `hpo.num_workers` |     (int) 0     | The number of threads to concurrently attempt different hyperparameter configurations.         | Multi-threading is banned in current version. |
+| `hpo.init_cand_num` |     (int) 16     | The number of initial hyperparameter configurations sampled from the search space.         | - |
+| `hpo.larger_better` |     (bool) False     | The indicator of whether the larger metric is better.         | - |
+| `hpo.scheduler` |     (string) 'rs' </br> Choices: {'rs', 'sha', 'wrap_sha'}     | Which algorithm to use.         | - |
+| `hpo.metric` |     (string) 'client_summarized_weighted_avg.val_loss'     | Metric to be optimized.         | - |
+
+#### Successive Halving Algorithm (SHA)
+
+| Name | (Type) Default Value | Description                                | Note |
+|:----:|:--------------------:|:-------------------------------------------|:-----|
+| `hpo.sha.elim_rate` |     (int) 3     | Reserve only top 1/`hpo.sha.elim_rate` hyperparameter configurations in each state.        | - |
+| `hpo.sha.budgets` |     (list of int) []     | Budgets for each SHA stage.        | - |
+
+
+#### FedEx
+
+| Name | (Type) Default Value | Description                                | Note |
+|:----:|:--------------------:|:-------------------------------------------|:-----|
+| `hpo.fedex.use` |     (bool) False     | Whether to use FedEx.        | - |
+| `hpo.fedex.ss` |     (striing) ''     | Path of the .yaml specifying the search space to be explored.        | - |
+| `hpo.fedex.flatten_ss` |     (bool) True     | Whether the search space has been flattened.        | - |
+| `hpo.fedex.eta0` |     (float) -1.0     | Initial learning rate.        | -1.0 means automatically determine the learning rate based on the size of search space. |
+| `hpo.fedex.sched` |     (string) 'auto' </br> Choices: {'auto', 'adaptive', 'aggressive', 'constant', 'scale' } | The strategy to update step sizes    | - |
+| `hpo.fedex.cutoff` |     (float) 0.0 | The entropy level below which to stop updating the config.        | - |
+| `hpo.fedex.gamma` |     (float) 0.0 | The discount factor; 0.0 is most recent, 1.0 is mean.        | - |
+| `hpo.fedex.diff` |     (bool) False | Whether to use the difference of validation losses before and after the local update as the reward signal.        | - |
+
+#### Wrappers for FedEx 
+
+| Name | (Type) Default Value | Description                                | Note |
+|:----:|:--------------------:|:-------------------------------------------|:-----|
+| `hpo.table.eps` |     (float) 0.1 | The probability to make local perturbation.        | Larger values lead to drastically different arms of the bandit FedEx attempts to solve. |
+| `hpo.table.num` |     (int) 27 | The number of arms of the bandit FedEx attempts to solve.        | - |
+| `hpo.table.idx` |     (int) 0 | The key (i.e., name) of the hyperparameter wrapper considers.        | No need to change this argument. |

federatedscope/core/configs/cfg_hpo.py

@@ -11,18 +11,13 @@ def extend_hpo_cfg(cfg):
     cfg.hpo.working_folder = 'hpo'
     cfg.hpo.ss = ''
     cfg.hpo.num_workers = 0
-    # cfg.hpo.init_strategy = 'random'
     cfg.hpo.init_cand_num = 16
-    cfg.hpo.log_scale = False
     cfg.hpo.larger_better = False
     cfg.hpo.scheduler = 'rs'
-    # plot the performance
-    cfg.hpo.plot_interval = 1
     cfg.hpo.metric = 'client_summarized_weighted_avg.val_loss'
 
     # SHA
     cfg.hpo.sha = CN()
-    cfg.hpo.sha.elim_round_num = 3
     cfg.hpo.sha.elim_rate = 3
     cfg.hpo.sha.budgets = []
 
@@ -44,15 +39,12 @@ def extend_hpo_cfg(cfg):
     cfg.hpo.fedex.cutoff = .0
     # discount factor; 0.0 is most recent, 1.0 is mean
     cfg.hpo.fedex.gamma = .0
-    cfg.hpo.fedex.num_arms = 16
     cfg.hpo.fedex.diff = False
 
     # Table
     cfg.hpo.table = CN()
-    cfg.hpo.table.ss = ''
     cfg.hpo.table.eps = 0.1
     cfg.hpo.table.num = 27
-    # cfg.hpo.table.cand = 81
     cfg.hpo.table.idx = 0
 
 
@@ -68,9 +60,7 @@ def assert_hpo_cfg(cfg):
                                      cfg.hpo.scheduler)
     assert cfg.hpo.num_workers >= 0, "#worker should be non-negative but " \
                                      "given {}".format(cfg.hpo.num_workers)
-    assert len(cfg.hpo.sha.budgets) == 0 or len(
-        cfg.hpo.sha.budgets
-    ) == cfg.hpo.sha.elim_round_num, \
+    assert len(cfg.hpo.sha.budgets) > 0, \
         "Either do NOT specify the budgets or specify the budget for each " \
         "SHA iteration, but the given budgets is {}".format(
             cfg.hpo.sha.budgets)