diff --git a/federatedscope/core/configs/README.md b/federatedscope/core/configs/README.md index 427bdcc55..4266d6eb5 100644 --- a/federatedscope/core/configs/README.md +++ b/federatedscope/core/configs/README.md @@ -154,7 +154,7 @@ The configurations related to NbAFL method. | `nbafl.w_clip` | (float) 1. | The threshold used for weight clipping. | - | | `nbafl.constant` | (float) 30. | The constant used in NbAFL. | - | -# SGDMF +#### SGDMF The configurations related to SGDMF method (only used in matrix factorization tasks). | Name | (Type) Default Value | Description | Note | @@ -165,3 +165,48 @@ The configurations related to SGDMF method (only used in matrix factorization ta | `sgdmf.delta` | (float) 0.5 | The $\delta$ used in DP. | - | | `sgdmf.constant` | (float) 1. | The constant in SGDMF | - | | `sgdmf.theta` | (int) -1 | - | -1 means per-rating privacy, otherwise per-user privacy | + +### Auto-tuning Components + +These arguments are exposed for customizing our provided auto-tuning components. + +#### General + +| Name | (Type) Default Value | Description | Note | +|:----:|:--------------------:|:-------------------------------------------|:-----| +| `hpo.working_folder` | (string) 'hpo' | Save model checkpoints and search space configurations to this folder. | Trials in the next stage of an iterative HPO algorithm can restore from the checkpoints of their corresponding last trials. | +| `hpo.ss` | (string) 'hpo' | File path of the .yaml that specifying the search space. | - | +| `hpo.num_workers` | (int) 0 | The number of threads to concurrently attempt different hyperparameter configurations. | Multi-threading is banned in current version. | +| `hpo.init_cand_num` | (int) 16 | The number of initial hyperparameter configurations sampled from the search space. | - | +| `hpo.larger_better` | (bool) False | The indicator of whether the larger metric is better. | - | +| `hpo.scheduler` | (string) 'rs'
Choices: {'rs', 'sha', 'wrap_sha'} | Which algorithm to use. | - | +| `hpo.metric` | (string) 'client_summarized_weighted_avg.val_loss' | Metric to be optimized. | - | + +#### Successive Halving Algorithm (SHA) + +| Name | (Type) Default Value | Description | Note | +|:----:|:--------------------:|:-------------------------------------------|:-----| +| `hpo.sha.elim_rate` | (int) 3 | Reserve only top 1/`hpo.sha.elim_rate` hyperparameter configurations in each state. | - | +| `hpo.sha.budgets` | (list of int) [] | Budgets for each SHA stage. | - | + + +#### FedEx + +| Name | (Type) Default Value | Description | Note | +|:----:|:--------------------:|:-------------------------------------------|:-----| +| `hpo.fedex.use` | (bool) False | Whether to use FedEx. | - | +| `hpo.fedex.ss` | (striing) '' | Path of the .yaml specifying the search space to be explored. | - | +| `hpo.fedex.flatten_ss` | (bool) True | Whether the search space has been flattened. | - | +| `hpo.fedex.eta0` | (float) -1.0 | Initial learning rate. | -1.0 means automatically determine the learning rate based on the size of search space. | +| `hpo.fedex.sched` | (string) 'auto'
Choices: {'auto', 'adaptive', 'aggressive', 'constant', 'scale' } | The strategy to update step sizes | - | +| `hpo.fedex.cutoff` | (float) 0.0 | The entropy level below which to stop updating the config. | - | +| `hpo.fedex.gamma` | (float) 0.0 | The discount factor; 0.0 is most recent, 1.0 is mean. | - | +| `hpo.fedex.diff` | (bool) False | Whether to use the difference of validation losses before and after the local update as the reward signal. | - | + +#### Wrappers for FedEx + +| Name | (Type) Default Value | Description | Note | +|:----:|:--------------------:|:-------------------------------------------|:-----| +| `hpo.table.eps` | (float) 0.1 | The probability to make local perturbation. | Larger values lead to drastically different arms of the bandit FedEx attempts to solve. | +| `hpo.table.num` | (int) 27 | The number of arms of the bandit FedEx attempts to solve. | - | +| `hpo.table.idx` | (int) 0 | The key (i.e., name) of the hyperparameter wrapper considers. | No need to change this argument. | \ No newline at end of file diff --git a/federatedscope/core/configs/cfg_hpo.py b/federatedscope/core/configs/cfg_hpo.py index d5c7b051b..72df79d1b 100644 --- a/federatedscope/core/configs/cfg_hpo.py +++ b/federatedscope/core/configs/cfg_hpo.py @@ -11,18 +11,13 @@ def extend_hpo_cfg(cfg): cfg.hpo.working_folder = 'hpo' cfg.hpo.ss = '' cfg.hpo.num_workers = 0 - # cfg.hpo.init_strategy = 'random' cfg.hpo.init_cand_num = 16 - cfg.hpo.log_scale = False cfg.hpo.larger_better = False cfg.hpo.scheduler = 'rs' - # plot the performance - cfg.hpo.plot_interval = 1 cfg.hpo.metric = 'client_summarized_weighted_avg.val_loss' # SHA cfg.hpo.sha = CN() - cfg.hpo.sha.elim_round_num = 3 cfg.hpo.sha.elim_rate = 3 cfg.hpo.sha.budgets = [] @@ -44,15 +39,12 @@ def extend_hpo_cfg(cfg): cfg.hpo.fedex.cutoff = .0 # discount factor; 0.0 is most recent, 1.0 is mean cfg.hpo.fedex.gamma = .0 - cfg.hpo.fedex.num_arms = 16 cfg.hpo.fedex.diff = False # Table cfg.hpo.table = CN() - cfg.hpo.table.ss = '' cfg.hpo.table.eps = 0.1 cfg.hpo.table.num = 27 - # cfg.hpo.table.cand = 81 cfg.hpo.table.idx = 0 @@ -68,9 +60,7 @@ def assert_hpo_cfg(cfg): cfg.hpo.scheduler) assert cfg.hpo.num_workers >= 0, "#worker should be non-negative but " \ "given {}".format(cfg.hpo.num_workers) - assert len(cfg.hpo.sha.budgets) == 0 or len( - cfg.hpo.sha.budgets - ) == cfg.hpo.sha.elim_round_num, \ + assert len(cfg.hpo.sha.budgets) > 0, \ "Either do NOT specify the budgets or specify the budget for each " \ "SHA iteration, but the given budgets is {}".format( cfg.hpo.sha.budgets)