diff --git a/benchmark/FedHPOB/README.md b/benchmark/FedHPOB/README.md index ee61004ac..eaa883192 100644 --- a/benchmark/FedHPOB/README.md +++ b/benchmark/FedHPOB/README.md @@ -53,7 +53,7 @@ The naming pattern of the url of data files obeys the rule: https://federatedscope.oss-cn-beijing.aliyuncs.com/fedhpob_{BENCHMARK}_{MODE}.zip ``` -where `{BENCHMARK}` should be one of `cnn`, `bert`, `gcn`, `lr`, or `mlp`, and `{MODE}` should be one of `tabular` or `surrogate`. +where `{BENCHMARK}` should be one of `cnn`, `bert`, `gcn`, `lr`, `mlp` or `cross_device`, and `{MODE}` should be one of `tabular` or `surrogate`. Then unzip the data file, move data under tabular mode to `~/data/tabular_data/` and data under surrogate mode to `~/data/surrogate_model`. @@ -122,45 +122,46 @@ If you find FedHPO-B useful for your research or development, please cite the fo Available tabular triplets look-up table: -| Model | Dataset | FedAlgo | -| ----- | ------------------------- | ------- | -| cnn | femnist | avg | -| cnn | cifar10@torchvision | avg | -| bert | cola@huggingface_datasets | avg | -| bert | cola@huggingface_datasets | opt | -| bert | sst2@huggingface_datasets | avg | -| bert | sst2@huggingface_datasets | opt | -| gcn | cora | avg | -| gcn | cora | opt | -| gcn | citeseer | avg | -| gcn | citeseer | opt | -| gcn | pubmed | avg | -| gcn | pubmed | opt | -| lr | 31@openml | avg | -| lr | 31@openml | opt | -| lr | 53@openml | avg | -| lr | 53@openml | opt | -| lr | 3917@openml | avg | -| lr | 3917@openml | opt | -| lr | 10101@openml | avg | -| lr | 10101@openml | opt | -| lr | 146818@openml | avg | -| lr | 146818@openml | opt | -| lr | 146821@openml | avg | -| lr | 146821@openml | opt | -| lr | 146822@openml | avg | -| lr | 146822@openml | opt | -| mlp | 31@openml | avg | -| mlp | 31@openml | opt | -| mlp | 53@openml | avg | -| mlp | 53@openml | opt | -| mlp | 3917@openml | avg | -| mlp | 3917@openml | opt | -| mlp | 10101@openml | avg | -| mlp | 10101@openml | opt | -| mlp | 146818@openml | avg | -| mlp | 146818@openml | opt | -| mlp | 146821@openml | avg | -| mlp | 146821@openml | opt | -| mlp | 146822@openml | avg | -| mlp | 146822@openml | opt | \ No newline at end of file +| Model | Dataset | FedAlgo | +| ------------ | ------------------------- | ------- | +| cnn | femnist | avg | +| cnn | cifar10@torchvision | avg | +| bert | cola@huggingface_datasets | avg | +| bert | cola@huggingface_datasets | opt | +| bert | sst2@huggingface_datasets | avg | +| bert | sst2@huggingface_datasets | opt | +| gcn | cora | avg | +| gcn | cora | opt | +| gcn | citeseer | avg | +| gcn | citeseer | opt | +| gcn | pubmed | avg | +| gcn | pubmed | opt | +| lr | 31@openml | avg | +| lr | 31@openml | opt | +| lr | 53@openml | avg | +| lr | 53@openml | opt | +| lr | 3917@openml | avg | +| lr | 3917@openml | opt | +| lr | 10101@openml | avg | +| lr | 10101@openml | opt | +| lr | 146818@openml | avg | +| lr | 146818@openml | opt | +| lr | 146821@openml | avg | +| lr | 146821@openml | opt | +| lr | 146822@openml | avg | +| lr | 146822@openml | opt | +| mlp | 31@openml | avg | +| mlp | 31@openml | opt | +| mlp | 53@openml | avg | +| mlp | 53@openml | opt | +| mlp | 3917@openml | avg | +| mlp | 3917@openml | opt | +| mlp | 10101@openml | avg | +| mlp | 10101@openml | opt | +| mlp | 146818@openml | avg | +| mlp | 146818@openml | opt | +| mlp | 146821@openml | avg | +| mlp | 146821@openml | opt | +| mlp | 146822@openml | avg | +| mlp | 146822@openml | opt | +| cross_device | twitter | avg | diff --git a/benchmark/FedHPOB/fedhpob/benchmarks/base_benchmark.py b/benchmark/FedHPOB/fedhpob/benchmarks/base_benchmark.py index 13fadc153..2c7cb79cf 100644 --- a/benchmark/FedHPOB/fedhpob/benchmarks/base_benchmark.py +++ b/benchmark/FedHPOB/fedhpob/benchmarks/base_benchmark.py @@ -23,6 +23,7 @@ def __init__(self, model, dname, algo, rng=None, **kwargs): # Load data and modify cfg of FS. self.cfg = global_cfg.clone() + self.cfg.set_new_allowed(True) filepath = os.path.join('scripts', model, f'{dname}.yaml') self.cfg.merge_from_file(filepath) self.cfg.data.type = dname diff --git a/benchmark/FedHPOB/fedhpob/config.py b/benchmark/FedHPOB/fedhpob/config.py index b10ed56df..ed90f04a9 100644 --- a/benchmark/FedHPOB/fedhpob/config.py +++ b/benchmark/FedHPOB/fedhpob/config.py @@ -12,7 +12,26 @@ def get_cs(dname, model, mode, alg='avg'): configuration_space = CS.ConfigurationSpace() fidelity_space = CS.ConfigurationSpace() # configuration_space - if dname in ['cora', 'citeseer', 'pubmed']: + if dname == 'twitter': + fidelity_space.add_hyperparameter( + CS.CategoricalHyperparameter('round', + choices=[x for x in range(500)])) + fidelity_space.add_hyperparameter( + CS.CategoricalHyperparameter('sample_rate', choices=[0.01])) + configuration_space.add_hyperparameter( + CS.CategoricalHyperparameter( + 'lr', choices=[0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0])) + configuration_space.add_hyperparameter( + CS.CategoricalHyperparameter('wd', choices=[0.0, 0.001, 0.01, + 0.1])) + configuration_space.add_hyperparameter( + CS.CategoricalHyperparameter('dropout', choices=[0.0])) + configuration_space.add_hyperparameter( + CS.CategoricalHyperparameter('step', choices=[1, 2, 3, 4])) + configuration_space.add_hyperparameter( + CS.CategoricalHyperparameter('batch_size', choices=[64])) + + elif dname in ['cora', 'citeseer', 'pubmed']: # GNN tabular, raw and surrogate fidelity_space.add_hyperparameter( CS.CategoricalHyperparameter('round', diff --git a/benchmark/FedHPOB/scripts/lr/run_hpo_twitter_lr.sh b/benchmark/FedHPOB/scripts/cross_device/run_hpo_twitter_lr.sh similarity index 100% rename from benchmark/FedHPOB/scripts/lr/run_hpo_twitter_lr.sh rename to benchmark/FedHPOB/scripts/cross_device/run_hpo_twitter_lr.sh diff --git a/benchmark/FedHPOB/scripts/lr/twitter.yaml b/benchmark/FedHPOB/scripts/cross_device/twitter.yaml similarity index 100% rename from benchmark/FedHPOB/scripts/lr/twitter.yaml rename to benchmark/FedHPOB/scripts/cross_device/twitter.yaml