Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactored the logger by reducing its redundancy and fixed some minor issues #29

Merged
merged 21 commits into from
Apr 24, 2022
Merged
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
6e8552e
Reducing the redundancy of the logger:
yxdyc Apr 20, 2022
1831be6
code reformat using the repo linter
yxdyc Apr 20, 2022
0b1bc61
Merge remote-tracking branch 'origin/master' into feature/logging_imp…
yxdyc Apr 20, 2022
1c31a55
improved the best_res_update logging
yxdyc Apr 20, 2022
b2a9324
improved the best_res_update logging
yxdyc Apr 21, 2022
439bc86
Merge remote-tracking branch 'origin/master' into feature/logging_imp…
yxdyc Apr 21, 2022
035d1ab
modified the usage of logger as non-root logger
yxdyc Apr 21, 2022
0654379
modified the global_cfg to distinct init_cfg in unit-test
yxdyc Apr 21, 2022
8549e18
re-activate the jenkins build
yxdyc Apr 21, 2022
be109c9
modified according to weirui's commment
yxdyc Apr 21, 2022
e97c575
modified according to weirui's commment
yxdyc Apr 21, 2022
ecdf0b7
Reducing the redundancy of the logger:
yxdyc Apr 20, 2022
dcb6dbd
code reformat using the repo linter
yxdyc Apr 20, 2022
34bd49a
improved the best_res_update logging
yxdyc Apr 20, 2022
aeda6c3
modified the global_cfg to distinct init_cfg in unit-test
yxdyc Apr 21, 2022
adaf769
re-activate the jenkins build
yxdyc Apr 21, 2022
2bbeda0
modified according to weirui's commment
yxdyc Apr 21, 2022
8dd87c5
modified according to weirui's commment
yxdyc Apr 21, 2022
d9c4b15
Merge branch 'Feature/logging_improve' of https://github.com/yxdyc/Fe…
yxdyc Apr 22, 2022
3dac6f1
bug fix for testing vertical FL and attacker
yxdyc Apr 22, 2022
2dde55b
Merge pull request #35 from yxdyc/Feature/logging_improve
yxdyc Apr 22, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
modified according to weirui's commment
yxdyc committed Apr 22, 2022
commit 2bbeda0ebcb5f52b08acd758a30d09ee44c30a69
2 changes: 1 addition & 1 deletion federatedscope/main.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os
import sys

DEV_MODE = True # simplify the federatedscope re-setup everytime we change the source codes of federatedscope
DEV_MODE = False # simplify the federatedscope re-setup everytime we change the source codes of federatedscope
if DEV_MODE:
file_dir = os.path.join(os.path.dirname(__file__), '..')
sys.path.append(file_dir)
23 changes: 12 additions & 11 deletions tests/test_MIA_gradient_ascent.py
Original file line number Diff line number Diff line change
@@ -50,18 +50,19 @@ def set_config_femnist(self, cfg):
return backup_cfg

def test_GradAscent_femnist_standalone(self):
backup_cfg = self.set_config_femnist(global_cfg)
setup_seed(global_cfg.seed)
update_logger(global_cfg)
init_cfg = global_cfg.clone()
backup_cfg = self.set_config_femnist(init_cfg)
setup_seed(init_cfg.seed)
update_logger(init_cfg)

data, modified_cfg = get_data(global_cfg.clone())
global_cfg.merge_from_other_cfg(modified_cfg)
data, modified_cfg = get_data(init_cfg.clone())
init_cfg.merge_from_other_cfg(modified_cfg)
self.assertIsNotNone(data)

Fed_runner = FedRunner(data=data,
server_class=get_server_cls(global_cfg),
client_class=get_client_cls(global_cfg),
config=global_cfg.clone())
server_class=get_server_cls(init_cfg),
client_class=get_client_cls(init_cfg),
config=init_cfg.clone())
self.assertIsNotNone(Fed_runner)
test_best_results = Fed_runner.run()
print(test_best_results)
@@ -72,11 +73,11 @@ def test_GradAscent_femnist_standalone(self):
600)
# print(Fed_runner.client.keys())
target_data_loss = Fed_runner.client[
global_cfg.attack.attacker_id].trainer.ctx.target_data_loss
init_cfg.attack.attacker_id].trainer.ctx.target_data_loss
self.assertIsNotNone(target_data_loss)
self.assertIn(global_cfg.attack.attacker_id, Fed_runner.client.keys())
self.assertIn(init_cfg.attack.attacker_id, Fed_runner.client.keys())

global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)


if __name__ == '__main__':
2 changes: 1 addition & 1 deletion tests/test_ditto.py
Original file line number Diff line number Diff line change
@@ -66,7 +66,7 @@ def test_femnist_standalone(self):
self.assertIsNotNone(Fed_runner)
test_best_results = Fed_runner.run()
print(test_best_results)
global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)
self.assertLess(
test_best_results["client_summarized_weighted_avg"]
['test_avg_loss'], 10)
8 changes: 4 additions & 4 deletions tests/test_efficient_simulation.py
Original file line number Diff line number Diff line change
@@ -22,8 +22,8 @@ def test_toy_example_standalone_cmp_sim_impl(self):

data, _ = get_data(case_cfg.clone())
Fed_runner = FedRunner(data=data,
server_class=get_server_cls(global_cfg),
client_class=get_client_cls(global_cfg),
server_class=get_server_cls(case_cfg),
client_class=get_client_cls(case_cfg),
config=case_cfg.clone())
efficient_test_results = Fed_runner.run()

@@ -34,8 +34,8 @@ def test_toy_example_standalone_cmp_sim_impl(self):
])
data, _ = get_data(case_cfg.clone())
Fed_runner = FedRunner(data=data,
server_class=get_server_cls(global_cfg),
client_class=get_client_cls(global_cfg),
server_class=get_server_cls(case_cfg),
client_class=get_client_cls(case_cfg),
config=case_cfg.clone())
ordinary_test_results = Fed_runner.run()
gap = efficient_test_results["client_summarized_weighted_avg"][
17 changes: 9 additions & 8 deletions tests/test_fedem.py
Original file line number Diff line number Diff line change
@@ -49,22 +49,23 @@ def set_config_femnist(self, cfg):
return backup_cfg

def test_femnist_standalone(self):
backup_cfg = self.set_config_femnist(global_cfg)
setup_seed(global_cfg.seed)
update_logger(global_cfg)
init_cfg = global_cfg.clone()
backup_cfg = self.set_config_femnist(init_cfg)
setup_seed(init_cfg.seed)
update_logger(init_cfg)

data, modified_cfg = get_data(global_cfg.clone())
global_cfg.merge_from_other_cfg(modified_cfg)
data, modified_cfg = get_data(init_cfg.clone())
init_cfg.merge_from_other_cfg(modified_cfg)
self.assertIsNotNone(data)

Fed_runner = FedRunner(data=data,
server_class=get_server_cls(global_cfg),
client_class=get_client_cls(global_cfg),
server_class=get_server_cls(init_cfg),
client_class=get_client_cls(init_cfg),
config=global_cfg.clone())
self.assertIsNotNone(Fed_runner)
test_best_results = Fed_runner.run()
print(test_best_results)
global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)
self.assertLess(
test_best_results["client_summarized_weighted_avg"]['test_loss'],
600)
2 changes: 1 addition & 1 deletion tests/test_fedprox.py
Original file line number Diff line number Diff line change
@@ -63,7 +63,7 @@ def test_fedprox_standalone(self):
config=init_cfg.clone())
self.assertIsNotNone(Fed_runner)
test_results = Fed_runner.run()
global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)

self.assertLess(
test_results['client_summarized_weighted_avg']['test_loss'], 600)
17 changes: 9 additions & 8 deletions tests/test_fedsageplus.py
Original file line number Diff line number Diff line change
@@ -50,22 +50,23 @@ def set_config_fedsageplus(self, cfg):
return backup_cfg

def test_fedsageplus_standalone(self):
backup_cfg = self.set_config_fedsageplus(global_cfg)
setup_seed(global_cfg.seed)
update_logger(global_cfg)
init_cfg = global_cfg.clone()
backup_cfg = self.set_config_fedsageplus(init_cfg)
setup_seed(init_cfg.seed)
update_logger(init_cfg)

data, modified_cfg = get_data(global_cfg.clone())
global_cfg.merge_from_other_cfg(modified_cfg)
data, modified_cfg = get_data(init_cfg.clone())
init_cfg.merge_from_other_cfg(modified_cfg)

self.assertIsNotNone(data)

Fed_runner = FedRunner(data=data,
server_class=get_server_cls(global_cfg),
client_class=get_client_cls(global_cfg),
server_class=get_server_cls(init_cfg),
client_class=get_client_cls(init_cfg),
config=global_cfg.clone())
self.assertIsNotNone(Fed_runner)
test_best_results = Fed_runner.run()
global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)
self.assertGreater(test_best_results["server_global_eval"]['test_acc'],
0.7)

12 changes: 6 additions & 6 deletions tests/test_femnist.py
Original file line number Diff line number Diff line change
@@ -51,18 +51,18 @@ def test_femnist_standalone(self):
setup_seed(init_cfg.seed)
update_logger(init_cfg)

data, modified_cfg = get_data(global_cfg.clone())
global_cfg.merge_from_other_cfg(modified_cfg)
data, modified_cfg = get_data(init_cfg.clone())
init_cfg.merge_from_other_cfg(modified_cfg)
self.assertIsNotNone(data)

Fed_runner = FedRunner(data=data,
server_class=get_server_cls(global_cfg),
client_class=get_client_cls(global_cfg),
config=global_cfg.clone())
server_class=get_server_cls(init_cfg),
client_class=get_client_cls(init_cfg),
config=init_cfg.clone())
self.assertIsNotNone(Fed_runner)
test_best_results = Fed_runner.run()
print(test_best_results)
global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)
self.assertLess(
test_best_results["client_summarized_weighted_avg"]['test_loss'],
600)
19 changes: 10 additions & 9 deletions tests/test_mf.py
Original file line number Diff line number Diff line change
@@ -43,21 +43,22 @@ def set_config_movielens1m(self, cfg):
return backup_cfg

def test_mf_standalone(self):
backup_cfg = self.set_config_movielens1m(global_cfg)
setup_seed(global_cfg.seed)
update_logger(global_cfg)
init_cfg = global_cfg.clone()
backup_cfg = self.set_config_movielens1m(init_cfg)
setup_seed(init_cfg.seed)
update_logger(init_cfg)

data, modified_cfg = get_data(global_cfg.clone())
global_cfg.merge_from_other_cfg(modified_cfg)
data, modified_cfg = get_data(init_cfg.clone())
init_cfg.merge_from_other_cfg(modified_cfg)
self.assertIsNotNone(data)

Fed_runner = FedRunner(data=data,
server_class=get_server_cls(global_cfg),
client_class=get_client_cls(global_cfg),
config=global_cfg.clone())
server_class=get_server_cls(init_cfg),
client_class=get_client_cls(init_cfg),
config=init_cfg.clone())
self.assertIsNotNone(Fed_runner)
test_results = Fed_runner.run()
global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)

self.assertLess(test_results['client_individual']['test_avg_loss'], 20)

5 changes: 3 additions & 2 deletions tests/test_nbafl.py
Original file line number Diff line number Diff line change
@@ -51,6 +51,7 @@ def set_config_femnist(self, cfg):
return backup_cfg

def test_nbafl_standalone(self):
# TODO, no need to backup the cfg in all test_xxx.py, as we are now using the init_cfg style
init_cfg = global_cfg.clone()
backup_cfg = self.set_config_femnist(init_cfg)
setup_seed(init_cfg.seed)
@@ -65,11 +66,11 @@ def test_nbafl_standalone(self):
Fed_runner = FedRunner(data=data,
server_class=get_server_cls(init_cfg),
client_class=get_client_cls(init_cfg),
config=global_cfg.clone())
config=init_cfg.clone())
self.assertIsNotNone(Fed_runner)
test_best_results = Fed_runner.run()
print(test_best_results)
global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)
self.assertLess(
test_best_results["client_summarized_weighted_avg"]['test_loss'],
500)
19 changes: 10 additions & 9 deletions tests/test_pfedme.py
Original file line number Diff line number Diff line change
@@ -51,22 +51,23 @@ def set_config_femnist(self, cfg):
return backup_cfg

def test_femnist_standalone(self):
backup_cfg = self.set_config_femnist(global_cfg)
setup_seed(global_cfg.seed)
update_logger(global_cfg)
init_cfg = global_cfg.clone()
backup_cfg = self.set_config_femnist(init_cfg)
setup_seed(init_cfg.seed)
update_logger(init_cfg)

data, modified_cfg = get_data(global_cfg.clone())
global_cfg.merge_from_other_cfg(modified_cfg)
data, modified_cfg = get_data(init_cfg.clone())
init_cfg.merge_from_other_cfg(modified_cfg)
self.assertIsNotNone(data)

Fed_runner = FedRunner(data=data,
server_class=get_server_cls(global_cfg),
client_class=get_client_cls(global_cfg),
config=global_cfg.clone())
server_class=get_server_cls(init_cfg),
client_class=get_client_cls(init_cfg),
config=init_cfg.clone())
self.assertIsNotNone(Fed_runner)
test_best_results = Fed_runner.run()
print(test_best_results)
global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)
self.assertLess(
test_best_results["client_summarized_weighted_avg"]
['test_avg_loss'], 10)
6 changes: 3 additions & 3 deletions tests/test_rec_IG_opt_attack.py
Original file line number Diff line number Diff line change
@@ -62,12 +62,12 @@ def test_IG_rec_femnist_standalone(self):

Fed_runner = FedRunner(data=data,
server_class=get_server_cls(init_cfg),
client_class=get_client_cls(global_cfg),
config=global_cfg.clone())
client_class=get_client_cls(init_cfg),
config=init_cfg.clone())
self.assertIsNotNone(Fed_runner)
test_best_results = Fed_runner.run()
print(test_best_results)
global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)
self.assertLess(
test_best_results["client_summarized_weighted_avg"]['test_loss'],
600)
4 changes: 2 additions & 2 deletions tests/test_rec_opt_attack.py
Original file line number Diff line number Diff line change
@@ -69,11 +69,11 @@ def test_rec_femnist_standalone(self):
Fed_runner = FedRunner(data=data,
server_class=get_server_cls(init_cfg),
client_class=get_client_cls(init_cfg),
config=global_cfg.clone())
config=init_cfg.clone())
self.assertIsNotNone(Fed_runner)
test_best_results = Fed_runner.run()
print(test_best_results)
global_cfg.merge_from_other_cfg(backup_cfg)
init_cfg.merge_from_other_cfg(backup_cfg)
self.assertLess(
test_best_results["client_summarized_weighted_avg"]['test_loss'],
600)