diff --git a/README.md b/README.md index 8f33dc3fc..f824c3113 100644 --- a/README.md +++ b/README.md @@ -162,13 +162,13 @@ Note that FederatedScope provides a unified interface for both standalone mode a The standalone mode in FederatedScope means to simulate multiple participants (servers and clients) in a single device, while participants' data are isolated from each other and their models might be shared via message passing. -Here we demonstrate how to run a standard FL task with FederatedScope, with setting `cfg.data.type = 'FEMNIST'`and `cfg.model.type = 'ConvNet2'` to run vanilla FedAvg for an image classification task. Users can customize training configurations, such as `cfg.federated.total_round_num`, `cfg.data.batch_size`, and `cfg.train.optimizer.lr`, in the configuration (a .yaml file), and run a standard FL task as: +Here we demonstrate how to run a standard FL task with FederatedScope, with setting `cfg.data.type = 'FEMNIST'`and `cfg.model.type = 'ConvNet2'` to run vanilla FedAvg for an image classification task. Users can customize training configurations, such as `cfg.federated.total_round_num`, `cfg.dataloader.batch_size`, and `cfg.train.optimizer.lr`, in the configuration (a .yaml file), and run a standard FL task as: ```bash # Run with default configurations python federatedscope/main.py --cfg scripts/example_configs/femnist.yaml # Or with custom configurations -python federatedscope/main.py --cfg scripts/example_configs/femnist.yaml federate.total_round_num 50 data.batch_size 128 +python federatedscope/main.py --cfg scripts/example_configs/femnist.yaml federate.total_round_num 50 dataloader.batch_size 128 ``` Then you can observe some monitored metrics during the training process as: diff --git a/environment/extra_dependencies_torch1.10-application.sh b/environment/extra_dependencies_torch1.10-application.sh index eaffc5844..4657afe7e 100644 --- a/environment/extra_dependencies_torch1.10-application.sh +++ b/environment/extra_dependencies_torch1.10-application.sh @@ -9,3 +9,6 @@ conda install -y nltk conda install -y sentencepiece textgrid typeguard -c conda-forge conda install -y transformers==4.16.2 tokenizers==0.10.3 datasets -c huggingface -c conda-forge conda install -y torchtext -c pytorch + +# Tabular +conda install -y openml==0.12.2 diff --git a/federatedscope/__init__.py b/federatedscope/__init__.py index d6e7a36f4..f15846754 100644 --- a/federatedscope/__init__.py +++ b/federatedscope/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, division, print_function -__version__ = '0.2.0' +__version__ = '0.2.1' def _setup_logger(): diff --git a/federatedscope/attack/auxiliary/poisoning_data.py b/federatedscope/attack/auxiliary/poisoning_data.py index ca3949ae5..0d0a9581e 100644 --- a/federatedscope/attack/auxiliary/poisoning_data.py +++ b/federatedscope/attack/auxiliary/poisoning_data.py @@ -50,9 +50,9 @@ def load_poisoned_dataset_edgeset(data, ctx, mode): poison_testset.append((transforms_funcs(sample), label)) data['poison_' + mode] = DataLoader( poison_testset, - batch_size=ctx.data.batch_size, + batch_size=ctx.dataloader.batch_size, shuffle=False, - num_workers=ctx.data.num_workers) + num_workers=ctx.dataloader.num_workers) elif "CIFAR10" in ctx.data.type: target_label = int(ctx.attack.target_label_ind) @@ -91,9 +91,9 @@ def load_poisoned_dataset_edgeset(data, ctx, mode): poison_testset.append((transforms_funcs(sample), label)) data['poison_' + mode] = DataLoader( poison_testset, - batch_size=ctx.data.batch_size, + batch_size=ctx.dataloader.batch_size, shuffle=False, - num_workers=ctx.data.num_workers) + num_workers=ctx.dataloader.num_workers) else: raise RuntimeError( @@ -213,9 +213,9 @@ def load_poisoned_dataset_pixel(data, ctx, mode): poisoned_dataset[iii] = (transforms_funcs(sample), label) data[mode] = DataLoader(poisoned_dataset, - batch_size=ctx.data.batch_size, + batch_size=ctx.dataloader.batch_size, shuffle=True, - num_workers=ctx.data.num_workers) + num_workers=ctx.dataloader.num_workers) if mode == MODE.TEST or mode == MODE.VAL: poisoned_dataset = addTrigger(data[mode].dataset, @@ -234,10 +234,11 @@ def load_poisoned_dataset_pixel(data, ctx, mode): # (channel, height, width) = sample.shape #(c,h,w) poisoned_dataset[iii] = (transforms_funcs(sample), label) - data['poison_' + mode] = DataLoader(poisoned_dataset, - batch_size=ctx.data.batch_size, - shuffle=False, - num_workers=ctx.data.num_workers) + data['poison_' + mode] = DataLoader( + poisoned_dataset, + batch_size=ctx.dataloader.batch_size, + shuffle=False, + num_workers=ctx.dataloader.num_workers) return data diff --git a/federatedscope/attack/trainer/MIA_invert_gradient_trainer.py b/federatedscope/attack/trainer/MIA_invert_gradient_trainer.py index 3a7e96d5e..981d4c875 100644 --- a/federatedscope/attack/trainer/MIA_invert_gradient_trainer.py +++ b/federatedscope/attack/trainer/MIA_invert_gradient_trainer.py @@ -4,7 +4,7 @@ import torch from federatedscope.core.trainers import GeneralTorchTrainer -from federatedscope.core.auxiliaries.dataloader_builder import WrapDataset +from federatedscope.core.data.wrap_dataset import WrapDataset from federatedscope.attack.auxiliary.MIA_get_target_data import get_target_data logger = logging.getLogger(__name__) diff --git a/federatedscope/attack/trainer/benign_trainer.py b/federatedscope/attack/trainer/benign_trainer.py index 800c93ed2..e6bd30c24 100644 --- a/federatedscope/attack/trainer/benign_trainer.py +++ b/federatedscope/attack/trainer/benign_trainer.py @@ -1,15 +1,8 @@ -from calendar import c import logging from typing import Type -import torch import numpy as np from federatedscope.core.trainers import GeneralTorchTrainer -from federatedscope.core.auxiliaries.transform_builder import get_transform -from federatedscope.attack.auxiliary.backdoor_utils import normalize -from federatedscope.core.auxiliaries.dataloader_builder import WrapDataset -from federatedscope.core.auxiliaries.dataloader_builder import get_dataloader -from federatedscope.core.auxiliaries.ReIterator import ReIterator logger = logging.getLogger(__name__) diff --git a/federatedscope/contrib/data/example.py b/federatedscope/contrib/data/example.py index ef800f584..b896bdf7f 100644 --- a/federatedscope/contrib/data/example.py +++ b/federatedscope/contrib/data/example.py @@ -1,7 +1,7 @@ from federatedscope.register import register_data -def MyData(config): +def MyData(config, client_cfgs=None): r""" Returns: data: @@ -17,12 +17,13 @@ def MyData(config): """ data = None config = config + client_cfgs = client_cfgs return data, config -def call_my_data(config): +def call_my_data(config, client_cfgs): if config.data.type == "mydata": - data, modified_config = MyData(config) + data, modified_config = MyData(config, client_cfgs) return data, modified_config diff --git a/federatedscope/contrib/trainer/torch_example.py b/federatedscope/contrib/trainer/torch_example.py new file mode 100644 index 000000000..18cd5d7a0 --- /dev/null +++ b/federatedscope/contrib/trainer/torch_example.py @@ -0,0 +1,104 @@ +import inspect +from federatedscope.register import register_trainer +from federatedscope.core.trainers import BaseTrainer + +# An example for converting torch training process to FS training process + +# Refer to `federatedscope.core.trainers.BaseTrainer` for interface. + +# Try with FEMNIST: +# python federatedscope/main.py --cfg scripts/example_configs/femnist.yaml \ +# trainer.type mytorchtrainer federate.sample_client_rate 0.01 \ +# federate.total_round_num 5 eval.best_res_update_round_wise_key test_loss + + +class MyTorchTrainer(BaseTrainer): + def __init__(self, model, data, device, **kwargs): + import torch + # NN modules + self.model = model + # FS `ClientData` or your own data + self.data = data + # Device name + self.device = device + # kwargs + self.kwargs = kwargs + # Criterion & Optimizer + self.criterion = torch.nn.CrossEntropyLoss() + self.optimizer = torch.optim.SGD(self.model.parameters(), + lr=0.001, + momentum=0.9, + weight_decay=1e-4) + + def train(self): + # _hook_on_fit_start_init + self.model.to(self.device) + self.model.train() + + total_loss = num_samples = 0 + # _hook_on_batch_start_init + for x, y in self.data['train']: + # _hook_on_batch_forward + x, y = x.to(self.device), y.to(self.device) + outputs = self.model(x) + loss = self.criterion(outputs, y) + + # _hook_on_batch_backward + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + # _hook_on_batch_end + total_loss += loss.item() * y.shape[0] + num_samples += y.shape[0] + + # _hook_on_fit_end + return num_samples, self.model.cpu().state_dict(), \ + {'loss_total': total_loss, 'avg_loss': total_loss/float( + num_samples)} + + def evaluate(self, target_data_split_name='test'): + import torch + with torch.no_grad(): + self.model.to(self.device) + self.model.eval() + total_loss = num_samples = 0 + # _hook_on_batch_start_init + for x, y in self.data[target_data_split_name]: + # _hook_on_batch_forward + x, y = x.to(self.device), y.to(self.device) + pred = self.model(x) + loss = self.criterion(pred, y) + + # _hook_on_batch_end + total_loss += loss.item() * y.shape[0] + num_samples += y.shape[0] + + # _hook_on_fit_end + return { + f'{target_data_split_name}_loss': total_loss, + f'{target_data_split_name}_total': num_samples, + f'{target_data_split_name}_avg_loss': total_loss / + float(num_samples) + } + + def update(self, model_parameters, strict=False): + self.model.load_state_dict(model_parameters, strict) + + def get_model_para(self): + return self.model.cpu().state_dict() + + def print_trainer_meta_info(self): + sign = inspect.signature(self.__init__).parameters.values() + meta_info = tuple([(val.name, getattr(self, val.name)) + for val in sign]) + return f'{self.__class__.__name__}{meta_info}' + + +def call_my_torch_trainer(trainer_type): + if trainer_type == 'mytorchtrainer': + trainer_builder = MyTorchTrainer + return trainer_builder + + +register_trainer('mytorchtrainer', call_my_torch_trainer) diff --git a/federatedscope/core/auxiliaries/data_builder.py b/federatedscope/core/auxiliaries/data_builder.py index 360941546..617d9689e 100644 --- a/federatedscope/core/auxiliaries/data_builder.py +++ b/federatedscope/core/auxiliaries/data_builder.py @@ -1,12 +1,8 @@ -import copy -import os -import pickle import logging -from random import shuffle - -import numpy as np -from collections import defaultdict +from importlib import import_module +from federatedscope.core.data.utils import RegexInverseMap, load_dataset, \ + convert_data_mode from federatedscope.core.auxiliaries.utils import setup_seed import federatedscope.register as register @@ -20,786 +16,55 @@ f'{error} in `federatedscope.contrib.data`, some modules are not ' f'available.') - -def load_toy_data(config=None): - generate = config.federate.mode.lower() == 'standalone' - - def _generate_data(client_num=5, - instance_num=1000, - feature_num=5, - save_data=False): - """ - Generate data in FedRunner format - Args: - client_num: - instance_num: - feature_num: - save_data: - - Returns: - { - '{client_id}': { - 'train': { - 'x': ..., - 'y': ... - }, - 'test': { - 'x': ..., - 'y': ... - }, - 'val': { - 'x': ..., - 'y': ... - } - } - } - - """ - weights = np.random.normal(loc=0.0, scale=1.0, size=feature_num) - bias = np.random.normal(loc=0.0, scale=1.0) - data = dict() - for each_client in range(1, client_num + 1): - data[each_client] = dict() - client_x = np.random.normal(loc=0.0, - scale=0.5 * each_client, - size=(instance_num, feature_num)) - client_y = np.sum(client_x * weights, axis=-1) + bias - client_y = np.expand_dims(client_y, -1) - client_data = {'x': client_x, 'y': client_y} - data[each_client]['train'] = client_data - - # test data - test_x = np.random.normal(loc=0.0, - scale=1.0, - size=(instance_num, feature_num)) - test_y = np.sum(test_x * weights, axis=-1) + bias - test_y = np.expand_dims(test_y, -1) - test_data = {'x': test_x, 'y': test_y} - for each_client in range(1, client_num + 1): - data[each_client]['test'] = test_data - - # val data - val_x = np.random.normal(loc=0.0, - scale=1.0, - size=(instance_num, feature_num)) - val_y = np.sum(val_x * weights, axis=-1) + bias - val_y = np.expand_dims(val_y, -1) - val_data = {'x': val_x, 'y': val_y} - for each_client in range(1, client_num + 1): - data[each_client]['val'] = val_data - - # server_data - data[0] = dict() - data[0]['train'] = None - data[0]['val'] = val_data - data[0]['test'] = test_data - - if save_data: - # server_data = dict() - save_client_data = dict() - - for client_idx in range(0, client_num + 1): - if client_idx == 0: - filename = 'data/server_data' - else: - filename = 'data/client_{:d}_data'.format(client_idx) - with open(filename, 'wb') as f: - save_client_data['train'] = { - k: v.tolist() - for k, v in data[client_idx]['train'].items() - } - save_client_data['val'] = { - k: v.tolist() - for k, v in data[client_idx]['val'].items() - } - save_client_data['test'] = { - k: v.tolist() - for k, v in data[client_idx]['test'].items() - } - pickle.dump(save_client_data, f) - - return data - - if generate: - data = _generate_data(client_num=config.federate.client_num, - save_data=config.data.save_data) - else: - with open(config.distribute.data_file, 'rb') as f: - data = pickle.load(f) - for key in data.keys(): - data[key] = {k: np.asarray(v) - for k, v in data[key].items() - } if data[key] is not None else None - - return data, config - - -def load_external_data(config=None): - r""" Based on the configuration file, this function imports external - datasets and applies train/valid/test splits and split by some specific - `splitter` into the standard FederatedScope input data format. - - Args: - config: `CN` from `federatedscope/core/configs/config.py` - - Returns: - data_local_dict: dict of split dataloader. - Format: - { - 'client_id': { - 'train': DataLoader(), - 'test': DataLoader(), - 'val': DataLoader() - } - } - modified_config: `CN` from `federatedscope/core/configs/config.py`, - which might be modified in the function. - - """ - - import torch - import inspect - from importlib import import_module - from torch.utils.data import DataLoader - from federatedscope.core.auxiliaries.splitter_builder import get_splitter - from federatedscope.core.auxiliaries.transform_builder import get_transform - - def get_func_args(func): - sign = inspect.signature(func).parameters.values() - sign = set([val.name for val in sign]) - return sign - - def filter_dict(func, kwarg): - sign = get_func_args(func) - common_args = sign.intersection(kwarg.keys()) - filtered_dict = {key: kwarg[key] for key in common_args} - return filtered_dict - - def load_torchvision_data(name, splits=None, config=None): - dataset_func = getattr(import_module('torchvision.datasets'), name) - transform_funcs = get_transform(config, 'torchvision') - if config.data.args: - raw_args = config.data.args[0] - else: - raw_args = {} - if 'download' not in raw_args.keys(): - raw_args.update({'download': True}) - filtered_args = filter_dict(dataset_func.__init__, raw_args) - func_args = get_func_args(dataset_func.__init__) - - # Perform split on different dataset - if 'train' in func_args: - # Split train to (train, val) - dataset_train = dataset_func(root=config.data.root, - train=True, - **filtered_args, - **transform_funcs) - dataset_val = None - dataset_test = dataset_func(root=config.data.root, - train=False, - **filtered_args, - **transform_funcs) - if splits: - train_size = int(splits[0] * len(dataset_train)) - val_size = len(dataset_train) - train_size - lengths = [train_size, val_size] - dataset_train, dataset_val = \ - torch.utils.data.dataset.random_split(dataset_train, - lengths) - - elif 'split' in func_args: - # Use raw split - dataset_train = dataset_func(root=config.data.root, - split='train', - **filtered_args, - **transform_funcs) - dataset_val = dataset_func(root=config.data.root, - split='valid', - **filtered_args, - **transform_funcs) - dataset_test = dataset_func(root=config.data.root, - split='test', - **filtered_args, - **transform_funcs) - elif 'classes' in func_args: - # Use raw split - dataset_train = dataset_func(root=config.data.root, - classes='train', - **filtered_args, - **transform_funcs) - dataset_val = dataset_func(root=config.data.root, - classes='valid', - **filtered_args, - **transform_funcs) - dataset_test = dataset_func(root=config.data.root, - classes='test', - **filtered_args, - **transform_funcs) - else: - # Use config.data.splits - dataset = dataset_func(root=config.data.root, - **filtered_args, - **transform_funcs) - train_size = int(splits[0] * len(dataset)) - val_size = int(splits[1] * len(dataset)) - test_size = len(dataset) - train_size - val_size - lengths = [train_size, val_size, test_size] - dataset_train, dataset_val, dataset_test = \ - torch.utils.data.dataset.random_split(dataset, lengths) - - data_dict = { - 'train': dataset_train, - 'val': dataset_val, - 'test': dataset_test - } - - return data_dict - - def load_torchtext_data(name, splits=None, config=None): - from torch.nn.utils.rnn import pad_sequence - from federatedscope.nlp.dataset.utils import label_to_index - - dataset_func = getattr(import_module('torchtext.datasets'), name) - if config.data.args: - raw_args = config.data.args[0] - else: - raw_args = {} - assert 'max_len' in raw_args, "Miss key 'max_len' in " \ - "`config.data.args`." - filtered_args = filter_dict(dataset_func.__init__, raw_args) - dataset = dataset_func(root=config.data.root, **filtered_args) - - # torchtext.transforms requires >= 0.12.0 and torch = 1.11.0, - # so we do not use `get_transform` in torchtext. - - # Merge all data and tokenize - x_list = [] - y_list = [] - for data_iter in dataset: - data, targets = [], [] - for i, item in enumerate(data_iter): - data.append(item[1]) - targets.append(item[0]) - x_list.append(data) - y_list.append(targets) - - x_all, y_all = [], [] - for i in range(len(x_list)): - x_all += x_list[i] - y_all += y_list[i] - - if config.model.type.endswith('transformers'): - from transformers import AutoTokenizer - cache_path = os.path.join(os.getcwd(), "huggingface") - try: - tokenizer = AutoTokenizer.from_pretrained( - config.model.type.split('@')[0], - local_files_only=True, - cache_dir=cache_path) - except Exception as e: - logging.error(f"When loading cached file form " - f"{cache_path}, we faced the exception: \n " - f"{str(e)}") - - x_all = tokenizer(x_all, - return_tensors='pt', - padding=True, - truncation=True, - max_length=raw_args['max_len']) - data = [{key: value[i] - for key, value in x_all.items()} - for i in range(len(next(iter(x_all.values()))))] - if 'classification' in config.model.task.lower(): - targets = label_to_index(y_all) - else: - y_all = tokenizer(y_all, - return_tensors='pt', - padding=True, - truncation=True, - max_length=raw_args['max_len']) - targets = [{key: value[i] - for key, value in y_all.items()} - for i in range(len(next(iter(y_all.values()))))] - else: - from torchtext.data import get_tokenizer - tokenizer = get_tokenizer("basic_english") - if len(config.data.transform) == 0: - raise ValueError( - "`transform` must be one pretrained Word Embeddings from \ - ['GloVe', 'FastText', 'CharNGram']") - if len(config.data.transform) == 1: - config.data.transform.append({}) - vocab = getattr(import_module('torchtext.vocab'), - config.data.transform[0])( - dim=config.model.in_channels, - **config.data.transform[1]) - - if 'classification' in config.model.task.lower(): - data = [ - vocab.get_vecs_by_tokens(tokenizer(x), - lower_case_backup=True) - for x in x_all - ] - targets = label_to_index(y_all) - else: - data = [ - vocab.get_vecs_by_tokens(tokenizer(x), - lower_case_backup=True) - for x in x_all - ] - targets = [ - vocab.get_vecs_by_tokens(tokenizer(y), - lower_case_backup=True) - for y in y_all - ] - targets = pad_sequence(targets).transpose( - 0, 1)[:, :raw_args['max_len'], :] - data = pad_sequence(data).transpose(0, - 1)[:, :raw_args['max_len'], :] - # Split data to raw - num_items = [len(ds) for ds in x_list] - data_list, cnt = [], 0 - for num in num_items: - data_list.append([ - (x, y) - for x, y in zip(data[cnt:cnt + num], targets[cnt:cnt + num]) - ]) - cnt += num - - if len(data_list) == 3: - # Use raw splits - data_dict = { - 'train': data_list[0], - 'val': data_list[1], - 'test': data_list[2] - } - elif len(data_list) == 2: - # Split train to (train, val) - data_dict = { - 'train': data_list[0], - 'val': None, - 'test': data_list[1] - } - if splits: - train_size = int(splits[0] * len(data_dict['train'])) - val_size = len(data_dict['train']) - train_size - lengths = [train_size, val_size] - data_dict['train'], data_dict[ - 'val'] = torch.utils.data.dataset.random_split( - data_dict['train'], lengths) - else: - # Use config.data.splits - data_dict = {} - train_size = int(splits[0] * len(data_list[0])) - val_size = int(splits[1] * len(data_list[0])) - test_size = len(data_list[0]) - train_size - val_size - lengths = [train_size, val_size, test_size] - data_dict['train'], data_dict['val'], data_dict[ - 'test'] = torch.utils.data.dataset.random_split( - data_list[0], lengths) - - return data_dict - - def load_torchaudio_data(name, splits=None, config=None): - import torchaudio - - # dataset_func = getattr(import_module('torchaudio.datasets'), name) - raise NotImplementedError - - def load_torch_geometric_data(name, splits=None, config=None): - import torch_geometric - - # dataset_func = getattr(import_module('torch_geometric.datasets'), - # name) - raise NotImplementedError - - def load_huggingface_datasets_data(name, splits=None, config=None): - from datasets import load_dataset, load_from_disk - - if config.data.args: - raw_args = config.data.args[0] - else: - raw_args = {} - assert 'max_len' in raw_args, "Miss key 'max_len' in " \ - "`config.data.args`." - filtered_args = filter_dict(load_dataset, raw_args) - logger.info("Begin to load huggingface dataset") - if "hg_cache_dir" in raw_args: - hugging_face_path = raw_args["hg_cache_dir"] - else: - hugging_face_path = os.getcwd() - - if "load_disk_dir" in raw_args: - load_path = raw_args["load_disk_dir"] - try: - dataset = load_from_disk(load_path) - except Exception as e: - logging.error(f"When loading cached dataset form " - f"{load_path}, we faced the exception: \n " - f"{str(e)}") - else: - dataset = load_dataset(path=config.data.root, - name=name, - **filtered_args) - if config.model.type.endswith('transformers'): - os.environ["TOKENIZERS_PARALLELISM"] = "false" - from transformers import AutoTokenizer - logger.info("To load huggingface tokenizer") - tokenizer = AutoTokenizer.from_pretrained( - config.model.type.split('@')[0], - local_files_only=True, - cache_dir=os.path.join(hugging_face_path, "transformers")) - - for split in dataset: - x_all = [i['sentence'] for i in dataset[split]] - targets = [i['label'] for i in dataset[split]] - - if split == "train" and "used_train_ratio" in raw_args and \ - 1 > raw_args['used_train_ratio'] > 0: - selected_idx = [i for i in range(len(dataset[split]))] - shuffle(selected_idx) - selected_idx = selected_idx[:int( - len(selected_idx) * raw_args['used_train_ratio'])] - x_all = [ - element for i, element in enumerate(x_all) - if i in selected_idx - ] - targets = [ - element for i, element in enumerate(targets) - if i in selected_idx - ] - - x_all = tokenizer(x_all, - return_tensors='pt', - padding=True, - truncation=True, - max_length=raw_args['max_len']) - data = [{key: value[i] - for key, value in x_all.items()} - for i in range(len(next(iter(x_all.values()))))] - dataset[split] = (data, targets) - data_dict = { - 'train': [(x, y) - for x, y in zip(dataset['train'][0], dataset['train'][1]) - ], - 'val': [(x, y) for x, y in zip(dataset['validation'][0], - dataset['validation'][1])], - 'test': [ - (x, y) for x, y in zip(dataset['test'][0], dataset['test'][1]) - ] if (set(dataset['test'][1]) - set([-1])) else None, - } - original_train_size = len(data_dict["train"]) - - if "half_val_dummy_test" in raw_args and raw_args[ - "half_val_dummy_test"]: - # since the "test" set from GLUE dataset may be masked, we need to - # submit to get the ground-truth, for fast FL experiments, - # we split the validation set into two parts with the same size as - # new test/val data - original_val = [(x, y) for x, y in zip(dataset['validation'][0], - dataset['validation'][1])] - data_dict["val"], data_dict[ - "test"] = original_val[:len(original_val) // - 2], original_val[len(original_val) // - 2:] - if "val_as_dummy_test" in raw_args and raw_args["val_as_dummy_test"]: - # use the validation set as tmp test set, - # and partial training set as validation set - data_dict["test"] = data_dict["val"] - data_dict["val"] = [] - if "part_train_dummy_val" in raw_args and 1 > raw_args[ - "part_train_dummy_val"] > 0: - new_val_part = int(original_train_size * - raw_args["part_train_dummy_val"]) - data_dict["val"].extend(data_dict["train"][:new_val_part]) - data_dict["train"] = data_dict["train"][new_val_part:] - if "part_train_dummy_test" in raw_args and 1 > raw_args[ - "part_train_dummy_test"] > 0: - new_test_part = int(original_train_size * - raw_args["part_train_dummy_test"]) - data_dict["test"] = data_dict["val"] - if data_dict["test"] is not None: - data_dict["test"].extend(data_dict["train"][:new_test_part]) - else: - data_dict["test"] = (data_dict["train"][:new_test_part]) - data_dict["train"] = data_dict["train"][new_test_part:] - - return data_dict - - def load_openml_data(tid, splits=None, config=None): - import openml - from sklearn.model_selection import train_test_split - - task = openml.tasks.get_task(int(tid)) - did = task.dataset_id - dataset = openml.datasets.get_dataset(did) - data, targets, _, _ = dataset.get_data( - dataset_format="array", target=dataset.default_target_attribute) - - train_data, test_data, train_targets, test_targets = train_test_split( - data, targets, train_size=splits[0], random_state=config.seed) - val_data, test_data, val_targets, test_targets = train_test_split( - test_data, - test_targets, - train_size=splits[1] / (1. - splits[0]), - random_state=config.seed) - data_dict = { - 'train': [(x, y) for x, y in zip(train_data, train_targets)], - 'val': [(x, y) for x, y in zip(val_data, val_targets)], - 'test': [(x, y) for x, y in zip(test_data, test_targets)] - } - return data_dict - - DATA_LOAD_FUNCS = { - 'torchvision': load_torchvision_data, - 'torchtext': load_torchtext_data, - 'torchaudio': load_torchaudio_data, - 'torch_geometric': load_torch_geometric_data, - 'huggingface_datasets': load_huggingface_datasets_data, - 'openml': load_openml_data - } - - modified_config = config.clone() - - # Load dataset - splits = modified_config.data.splits - name, package = modified_config.data.type.split('@') - - dataset = DATA_LOAD_FUNCS[package.lower()](name, splits, modified_config) - splitter = get_splitter(modified_config) - - data_local_dict = { - x: {} - for x in range(1, modified_config.federate.client_num + 1) - } - - # Build dict of Dataloader - train_label_distribution = None - for split in dataset: - if dataset[split] is None: - continue - train_labels = list() - for i, ds in enumerate( - splitter(dataset[split], prior=train_label_distribution)): - labels = [x[1] for x in ds] - if split == 'train': - train_labels.append(labels) - data_local_dict[i + 1][split] = DataLoader( - ds, - batch_size=modified_config.data.batch_size, - shuffle=True, - num_workers=modified_config.data.num_workers) - else: - data_local_dict[i + 1][split] = DataLoader( - ds, - batch_size=modified_config.data.batch_size, - shuffle=False, - num_workers=modified_config.data.num_workers) - - if modified_config.data.consistent_label_distribution and len( - train_labels) > 0: - train_label_distribution = train_labels - - return data_local_dict, modified_config - - -def get_data(config): - """Instantiate the dataset and update the configuration accordingly if +# TODO: Add PyGNodeDataTranslator and PyGLinkDataTranslator +# TODO: move splitter to PyGNodeDataTranslator and PyGLinkDataTranslator +TRANS_DATA_MAP = { + 'BaseDataTranslator': [ + '.*?@.*?', 'hiv', 'proteins', 'imdb-binary', 'bbbp', 'tox21', 'bace', + 'sider', 'clintox', 'esol', 'freesolv', 'lipo' + ], + 'DummyDataTranslator': [ + 'toy', 'quadratic', 'femnist', 'celeba', 'shakespeare', 'twitter', + 'subreddit', 'synthetic', 'ciao', 'epinions', '.*?vertical_fl_data.*?', + '.*?movielens.*?', '.*?cikmcup.*?', 'graph_multi_domain.*?', 'cora', + 'citeseer', 'pubmed', 'dblp_conf', 'dblp_org', 'csbm.*?', 'fb15k-237', + 'wn18' + ], # Dummy for FL dataset +} +DATA_TRANS_MAP = RegexInverseMap(TRANS_DATA_MAP, None) + + +def get_data(config, client_cfgs=None): + """Instantiate the data and update the configuration accordingly if necessary. Arguments: - config (obj): a cfg node object. + config: a cfg node object. + client_cfgs: dict of client-specific cfg node object. Returns: obj: The dataset object. cfg.node: The updated configuration. """ - # fix the seed for data generation, - # will restore the user-specified on after the generation + # Fix the seed for data generation setup_seed(12345) + for func in register.data_dict.values(): - data_and_config = func(config) + data_and_config = func(config, client_cfgs) if data_and_config is not None: return data_and_config - if config.data.type.lower() == 'toy': - data, modified_config = load_toy_data(config) - elif config.data.type.lower() == 'quadratic': - from federatedscope.tabular.dataloader import load_quadratic_dataset - data, modified_config = load_quadratic_dataset(config) - elif config.data.type.lower() in ['femnist', 'celeba']: - from federatedscope.cv.dataloader import load_cv_dataset - data, modified_config = load_cv_dataset(config) - elif config.data.type.lower() in [ - 'shakespeare', 'twitter', 'subreddit', 'synthetic' - ]: - from federatedscope.nlp.dataloader import load_nlp_dataset - data, modified_config = load_nlp_dataset(config) - elif config.data.type.lower() in [ - 'cora', - 'citeseer', - 'pubmed', - 'dblp_conf', - 'dblp_org', - ] or config.data.type.lower().startswith('csbm'): - from federatedscope.gfl.dataloader import load_nodelevel_dataset - data, modified_config = load_nodelevel_dataset(config) - elif config.data.type.lower() in ['ciao', 'epinions', 'fb15k-237', 'wn18']: - from federatedscope.gfl.dataloader import load_linklevel_dataset - data, modified_config = load_linklevel_dataset(config) - elif config.data.type.lower() in [ - 'hiv', 'proteins', 'imdb-binary', 'bbbp', 'tox21', 'bace', 'sider', - 'clintox', 'esol', 'freesolv', 'lipo' - ] or config.data.type.startswith('graph_multi_domain'): - from federatedscope.gfl.dataloader import load_graphlevel_dataset - data, modified_config = load_graphlevel_dataset(config) - elif config.data.type.lower() == 'vertical_fl_data': - from federatedscope.vertical_fl.dataloader import load_vertical_data - data, modified_config = load_vertical_data(config, generate=True) - elif 'movielens' in config.data.type.lower( - ) or 'netflix' in config.data.type.lower(): - from federatedscope.mf.dataloader import load_mf_dataset - data, modified_config = load_mf_dataset(config) - elif '@' in config.data.type.lower(): - data, modified_config = load_external_data(config) - elif 'cikmcup' in config.data.type.lower(): - from federatedscope.gfl.dataset.cikm_cup import load_cikmcup_data - data, modified_config = load_cikmcup_data(config) - elif config.data.type is None or config.data.type == "": - # The participant (only for server in this version) does not own data - data = None - modified_config = config - else: - raise ValueError('Data {} not found.'.format(config.data.type)) - - if 'backdoor' in config.attack.attack_method and 'edge' in \ - config.attack.trigger_type: - import os - import torch - from federatedscope.attack.auxiliary import \ - create_ardis_poisoned_dataset, create_ardis_test_dataset - if not os.path.exists(config.attack.edge_path): - os.makedirs(config.attack.edge_path) - poisoned_edgeset = create_ardis_poisoned_dataset( - data_path=config.attack.edge_path) - - ardis_test_dataset = create_ardis_test_dataset( - config.attack.edge_path) - logger.info("Writing poison_data to: {}".format( - config.attack.edge_path)) + # Load dataset from source files + dataset, modified_config = load_dataset(config) - with open(config.attack.edge_path + "poisoned_edgeset_training", - "wb") as saved_data_file: - torch.save(poisoned_edgeset, saved_data_file) + # Perform translator to non-FL dataset + translator = getattr(import_module('federatedscope.core.data'), + DATA_TRANS_MAP[config.data.type.lower()])( + modified_config, client_cfgs) + data = translator(dataset) - with open(config.attack.edge_path + "ardis_test_dataset.pt", - "wb") as ardis_data_file: - torch.save(ardis_test_dataset, ardis_data_file) - logger.warning('please notice: downloading the poisoned dataset \ - on cifar-10 from \ - https://github.com/ksreenivasan/OOD_Federated_Learning') - - if 'backdoor' in config.attack.attack_method: - from federatedscope.attack.auxiliary import poisoning - poisoning(data, modified_config) + # Convert `StandaloneDataDict` to `ClientData` when in distribute mode + data = convert_data_mode(data, modified_config) + # Restore the user-specified seed after the data generation setup_seed(config.seed) - if config.federate.mode.lower() == 'standalone': - return data, modified_config - else: - # Invalid data_idx - if config.distribute.data_idx == -1: - return data, config - elif config.distribute.data_idx not in data.keys(): - data_idx = np.random.choice(list(data.keys())) - logger.warning( - f"The provided data_idx={config.distribute.data_idx} is " - f"invalid, so that we randomly sample a data_idx as {data_idx}" - ) - else: - data_idx = config.distribute.data_idx - return data[data_idx], config - - setup_seed(config.seed) - - -def merge_data(all_data, merged_max_data_id=None, specified_dataset_name=None): - """ - Merge data from client 1 to `merged_max_data_id` contained in given - `all_data`. - - :param all_data: - :param merged_max_data_id: - :param specified_dataset_name: - :return: - """ - if merged_max_data_id is None: - merged_max_data_id = len(all_data) - 1 - assert merged_max_data_id >= 1 - - if specified_dataset_name is None: - dataset_names = list(all_data[1].keys()) # e.g., train, test, val - else: - if not isinstance(specified_dataset_name, list): - specified_dataset_name = [specified_dataset_name] - dataset_names = specified_dataset_name - - import torch.utils.data - assert len(dataset_names) >= 1, \ - "At least one sub-dataset is required in client 1" - data_name = "test" if "test" in dataset_names else dataset_names[0] - id_contain_all_dataset_key = -1 - # check the existence of the data to be merged - for client_id in range(1, merged_max_data_id + 1): - contain_all_dataset_key = True - for dataset_name in dataset_names: - if dataset_name not in all_data[client_id]: - contain_all_dataset_key = False - logger.warning(f'Client {client_id} does not contain ' - f'dataset key {dataset_name}.') - if id_contain_all_dataset_key == -1 and contain_all_dataset_key: - id_contain_all_dataset_key = client_id - assert id_contain_all_dataset_key != -1, \ - "At least one client within [1, merged_max_data_id] should contain " \ - "all the key for expected dataset names." - - if isinstance(all_data[id_contain_all_dataset_key][data_name], dict): - data_elem_names = list(all_data[id_contain_all_dataset_key] - [data_name].keys()) # e.g., x, y - # init with empty list for each dataset_name - merged_data = {name: defaultdict(list) for name in dataset_names} - for data_id in range(1, merged_max_data_id + 1): - for d_name in dataset_names: - if d_name not in all_data[data_id]: - continue - for elem_name in data_elem_names: - merged_data[d_name][elem_name].append( - all_data[data_id][d_name][elem_name]) - for d_name in dataset_names: - for elem_name in data_elem_names: - merged_data[d_name][elem_name] = np.concatenate( - merged_data[d_name][elem_name]) - elif issubclass(type(all_data[id_contain_all_dataset_key][data_name]), - torch.utils.data.DataLoader): - # init with the same data as - # the one of client `id_contain_all_dataset_key` - merged_data = { - name: copy.deepcopy(all_data[id_contain_all_dataset_key][name]) - for name in dataset_names - } - for data_id in range(1, merged_max_data_id + 1): - if data_id == id_contain_all_dataset_key: - continue - for d_name in dataset_names: - if d_name not in all_data[data_id]: - continue - merged_data[d_name].dataset.extend( - all_data[data_id][d_name].dataset) - else: - raise NotImplementedError( - "Un-supported type when merging data across different clients." - f"Your data type is " - f"{type(all_data[id_contain_all_dataset_key][data_name])}. " - f"Currently we only support the following forms: " - " 1): {data_id: {train: {x:ndarray, y:ndarray}} }" - " 2): {data_id: {train: DataLoader }") - return merged_data + return data, modified_config diff --git a/federatedscope/core/auxiliaries/dataloader_builder.py b/federatedscope/core/auxiliaries/dataloader_builder.py index 858ba1a03..a412d76d2 100644 --- a/federatedscope/core/auxiliaries/dataloader_builder.py +++ b/federatedscope/core/auxiliaries/dataloader_builder.py @@ -1,3 +1,5 @@ +from federatedscope.core.data.utils import filter_dict + try: import torch from torch.utils.data import Dataset @@ -6,36 +8,67 @@ Dataset = object -def get_dataloader(dataset, config): - if config.backend == 'torch': - from torch.utils.data import DataLoader - dataloader = DataLoader(dataset, - batch_size=config.data.batch_size, - shuffle=config.data.shuffle, - num_workers=config.data.num_workers, - pin_memory=True) - return dataloader - else: - return None - +def get_dataloader(dataset, config, split='train'): + """ + Instantiate a DataLoader via config. -class WrapDataset(Dataset): - """Wrap raw data into pytorch Dataset + Args: + dataset: dataset from which to load the data. + config: configs containing batch_size, shuffle, etc. + split: current split (default: 'train'), if split is 'test', shuffle + will be `False`. And in PyG, 'test' split will use + `NeighborSampler` by default. - Arguments: - data (dict): raw data dictionary contains "x" and "y" + Returns: + dataloader: Instance of specific DataLoader configured by config. """ - def __init__(self, data): - super(WrapDataset, self).__init__() - self.data = data - - def __getitem__(self, idx): - if not isinstance(self.data["x"][idx], torch.Tensor): - return torch.from_numpy( - self.data["x"][idx]).float(), torch.from_numpy( - self.data["y"][idx]).float() - return self.data["x"][idx], self.data["y"][idx] - - def __len__(self): - return len(self.data["y"]) + # DataLoader builder only support torch backend now. + if config.backend != 'torch': + return None + + if config.dataloader.type == 'base': + from torch.utils.data import DataLoader + loader_cls = DataLoader + elif config.dataloader.type == 'raw': + # No DataLoader + return dataset + elif config.dataloader.type == 'pyg': + from torch_geometric.loader import DataLoader as PyGDataLoader + loader_cls = PyGDataLoader + elif config.dataloader.type == 'graphsaint-rw': + if split == 'train': + from torch_geometric.loader import GraphSAINTRandomWalkSampler + loader_cls = GraphSAINTRandomWalkSampler + else: + from torch_geometric.loader import NeighborSampler + loader_cls = NeighborSampler + elif config.dataloader.type == 'neighbor': + from torch_geometric.loader import NeighborSampler + loader_cls = NeighborSampler + elif config.dataloader.type == 'mf': + from federatedscope.mf.dataloader import MFDataLoader + loader_cls = MFDataLoader + else: + raise ValueError(f'data.loader.type {config.data.loader.type} ' + f'not found!') + + raw_args = dict(config.dataloader) + if split != 'train': + raw_args['shuffle'] = False + raw_args['sizes'] = [-1] + raw_args['drop_last'] = False + # For evaluation in GFL + if config.dataloader.type in ['graphsaint-rw', 'neighbor']: + raw_args['batch_size'] = 4096 + dataset = dataset[0].edge_index + else: + if config.dataloader.type in ['graphsaint-rw']: + # Raw graph + dataset = dataset[0] + elif config.dataloader.type in ['neighbor']: + # edge_index of raw graph + dataset = dataset[0].edge_index + filtered_args = filter_dict(loader_cls.__init__, raw_args) + dataloader = loader_cls(dataset, **filtered_args) + return dataloader diff --git a/federatedscope/core/auxiliaries/model_builder.py b/federatedscope/core/auxiliaries/model_builder.py index 0d80e015d..522159153 100644 --- a/federatedscope/core/auxiliaries/model_builder.py +++ b/federatedscope/core/auxiliaries/model_builder.py @@ -1,4 +1,7 @@ import logging + +import numpy as np + import federatedscope.register as register logger = logging.getLogger(__name__) @@ -16,7 +19,7 @@ def get_shape_from_data(data, model_config, backend='torch'): Extract the input shape from the given data, which can be used to build the data. Users can also use `data.input_shape` to specify the shape Arguments: - data (object): the data used for local training or evaluation + data (`ClientData`): the data used for local training or evaluation The expected data format: 1): {train/val/test: {x:ndarray, y:ndarray}}} 2): {train/val/test: DataLoader} @@ -31,7 +34,7 @@ def get_shape_from_data(data, model_config, backend='torch'): 'gcn', 'sage', 'gpr', 'gat', 'gin', 'mpnn' ] or model_config.type.startswith('gnn_'): num_label = data['num_label'] if 'num_label' in data else None - num_edge_features = data[ + num_edge_features = data['data'][ 'num_edge_features'] if model_config.type == 'mpnn' else None if model_config.task.startswith('graph'): # graph-level task @@ -39,12 +42,14 @@ def get_shape_from_data(data, model_config, backend='torch'): return (data_representative.x.shape, num_label, num_edge_features) else: # node/link-level task - return (data.x.shape, num_label, num_edge_features) + return (data['data'].x.shape, num_label, num_edge_features) if isinstance(data, dict): keys = list(data.keys()) if 'test' in keys: key_representative = 'test' + elif 'val' in keys: + key_representative = 'val' elif 'train' in keys: key_representative = 'train' elif 'data' in keys: diff --git a/federatedscope/core/auxiliaries/splitter_builder.py b/federatedscope/core/auxiliaries/splitter_builder.py index 14ae5ede2..fe2a138d7 100644 --- a/federatedscope/core/auxiliaries/splitter_builder.py +++ b/federatedscope/core/auxiliaries/splitter_builder.py @@ -43,6 +43,7 @@ def get_splitter(config): from federatedscope.core.splitters.generic import IIDSplitter splitter = IIDSplitter(client_num) else: - logger.warning(f'Splitter {config.data.splitter} not found.') + logger.warning(f'Splitter {config.data.splitter} not found or not ' + f'used.') splitter = None return splitter diff --git a/federatedscope/core/configs/README.md b/federatedscope/core/configs/README.md index 2c55ec7d1..815048234 100644 --- a/federatedscope/core/configs/README.md +++ b/federatedscope/core/configs/README.md @@ -14,33 +14,33 @@ We summarize all the customizable configurations: ### Data The configurations related to the data/dataset are defined in `cfg_data.py`. -| Name | (Type) Default Value | Description | Note | -|:----:|:-----:|:---------- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `data.root` | (string) 'data' | The folder where the data file located. `data.root` would be used together with `data.type` to load the dataset. | - | -| `data.type` | (string) 'toy' | Dataset name | CV: 'femnist', 'celeba' ; NLP: 'shakespeare', 'subreddit', 'twitter'; Graph: 'cora', 'citeseer', 'pubmed', 'dblp_conf', 'dblp_org', 'csbm', 'epinions', 'ciao', 'fb15k-237', 'wn18', 'fb15k' , 'MUTAG', 'BZR', 'COX2', 'DHFR', 'PTC_MR', 'AIDS', 'NCI1', 'ENZYMES', 'DD', 'PROTEINS', 'COLLAB', 'IMDB-BINARY', 'IMDB-MULTI', 'REDDIT-BINARY', 'IMDB-BINARY', 'IMDB-MULTI', 'HIV', 'ESOL', 'FREESOLV', 'LIPO', 'PCBA', 'MUV', 'BACE', 'BBBP', 'TOX21', 'TOXCAST', 'SIDER', 'CLINTOX', 'graph_multi_domain_mol', 'graph_multi_domain_small', 'graph_multi_domain_mix', 'graph_multi_domain_biochem'; MF: 'vflmovielens1m', 'vflmovielens10m', 'hflmovielens1m', 'hflmovielens10m', 'vflnetflix', 'hflnetflix'; Tabular: 'toy', 'synthetic'; External dataset: 'DNAME@torchvision', 'DNAME@torchtext', 'DNAME@huggingface_datasets', 'DNAME@openml'. | -| `data.args` | (list) [] | Args for the external dataset | Used for external dataset, eg. `[{'download': False}]` | -| `data.save_data` | (bool) False | Whether to save the generated toy data | - | -| `data.splitter` | (string) '' | Splitter name for standalone dataset | Generic splitter: 'lda'; Graph splitter: 'louvain', 'random', 'rel_type', 'scaffold', 'scaffold_lda', 'rand_chunk' | -| `data.splitter_args` | (list) [] | Args for splitter. | Used for splitter, eg. `[{'alpha': 0.5}]` | -| `data.transform` | (list) [] | Transform for x of data | Used in `get_item` in torch.dataset, eg. `[['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]]` | -| `data.target_transform` | (list) [] | Transform for y of data | Use as `data.transform` | -| `data.pre_transform` | (list) [] | Pre_transform for `torch_geometric` dataset | Use as `data.transform` | -| `data.batch_size` | (int) 64 | batch_size for DataLoader | - | -| `data.drop_last` | (bool) False | Whether drop last batch (if the number of last batch is smaller than batch_size) in DataLoader | - | -| `data.sizes` | (list) [10, 5] | Sample size for graph DataLoader | The length of `data.sizes` must meet the layer of GNN models. | -| `data.shuffle` | (bool) True | Shuffle train DataLoader | - | -| `data.server_holds_all` | (bool) False | Only use in global mode, whether the server (workers with idx 0) holds all data, useful in global training/evaluation case | - | -| `data.subsample` | (float) 1.0 |  Only used in LEAF datasets, subsample clients from all clients | - | -| `data.splits` | (list) [0.8, 0.1, 0.1] | Train, valid, test splits | - | +| Name | (Type) Default Value | Description | Note | +|:--------------------------------------------:|:-----:|:---------- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `data.root` | (string) 'data' | The folder where the data file located. `data.root` would be used together with `data.type` to load the dataset. | - | +| `data.type` | (string) 'toy' | Dataset name | CV: 'femnist', 'celeba' ; NLP: 'shakespeare', 'subreddit', 'twitter'; Graph: 'cora', 'citeseer', 'pubmed', 'dblp_conf', 'dblp_org', 'csbm', 'epinions', 'ciao', 'fb15k-237', 'wn18', 'fb15k' , 'MUTAG', 'BZR', 'COX2', 'DHFR', 'PTC_MR', 'AIDS', 'NCI1', 'ENZYMES', 'DD', 'PROTEINS', 'COLLAB', 'IMDB-BINARY', 'IMDB-MULTI', 'REDDIT-BINARY', 'IMDB-BINARY', 'IMDB-MULTI', 'HIV', 'ESOL', 'FREESOLV', 'LIPO', 'PCBA', 'MUV', 'BACE', 'BBBP', 'TOX21', 'TOXCAST', 'SIDER', 'CLINTOX', 'graph_multi_domain_mol', 'graph_multi_domain_small', 'graph_multi_domain_mix', 'graph_multi_domain_biochem'; MF: 'vflmovielens1m', 'vflmovielens10m', 'hflmovielens1m', 'hflmovielens10m', 'vflnetflix', 'hflnetflix'; Tabular: 'toy', 'synthetic'; External dataset: 'DNAME@torchvision', 'DNAME@torchtext', 'DNAME@huggingface_datasets', 'DNAME@openml'. | +| `data.args` | (list) [] | Args for the external dataset | Used for external dataset, eg. `[{'download': False}]` | +| `data.save_data` | (bool) False | Whether to save the generated toy data | - | +| `data.splitter` | (string) '' | Splitter name for standalone dataset | Generic splitter: 'lda'; Graph splitter: 'louvain', 'random', 'rel_type', 'graph_type', 'scaffold', 'scaffold_lda', 'rand_chunk' | +| `data.splitter_args` | (list) [] | Args for splitter. | Used for splitter, eg. `[{'alpha': 0.5}]` | +| `data.transform` | (list) [] | Transform for x of data | Used in `get_item` in torch.dataset, eg. `[['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]]` | +| `data.target_transform` | (list) [] | Transform for y of data | Use as `data.transform` | +| `data.pre_transform` | (list) [] | Pre_transform for `torch_geometric` dataset | Use as `data.transform` | +| `dataloader.batch_size` | (int) 64 | batch_size for DataLoader | - | +| `dataloader.drop_last` | (bool) False | Whether drop last batch (if the number of last batch is smaller than batch_size) in DataLoader | - | +| `dataloader.sizes` | (list) [10, 5] | Sample size for graph DataLoader | The length of `dataloader.sizes` must meet the layer of GNN models. | +| `dataloader.shuffle` | (bool) True | Shuffle train DataLoader | - | +| `data.server_holds_all` | (bool) False | Only use in global mode, whether the server (workers with idx 0) holds all data, useful in global training/evaluation case | - | +| `data.subsample` | (float) 1.0 |  Only used in LEAF datasets, subsample clients from all clients | - | +| `data.splits` | (list) [0.8, 0.1, 0.1] | Train, valid, test splits | - | | `data.`
`consistent_label_distribution` | (bool) False | Make label distribution of train/val/test set over clients keep consistent during splitting | - | -| `data.cSBM_phi` | (list) [0.5, 0.5, 0.5] | Phi for cSBM graph dataset | - | -| `data.loader` | (string) '' | Graph sample name, used in minibatch trainer | 'graphsaint-rw': use `GraphSAINTRandomWalkSampler` as DataLoader; 'neighbor': use `NeighborSampler` as DataLoader. | -| `data.num_workers` | (int) 0 | num_workers in DataLoader | - | -| `data.graphsaint.walk_length` | (int) 2 | The length of each random walk in graphsaint. | - | -| `data.graphsaint.num_steps` | (int) 30 | The number of iterations per epoch in graphsaint. | - | -| `data.quadratic.dim` | (int) 1 | Dim of synthetic quadratic  dataset | - | -| `data.quadratic.min_curv` | (float) 0.02 | Min_curve of synthetic quadratic dataset | - | -| `data.quadratic.max_curv` | (float) 12.5 | Max_cur of synthetic quadratic dataset | - | +| `data.cSBM_phi` | (list) [0.5, 0.5, 0.5] | Phi for cSBM graph dataset | - | +| `data.loader` | (string) '' | Graph sample name, used in minibatch trainer | 'graphsaint-rw': use `GraphSAINTRandomWalkSampler` as DataLoader; 'neighbor': use `NeighborSampler` as DataLoader. | +| `dataloader.num_workers` | (int) 0 | num_workers in DataLoader | - | +| `dataloader.walk_length` | (int) 2 | The length of each random walk in graphsaint. | - | +| `dataloader.num_steps` | (int) 30 | The number of iterations per epoch in graphsaint. | - | +| `data.quadratic.dim` | (int) 1 | Dim of synthetic quadratic  dataset | - | +| `data.quadratic.min_curv` | (float) 0.02 | Min_curve of synthetic quadratic dataset | - | +| `data.quadratic.max_curv` | (float) 12.5 | Max_cur of synthetic quadratic dataset | - | ### Model @@ -292,14 +292,14 @@ The configurations related to NbAFL method. #### SGDMF The configurations related to SGDMF method (only used in matrix factorization tasks). -| Name | (Type) Default Value | Description | Note | -|:---------------:|:--------------------:|:-----------------------------------|:--------------------------------------------------------| -| `sgdmf.use` | (bool) False | The indicator of the SGDMF method. | | -| `sgdmf.R` | (float) 5. | The upper bound of rating. | - | -| `sgdmf.epsilon` | (float) 4. | The $\epsilon$ used in DP. | - | -| `sgdmf.delta` | (float) 0.5 | The $\delta$ used in DP. | - | -| `sgdmf.constant` | (float) 1. | The constant in SGDMF | - | -| `sgdmf.theta` | (int) -1 | - | -1 means per-rating privacy, otherwise per-user privacy | +| Name | (Type) Default Value | Description | Note | +|:------------------:|:--------------------:|:-----------------------------------|:--------------------------------------------------------| +| `sgdmf.use` | (bool) False | The indicator of the SGDMF method. | | +| `sgdmf.R` | (float) 5. | The upper bound of rating. | - | +| `sgdmf.epsilon` | (float) 4. | The $\epsilon$ used in DP. | - | +| `sgdmf.delta` | (float) 0.5 | The $\delta$ used in DP. | - | +| `sgdmf.constant` | (float) 1. | The constant in SGDMF | - | +| `dagaloader.theta` | (int) -1 | - | -1 means per-rating privacy, otherwise per-user privacy | ### Auto-tuning Components diff --git a/federatedscope/core/configs/cfg_data.py b/federatedscope/core/configs/cfg_data.py index dea7c2091..b9eb0d48c 100644 --- a/federatedscope/core/configs/cfg_data.py +++ b/federatedscope/core/configs/cfg_data.py @@ -1,6 +1,10 @@ +import logging + from federatedscope.core.configs.config import CN from federatedscope.register import register_config +logger = logging.getLogger(__name__) + def extend_data_cfg(cfg): # ---------------------------------------------------------------------- # @@ -20,10 +24,6 @@ def extend_data_cfg(cfg): cfg.data.target_transform = [] # target_transform for y, use as above cfg.data.pre_transform = [ ] # pre_transform for `torch_geometric` dataset, use as above - cfg.data.batch_size = 64 - cfg.data.drop_last = False - cfg.data.sizes = [10, 5] - cfg.data.shuffle = True cfg.data.server_holds_all = False # whether the server (workers with # idx 0) holds all data, useful in global training/evaluation case cfg.data.subsample = 1.0 @@ -32,11 +32,22 @@ def extend_data_cfg(cfg): # distributions of train/val/test set over clients will be kept # consistent during splitting cfg.data.cSBM_phi = [0.5, 0.5, 0.5] - cfg.data.loader = '' - cfg.data.num_workers = 0 - cfg.data.graphsaint = CN() - cfg.data.graphsaint.walk_length = 2 - cfg.data.graphsaint.num_steps = 30 + + # DataLoader related args + cfg.dataloader = CN() + cfg.dataloader.type = 'base' + cfg.dataloader.batch_size = 64 + cfg.dataloader.shuffle = True + cfg.dataloader.num_workers = 0 + cfg.dataloader.drop_last = False + cfg.dataloader.pin_memory = True + # GFL: graphsaint DataLoader + cfg.dataloader.walk_length = 2 + cfg.dataloader.num_steps = 30 + # GFL: neighbor sampler DataLoader + cfg.dataloader.sizes = [10, 5] + # DP: -1 means per-rating privacy, otherwise per-user privacy + cfg.dataloader.theta = -1 # quadratic cfg.data.quadratic = CN() @@ -44,23 +55,71 @@ def extend_data_cfg(cfg): cfg.data.quadratic.min_curv = 0.02 cfg.data.quadratic.max_curv = 12.5 + # --------------- outdated configs --------------- + # TODO: delete this code block + cfg.data.loader = '' + cfg.data.batch_size = 64 + cfg.data.shuffle = True + cfg.data.num_workers = 0 + cfg.data.drop_last = False + cfg.data.walk_length = 2 + cfg.data.num_steps = 30 + cfg.data.sizes = [10, 5] + # --------------- register corresponding check function ---------- cfg.register_cfg_check_fun(assert_data_cfg) def assert_data_cfg(cfg): - if cfg.data.loader == 'graphsaint-rw': - assert cfg.model.layer == cfg.data.graphsaint.walk_length, 'Sample ' \ + if cfg.dataloader.type == 'graphsaint-rw': + assert cfg.model.layer == cfg.dataloader.walk_length, 'Sample ' \ 'size ' \ 'mismatch' - if cfg.data.loader == 'neighbor': - assert cfg.model.layer == len(cfg.data.sizes), 'Sample size mismatch' + if cfg.dataloader.type == 'neighbor': + assert cfg.model.layer == len( + cfg.dataloader.sizes), 'Sample size mismatch' if '@' in cfg.data.type: assert cfg.federate.client_num > 0, '`federate.client_num` should ' \ 'be greater than 0 when using ' \ 'external data' assert cfg.data.splitter, '`data.splitter` should not be empty when ' \ 'using external data' + # -------------------------------------------------------------------- + # For compatibility with older versions of FS + # TODO: delete this code block + if cfg.data.loader != '': + logger.warning('config `cfg.data.loader` will be remove in the ' + 'future, use `cfg.dataloader.type` instead.') + cfg.dataloader.type = cfg.data.loader + if cfg.data.batch_size != 64: + logger.warning('config `cfg.data.batch_size` will be remove in the ' + 'future, use `cfg.dataloader.batch_size` instead.') + cfg.dataloader.batch_size = cfg.data.batch_size + if not cfg.data.shuffle: + logger.warning('config `cfg.data.shuffle` will be remove in the ' + 'future, use `cfg.dataloader.shuffle` instead.') + cfg.dataloader.shuffle = cfg.data.shuffle + if cfg.data.num_workers != 0: + logger.warning('config `cfg.data.num_workers` will be remove in the ' + 'future, use `cfg.dataloader.num_workers` instead.') + cfg.dataloader.num_workers = cfg.data.num_workers + if cfg.data.drop_last: + logger.warning('config `cfg.data.drop_last` will be remove in the ' + 'future, use `cfg.dataloader.drop_last` instead.') + cfg.dataloader.drop_last = cfg.data.drop_last + if cfg.data.walk_length != 2: + logger.warning('config `cfg.data.walk_length` will be remove in the ' + 'future, use `cfg.dataloader.walk_length` instead.') + cfg.dataloader.walk_length = cfg.data.walk_length + if cfg.data.num_steps != 30: + logger.warning('config `cfg.data.num_steps` will be remove in the ' + 'future, use `cfg.dataloader.num_steps` instead.') + cfg.dataloader.num_steps = cfg.data.num_steps + if cfg.data.sizes != [10, 5]: + logger.warning('config `cfg.data.sizes` will be remove in the ' + 'future, use `cfg.dataloader.sizes` instead.') + cfg.dataloader.sizes = cfg.data.sizes + # -------------------------------------------------------------------- register_config("data", extend_data_cfg) diff --git a/federatedscope/core/configs/cfg_differential_privacy.py b/federatedscope/core/configs/cfg_differential_privacy.py index 9f90ad1de..7a6ae3e41 100644 --- a/federatedscope/core/configs/cfg_differential_privacy.py +++ b/federatedscope/core/configs/cfg_differential_privacy.py @@ -25,8 +25,6 @@ def extend_dp_cfg(cfg): cfg.sgdmf.epsilon = 4. # \epsilon in dp cfg.sgdmf.delta = 0.5 # \delta in dp cfg.sgdmf.constant = 1. # constant - cfg.sgdmf.theta = -1 # -1 means per-rating privacy, otherwise per-user - # privacy # --------------- register corresponding check function ---------- cfg.register_cfg_check_fun(assert_dp_cfg) diff --git a/federatedscope/core/data/README.md b/federatedscope/core/data/README.md new file mode 100644 index 000000000..e63e062df --- /dev/null +++ b/federatedscope/core/data/README.md @@ -0,0 +1,155 @@ +# DataZoo + +FederatedScope provides a rich collection of federated datasets for researchers, including images, texts, graphs, recommendation systems, and speeches, as well as utility classes `BaseDataTranslator` for building your own FS datasets. + +## Built-in FS data + +All datasets can be accessed from [`federatedscope.core.auxiliaries.data_builder.get_data`](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/core/auxiliaries/data_builder.py), which are built to [`federatedscope.core.data.StandaloneDataDict`](https://github.com/alibaba/FederatedScope/tree/master/federatedscope/core/data/base_data.py) (for more details, see [[DataZoo advanced]](#advanced)). By setting `cfg.data.type = DATASET_NAME`, FS would download and pre-process a specific dataset to be passed to `FedRunner`. For example: + +```python +# Source: federatedscope/main.py + +data, cfg = get_data(cfg) +runner = FedRunner(data=data, + server_class=get_server_cls(cfg), + client_class=get_client_cls(cfg), + config=cfg.clone()) +``` + +We provide a **look-up table** for you to get started with our DataZoo: + +| `cfg.data.type` | Domain | +| ------------------------------------------------------------ | ------------------- | +| [FEMNIST](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/cv/dataset/leaf_cv.py) | CV | +| [Celeba](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/cv/dataset/leaf_cv.py) | CV | +| [{DNAME}@torchvision](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/core/auxiliaries/data_builder.py) | CV | +| [Shakespeare](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/nlp/dataset/leaf_nlp.py) | NLP | +| [SubReddit](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/nlp/dataset/leaf_nlp.py) | NLP | +| [Twitter (Sentiment140)](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/nlp/dataset/leaf_twitter.py) | NLP | +| [{DNAME}@torchtext](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/core/auxiliaries/data_builder.py) | NLP | +| [{DNAME}@huggingface_datasets](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/core/auxiliaries/data_builder.py) | NLP | +| [Cora](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_node.py) | Graph (node-level) | +| [CiteSeer](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_node.py) | Graph (node-level) | +| [PubMed](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_node.py) | Graph (node-level) | +| [DBLP_conf](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataset/dblp_new.py) | Graph (node-level) | +| [DBLP_org](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataset/dblp_new.py) | Graph (node-level) | +| [csbm](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataset/cSBM_dataset.py) | Graph (node-level) | +| [Epinions](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataset/recsys.py) | Graph (link-level) | +| [Ciao](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataset/recsys.py) | Graph (link-level) | +| [FB15k](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_link.py) | Graph (link-level) | +| [FB15k-237](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_link.py) | Graph (link-level) | +| [WN18](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_link.py) | Graph (link-level) | +| [MUTAG](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [BZR](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [COX2](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [DHFR](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [PTC_MR](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [AIDS](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [NCI1](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [ENZYMES](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [DD](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [PROTEINS](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [COLLAB](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [IMDB-BINARY](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [IMDB-MULTI](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [REDDIT-BINARY](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [HIV](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [ESOL](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [FREESOLV](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [LIPO](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [PCBA](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [MUV](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [BACE](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [BBBP](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [TOX21](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [TOXCAST](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [SIDER](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [CLINTOX](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [graph_multi_domain_mol](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [graph_multi_domain_small](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [graph_multi_domain_biochem](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataloader/dataloader_graph.py) | Graph (graph-level) | +| [cikmcup](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/gfl/dataset/cikm_cup.py) | Graph (graph-level) | +| [toy](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/core/auxiliaries/data_builder.py) | Tabular | +| [synthetic](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/nlp/dataset/leaf_synthetic.py) | Tabular | +| [quadratic](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/tabular/dataloader/quadratic.py) | Tabular | +| [{DNAME}openml](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/core/auxiliaries/data_builder.py) | Tabular | +| [vertical_fl_data](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/vertical_fl/dataloader/dataloader.py) | Tabular(vertical) | +| [VFLMovieLens1M](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/mf/dataset/movielens.py) | Recommendation | +| [VFLMovieLens10M](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/mf/dataset/movielens.py) | Recommendation | +| [HFLMovieLens1M](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/mf/dataset/movielens.py) | Recommendation | +| [HFLMovieLens10M](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/mf/dataset/movielens.py) | Recommendation | +| [VFLNetflix](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/mf/dataset/netflix.py) | Recommendation | +| [HFLNetflix](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/mf/dataset/netflix.py) | Recommendation | + +## DataZoo Advanced + +In this section, we will introduce key concepts and tools to help you understand how FS data works and how to use it to build your own data in FS. + +Concepts: + +* [`federatedscope.core.data.ClientData`](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/core/data/base_data.py) + + * `ClientData` is a subclass of `dict`. In federated learning, each client (server) owns a `ClientData` for training, validating, or testing. Thus, each `ClientData` has one or more of `train`, `val`, and `test` as keys, and `DataLoader` accordingly. + + * The `DataLoader` of each key is created by `setup()` method, which specifies the arguments of `DataLoader`, such as `batch_size`, `shuffle` of `cfg`. + + Example: + + ```python + # Instantiate client_data for each Client + client_data = ClientData(DataLoader, + cfg, + train=train_data, + val=None, + test=test_data) + # other_cfg with different batch size + client_data.setup(other_cfg) + print(client_data) + + >> {'train': DataLoader(train_data), 'test': DataLoader(test_data)} + ``` + +* [`federatedscope.core.data.StandaloneDataDict`](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/core/data/base_data.py) + * `StandaloneDataDict` is a subclass of `dict`. As the name implies, `StandaloneDataDict` consists of all `ClientData` with client index as key (`0`, `1`, `2`, ...) in standalone mode. The key `0` is the data of the server for global evaluation or other usages. + * The method `preprocess()` in `StandaloneDataDict` makes changes to inner `ClientData` when `cfg` changes, such as in global mode, we set `cfg.federate.method == "global"`, and `StandaloneDataDict` will merge all `ClientData` to one client to perform global training. + +Tools + +* [`federatedscope.core.data.BaseDataTranslator`](https://github.com/alibaba/FederatedScope/blob/master/federatedscope/core/data/base_translator.py) + + * `BaseDataTranslator` converts [`torch.utils.data.Dataset`](https://pytorch.org/docs/stable/data.html#torch.utils.data.Dataset) or `dict` of data split to `StandaloneDataDict` according to `cfg`. After translating, it can be directly passed to `FedRunner` to launch a FL course. + + * `BaseDataTranslator` will split data to `train`, `val,` and `test` by `cfg.data.splits` (**ML split**). And using `Splitter` to split each data split to each client (**FL split**). In order to use `BaseDataTranslator`, `cfg.data.splitter`, `cfg.federate.client_num,` and other arguments of `Splitter` must be specified. + + Example: + + ```python + cfg.data.splitter = 'lda' + cfg.federate.client_num = 5 + cfg.data.splitter_args = [{'alpha': 0.2}] + + translator = BaseDataTranslator(global_cfg, DataLoader) + raw_data = CIFAR10() + fs_data = translator(raw_data) + + runner = FedRunner(data=fs_data, + server_class=get_server_cls(cfg), + client_class=get_client_cls(cfg), + config=cfg.clone()) + ``` + +* [`federatedscope.core.splitters`](federatedscope.core.splitters) + + * To generate simulated federation datasets, we provide `splitter` who are responsible for dispersing a given standalone dataset into multiple clients, with configurable statistical heterogeneity among them. + + We provide a **look-up table** for you to get started with our `Splitter`: + + | `cfg.data.splitter` | Domain | Arguments | + | :------------------ | ------------------- | :----------------------------------------------- | + | LDA | Generic | `alpha` | + | Louvain | Graph (node-level) | `delta` | + | Random | Graph (node-level) | `sampling_rate`, `overlapping_rate`, `drop_edge` | + | rel_type | Graph (link-level) | `alpha` | + | Scaffold | Molecular | - | + | Scaffold_lda | Molecular | `alpha` | + | Rand_chunk | Graph (graph-level) | - | \ No newline at end of file diff --git a/federatedscope/core/data/__init__.py b/federatedscope/core/data/__init__.py new file mode 100644 index 000000000..be0c01451 --- /dev/null +++ b/federatedscope/core/data/__init__.py @@ -0,0 +1,8 @@ +from federatedscope.core.data.base_data import StandaloneDataDict, ClientData +from federatedscope.core.data.base_translator import BaseDataTranslator +from federatedscope.core.data.dummy_translator import DummyDataTranslator + +__all__ = [ + 'StandaloneDataDict', 'ClientData', 'BaseDataTranslator', + 'DummyDataTranslator' +] diff --git a/federatedscope/core/data/base_data.py b/federatedscope/core/data/base_data.py new file mode 100644 index 000000000..c219f5049 --- /dev/null +++ b/federatedscope/core/data/base_data.py @@ -0,0 +1,174 @@ +import logging +from federatedscope.core.data.utils import merge_data +from federatedscope.core.auxiliaries.dataloader_builder import get_dataloader + +logger = logging.getLogger(__name__) + + +class StandaloneDataDict(dict): + """ + `StandaloneDataDict` maintain several `ClientData`. + """ + def __init__(self, datadict, global_cfg): + """ + + Args: + datadict: `Dict` with `client_id` as key, `ClientData` as value. + global_cfg: global CfgNode + """ + self.global_cfg = global_cfg + self.client_cfgs = None + datadict = self.preprocess(datadict) + super(StandaloneDataDict, self).__init__(datadict) + + def resetup(self, global_cfg, client_cfgs=None): + """ + Resetup new configs for `ClientData`, which might be used in HPO. + + Args: + global_cfg: enable new config for `ClientData` + client_cfgs: enable new client-specific config for `ClientData` + """ + self.global_cfg, self.client_cfgs = global_cfg, client_cfgs + for client_id, client_data in self.items(): + if isinstance(client_data, ClientData): + if client_cfgs is not None: + client_cfg = global_cfg.clone() + client_cfg.merge_from_other_cfg( + client_cfgs.get(f'client_{client_id}')) + else: + client_cfg = global_cfg + client_data.setup(client_cfg) + else: + logger.warning('`client_data` is not subclass of ' + '`ClientData`, and cannot re-setup ' + 'DataLoader with new configs.') + + def preprocess(self, datadict): + """ + Preprocess for StandaloneDataDict for: + 1. Global evaluation (merge test data). + 2. Global mode (train with centralized setting, merge all data). + + Args: + datadict: dict with `client_id` as key, `ClientData` as value. + """ + if self.global_cfg.federate.merge_test_data: + server_data = merge_data( + all_data=datadict, + merged_max_data_id=self.global_cfg.federate.client_num, + specified_dataset_name=['test']) + # `0` indicate Server + datadict[0] = server_data + + if self.global_cfg.federate.method == "global": + if self.global_cfg.federate.client_num != 1: + if self.global_cfg.data.server_holds_all: + assert datadict[0] is not None \ + and len(datadict[0]) != 0, \ + "You specified cfg.data.server_holds_all=True " \ + "but data[0] is None. Please check whether you " \ + "pre-process the data[0] correctly" + datadict[1] = datadict[0] + else: + logger.info(f"Will merge data from clients whose ids in " + f"[1, {self.global_cfg.federate.client_num}]") + datadict[1] = merge_data( + all_data=datadict, + merged_max_data_id=self.global_cfg.federate.client_num) + datadict = self.attack(datadict) + return datadict + + def attack(self, datadict): + """ + Apply attack to `StandaloneDataDict`. + + """ + if 'backdoor' in self.global_cfg.attack.attack_method and 'edge' in \ + self.global_cfg.attack.trigger_type: + import os + import torch + from federatedscope.attack.auxiliary import \ + create_ardis_poisoned_dataset, create_ardis_test_dataset + if not os.path.exists(self.global_cfg.attack.edge_path): + os.makedirs(self.global_cfg.attack.edge_path) + poisoned_edgeset = create_ardis_poisoned_dataset( + data_path=self.global_cfg.attack.edge_path) + + ardis_test_dataset = create_ardis_test_dataset( + self.global_cfg.attack.edge_path) + + logger.info("Writing poison_data to: {}".format( + self.global_cfg.attack.edge_path)) + + with open( + self.global_cfg.attack.edge_path + + "poisoned_edgeset_training", "wb") as saved_data_file: + torch.save(poisoned_edgeset, saved_data_file) + + with open( + self.global_cfg.attack.edge_path + + "ardis_test_dataset.pt", "wb") as ardis_data_file: + torch.save(ardis_test_dataset, ardis_data_file) + logger.warning( + 'please notice: downloading the poisoned dataset \ + on cifar-10 from \ + https://github.com/ksreenivasan/OOD_Federated_Learning' + ) + + if 'backdoor' in self.global_cfg.attack.attack_method: + from federatedscope.attack.auxiliary import poisoning + poisoning(datadict, self.global_cfg) + return datadict + + +class ClientData(dict): + """ + `ClientData` converts dataset to train/val/test DataLoader. + Key `data` in `ClientData` is the raw dataset. + """ + def __init__(self, client_cfg, train=None, val=None, test=None, **kwargs): + """ + + Args: + loader: Dataloader class or data dict which have been built + client_cfg: client-specific CfgNode + data: raw dataset, which will stay raw + train: train dataset, which will be converted to DataLoader + val: valid dataset, which will be converted to DataLoader + test: test dataset, which will be converted to DataLoader + """ + self.client_cfg = None + self.train = train + self.val = val + self.test = test + self.setup(client_cfg) + if kwargs is not None: + for key in kwargs: + self[key] = kwargs[key] + super(ClientData, self).__init__() + + def setup(self, new_client_cfg=None): + """ + + Args: + new_client_cfg: new client-specific CfgNode + + Returns: + Status: indicate whether the client_cfg is updated + """ + # if `batch_size` or `shuffle` change, reinstantiate DataLoader + if self.client_cfg is not None: + if dict(self.client_cfg.dataloader) == dict( + new_client_cfg.dataloader): + return False + + self.client_cfg = new_client_cfg + if self.train is not None: + self['train'] = get_dataloader(self.train, self.client_cfg, + 'train') + if self.val is not None: + self['val'] = get_dataloader(self.val, self.client_cfg, 'val') + if self.test is not None: + self['test'] = get_dataloader(self.test, self.client_cfg, 'test') + return True diff --git a/federatedscope/core/data/base_translator.py b/federatedscope/core/data/base_translator.py new file mode 100644 index 000000000..4fc82a0d3 --- /dev/null +++ b/federatedscope/core/data/base_translator.py @@ -0,0 +1,129 @@ +import logging +import numpy as np + +from federatedscope.core.auxiliaries.splitter_builder import get_splitter +from federatedscope.core.data import ClientData, StandaloneDataDict + +logger = logging.getLogger(__name__) + + +class BaseDataTranslator: + """ + Perform process: + Dataset -> ML split -> FL split -> Data (passed to FedRunner) + + """ + def __init__(self, global_cfg, client_cfgs=None): + """ + Convert data to `StandaloneDataDict`. + + Args: + global_cfg: global CfgNode + client_cfgs: client cfg `Dict` + """ + self.global_cfg = global_cfg + self.client_cfgs = client_cfgs + self.splitter = get_splitter(global_cfg) + + def __call__(self, dataset): + """ + + Args: + dataset: `torch.utils.data.Dataset`, `List` of (feature, label) + or split dataset tuple of (train, val, test) or Tuple of + split dataset with [train, val, test] + + Returns: + datadict: instance of `StandaloneDataDict`, which is a subclass of + `dict`. + """ + datadict = self.split(dataset) + datadict = StandaloneDataDict(datadict, self.global_cfg) + + return datadict + + def split(self, dataset): + """ + Perform ML split and FL split. + + Returns: + dict of `ClientData` with client_idx as key. + + """ + train, val, test = self.split_train_val_test(dataset) + datadict = self.split_to_client(train, val, test) + return datadict + + def split_train_val_test(self, dataset): + """ + Split dataset to train, val, test if not provided. + + Returns: + split_data (List): List of split dataset, [train, val, test] + + """ + splits = self.global_cfg.data.splits + if isinstance(dataset, tuple): + # No need to split train/val/test for tuple dataset. + error_msg = 'If dataset is tuple, it must contains ' \ + 'train, valid and test split.' + assert len(dataset) == len(['train', 'val', 'test']), error_msg + return [dataset[0], dataset[1], dataset[2]] + + index = np.random.permutation(np.arange(len(dataset))) + train_size = int(splits[0] * len(dataset)) + val_size = int(splits[1] * len(dataset)) + + train_dataset = [dataset[x] for x in index[:train_size]] + val_dataset = [ + dataset[x] for x in index[train_size:train_size + val_size] + ] + test_dataset = [dataset[x] for x in index[train_size + val_size:]] + return train_dataset, val_dataset, test_dataset + + def split_to_client(self, train, val, test): + """ + Split dataset to clients and build `ClientData`. + + Returns: + data_dict (dict): dict of `ClientData` with client_idx as key. + + """ + + # Initialization + client_num = self.global_cfg.federate.client_num + split_train, split_val, split_test = [[None] * client_num] * 3 + train_label_distribution = None + + # Split train/val/test to client + if len(train) > 0: + split_train = self.splitter(train) + if self.global_cfg.data.consistent_label_distribution: + try: + train_label_distribution = [[j[1] for j in x] + for x in split_train] + except: + logger.warning( + 'Cannot access train label distribution for ' + 'splitter.') + if len(val) > 0: + split_val = self.splitter(val, prior=train_label_distribution) + if len(test) > 0: + split_test = self.splitter(test, prior=train_label_distribution) + + # Build data dict with `ClientData`, key `0` for server. + data_dict = { + 0: ClientData(self.global_cfg, train=train, val=val, test=test) + } + for client_id in range(1, client_num + 1): + if self.client_cfgs is not None: + client_cfg = self.global_cfg.clone() + client_cfg.merge_from_other_cfg( + self.client_cfgs.get(f'client_{client_id}')) + else: + client_cfg = self.global_cfg + data_dict[client_id] = ClientData(client_cfg, + train=split_train[client_id - 1], + val=split_val[client_id - 1], + test=split_test[client_id - 1]) + return data_dict diff --git a/federatedscope/core/data/dummy_translator.py b/federatedscope/core/data/dummy_translator.py new file mode 100644 index 000000000..640a80ec3 --- /dev/null +++ b/federatedscope/core/data/dummy_translator.py @@ -0,0 +1,38 @@ +from federatedscope.core.data.base_translator import BaseDataTranslator +from federatedscope.core.data.base_data import ClientData + + +class DummyDataTranslator(BaseDataTranslator): + """ + DummyDataTranslator convert FL dataset to DataLoader. + Do not perform FL split. + """ + def split(self, dataset): + if not isinstance(dataset, dict): + raise TypeError(f'Not support data type {type(dataset)}') + datadict = {} + for client_id in dataset.keys(): + if self.client_cfgs is not None: + client_cfg = self.global_cfg.clone() + client_cfg.merge_from_other_cfg( + self.client_cfgs.get(f'client_{client_id}')) + else: + client_cfg = self.global_cfg + + if isinstance(dataset[client_id], dict): + datadict[client_id] = ClientData(client_cfg, + **dataset[client_id]) + else: + # Do not have train/val/test + train, val, test = self.split_train_val_test( + dataset[client_id]) + tmp_dict = dict(train=train, val=val, test=test) + # Only for graph-level task, get number of graph labels + if client_cfg.model.task.startswith('graph') and \ + client_cfg.model.out_channels == 0: + s = set() + for g in dataset[client_id]: + s.add(g.y.item()) + tmp_dict['num_label'] = len(s) + datadict[client_id] = ClientData(client_cfg, **tmp_dict) + return datadict diff --git a/federatedscope/core/data/utils.py b/federatedscope/core/data/utils.py new file mode 100644 index 000000000..1b46cac67 --- /dev/null +++ b/federatedscope/core/data/utils.py @@ -0,0 +1,617 @@ +import copy +import inspect +import logging +import os +import re +from collections import defaultdict + +import numpy as np +from random import shuffle + +logger = logging.getLogger(__name__) + + +class RegexInverseMap: + def __init__(self, n_dic, val): + self._items = {} + for key, values in n_dic.items(): + for value in values: + self._items[value] = key + self.__val = val + + def __getitem__(self, key): + for regex in self._items.keys(): + if re.compile(regex).match(key): + return self._items[regex] + return self.__val + + def __repr__(self): + return str(self._items.items()) + + +def load_dataset(config): + if config.data.type.lower() == 'toy': + from federatedscope.tabular.dataloader.toy import load_toy_data + dataset, modified_config = load_toy_data(config) + elif config.data.type.lower() == 'quadratic': + from federatedscope.tabular.dataloader import load_quadratic_dataset + dataset, modified_config = load_quadratic_dataset(config) + elif config.data.type.lower() in ['femnist', 'celeba']: + from federatedscope.cv.dataloader import load_cv_dataset + dataset, modified_config = load_cv_dataset(config) + elif config.data.type.lower() in [ + 'shakespeare', 'twitter', 'subreddit', 'synthetic' + ]: + from federatedscope.nlp.dataloader import load_nlp_dataset + dataset, modified_config = load_nlp_dataset(config) + elif config.data.type.lower() in [ + 'cora', + 'citeseer', + 'pubmed', + 'dblp_conf', + 'dblp_org', + ] or config.data.type.lower().startswith('csbm'): + from federatedscope.gfl.dataloader import load_nodelevel_dataset + dataset, modified_config = load_nodelevel_dataset(config) + elif config.data.type.lower() in ['ciao', 'epinions', 'fb15k-237', 'wn18']: + from federatedscope.gfl.dataloader import load_linklevel_dataset + dataset, modified_config = load_linklevel_dataset(config) + elif config.data.type.lower() in [ + 'hiv', 'proteins', 'imdb-binary', 'bbbp', 'tox21', 'bace', 'sider', + 'clintox', 'esol', 'freesolv', 'lipo', 'cikmcup' + ] or config.data.type.startswith('graph_multi_domain'): + from federatedscope.gfl.dataloader import load_graphlevel_dataset + dataset, modified_config = load_graphlevel_dataset(config) + elif config.data.type.lower() == 'vertical_fl_data': + from federatedscope.vertical_fl.dataloader import load_vertical_data + dataset, modified_config = load_vertical_data(config, generate=True) + elif 'movielens' in config.data.type.lower( + ) or 'netflix' in config.data.type.lower(): + from federatedscope.mf.dataloader import load_mf_dataset + dataset, modified_config = load_mf_dataset(config) + elif '@' in config.data.type.lower(): + from federatedscope.core.data.utils import load_external_data + dataset, modified_config = load_external_data(config) + elif config.data.type is None or config.data.type == "": + # The participant (only for server in this version) does not own data + dataset = None + modified_config = config + else: + raise ValueError('Dataset {} not found.'.format(config.data.type)) + return dataset, modified_config + + +def load_external_data(config=None): + r""" Based on the configuration file, this function imports external + datasets and applies train/valid/test splits and split by some specific + `splitter` into the standard FederatedScope input data format. + + Args: + config: `CN` from `federatedscope/core/configs/config.py` + + Returns: + data_local_dict: dict of split dataloader. + Format: + { + 'client_id': { + 'train': DataLoader(), + 'test': DataLoader(), + 'val': DataLoader() + } + } + modified_config: `CN` from `federatedscope/core/configs/config.py`, + which might be modified in the function. + + """ + + import torch + from importlib import import_module + from torch.utils.data import DataLoader + from federatedscope.core.auxiliaries.transform_builder import get_transform + + def load_torchvision_data(name, splits=None, config=None): + dataset_func = getattr(import_module('torchvision.datasets'), name) + transform_funcs = get_transform(config, 'torchvision') + if config.data.args: + raw_args = config.data.args[0] + else: + raw_args = {} + if 'download' not in raw_args.keys(): + raw_args.update({'download': True}) + filtered_args = filter_dict(dataset_func.__init__, raw_args) + func_args = get_func_args(dataset_func.__init__) + + # Perform split on different dataset + if 'train' in func_args: + # Split train to (train, val) + dataset_train = dataset_func(root=config.data.root, + train=True, + **filtered_args, + **transform_funcs) + dataset_val = None + dataset_test = dataset_func(root=config.data.root, + train=False, + **filtered_args, + **transform_funcs) + if splits: + train_size = int(splits[0] * len(dataset_train)) + val_size = len(dataset_train) - train_size + lengths = [train_size, val_size] + dataset_train, dataset_val = \ + torch.utils.data.dataset.random_split(dataset_train, + lengths) + + elif 'split' in func_args: + # Use raw split + dataset_train = dataset_func(root=config.data.root, + split='train', + **filtered_args, + **transform_funcs) + dataset_val = dataset_func(root=config.data.root, + split='valid', + **filtered_args, + **transform_funcs) + dataset_test = dataset_func(root=config.data.root, + split='test', + **filtered_args, + **transform_funcs) + elif 'classes' in func_args: + # Use raw split + dataset_train = dataset_func(root=config.data.root, + classes='train', + **filtered_args, + **transform_funcs) + dataset_val = dataset_func(root=config.data.root, + classes='valid', + **filtered_args, + **transform_funcs) + dataset_test = dataset_func(root=config.data.root, + classes='test', + **filtered_args, + **transform_funcs) + else: + # Use config.data.splits + dataset = dataset_func(root=config.data.root, + **filtered_args, + **transform_funcs) + train_size = int(splits[0] * len(dataset)) + val_size = int(splits[1] * len(dataset)) + test_size = len(dataset) - train_size - val_size + lengths = [train_size, val_size, test_size] + dataset_train, dataset_val, dataset_test = \ + torch.utils.data.dataset.random_split(dataset, lengths) + + data_split_dict = { + 'train': dataset_train, + 'val': dataset_val, + 'test': dataset_test + } + + return data_split_dict + + def load_torchtext_data(name, splits=None, config=None): + from torch.nn.utils.rnn import pad_sequence + from federatedscope.nlp.dataset.utils import label_to_index + + dataset_func = getattr(import_module('torchtext.datasets'), name) + if config.data.args: + raw_args = config.data.args[0] + else: + raw_args = {} + assert 'max_len' in raw_args, "Miss key 'max_len' in " \ + "`config.data.args`." + filtered_args = filter_dict(dataset_func.__init__, raw_args) + dataset = dataset_func(root=config.data.root, **filtered_args) + + # torchtext.transforms requires >= 0.12.0 and torch = 1.11.0, + # so we do not use `get_transform` in torchtext. + + # Merge all data and tokenize + x_list = [] + y_list = [] + for data_iter in dataset: + data, targets = [], [] + for i, item in enumerate(data_iter): + data.append(item[1]) + targets.append(item[0]) + x_list.append(data) + y_list.append(targets) + + x_all, y_all = [], [] + for i in range(len(x_list)): + x_all += x_list[i] + y_all += y_list[i] + + if config.model.type.endswith('transformers'): + from transformers import AutoTokenizer + cache_path = os.path.join(os.getcwd(), "huggingface") + try: + tokenizer = AutoTokenizer.from_pretrained( + config.model.type.split('@')[0], + local_files_only=True, + cache_dir=cache_path) + except Exception as e: + logging.error(f"When loading cached file form " + f"{cache_path}, we faced the exception: \n " + f"{str(e)}") + + x_all = tokenizer(x_all, + return_tensors='pt', + padding=True, + truncation=True, + max_length=raw_args['max_len']) + data = [{key: value[i] + for key, value in x_all.items()} + for i in range(len(next(iter(x_all.values()))))] + if 'classification' in config.model.task.lower(): + targets = label_to_index(y_all) + else: + y_all = tokenizer(y_all, + return_tensors='pt', + padding=True, + truncation=True, + max_length=raw_args['max_len']) + targets = [{key: value[i] + for key, value in y_all.items()} + for i in range(len(next(iter(y_all.values()))))] + else: + from torchtext.data import get_tokenizer + tokenizer = get_tokenizer("basic_english") + if len(config.data.transform) == 0: + raise ValueError( + "`transform` must be one pretrained Word Embeddings from \ + ['GloVe', 'FastText', 'CharNGram']") + if len(config.data.transform) == 1: + config.data.transform.append({}) + vocab = getattr(import_module('torchtext.vocab'), + config.data.transform[0])( + dim=config.model.in_channels, + **config.data.transform[1]) + + if 'classification' in config.model.task.lower(): + data = [ + vocab.get_vecs_by_tokens(tokenizer(x), + lower_case_backup=True) + for x in x_all + ] + targets = label_to_index(y_all) + else: + data = [ + vocab.get_vecs_by_tokens(tokenizer(x), + lower_case_backup=True) + for x in x_all + ] + targets = [ + vocab.get_vecs_by_tokens(tokenizer(y), + lower_case_backup=True) + for y in y_all + ] + targets = pad_sequence(targets).transpose( + 0, 1)[:, :raw_args['max_len'], :] + data = pad_sequence(data).transpose(0, + 1)[:, :raw_args['max_len'], :] + # Split data to raw + num_items = [len(ds) for ds in x_list] + data_list, cnt = [], 0 + for num in num_items: + data_list.append([ + (x, y) + for x, y in zip(data[cnt:cnt + num], targets[cnt:cnt + num]) + ]) + cnt += num + + if len(data_list) == 3: + # Use raw splits + data_split_dict = { + 'train': data_list[0], + 'val': data_list[1], + 'test': data_list[2] + } + elif len(data_list) == 2: + # Split train to (train, val) + data_split_dict = { + 'train': data_list[0], + 'val': None, + 'test': data_list[1] + } + if splits: + train_size = int(splits[0] * len(data_split_dict['train'])) + val_size = len(data_split_dict['train']) - train_size + lengths = [train_size, val_size] + data_split_dict['train'], data_split_dict[ + 'val'] = torch.utils.data.dataset.random_split( + data_split_dict['train'], lengths) + else: + # Use config.data.splits + data_split_dict = {} + train_size = int(splits[0] * len(data_list[0])) + val_size = int(splits[1] * len(data_list[0])) + test_size = len(data_list[0]) - train_size - val_size + lengths = [train_size, val_size, test_size] + data_split_dict['train'], data_split_dict['val'], data_split_dict[ + 'test'] = torch.utils.data.dataset.random_split( + data_list[0], lengths) + + return data_split_dict + + def load_torchaudio_data(name, splits=None, config=None): + + # dataset_func = getattr(import_module('torchaudio.datasets'), name) + raise NotImplementedError + + def load_huggingface_datasets_data(name, splits=None, config=None): + from datasets import load_dataset, load_from_disk + + if config.data.args: + raw_args = config.data.args[0] + else: + raw_args = {} + assert 'max_len' in raw_args, "Miss key 'max_len' in " \ + "`config.data.args`." + filtered_args = filter_dict(load_dataset, raw_args) + logger.info("Begin to load huggingface dataset") + if "hg_cache_dir" in raw_args: + hugging_face_path = raw_args["hg_cache_dir"] + else: + hugging_face_path = os.getcwd() + + if "load_disk_dir" in raw_args: + load_path = raw_args["load_disk_dir"] + try: + dataset = load_from_disk(load_path) + except Exception as e: + logging.error(f"When loading cached dataset form " + f"{load_path}, we faced the exception: \n " + f"{str(e)}") + else: + dataset = load_dataset(path=config.data.root, + name=name, + **filtered_args) + if config.model.type.endswith('transformers'): + os.environ["TOKENIZERS_PARALLELISM"] = "false" + from transformers import AutoTokenizer + logger.info("To load huggingface tokenizer") + tokenizer = AutoTokenizer.from_pretrained( + config.model.type.split('@')[0], + local_files_only=True, + cache_dir=os.path.join(hugging_face_path, "transformers")) + + for split in dataset: + x_all = [i['sentence'] for i in dataset[split]] + targets = [i['label'] for i in dataset[split]] + + if split == "train" and "used_train_ratio" in raw_args and \ + 1 > raw_args['used_train_ratio'] > 0: + selected_idx = [i for i in range(len(dataset[split]))] + shuffle(selected_idx) + selected_idx = selected_idx[:int( + len(selected_idx) * raw_args['used_train_ratio'])] + x_all = [ + element for i, element in enumerate(x_all) + if i in selected_idx + ] + targets = [ + element for i, element in enumerate(targets) + if i in selected_idx + ] + + x_all = tokenizer(x_all, + return_tensors='pt', + padding=True, + truncation=True, + max_length=raw_args['max_len']) + data = [{key: value[i] + for key, value in x_all.items()} + for i in range(len(next(iter(x_all.values()))))] + dataset[split] = (data, targets) + data_split_dict = { + 'train': [(x, y) + for x, y in zip(dataset['train'][0], dataset['train'][1]) + ], + 'val': [(x, y) for x, y in zip(dataset['validation'][0], + dataset['validation'][1])], + 'test': [ + (x, y) for x, y in zip(dataset['test'][0], dataset['test'][1]) + ] if (set(dataset['test'][1]) - set([-1])) else None, + } + original_train_size = len(data_split_dict["train"]) + + if "half_val_dummy_test" in raw_args and raw_args[ + "half_val_dummy_test"]: + # since the "test" set from GLUE dataset may be masked, we need to + # submit to get the ground-truth, for fast FL experiments, + # we split the validation set into two parts with the same size as + # new test/val data + original_val = [(x, y) for x, y in zip(dataset['validation'][0], + dataset['validation'][1])] + data_split_dict["val"], data_split_dict[ + "test"] = original_val[:len(original_val) // + 2], original_val[len(original_val) // + 2:] + if "val_as_dummy_test" in raw_args and raw_args["val_as_dummy_test"]: + # use the validation set as tmp test set, + # and partial training set as validation set + data_split_dict["test"] = data_split_dict["val"] + data_split_dict["val"] = [] + if "part_train_dummy_val" in raw_args and 1 > raw_args[ + "part_train_dummy_val"] > 0: + new_val_part = int(original_train_size * + raw_args["part_train_dummy_val"]) + data_split_dict["val"].extend( + data_split_dict["train"][:new_val_part]) + data_split_dict["train"] = data_split_dict["train"][new_val_part:] + if "part_train_dummy_test" in raw_args and 1 > raw_args[ + "part_train_dummy_test"] > 0: + new_test_part = int(original_train_size * + raw_args["part_train_dummy_test"]) + data_split_dict["test"] = data_split_dict["val"] + if data_split_dict["test"] is not None: + data_split_dict["test"].extend( + data_split_dict["train"][:new_test_part]) + else: + data_split_dict["test"] = ( + data_split_dict["train"][:new_test_part]) + data_split_dict["train"] = data_split_dict["train"][new_test_part:] + + return data_split_dict + + def load_openml_data(tid, splits=None, config=None): + import openml + from sklearn.model_selection import train_test_split + + task = openml.tasks.get_task(int(tid)) + did = task.dataset_id + dataset = load_dataset(did) + data, targets, _, _ = dataset.get_data( + dataset_format="array", target=dataset.default_target_attribute) + + train_data, test_data, train_targets, test_targets = train_test_split( + data, targets, train_size=splits[0], random_state=config.seed) + val_data, test_data, val_targets, test_targets = train_test_split( + test_data, + test_targets, + train_size=splits[1] / (1. - splits[0]), + random_state=config.seed) + data_split_dict = { + 'train': [(x, y) for x, y in zip(train_data, train_targets)], + 'val': [(x, y) for x, y in zip(val_data, val_targets)], + 'test': [(x, y) for x, y in zip(test_data, test_targets)] + } + return data_split_dict + + DATA_LOAD_FUNCS = { + 'torchvision': load_torchvision_data, + 'torchtext': load_torchtext_data, + 'torchaudio': load_torchaudio_data, + 'huggingface_datasets': load_huggingface_datasets_data, + 'openml': load_openml_data + } + + modified_config = config.clone() + + # Load dataset + splits = modified_config.data.splits + name, package = modified_config.data.type.split('@') + + # Comply with the original train/val/test + dataset = DATA_LOAD_FUNCS[package.lower()](name, splits, modified_config) + data_split_tuple = (dataset.get('train'), dataset.get('val'), + dataset.get('test')) + + return data_split_tuple, modified_config + + +def convert_data_mode(data, config): + if config.federate.mode.lower() == 'standalone': + return data + else: + # Invalid data_idx + if config.distribute.data_idx == -1: + return data + elif config.distribute.data_idx not in data.keys(): + data_idx = np.random.choice(list(data.keys())) + logger.warning( + f"The provided data_idx={config.distribute.data_idx} is " + f"invalid, so that we randomly sample a data_idx as {data_idx}" + ) + else: + data_idx = config.distribute.data_idx + return data[data_idx] + + +def get_func_args(func): + sign = inspect.signature(func).parameters.values() + sign = set([val.name for val in sign]) + return sign + + +def filter_dict(func, kwarg): + sign = get_func_args(func) + common_args = sign.intersection(kwarg.keys()) + filtered_dict = {key: kwarg[key] for key in common_args} + return filtered_dict + + +def merge_data(all_data, merged_max_data_id=None, specified_dataset_name=None): + """ + Merge data from client 1 to `merged_max_data_id` contained in given + `all_data`. + :param all_data: + :param merged_max_data_id: + :param specified_dataset_name: + :return: + """ + import torch.utils.data + from federatedscope.core.data.wrap_dataset import WrapDataset + + # Assert + if merged_max_data_id is None: + merged_max_data_id = len(all_data) - 1 + assert merged_max_data_id >= 1 + if specified_dataset_name is None: + dataset_names = list(all_data[1].keys()) # e.g., train, test, val + else: + if not isinstance(specified_dataset_name, list): + specified_dataset_name = [specified_dataset_name] + dataset_names = specified_dataset_name + assert len(dataset_names) >= 1, \ + "At least one sub-dataset is required in client 1" + + data_name = "test" if "test" in dataset_names else dataset_names[0] + id_contain_all_dataset_key = -1 + # check the existence of the data to be merged + for client_id in range(1, merged_max_data_id + 1): + contain_all_dataset_key = True + for dataset_name in dataset_names: + if dataset_name not in all_data[client_id]: + contain_all_dataset_key = False + logger.warning(f'Client {client_id} does not contain ' + f'dataset key {dataset_name}.') + if id_contain_all_dataset_key == -1 and contain_all_dataset_key: + id_contain_all_dataset_key = client_id + assert id_contain_all_dataset_key != -1, \ + "At least one client within [1, merged_max_data_id] should contain " \ + "all the key for expected dataset names." + + if issubclass(type(all_data[id_contain_all_dataset_key][data_name]), + torch.utils.data.DataLoader): + if isinstance(all_data[id_contain_all_dataset_key][data_name].dataset, + WrapDataset): + data_elem_names = list(all_data[id_contain_all_dataset_key] + [data_name].dataset.dataset.keys()) # + # e.g., x, y + merged_data = {name: defaultdict(list) for name in dataset_names} + for data_id in range(1, merged_max_data_id + 1): + for d_name in dataset_names: + if d_name not in all_data[data_id]: + continue + for elem_name in data_elem_names: + merged_data[d_name][elem_name].append( + all_data[data_id] + [d_name].dataset.dataset[elem_name]) + for d_name in dataset_names: + for elem_name in data_elem_names: + merged_data[d_name][elem_name] = np.concatenate( + merged_data[d_name][elem_name]) + for name in all_data[id_contain_all_dataset_key]: + all_data[id_contain_all_dataset_key][ + name].dataset.dataset = merged_data[name] + else: + merged_data = copy.deepcopy(all_data[id_contain_all_dataset_key]) + for data_id in range(1, merged_max_data_id + 1): + if data_id == id_contain_all_dataset_key: + continue + for d_name in dataset_names: + if d_name not in all_data[data_id]: + continue + merged_data[d_name].dataset.extend( + all_data[data_id][d_name].dataset) + else: + raise NotImplementedError( + "Un-supported type when merging data across different clients." + f"Your data type is " + f"{type(all_data[id_contain_all_dataset_key][data_name])}. " + f"Currently we only support the following forms: " + " 1): {data_id: {train: {x:ndarray, y:ndarray}} }" + " 2): {data_id: {train: DataLoader }") + return merged_data diff --git a/federatedscope/core/data/wrap_dataset.py b/federatedscope/core/data/wrap_dataset.py new file mode 100644 index 000000000..72ed832fa --- /dev/null +++ b/federatedscope/core/data/wrap_dataset.py @@ -0,0 +1,31 @@ +import torch +import numpy as np +from torch.utils.data import Dataset + + +class WrapDataset(Dataset): + """Wrap raw data into pytorch Dataset + + Arguments: + dataset (dict): raw data dictionary contains "x" and "y" + + """ + def __init__(self, dataset): + super(WrapDataset, self).__init__() + self.dataset = dataset + + def __getitem__(self, idx): + if isinstance(self.dataset["x"][idx], torch.Tensor): + return self.dataset["x"][idx], self.dataset["y"][idx] + elif isinstance(self.dataset["x"][idx], np.ndarray): + return torch.from_numpy( + self.dataset["x"][idx]).float(), torch.from_numpy( + self.dataset["y"][idx]).float() + elif isinstance(self.dataset["x"][idx], list): + return torch.FloatTensor(self.dataset["x"][idx]), \ + torch.FloatTensor(self.dataset["y"][idx]) + else: + raise TypeError + + def __len__(self): + return len(self.dataset["y"]) diff --git a/federatedscope/core/fed_runner.py b/federatedscope/core/fed_runner.py index 795cf0d32..dac9e6649 100644 --- a/federatedscope/core/fed_runner.py +++ b/federatedscope/core/fed_runner.py @@ -8,8 +8,8 @@ from federatedscope.core.workers import Server, Client from federatedscope.core.gpu_manager import GPUManager from federatedscope.core.auxiliaries.model_builder import get_model -from federatedscope.core.auxiliaries.data_builder import merge_data from federatedscope.core.auxiliaries.utils import get_resource_info +from federatedscope.core.data.utils import merge_data logger = logging.getLogger(__name__) @@ -28,14 +28,14 @@ class FedRunner(object): client_class: The client class is used for instantiating a ( customized) client. config: The configurations of the FL course. - client_config: The clients' configurations. + client_configs: The clients' configurations. """ def __init__(self, data, server_class=Server, client_class=Client, config=None, - client_config=None): + client_configs=None): self.data = data self.server_class = server_class self.client_class = client_class @@ -44,7 +44,7 @@ def __init__(self, if not config.is_ready_for_run: config.ready_for_run() self.cfg = config - self.client_cfg = client_config + self.client_cfgs = client_configs self.mode = self.cfg.federate.mode.lower() self.gpu_manager = GPUManager(gpu_available=self.cfg.use_gpu, @@ -89,24 +89,10 @@ def _setup_for_standalone(self): "specify a non-zero value for client_num" if self.cfg.federate.method == "global": - if self.cfg.federate.client_num != 1: - if self.cfg.data.server_holds_all: - assert self.data[0] is not None \ - and len(self.data[0]) != 0, \ - "You specified cfg.data.server_holds_all=True " \ - "but data[0] is None. Please check whether you " \ - "pre-process the data[0] correctly" - self.data[1] = self.data[0] - else: - logger.info(f"Will merge data from clients whose ids in " - f"[1, {self.cfg.federate.client_num}]") - self.data[1] = merge_data( - all_data=self.data, - merged_max_data_id=self.cfg.federate.client_num) - self.cfg.defrost() - self.cfg.federate.client_num = 1 - self.cfg.federate.sample_client_num = 1 - self.cfg.freeze() + self.cfg.defrost() + self.cfg.federate.client_num = 1 + self.cfg.federate.sample_client_num = 1 + self.cfg.freeze() # sample resource information if self.resource_info is not None: @@ -286,15 +272,7 @@ def _setup_server(self, resource_info=None, client_resource_info=None): """ self.server_id = 0 if self.mode == 'standalone': - if self.cfg.federate.merge_test_data: - server_data = merge_data( - all_data=self.data, - merged_max_data_id=self.cfg.federate.client_num, - specified_dataset_name=['test']) - model = get_model(self.cfg.model, - server_data, - backend=self.cfg.backend) - elif self.server_id in self.data: + if self.server_id in self.data: server_data = self.data[self.server_id] model = get_model(self.cfg.model, server_data, @@ -375,10 +353,10 @@ def _setup_client(self, if self.client_class: client_specific_config = self.cfg.clone() - if self.client_cfg: + if self.client_cfgs: client_specific_config.defrost() client_specific_config.merge_from_other_cfg( - self.client_cfg.get('client_{}'.format(client_id))) + self.client_cfgs.get('client_{}'.format(client_id))) client_specific_config.freeze() client_device = self._server_device if \ self.cfg.federate.share_local_model else \ diff --git a/federatedscope/core/monitors/monitor.py b/federatedscope/core/monitors/monitor.py index e5f33a323..0f4832164 100644 --- a/federatedscope/core/monitors/monitor.py +++ b/federatedscope/core/monitors/monitor.py @@ -316,17 +316,21 @@ def format_eval_res(self, 3356, 'test_loss': 2892, 'test_total': 5000 }, 'Results_fairness': { - 'test_total': 33.99, 'test_correct': 27.185, 'test_avg_loss_std': 0.433551, - 'test_avg_loss_bottom_decile': 0.356503, 'test_avg_loss_top_decile': 1.212492, + 'test_total': 33.99, 'test_correct': 27.185, + 'test_avg_loss_std': 0.433551, + 'test_avg_loss_bottom_decile': 0.356503, + 'test_avg_loss_top_decile': 1.212492, 'test_avg_loss_min': 0.198317, 'test_avg_loss_max': 3.603567, - 'test_avg_loss_bottom10%': 0.276681, 'test_avg_loss_top10%': 1.686649, + 'test_avg_loss_bottom10%': 0.276681, 'test_avg_loss_top10%': + 1.686649, 'test_avg_loss_cos1': 0.867932, 'test_avg_loss_entropy': 5.164172, 'test_loss_std': 13.686828, 'test_loss_bottom_decile': 11.822035, 'test_loss_top_decile': 39.727236, 'test_loss_min': 7.337724, 'test_loss_max': 100.899873, 'test_loss_bottom10%': 9.618685, 'test_loss_top10%': 54.96769, 'test_loss_cos1': 0.880356, 'test_loss_entropy': 5.175803, 'test_acc_std': 0.123823, - 'test_acc_bottom_decile': 0.676471, 'test_acc_top_decile': 0.916667, + 'test_acc_bottom_decile': 0.676471, 'test_acc_top_decile': + 0.916667, 'test_acc_min': 0.071429, 'test_acc_max': 0.972973, 'test_acc_bottom10%': 0.527482, 'test_acc_top10%': 0.94486, 'test_acc_cos1': 0.988134, 'test_acc_entropy': 5.283755 @@ -387,16 +391,22 @@ def format_eval_res(self, # min and max new_results[f"{key}_min"] = all_res[0] new_results[f"{key}_max"] = all_res[-1] - #bottom and top 10% - new_results[f"{key}_bottom10%"]=np.mean(all_res[: - all_res.size // 10]) - new_results[f"{key}_top10%"]=np.mean(all_res[ - all_res.size * 9 // 10:]) - #cosine similarity between the performance distribution and 1 - new_results[f"{key}_cos1"]=np.mean(all_res)/(np.sqrt(np.mean(all_res**2))) - #entropy of performance distribution - all_res_preprocessed=all_res+1e-9 - new_results[f"{key}_entropy"]=np.sum(-all_res_preprocessed/np.sum(all_res_preprocessed)*(np.log((all_res_preprocessed)/np.sum(all_res_preprocessed)))) + # bottom and top 10% + new_results[f"{key}_bottom10%"] = np.mean( + all_res[:all_res.size // 10]) + new_results[f"{key}_top10%"] = np.mean( + all_res[all_res.size * 9 // 10:]) + # cosine similarity between the performance + # distribution and 1 + new_results[f"{key}_cos1"] = np.mean(all_res) / ( + np.sqrt(np.mean(all_res**2))) + # entropy of performance distribution + all_res_preprocessed = all_res + 1e-9 + new_results[f"{key}_entropy"] = np.sum( + -all_res_preprocessed / + np.sum(all_res_preprocessed) * (np.log( + (all_res_preprocessed) / + np.sum(all_res_preprocessed)))) round_formatted_results[f'Results_{form}'] = new_results with open(os.path.join(self.outdir, "eval_results.raw"), diff --git a/federatedscope/core/splitters/generic/lda_splitter.py b/federatedscope/core/splitters/generic/lda_splitter.py index c8d4789c7..b32d5e07f 100644 --- a/federatedscope/core/splitters/generic/lda_splitter.py +++ b/federatedscope/core/splitters/generic/lda_splitter.py @@ -9,7 +9,7 @@ def __init__(self, client_num, alpha=0.5): self.alpha = alpha super(LDASplitter, self).__init__(client_num) - def __call__(self, dataset, prior=None): + def __call__(self, dataset, prior=None, **kwargs): dataset = [ds for ds in dataset] label = np.array([y for x, y in dataset]) idx_slice = dirichlet_distribution_noniid_slice(label, diff --git a/federatedscope/core/splitters/graph/louvain_splitter.py b/federatedscope/core/splitters/graph/louvain_splitter.py index 9ed0e5f4c..908ae9a43 100644 --- a/federatedscope/core/splitters/graph/louvain_splitter.py +++ b/federatedscope/core/splitters/graph/louvain_splitter.py @@ -22,7 +22,7 @@ def __init__(self, client_num, delta=20): self.delta = delta BaseSplitter.__init__(self, client_num) - def __call__(self, data): + def __call__(self, data, **kwargs): data.index_orig = torch.arange(data.num_nodes) G = to_networkx( data, diff --git a/federatedscope/core/splitters/graph/randchunk_splitter.py b/federatedscope/core/splitters/graph/randchunk_splitter.py index 6a8f5ac66..07e2e93cb 100644 --- a/federatedscope/core/splitters/graph/randchunk_splitter.py +++ b/federatedscope/core/splitters/graph/randchunk_splitter.py @@ -1,13 +1,14 @@ import numpy as np from torch_geometric.transforms import BaseTransform +from federatedscope.core.splitters import BaseSplitter -class RandChunkSplitter(BaseTransform): +class RandChunkSplitter(BaseTransform, BaseSplitter): def __init__(self, client_num): - super(RandChunkSplitter, self).__init__(client_num) + BaseSplitter.__init__(self, client_num) - def __call__(self, dataset): + def __call__(self, dataset, **kwargs): r"""Split dataset via random chunk. Arguments: diff --git a/federatedscope/core/splitters/graph/random_splitter.py b/federatedscope/core/splitters/graph/random_splitter.py index 10faebcb8..a3c12be1e 100644 --- a/federatedscope/core/splitters/graph/random_splitter.py +++ b/federatedscope/core/splitters/graph/random_splitter.py @@ -52,7 +52,7 @@ def __init__(self, self.drop_edge = drop_edge - def __call__(self, data): + def __call__(self, data, **kwargs): data.index_orig = torch.arange(data.num_nodes) G = to_networkx( data, diff --git a/federatedscope/core/splitters/graph/reltype_splitter.py b/federatedscope/core/splitters/graph/reltype_splitter.py index abf0e011c..2452addbd 100644 --- a/federatedscope/core/splitters/graph/reltype_splitter.py +++ b/federatedscope/core/splitters/graph/reltype_splitter.py @@ -24,7 +24,7 @@ def __init__(self, client_num, alpha=0.5, realloc_mask=False): self.alpha = alpha self.realloc_mask = realloc_mask - def __call__(self, data): + def __call__(self, data, **kwargs): data_list = [] label = data.edge_type.numpy() idx_slice = dirichlet_distribution_noniid_slice( diff --git a/federatedscope/core/splitters/graph/scaffold_lda_splitter.py b/federatedscope/core/splitters/graph/scaffold_lda_splitter.py index 08c520d31..87b119a3b 100644 --- a/federatedscope/core/splitters/graph/scaffold_lda_splitter.py +++ b/federatedscope/core/splitters/graph/scaffold_lda_splitter.py @@ -53,7 +53,7 @@ def __init__(self): Chem.rdchem.BondStereo.STEREOTRANS, ] - def __call__(self, data): + def __call__(self, data, **kwargs): mol = Chem.MolFromSmiles(data.smiles) xs = [] diff --git a/federatedscope/core/splitters/graph/scaffold_splitter.py b/federatedscope/core/splitters/graph/scaffold_splitter.py index 77dfc60df..db41779f6 100644 --- a/federatedscope/core/splitters/graph/scaffold_splitter.py +++ b/federatedscope/core/splitters/graph/scaffold_splitter.py @@ -53,7 +53,7 @@ class ScaffoldSplitter(BaseSplitter): def __init__(self, client_num): super(ScaffoldSplitter, self).__init__(client_num) - def __call__(self, dataset): + def __call__(self, dataset, **kwargs): r"""Split dataset with smiles string into scaffold split Arguments: diff --git a/federatedscope/core/trainers/__init__.py b/federatedscope/core/trainers/__init__.py index a4645c251..de9221ee1 100644 --- a/federatedscope/core/trainers/__init__.py +++ b/federatedscope/core/trainers/__init__.py @@ -1,3 +1,4 @@ +from federatedscope.core.trainers.base_trainer import BaseTrainer from federatedscope.core.trainers.trainer import Trainer from federatedscope.core.trainers.torch_trainer import GeneralTorchTrainer from federatedscope.core.trainers.trainer_multi_model import \ @@ -13,5 +14,6 @@ __all__ = [ 'Trainer', 'Context', 'GeneralTorchTrainer', 'GeneralMultiModelTrainer', 'wrap_pFedMeTrainer', 'wrap_DittoTrainer', 'FedEMTrainer', - 'wrap_fedprox_trainer', 'wrap_nbafl_trainer', 'wrap_nbafl_server' + 'wrap_fedprox_trainer', 'wrap_nbafl_trainer', 'wrap_nbafl_server', + 'BaseTrainer' ] diff --git a/federatedscope/core/trainers/base_trainer.py b/federatedscope/core/trainers/base_trainer.py new file mode 100644 index 000000000..50bf57245 --- /dev/null +++ b/federatedscope/core/trainers/base_trainer.py @@ -0,0 +1,29 @@ +import abc + + +class BaseTrainer(abc.ABC): + def __init__(self, model, data, device, **kwargs): + self.model = model + self.data = data + self.device = device + self.kwargs = kwargs + + @abc.abstractmethod + def train(self): + raise NotImplementedError + + @abc.abstractmethod + def evaluate(self, target_data_split_name='test'): + raise NotImplementedError + + @abc.abstractmethod + def update(self, model_parameters, strict=False): + raise NotImplementedError + + @abc.abstractmethod + def get_model_para(self): + raise NotImplementedError + + @abc.abstractmethod + def print_trainer_meta_info(self): + raise NotImplementedError diff --git a/federatedscope/core/trainers/context.py b/federatedscope/core/trainers/context.py index 319044b93..e612339df 100644 --- a/federatedscope/core/trainers/context.py +++ b/federatedscope/core/trainers/context.py @@ -1,4 +1,3 @@ -import math import logging import collections @@ -8,6 +7,7 @@ from federatedscope.core.auxiliaries.regularizer_builder import get_regularizer from federatedscope.core.auxiliaries.enums import MODE from federatedscope.core.auxiliaries.utils import calculate_batch_epoch_num +from federatedscope.core.data import ClientData logger = logging.getLogger(__name__) @@ -128,6 +128,8 @@ def setup_vars(self): self.device) self.regularizer = get_regularizer(self.cfg.regularizer.type) self.grad_clip = self.cfg.grad.grad_clip + if isinstance(self.data, ClientData): + self.data.setup(self.cfg) elif self.cfg.backend == 'tensorflow': self.trainable_para_names = self.model.trainable_variables() self.criterion = None @@ -145,7 +147,8 @@ def setup_vars(self): calculate_batch_epoch_num( self.cfg.train.local_update_steps, self.cfg.train.batch_or_epoch, self.num_train_data, - self.cfg.data.batch_size, self.cfg.data.drop_last) + self.cfg.dataloader.batch_size, + self.cfg.dataloader.drop_last) # Process evaluation data for mode in ["val", "test"]: @@ -155,10 +158,10 @@ def setup_vars(self): setattr( self, "num_{}_batch".format(mode), getattr(self, "num_{}_data".format(mode)) // - self.cfg.data.batch_size + - int(not self.cfg.data.drop_last and bool( + self.cfg.dataloader.batch_size + + int(not self.cfg.dataloader.drop_last and bool( getattr(self, "num_{}_data".format(mode)) % - self.cfg.data.batch_size))) + self.cfg.dataloader.batch_size))) def track_mode(self, mode): self.mode_stack.append(mode) diff --git a/federatedscope/core/trainers/torch_trainer.py b/federatedscope/core/trainers/torch_trainer.py index 8ffa4ea14..a5c2a098a 100644 --- a/federatedscope/core/trainers/torch_trainer.py +++ b/federatedscope/core/trainers/torch_trainer.py @@ -16,7 +16,7 @@ from federatedscope.core.auxiliaries.scheduler_builder import get_scheduler from federatedscope.core.trainers.trainer import Trainer from federatedscope.core.trainers.context import CtxVar -from federatedscope.core.auxiliaries.dataloader_builder import WrapDataset +from federatedscope.core.data.wrap_dataset import WrapDataset from federatedscope.core.auxiliaries.dataloader_builder import get_dataloader from federatedscope.core.auxiliaries.ReIterator import ReIterator from federatedscope.core.auxiliaries.utils import param2tensor, \ @@ -173,7 +173,7 @@ def _hook_on_epoch_start(self, ctx): if ctx.get("{}_loader".format(ctx.cur_split)) is None: loader = get_dataloader( WrapDataset(ctx.get("{}_data".format(ctx.cur_split))), - self.cfg) + self.cfg, ctx.cur_split) setattr(ctx, "{}_loader".format(ctx.cur_split), ReIterator(loader)) elif not isinstance(ctx.get("{}_loader".format(ctx.cur_split)), ReIterator): diff --git a/federatedscope/core/trainers/trainer.py b/federatedscope/core/trainers/trainer.py index d0f77c1a1..409b6631f 100644 --- a/federatedscope/core/trainers/trainer.py +++ b/federatedscope/core/trainers/trainer.py @@ -2,6 +2,7 @@ import copy import logging +from federatedscope.core.trainers.base_trainer import BaseTrainer from federatedscope.core.auxiliaries.enums import MODE from federatedscope.core.auxiliaries.enums import LIFECYCLE from federatedscope.core.auxiliaries.decorators import use_diff @@ -23,7 +24,7 @@ logger = logging.getLogger(__name__) -class Trainer(object): +class Trainer(BaseTrainer): """ Register, organize and run the train/test/val procedures """ diff --git a/federatedscope/core/trainers/trainer_Ditto.py b/federatedscope/core/trainers/trainer_Ditto.py index e5ae8f4ef..e4e4a0200 100644 --- a/federatedscope/core/trainers/trainer_Ditto.py +++ b/federatedscope/core/trainers/trainer_Ditto.py @@ -99,8 +99,8 @@ def init_Ditto_ctx(base_trainer): calculate_batch_epoch_num(cfg_p_local_update_steps, cfg.train.batch_or_epoch, ctx.num_train_data, - cfg.data.batch_size, - cfg.data.drop_last) + cfg.dataloader.batch_size, + cfg.dataloader.drop_last) # In the first # 1. `num_train_batch` and `num_train_batch_last_epoch` diff --git a/federatedscope/core/workers/base_client.py b/federatedscope/core/workers/base_client.py new file mode 100644 index 000000000..8ec2d3b83 --- /dev/null +++ b/federatedscope/core/workers/base_client.py @@ -0,0 +1,121 @@ +import abc +from federatedscope.core.workers.base_worker import Worker + + +class BaseClient(Worker): + def __init__(self, ID, state, config, model, strategy): + super(BaseClient, self).__init__(ID, state, config, model, strategy) + self.msg_handlers = dict() + + def register_handlers(self, msg_type, callback_func): + """ + To bind a message type with a handling function. + + Arguments: + msg_type (str): The defined message type + callback_func: The handling functions to handle the received + message + """ + self.msg_handlers[msg_type] = callback_func + + def _register_default_handlers(self): + self.register_handlers('assign_client_id', + self.callback_funcs_for_assign_id) + self.register_handlers('ask_for_join_in_info', + self.callback_funcs_for_join_in_info) + self.register_handlers('address', self.callback_funcs_for_address) + self.register_handlers('model_para', + self.callback_funcs_for_model_para) + self.register_handlers('ss_model_para', + self.callback_funcs_for_model_para) + self.register_handlers('evaluate', self.callback_funcs_for_evaluate) + self.register_handlers('finish', self.callback_funcs_for_finish) + self.register_handlers('converged', self.callback_funcs_for_converged) + + @abc.abstractmethod + def run(self): + """ + To listen to the message and handle them accordingly (used for + distributed mode) + """ + raise NotImplementedError + + @abc.abstractmethod + def callback_funcs_for_model_para(self, message): + """ + The handling function for receiving model parameters, + which triggers the local training process. + This handling function is widely used in various FL courses. + + Arguments: + message: The received message, which includes sender, receiver, + state, and content. + More detail can be found in federatedscope.core.message + """ + raise NotImplementedError + + @abc.abstractmethod + def callback_funcs_for_assign_id(self, message): + """ + The handling function for receiving the client_ID assigned by the + server (during the joining process), + which is used in the distributed mode. + + Arguments: + message: The received message + """ + raise NotImplementedError + + @abc.abstractmethod + def callback_funcs_for_join_in_info(self, message): + """ + The handling function for receiving the request of join in information + (such as batch_size, num_of_samples) during the joining process. + + Arguments: + message: The received message + """ + raise NotImplementedError + + @abc.abstractmethod + def callback_funcs_for_address(self, message): + """ + The handling function for receiving other clients' IP addresses, + which is used for constructing a complex topology + + Arguments: + message: The received message + """ + raise NotImplementedError + + @abc.abstractmethod + def callback_funcs_for_evaluate(self, message): + """ + The handling function for receiving the request of evaluating + + Arguments: + message: The received message + """ + raise NotImplementedError + + @abc.abstractmethod + def callback_funcs_for_finish(self, message): + """ + The handling function for receiving the signal of finishing the FL + course. + + Arguments: + message: The received message + """ + raise NotImplementedError + + @abc.abstractmethod + def callback_funcs_for_converged(self, message): + """ + The handling function for receiving the signal that the FL course + converged + + Arguments: + message: The received message + """ + raise NotImplementedError diff --git a/federatedscope/core/workers/base_server.py b/federatedscope/core/workers/base_server.py new file mode 100644 index 000000000..10788bf0b --- /dev/null +++ b/federatedscope/core/workers/base_server.py @@ -0,0 +1,74 @@ +import abc +from federatedscope.core.workers import Worker + + +class BaseServer(Worker): + def __init__(self, ID, state, config, model, strategy): + super(BaseServer, self).__init__(ID, state, config, model, strategy) + self.msg_handlers = dict() + + def register_handlers(self, msg_type, callback_func): + """ + To bind a message type with a handling function. + + Arguments: + msg_type (str): The defined message type + callback_func: The handling functions to handle the received + message + """ + self.msg_handlers[msg_type] = callback_func + + def _register_default_handlers(self): + self.register_handlers('join_in', self.callback_funcs_for_join_in) + self.register_handlers('join_in_info', self.callback_funcs_for_join_in) + self.register_handlers('model_para', self.callback_funcs_model_para) + self.register_handlers('metrics', self.callback_funcs_for_metrics) + + @abc.abstractmethod + def run(self): + """ + To start the FL course, listen and handle messages (for distributed + mode). + """ + raise NotImplementedError + + @abc.abstractmethod + def callback_funcs_model_para(self, message): + """ + The handling function for receiving model parameters, which triggers + check_and_move_on (perform aggregation when enough feedback has + been received). + This handling function is widely used in various FL courses. + + Arguments: + message: The received message, which includes sender, receiver, + state, and content. More detail can be found in + federatedscope.core.message + """ + raise NotImplementedError + + @abc.abstractmethod + def callback_funcs_for_join_in(self, message): + """ + The handling function for receiving the join in information. The + server might request for some information (such as num_of_samples) + if necessary, assign IDs for the servers. + If all the clients have joined in, the training process will be + triggered. + + Arguments: + message: The received message + """ + raise NotImplementedError + + @abc.abstractmethod + def callback_funcs_for_metrics(self, message): + """ + The handling function for receiving the evaluation results, + which triggers check_and_move_on + (perform aggregation when enough feedback has been received). + + Arguments: + message: The received message + """ + raise NotImplementedError diff --git a/federatedscope/core/workers/client.py b/federatedscope/core/workers/client.py index 41a1ed980..7c145b99f 100644 --- a/federatedscope/core/workers/client.py +++ b/federatedscope/core/workers/client.py @@ -12,11 +12,12 @@ from federatedscope.core.secret_sharing import AdditiveSecretSharing from federatedscope.core.auxiliaries.utils import merge_dict, \ calculate_time_cost +from federatedscope.core.workers.base_client import BaseClient logger = logging.getLogger(__name__) -class Client(Worker): +class Client(BaseClient): """ The Client class, which describes the behaviors of client in an FL course. The behaviors are described by the handling functions (named as @@ -92,7 +93,6 @@ def __init__(self, self.msg_buffer = {'train': dict(), 'eval': dict()} # Register message handlers - self.msg_handlers = dict() self._register_default_handlers() # Communication and communication ability @@ -168,31 +168,6 @@ def _calculate_model_delta(self, init_model, updated_model): else: return model_deltas[0] - def register_handlers(self, msg_type, callback_func): - """ - To bind a message type with a handling function. - - Arguments: - msg_type (str): The defined message type - callback_func: The handling functions to handle the received - message - """ - self.msg_handlers[msg_type] = callback_func - - def _register_default_handlers(self): - self.register_handlers('assign_client_id', - self.callback_funcs_for_assign_id) - self.register_handlers('ask_for_join_in_info', - self.callback_funcs_for_join_in_info) - self.register_handlers('address', self.callback_funcs_for_address) - self.register_handlers('model_para', - self.callback_funcs_for_model_para) - self.register_handlers('ss_model_para', - self.callback_funcs_for_model_para) - self.register_handlers('evaluate', self.callback_funcs_for_evaluate) - self.register_handlers('finish', self.callback_funcs_for_finish) - self.register_handlers('converged', self.callback_funcs_for_converged) - def join_in(self): """ To send 'join_in' message to the server for joining in the FL course. @@ -414,14 +389,13 @@ def callback_funcs_for_join_in_info(self, message: Message): if requirement.lower() == 'num_sample': if self._cfg.train.batch_or_epoch == 'batch': num_sample = self._cfg.train.local_update_steps * \ - self._cfg.data.batch_size + self._cfg.dataloader.batch_size else: num_sample = self._cfg.train.local_update_steps * \ - self.trainer.ctx.num_train_batch + len(self.data['train']) join_in_info['num_sample'] = num_sample if self._cfg.trainer.type == 'nodefullbatch_trainer': - join_in_info['num_sample'] = \ - self.trainer.ctx.data.x.shape[0] + join_in_info['num_sample'] = self.data['data'].x.shape[0] elif requirement.lower() == 'client_resource': assert self.comm_bandwidth is not None and self.comp_speed \ is not None, "The requirement join_in_info " \ diff --git a/federatedscope/core/workers/server.py b/federatedscope/core/workers/server.py index 59c5a514a..0bb9a0d82 100644 --- a/federatedscope/core/workers/server.py +++ b/federatedscope/core/workers/server.py @@ -10,18 +10,18 @@ from federatedscope.core.message import Message from federatedscope.core.communication import StandaloneCommManager, \ gRPCCommManager -from federatedscope.core.workers import Worker from federatedscope.core.auxiliaries.aggregator_builder import get_aggregator from federatedscope.core.auxiliaries.sampler_builder import get_sampler from federatedscope.core.auxiliaries.utils import merge_dict, Timeout, \ merge_param_dict from federatedscope.core.auxiliaries.trainer_builder import get_trainer from federatedscope.core.secret_sharing import AdditiveSecretSharing +from federatedscope.core.workers.base_server import BaseServer logger = logging.getLogger(__name__) -class Server(Worker): +class Server(BaseServer): """ The Server class, which describes the behaviors of server in an FL course. The behaviors are described by the handled functions (named as @@ -166,7 +166,6 @@ def __init__(self, if 'client_resource_info' in kwargs else None # Register message handlers - self.msg_handlers = dict() self._register_default_handlers() # Initialize communication manager and message buffer @@ -206,23 +205,6 @@ def total_round_num(self, value): def register_noise_injector(self, func): self._noise_injector = func - def register_handlers(self, msg_type, callback_func): - """ - To bind a message type with a handling function. - - Arguments: - msg_type (str): The defined message type - callback_func: The handling functions to handle the received - message - """ - self.msg_handlers[msg_type] = callback_func - - def _register_default_handlers(self): - self.register_handlers('join_in', self.callback_funcs_for_join_in) - self.register_handlers('join_in_info', self.callback_funcs_for_join_in) - self.register_handlers('model_para', self.callback_funcs_model_para) - self.register_handlers('metrics', self.callback_funcs_for_metrics) - def run(self): """ To start the FL course, listen and handle messages (for distributed diff --git a/federatedscope/cv/baseline/fedavg_convnet2_on_celeba.yaml b/federatedscope/cv/baseline/fedavg_convnet2_on_celeba.yaml index a84531c6a..500d94f9d 100644 --- a/federatedscope/cv/baseline/fedavg_convnet2_on_celeba.yaml +++ b/federatedscope/cv/baseline/fedavg_convnet2_on_celeba.yaml @@ -10,10 +10,10 @@ data: root: data/ type: celeba splits: [0.6,0.2,0.2] - batch_size: 5 subsample: 0.1 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 5 model: type: convnet2 hidden: 2048 diff --git a/federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml b/federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml index 3a500e979..9a29857ce 100644 --- a/federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml +++ b/federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml @@ -11,10 +11,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 10 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 10 model: type: convnet2 hidden: 2048 diff --git a/federatedscope/cv/baseline/fedbn_convnet2_on_femnist.yaml b/federatedscope/cv/baseline/fedbn_convnet2_on_femnist.yaml index 67f49dc41..d925a1b49 100644 --- a/federatedscope/cv/baseline/fedbn_convnet2_on_femnist.yaml +++ b/federatedscope/cv/baseline/fedbn_convnet2_on_femnist.yaml @@ -11,10 +11,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 10 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 10 model: type: convnet2 hidden: 2048 diff --git a/federatedscope/cv/dataloader/dataloader.py b/federatedscope/cv/dataloader/dataloader.py index 6ca535f43..9ea27e492 100644 --- a/federatedscope/cv/dataloader/dataloader.py +++ b/federatedscope/cv/dataloader/dataloader.py @@ -1,5 +1,3 @@ -from torch.utils.data import DataLoader - from federatedscope.cv.dataset.leaf_cv import LEAF_CV from federatedscope.core.auxiliaries.transform_builder import get_transform @@ -18,7 +16,6 @@ def load_cv_dataset(config=None): path = config.data.root name = config.data.type.lower() - batch_size = config.data.batch_size transforms_funcs = get_transform(config, 'torchvision') if name in ['femnist', 'celeba']: @@ -36,25 +33,9 @@ def load_cv_dataset(config=None): ) if config.federate.client_num > 0 else len(dataset) config.merge_from_list(['federate.client_num', client_num]) - # get local dataset - data_local_dict = dict() - for client_idx in range(client_num): - dataloader = { - 'train': DataLoader(dataset[client_idx]['train'], - batch_size, - shuffle=config.data.shuffle, - num_workers=config.data.num_workers), - 'test': DataLoader(dataset[client_idx]['test'], - batch_size, - shuffle=False, - num_workers=config.data.num_workers) - } - if 'val' in dataset[client_idx]: - dataloader['val'] = DataLoader(dataset[client_idx]['val'], - batch_size, - shuffle=False, - num_workers=config.data.num_workers) - - data_local_dict[client_idx + 1] = dataloader + # Convert list to dict + data_dict = dict() + for client_idx in range(1, client_num + 1): + data_dict[client_idx] = dataset[client_idx - 1] - return data_local_dict, config + return data_dict, config diff --git a/federatedscope/gfl/__init__.py b/federatedscope/gfl/__init__.py index 7fd229a32..e69de29bb 100644 --- a/federatedscope/gfl/__init__.py +++ b/federatedscope/gfl/__init__.py @@ -1 +0,0 @@ -__version__ = '0.2.0' diff --git a/federatedscope/gfl/baseline/example.yaml b/federatedscope/gfl/baseline/example.yaml index a8f7a9f02..063cbdf58 100644 --- a/federatedscope/gfl/baseline/example.yaml +++ b/federatedscope/gfl/baseline/example.yaml @@ -23,6 +23,9 @@ data: type: cora # Use Louvain algorithm to split `Cora` splitter: 'louvain' +dataloader: + # Type of sampler + type: pyg # Use fullbatch training, batch_size should be `1` batch_size: 1 diff --git a/federatedscope/gfl/baseline/fedavg_gcn_fullbatch_on_dblpnew.yaml b/federatedscope/gfl/baseline/fedavg_gcn_fullbatch_on_dblpnew.yaml index 09285bf90..2878b70e1 100644 --- a/federatedscope/gfl/baseline/fedavg_gcn_fullbatch_on_dblpnew.yaml +++ b/federatedscope/gfl/baseline/fedavg_gcn_fullbatch_on_dblpnew.yaml @@ -10,8 +10,10 @@ federate: data: root: data/ type: dblp_conf - batch_size: 1 splits: [0.5, 0.2, 0.3] +dataloader: + type: pyg + batch_size: 1 model: type: gcn hidden: 1024 diff --git a/federatedscope/gfl/baseline/fedavg_gcn_fullbatch_on_kg.yaml b/federatedscope/gfl/baseline/fedavg_gcn_fullbatch_on_kg.yaml index b300f7d6f..efdce0813 100644 --- a/federatedscope/gfl/baseline/fedavg_gcn_fullbatch_on_kg.yaml +++ b/federatedscope/gfl/baseline/fedavg_gcn_fullbatch_on_kg.yaml @@ -13,6 +13,8 @@ data: type: wn18 splitter: rel_type pre_transform: ['Constant', {'value':1.0, 'cat':False}] +dataloader: + type: pyg model: type: gcn hidden: 64 diff --git a/federatedscope/gfl/baseline/fedavg_gcn_minibatch_on_hiv.yaml b/federatedscope/gfl/baseline/fedavg_gcn_minibatch_on_hiv.yaml index bbe98ec8d..b0b6c4927 100644 --- a/federatedscope/gfl/baseline/fedavg_gcn_minibatch_on_hiv.yaml +++ b/federatedscope/gfl/baseline/fedavg_gcn_minibatch_on_hiv.yaml @@ -12,6 +12,8 @@ data: root: data/ type: hiv splitter: scaffold +dataloader: + type: pyg model: type: gcn hidden: 64 diff --git a/federatedscope/gfl/baseline/fedavg_gin_minibatch_on_cikmcup.yaml b/federatedscope/gfl/baseline/fedavg_gin_minibatch_on_cikmcup.yaml index 957684f3d..c7865d4c8 100644 --- a/federatedscope/gfl/baseline/fedavg_gin_minibatch_on_cikmcup.yaml +++ b/federatedscope/gfl/baseline/fedavg_gin_minibatch_on_cikmcup.yaml @@ -12,6 +12,8 @@ federate: data: root: data/ type: cikmcup +dataloader: + type: pyg model: type: gin hidden: 64 diff --git a/federatedscope/gfl/baseline/fedavg_gnn_minibatch_on_multi_task.yaml b/federatedscope/gfl/baseline/fedavg_gnn_minibatch_on_multi_task.yaml index 7b15bdb67..a97aade78 100644 --- a/federatedscope/gfl/baseline/fedavg_gnn_minibatch_on_multi_task.yaml +++ b/federatedscope/gfl/baseline/fedavg_gnn_minibatch_on_multi_task.yaml @@ -12,6 +12,8 @@ data: root: data/ type: graph_multi_domain_mol pre_transform: ['Constant', {'value':1.0, 'cat':False}] +dataloader: + type: pyg model: type: gin hidden: 64 diff --git a/federatedscope/gfl/baseline/fedavg_gnn_minibatch_on_multi_task_total_samples_aggr.yaml b/federatedscope/gfl/baseline/fedavg_gnn_minibatch_on_multi_task_total_samples_aggr.yaml index ad99acca9..e8933c11f 100644 --- a/federatedscope/gfl/baseline/fedavg_gnn_minibatch_on_multi_task_total_samples_aggr.yaml +++ b/federatedscope/gfl/baseline/fedavg_gnn_minibatch_on_multi_task_total_samples_aggr.yaml @@ -13,6 +13,8 @@ data: root: data/ type: graph_multi_domain_mix pre_transform: ['Constant', {'value':1.0, 'cat':False}] +dataloader: + type: pyg model: type: gin hidden: 64 diff --git a/federatedscope/gfl/baseline/fedavg_gnn_node_fullbatch_citation.yaml b/federatedscope/gfl/baseline/fedavg_gnn_node_fullbatch_citation.yaml index d5bc2124c..965269e18 100644 --- a/federatedscope/gfl/baseline/fedavg_gnn_node_fullbatch_citation.yaml +++ b/federatedscope/gfl/baseline/fedavg_gnn_node_fullbatch_citation.yaml @@ -12,6 +12,8 @@ data: root: data/ type: cora splitter: 'louvain' +dataloader: + type: pyg batch_size: 1 model: type: gcn diff --git a/federatedscope/gfl/baseline/fedavg_on_cSBM.yaml b/federatedscope/gfl/baseline/fedavg_on_cSBM.yaml index 052b8f006..d0de6f9ba 100644 --- a/federatedscope/gfl/baseline/fedavg_on_cSBM.yaml +++ b/federatedscope/gfl/baseline/fedavg_on_cSBM.yaml @@ -12,6 +12,8 @@ data: type: 'csbm' #type: 'csbm_data_feb_07_2022-00:19' cSBM_phi: [0.1, 0.5, 0.9] +dataloader: + type: pyg batch_size: 1 model: type: gpr diff --git a/federatedscope/gfl/baseline/fedavg_sage_minibatch_on_dblpnew.yaml b/federatedscope/gfl/baseline/fedavg_sage_minibatch_on_dblpnew.yaml index 840196e98..29838bb21 100644 --- a/federatedscope/gfl/baseline/fedavg_sage_minibatch_on_dblpnew.yaml +++ b/federatedscope/gfl/baseline/fedavg_sage_minibatch_on_dblpnew.yaml @@ -9,9 +9,10 @@ federate: total_round_num: 400 data: root: data/ - loader: graphsaint-rw - batch_size: 256 type: dblp_conf +dataloader: + type: graphsaint-rw + batch_size: 256 model: type: sage hidden: 1024 diff --git a/federatedscope/gfl/baseline/fedavg_wpsn_on_cSBM.yaml b/federatedscope/gfl/baseline/fedavg_wpsn_on_cSBM.yaml index 1672c5c98..ef77a592c 100644 --- a/federatedscope/gfl/baseline/fedavg_wpsn_on_cSBM.yaml +++ b/federatedscope/gfl/baseline/fedavg_wpsn_on_cSBM.yaml @@ -12,6 +12,8 @@ data: type: 'csbm' #type: 'csbm_data_feb_05_2022-19:23' cSBM_phi: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] +dataloader: + type: pyg model: type: gpr hidden: 256 diff --git a/federatedscope/gfl/baseline/fedbn_gnn_minibatch_on_multi_task.yaml b/federatedscope/gfl/baseline/fedbn_gnn_minibatch_on_multi_task.yaml index 277575af4..9487f37fb 100644 --- a/federatedscope/gfl/baseline/fedbn_gnn_minibatch_on_multi_task.yaml +++ b/federatedscope/gfl/baseline/fedbn_gnn_minibatch_on_multi_task.yaml @@ -12,6 +12,8 @@ data: root: data/ type: graph_multi_domain_mix pre_transform: ['Constant', {'value':1.0, 'cat':False}] +dataloader: + type: pyg model: type: gin hidden: 64 diff --git a/federatedscope/gfl/baseline/isolated_gin_minibatch_on_cikmcup.yaml b/federatedscope/gfl/baseline/isolated_gin_minibatch_on_cikmcup.yaml index 7880f4f3b..3153c315d 100644 --- a/federatedscope/gfl/baseline/isolated_gin_minibatch_on_cikmcup.yaml +++ b/federatedscope/gfl/baseline/isolated_gin_minibatch_on_cikmcup.yaml @@ -14,6 +14,8 @@ data: batch_size: 64 root: data/ type: cikmcup +dataloader: + type: pyg model: type: gin hidden: 64 diff --git a/federatedscope/gfl/baseline/local_gnn_node_fullbatch_citation.yaml b/federatedscope/gfl/baseline/local_gnn_node_fullbatch_citation.yaml index 6c960bd89..c8a298f52 100644 --- a/federatedscope/gfl/baseline/local_gnn_node_fullbatch_citation.yaml +++ b/federatedscope/gfl/baseline/local_gnn_node_fullbatch_citation.yaml @@ -12,6 +12,9 @@ data: root: data/ type: cora splitter: 'louvain' +dataloader: + type: pyg + batch_size: 1 model: type: gcn hidden: 64 diff --git a/federatedscope/gfl/dataloader/dataloader_graph.py b/federatedscope/gfl/dataloader/dataloader_graph.py index 709a40e38..5be531918 100644 --- a/federatedscope/gfl/dataloader/dataloader_graph.py +++ b/federatedscope/gfl/dataloader/dataloader_graph.py @@ -1,18 +1,8 @@ -import numpy as np - from torch_geometric import transforms -from torch_geometric.loader import DataLoader from torch_geometric.datasets import TUDataset, MoleculeNet -from federatedscope.core.auxiliaries.splitter_builder import get_splitter from federatedscope.core.auxiliaries.transform_builder import get_transform - - -def get_numGraphLabels(dataset): - s = set() - for g in dataset: - s.add(g.y.item()) - return len(s) +from federatedscope.gfl.dataset.cikm_cup import CIKMCUPDataset def load_graphlevel_dataset(config=None): @@ -30,11 +20,6 @@ def load_graphlevel_dataset(config=None): splits = config.data.splits path = config.data.root name = config.data.type.upper() - client_num = config.federate.client_num - batch_size = config.data.batch_size - - # Splitter - splitter = get_splitter(config) # Transforms transforms_funcs = get_transform(config, 'torch_geometric') @@ -50,18 +35,13 @@ def load_graphlevel_dataset(config=None): transforms_funcs['pre_transform'] = transforms.Constant(value=1.0, cat=False) dataset = TUDataset(path, name, **transforms_funcs) - if splitter is None: - raise ValueError('Please set the graph.') - dataset = splitter(dataset) elif name in [ 'HIV', 'ESOL', 'FREESOLV', 'LIPO', 'PCBA', 'MUV', 'BACE', 'BBBP', 'TOX21', 'TOXCAST', 'SIDER', 'CLINTOX' ]: dataset = MoleculeNet(path, name, **transforms_funcs) - if splitter is None: - raise ValueError('Please set the graph.') - dataset = splitter(dataset) + return dataset, config elif name.startswith('graph_multi_domain'.upper()): """ The `graph_multi_domain` datasets follows GCFL @@ -102,6 +82,8 @@ def load_graphlevel_dataset(config=None): transform=transforms_funcs['transform'] if 'transform' in transforms_funcs else None) dataset.append(tmp_dataset) + elif name == 'CIKM': + dataset = CIKMCUPDataset(config.data.root) else: raise ValueError(f'No dataset named: {name}!') @@ -110,42 +92,7 @@ def load_graphlevel_dataset(config=None): config.merge_from_list(['federate.client_num', client_num]) # get local dataset - data_local_dict = dict() - - # Build train/valid/test dataloader - raw_train = [] - raw_valid = [] - raw_test = [] - for client_idx, gs in enumerate(dataset): - index = np.random.permutation(np.arange(len(gs))) - train_idx = index[:int(len(gs) * splits[0])] - valid_idx = index[int(len(gs) * - splits[0]):int(len(gs) * sum(splits[:2]))] - test_idx = index[int(len(gs) * sum(splits[:2])):] - dataloader = { - 'num_label': get_numGraphLabels(gs), - 'train': DataLoader([gs[idx] for idx in train_idx], - batch_size, - shuffle=True, - num_workers=config.data.num_workers), - 'val': DataLoader([gs[idx] for idx in valid_idx], - batch_size, - shuffle=False, - num_workers=config.data.num_workers), - 'test': DataLoader([gs[idx] for idx in test_idx], - batch_size, - shuffle=False, - num_workers=config.data.num_workers), - } - data_local_dict[client_idx + 1] = dataloader - raw_train = raw_train + [gs[idx] for idx in train_idx] - raw_valid = raw_valid + [gs[idx] for idx in valid_idx] - raw_test = raw_test + [gs[idx] for idx in test_idx] - if not name.startswith('graph_multi_domain'.upper()): - data_local_dict[0] = { - 'train': DataLoader(raw_train, batch_size, shuffle=True), - 'val': DataLoader(raw_valid, batch_size, shuffle=False), - 'test': DataLoader(raw_test, batch_size, shuffle=False), - } - - return data_local_dict, config + data_dict = dict() + for client_idx in range(1, len(dataset) + 1): + data_dict[client_idx] = dataset[client_idx - 1] + return data_dict, config diff --git a/federatedscope/gfl/dataloader/dataloader_link.py b/federatedscope/gfl/dataloader/dataloader_link.py index 168b32517..989dbbe35 100644 --- a/federatedscope/gfl/dataloader/dataloader_link.py +++ b/federatedscope/gfl/dataloader/dataloader_link.py @@ -1,58 +1,23 @@ import torch -from torch_geometric.data import Data -from torch_geometric.loader import GraphSAINTRandomWalkSampler, NeighborSampler +from torch_geometric.utils import add_self_loops, remove_self_loops, \ + to_undirected from federatedscope.core.auxiliaries.splitter_builder import get_splitter from federatedscope.core.auxiliaries.transform_builder import get_transform -def raw2loader(raw_data, config=None): - """Transform a graph into either dataloader for graph-sampling-based - mini-batch training - or still a graph for full-batch training. - Arguments: - raw_data (PyG.Data): a raw graph. - :returns: - sampler (object): a Dict containing loader and subgraph_sampler or - still a PyG.Data object. - """ - - if config.data.loader == '': - sampler = raw_data - elif config.data.loader == 'graphsaint-rw': - loader = GraphSAINTRandomWalkSampler( - raw_data, - batch_size=config.data.batch_size, - walk_length=config.data.graphsaint.walk_length, - num_steps=config.data.graphsaint.num_steps, - sample_coverage=0) - subgraph_sampler = NeighborSampler(raw_data.edge_index, - sizes=[-1], - batch_size=4096, - shuffle=False, - num_workers=config.data.num_workers) - sampler = dict(data=raw_data, - train=loader, - val=subgraph_sampler, - test=subgraph_sampler) - else: - raise TypeError('Unsupported DataLoader Type {}'.format( - config.data.loader)) - - return sampler - - def load_linklevel_dataset(config=None): r""" :returns: - data_local_dict + data_dict :rtype: (Dict): dict{'client_id': Data()} """ path = config.data.root name = config.data.type.lower() + # TODO: remove splitter # Splitter splitter = get_splitter(config) @@ -85,11 +50,18 @@ def load_linklevel_dataset(config=None): config.merge_from_list(['federate.client_num', client_num]) # get local dataset - data_local_dict = dict() - - for client_idx in range(len(dataset)): - local_data = raw2loader(dataset[client_idx], config) - data_local_dict[client_idx + 1] = local_data + data_dict = dict() + + for client_idx in range(1, len(dataset) + 1): + local_data = dataset[client_idx - 1] + data_dict[client_idx] = local_data + # To undirected and add self-loop + data_dict[client_idx] = { + 'data': local_data, + 'train': [local_data], + 'val': [local_data], + 'test': [local_data] + } if global_dataset is not None: # Recode train & valid & test mask for global data @@ -100,11 +72,8 @@ def load_linklevel_dataset(config=None): global_edge_index = torch.LongTensor([[], []]) global_edge_type = torch.LongTensor([]) - for client_sampler in data_local_dict.values(): - if isinstance(client_sampler, Data): - client_subgraph = client_sampler - else: - client_subgraph = client_sampler['data'] + for client_data in data_dict.values(): + client_subgraph = client_data['data'] orig_index = torch.zeros_like(client_subgraph.edge_index) orig_index[0] = client_subgraph.index_orig[ client_subgraph.edge_index[0]] @@ -125,6 +94,10 @@ def load_linklevel_dataset(config=None): global_graph.test_edge_mask = test_edge_mask global_graph.edge_index = global_edge_index global_graph.edge_type = global_edge_type - data_local_dict[0] = raw2loader(global_graph, config) - - return data_local_dict, config + data_dict[0] = data_dict[0] = { + 'data': global_graph, + 'train': [global_graph], + 'val': [global_graph], + 'test': [global_graph] + } + return data_dict, config diff --git a/federatedscope/gfl/dataloader/dataloader_node.py b/federatedscope/gfl/dataloader/dataloader_node.py index e79a4e964..feef8d5d5 100644 --- a/federatedscope/gfl/dataloader/dataloader_node.py +++ b/federatedscope/gfl/dataloader/dataloader_node.py @@ -5,7 +5,6 @@ from torch_geometric.utils import add_self_loops, remove_self_loops, \ to_undirected from torch_geometric.data import Data -from torch_geometric.loader import GraphSAINTRandomWalkSampler, NeighborSampler from federatedscope.core.auxiliaries.splitter_builder import get_splitter from federatedscope.core.auxiliaries.transform_builder import get_transform @@ -13,82 +12,21 @@ INF = np.iinfo(np.int64).max -def raw2loader(raw_data, config=None): - """Transform a graph into either dataloader for graph-sampling-based - mini-batch training - or still a graph for full-batch training. - Arguments: - raw_data (PyG.Data): a raw graph. - :returns: - sampler (object): a Dict containing loader and subgraph_sampler or - still a PyG.Data object. - """ - # change directed graph to undirected - raw_data.edge_index = to_undirected( - remove_self_loops(raw_data.edge_index)[0]) - - if config.data.loader == '': - sampler = raw_data - elif config.data.loader == 'graphsaint-rw': - # Sampler would crash if there was isolated node. - raw_data.edge_index = add_self_loops(raw_data.edge_index, - num_nodes=raw_data.x.shape[0])[0] - loader = GraphSAINTRandomWalkSampler( - raw_data, - batch_size=config.data.batch_size, - walk_length=config.data.graphsaint.walk_length, - num_steps=config.data.graphsaint.num_steps, - sample_coverage=0) - subgraph_sampler = NeighborSampler(raw_data.edge_index, - sizes=[-1], - batch_size=4096, - shuffle=False, - num_workers=config.data.num_workers) - sampler = dict(data=raw_data, - train=loader, - val=subgraph_sampler, - test=subgraph_sampler) - elif config.data.loader == 'neighbor': - # Sampler would crash if there was isolated node. - raw_data.edge_index = add_self_loops(raw_data.edge_index, - num_nodes=raw_data.x.shape[0])[0] - - train_idx = raw_data.train_mask.nonzero(as_tuple=True)[0] - loader = NeighborSampler(raw_data.edge_index, - node_idx=train_idx, - sizes=config.data.sizes, - batch_size=config.data.batch_size, - shuffle=config.data.shuffle, - num_workers=config.data.num_workers) - subgraph_sampler = NeighborSampler(raw_data.edge_index, - sizes=[-1], - batch_size=4096, - shuffle=False, - num_workers=config.data.num_workers) - sampler = dict(data=raw_data, - train=loader, - val=subgraph_sampler, - test=subgraph_sampler) - - return sampler - - def load_nodelevel_dataset(config=None): r""" :returns: - data_local_dict + data_dict :rtype: Dict: dict{'client_id': Data()} """ path = config.data.root name = config.data.type.lower() + # TODO: remove splitter # Splitter splitter = get_splitter(config) - # Transforms transforms_funcs = get_transform(config, 'torch_geometric') - # Dataset if name in ["cora", "citeseer", "pubmed"]: num_split = { @@ -96,7 +34,6 @@ def load_nodelevel_dataset(config=None): 'citeseer': [332, 665, INF], 'pubmed': [3943, 3943, INF], } - dataset = Planetoid(path, name, split='random', @@ -154,19 +91,27 @@ def load_nodelevel_dataset(config=None): config.merge_from_list(['federate.client_num', client_num]) # get local dataset - data_local_dict = dict() - - for client_idx in range(len(dataset)): - local_data = raw2loader(dataset[client_idx], config) - data_local_dict[client_idx + 1] = local_data - + data_dict = dict() + for client_idx in range(1, len(dataset) + 1): + local_data = dataset[client_idx - 1] + # To undirected and add self-loop + local_data.edge_index = add_self_loops( + to_undirected(remove_self_loops(local_data.edge_index)[0]), + num_nodes=local_data.x.shape[0])[0] + data_dict[client_idx] = { + 'data': local_data, + 'train': [local_data], + 'val': [local_data], + 'test': [local_data] + } + # Keep ML split consistent with local graphs if global_dataset is not None: global_graph = global_dataset[0] train_mask = torch.zeros_like(global_graph.train_mask) val_mask = torch.zeros_like(global_graph.val_mask) test_mask = torch.zeros_like(global_graph.test_mask) - for client_sampler in data_local_dict.values(): + for client_sampler in data_dict.values(): if isinstance(client_sampler, Data): client_subgraph = client_sampler else: @@ -181,6 +126,10 @@ def load_nodelevel_dataset(config=None): global_graph.val_mask = val_mask global_graph.test_mask = test_mask - data_local_dict[0] = raw2loader(global_graph, config) - - return data_local_dict, config + data_dict[0] = { + 'data': global_graph, + 'train': [global_graph], + 'val': [global_graph], + 'test': [global_graph] + } + return data_dict, config diff --git a/federatedscope/gfl/dataset/cikm_cup.py b/federatedscope/gfl/dataset/cikm_cup.py index 7ae47a3ad..c3c152e2b 100644 --- a/federatedscope/gfl/dataset/cikm_cup.py +++ b/federatedscope/gfl/dataset/cikm_cup.py @@ -45,39 +45,3 @@ def __getitem__(self, idx): if split_data: data[split] = split_data return data - - -def load_cikmcup_data(config): - from torch_geometric.loader import DataLoader - - # Build data - dataset = CIKMCUPDataset(config.data.root) - config.merge_from_list(['federate.client_num', len(dataset)]) - - data_dict = {} - # Build DataLoader dict - for client_idx in range(1, config.federate.client_num + 1): - logger.info(f'Loading CIKMCUP data for Client #{client_idx}.') - dataloader_dict = {} - tmp_dataset = [] - if 'train' in dataset[client_idx]: - dataloader_dict['train'] = DataLoader(dataset[client_idx]['train'], - config.data.batch_size, - shuffle=config.data.shuffle) - tmp_dataset += dataset[client_idx]['train'] - if 'val' in dataset[client_idx]: - dataloader_dict['val'] = DataLoader(dataset[client_idx]['val'], - config.data.batch_size, - shuffle=False) - tmp_dataset += dataset[client_idx]['val'] - if 'test' in dataset[client_idx]: - dataloader_dict['test'] = DataLoader(dataset[client_idx]['test'], - config.data.batch_size, - shuffle=False) - tmp_dataset += dataset[client_idx]['test'] - if tmp_dataset: - dataloader_dict['num_label'] = 0 - - data_dict[client_idx] = dataloader_dict - - return data_dict, config diff --git a/federatedscope/gfl/fedsageplus/fedsageplus_on_cora.yaml b/federatedscope/gfl/fedsageplus/fedsageplus_on_cora.yaml index 8d316ee7a..0413de06e 100644 --- a/federatedscope/gfl/fedsageplus/fedsageplus_on_cora.yaml +++ b/federatedscope/gfl/fedsageplus/fedsageplus_on_cora.yaml @@ -12,6 +12,8 @@ data: root: data/ type: 'cora' splitter: 'louvain' +dataloader: + type: pyg batch_size: 1 model: type: sage diff --git a/federatedscope/gfl/fedsageplus/trainer.py b/federatedscope/gfl/fedsageplus/trainer.py index 86d2334a8..b9f665fc7 100644 --- a/federatedscope/gfl/fedsageplus/trainer.py +++ b/federatedscope/gfl/fedsageplus/trainer.py @@ -144,5 +144,5 @@ def cal_grad(self, raw_data, model_para, embedding, true_missing): @torch.no_grad() def embedding(self): model = self.ctx.model.to(self.ctx.device) - data = self.ctx.data.to(self.ctx.device) + data = self.ctx.data['data'].to(self.ctx.device) return model.encoder_model(data).to('cpu') diff --git a/federatedscope/gfl/fedsageplus/worker.py b/federatedscope/gfl/fedsageplus/worker.py index f1812598d..467b6d867 100644 --- a/federatedscope/gfl/fedsageplus/worker.py +++ b/federatedscope/gfl/fedsageplus/worker.py @@ -8,6 +8,7 @@ from federatedscope.core.workers.server import Server from federatedscope.core.workers.client import Client from federatedscope.core.auxiliaries.utils import merge_dict +from federatedscope.core.data import ClientData from federatedscope.gfl.trainer.nodetrainer import NodeMiniBatchTrainer from federatedscope.gfl.model.fedsageplus import LocalSage_Plus, FedSage_Plus @@ -255,10 +256,17 @@ def __init__(self, self).__init__(ID, server_id, state, config, data, model, device, strategy, *args, **kwargs) self.data = data - self.hide_data = HideGraph(self._cfg.fedsageplus.hide_portion)(data) + self.hide_data = HideGraph(self._cfg.fedsageplus.hide_portion)( + data['data']) + # Convert to `ClientData` + self.hide_data = ClientData(self._cfg, + train=[self.hide_data], + val=[self.hide_data], + test=[self.hide_data], + data=self.hide_data) self.device = device self.sage_batch_size = 64 - self.gen = LocalSage_Plus(data.x.shape[-1], + self.gen = LocalSage_Plus(data['data'].x.shape[-1], self._cfg.model.out_channels, hidden=self._cfg.model.hidden, gen_hidden=self._cfg.fedsageplus.gen_hidden, @@ -304,15 +312,17 @@ def callback_funcs_for_local_pre_train(self, message: Message): sender=self.ID, receiver=[sender], state=self.state, - content=[gen_para, embedding, self.hide_data.num_missing])) + content=[ + gen_para, embedding, self.hide_data['data'].num_missing + ])) logger.info(f'\tClient #{self.ID} send gen_para to Server #{sender}.') def callback_funcs_for_gen_para(self, message: Message): round, sender, content = message.state, message.sender, message.content gen_para, embedding, label, ID = content - gen_grad = self.trainer_fedgen.cal_grad(self.data, gen_para, embedding, - label) + gen_grad = self.trainer_fedgen.cal_grad(self.data['data'], gen_para, + embedding, label) self.state = round self.comm_manager.send( Message(msg_type='gradient', @@ -335,32 +345,37 @@ def callback_funcs_for_gradient(self, message): sender=self.ID, receiver=[sender], state=self.state, - content=[gen_para, embedding, self.hide_data.num_missing])) + content=[ + gen_para, embedding, self.hide_data['data'].num_missing + ])) logger.info(f'\tClient #{self.ID}: send gen_para to Server #{sender}.') def callback_funcs_for_setup_fedsage(self, message: Message): round, sender, _ = message.state, message.sender, message.content - self.filled_data = GraphMender(model=self.fedgen, - impaired_data=self.hide_data.cpu(), - original_data=self.data) + self.filled_data = GraphMender( + model=self.fedgen, + impaired_data=self.hide_data['data'].cpu(), + original_data=self.data['data']) subgraph_sampler = NeighborSampler( self.filled_data.edge_index, sizes=[-1], batch_size=4096, shuffle=False, - num_workers=self._cfg.data.num_workers) + num_workers=self._cfg.dataloader.num_workers) fill_dataloader = { 'data': self.filled_data, - 'train': NeighborSampler(self.filled_data.edge_index, - node_idx=self.filled_data.train_idx, - sizes=self._cfg.data.sizes, - batch_size=self.sage_batch_size, - shuffle=self._cfg.data.shuffle, - num_workers=self._cfg.data.num_workers), + 'train': NeighborSampler( + self.filled_data.edge_index, + node_idx=self.filled_data.train_idx, + sizes=self._cfg.dataloader.sizes, + batch_size=self.sage_batch_size, + shuffle=self._cfg.dataloader.shuffle, + num_workers=self._cfg.dataloader.num_workers), 'val': subgraph_sampler, 'test': subgraph_sampler } - self._cfg.merge_from_list(['data.batch_size', self.sage_batch_size]) + self._cfg.merge_from_list( + ['dataloader.batch_size', self.sage_batch_size]) self.trainer_clf = NodeMiniBatchTrainer(self.clf, fill_dataloader, self.device, diff --git a/federatedscope/gfl/flitplus/fedalgo_cls.yaml b/federatedscope/gfl/flitplus/fedalgo_cls.yaml index d05ed6727..3ddefb7b7 100644 --- a/federatedscope/gfl/flitplus/fedalgo_cls.yaml +++ b/federatedscope/gfl/flitplus/fedalgo_cls.yaml @@ -9,9 +9,11 @@ federate: data: root: data/ splitter: scaffold_lda - batch_size: 64 transform: ['AddSelfLoops'] splitter_args: [{'alpha': 0.1}] +dataloader: + type: pyg + batch_size: 64 model: type: mpnn hidden: 64 diff --git a/federatedscope/gfl/flitplus/trainer.py b/federatedscope/gfl/flitplus/trainer.py index 27325345a..bc22ccb39 100644 --- a/federatedscope/gfl/flitplus/trainer.py +++ b/federatedscope/gfl/flitplus/trainer.py @@ -4,7 +4,7 @@ from federatedscope.core.auxiliaries.enums import LIFECYCLE from federatedscope.core.trainers.context import CtxVar from federatedscope.gfl.loss.vat import VATLoss -from federatedscope.core.trainers.trainer import GeneralTorchTrainer +from federatedscope.core.trainers import GeneralTorchTrainer class FLITTrainer(GeneralTorchTrainer): diff --git a/federatedscope/gfl/gcflplus/gcflplus_on_multi_task.yaml b/federatedscope/gfl/gcflplus/gcflplus_on_multi_task.yaml index 57b48d291..56c70144d 100644 --- a/federatedscope/gfl/gcflplus/gcflplus_on_multi_task.yaml +++ b/federatedscope/gfl/gcflplus/gcflplus_on_multi_task.yaml @@ -8,6 +8,8 @@ data: root: data/ type: graph_multi_domain_mix pre_transform: ['Constant', {'value':1.0, 'cat':False}] +dataloader: + type: pyg model: type: gin hidden: 64 diff --git a/federatedscope/gfl/model/mpnn.py b/federatedscope/gfl/model/mpnn.py index dd1117331..e804eab2a 100644 --- a/federatedscope/gfl/model/mpnn.py +++ b/federatedscope/gfl/model/mpnn.py @@ -40,8 +40,7 @@ def forward(self, data): x, edge_index, edge_attr, batch = data.x, data.edge_index, \ data.edge_attr, data.batch elif isinstance(data, tuple): - x, edge_index, edge_attr, batch = data.x, data.edge_index, \ - data.edge_attr, data.batch + x, edge_index, edge_attr, batch = data else: raise TypeError('Unsupported data type!') diff --git a/federatedscope/gfl/trainer/linktrainer.py b/federatedscope/gfl/trainer/linktrainer.py index 72ab9b9a3..e2885b1a4 100644 --- a/federatedscope/gfl/trainer/linktrainer.py +++ b/federatedscope/gfl/trainer/linktrainer.py @@ -1,7 +1,6 @@ import torch from torch.utils.data import DataLoader -from torch_geometric.data import Data from torch_geometric.loader import GraphSAINTRandomWalkSampler, NeighborSampler from federatedscope.core.auxiliaries.enums import LIFECYCLE @@ -42,24 +41,28 @@ def parse_data(self, data): """ init_dict = dict() - if isinstance(data, Data): + if isinstance(data, dict): for mode in ["train", "val", "test"]: - edges = data.edge_index.T[data[MODE2MASK[mode]]] + graph_data = data['data'] + edges = graph_data.edge_index.T[graph_data[MODE2MASK[mode]]] # Use an index loader - index_loader = DataLoader(range(edges.size(0)), - self.cfg.data.batch_size, - shuffle=self.cfg.data.shuffle - if mode == 'train' else False, - drop_last=self.cfg.data.drop_last - if mode == 'train' else False) + index_loader = DataLoader( + range(edges.size(0)), + self.cfg.dataloader.batch_size, + shuffle=self.cfg.dataloader.shuffle + if mode == 'train' else False, + drop_last=self.cfg.dataloader.drop_last + if mode == 'train' else False) init_dict["{}_loader".format(mode)] = index_loader init_dict["num_{}_data".format(mode)] = edges.size(0) init_dict["{}_data".format(mode)] = None else: - raise TypeError("Type of data should be PyG data.") + raise TypeError("Type of data should be dict.") return init_dict def _hook_on_epoch_start_data2device(self, ctx): + if isinstance(ctx.data, dict): + ctx.data = ctx.data['data'] ctx.data = ctx.data.to(ctx.device) # For handling different dict key if "input_edge_index" in ctx.data: @@ -159,7 +162,7 @@ def parse_data(self, data): data.get(mode) ] init_dict["num_{}_data".format( - mode)] = self.cfg.data.batch_size + mode)] = self.cfg.dataloader.batch_size else: raise TypeError("Type {} is not supported.".format( type(data.get(mode)))) @@ -187,7 +190,7 @@ def _hook_on_batch_forward(self, ctx): pred = [] for perm in DataLoader(range(edges.size(0)), - self.cfg.data.batch_size): + self.cfg.dataloader.batch_size): edge = edges[perm].T pred += [ctx.model.link_predictor(h, edge).squeeze()] pred = torch.cat(pred, dim=0) diff --git a/federatedscope/gfl/trainer/nodetrainer.py b/federatedscope/gfl/trainer/nodetrainer.py index e22341d07..7871bdb2c 100644 --- a/federatedscope/gfl/trainer/nodetrainer.py +++ b/federatedscope/gfl/trainer/nodetrainer.py @@ -1,7 +1,4 @@ import torch - -from torch_geometric.loader import DataLoader as PyGDataLoader -from torch_geometric.data import Data from torch_geometric.loader import GraphSAINTRandomWalkSampler, NeighborSampler from federatedscope.core.auxiliaries.enums import LIFECYCLE @@ -22,15 +19,14 @@ def parse_data(self, data): """ init_dict = dict() - if isinstance(data, Data): + if isinstance(data, dict): for mode in ["train", "val", "test"]: - init_dict["{}_loader".format(mode)] = PyGDataLoader([data]) + init_dict["{}_loader".format(mode)] = data.get(mode) init_dict["{}_data".format(mode)] = None # For node-level task dataloader contains one graph init_dict["num_{}_data".format(mode)] = 1 - else: - raise TypeError("Type of data should be PyG data.") + raise TypeError("Type of data should be dict.") return init_dict def _hook_on_batch_forward(self, ctx): @@ -119,7 +115,7 @@ def parse_data(self, data): data.get(mode) ] init_dict["num_{}_data".format( - mode)] = self.cfg.data.batch_size + mode)] = self.cfg.dataloader.batch_size else: raise TypeError("Type {} is not supported.".format( type(data.get(mode)))) @@ -154,9 +150,9 @@ def _hook_on_batch_forward(self, ctx): else: # For GraphSAINTRandomWalkSampler or PyGDataLoader batch = ctx.data_batch.to(ctx.device) - pred = ctx.model(batch.x, - batch.edge_index)[batch['{}_mask'.format( - ctx.cur_split)]] + pred = ctx.model( + (batch.x, + batch.edge_index))[batch['{}_mask'.format(ctx.cur_split)]] label = batch.y[batch['{}_mask'.format(ctx.cur_split)]] ctx.batch_size = torch.sum(ctx.data_batch['train_mask']).item() else: diff --git a/federatedscope/hpo.py b/federatedscope/hpo.py index 472bd4e15..c3b16cf9f 100644 --- a/federatedscope/hpo.py +++ b/federatedscope/hpo.py @@ -26,7 +26,7 @@ init_cfg.merge_from_file(args.cfg_file) init_cfg.merge_from_list(args.opts) - update_logger(init_cfg) + update_logger(init_cfg, clear_before_add=True) setup_seed(init_cfg.seed) assert not args.client_cfg_file, 'No support for client-wise config in ' \ diff --git a/federatedscope/main.py b/federatedscope/main.py index 40d8a0e94..0eae9c8c3 100644 --- a/federatedscope/main.py +++ b/federatedscope/main.py @@ -32,13 +32,14 @@ setup_seed(init_cfg.seed) # load clients' cfg file - client_cfg = CfgNode.load_cfg(open(args.client_cfg_file, - 'r')) if args.client_cfg_file else None + client_cfgs = CfgNode.load_cfg(open(args.client_cfg_file, + 'r')) if args.client_cfg_file else None # federated dataset might change the number of clients # thus, we allow the creation procedure of dataset to modify the global # cfg object - data, modified_cfg = get_data(config=init_cfg.clone()) + data, modified_cfg = get_data(config=init_cfg.clone(), + client_cfgs=client_cfgs) init_cfg.merge_from_other_cfg(modified_cfg) init_cfg.freeze() @@ -47,5 +48,5 @@ server_class=get_server_cls(init_cfg), client_class=get_client_cls(init_cfg), config=init_cfg.clone(), - client_config=client_cfg) + client_configs=client_cfgs) _ = runner.run() diff --git a/federatedscope/mf/baseline/hfl-sgdmf_fedavg_standalone_on_movielens1m.yaml b/federatedscope/mf/baseline/hfl-sgdmf_fedavg_standalone_on_movielens1m.yaml index a860c92d1..d94668156 100644 --- a/federatedscope/mf/baseline/hfl-sgdmf_fedavg_standalone_on_movielens1m.yaml +++ b/federatedscope/mf/baseline/hfl-sgdmf_fedavg_standalone_on_movielens1m.yaml @@ -9,8 +9,9 @@ federate: data: root: data/ type: HFLMovieLens1M - batch_size: 64 - num_workers: 0 +dataloader: + type: mf + theta: -1 model: type: HMFNet hidden: 20 @@ -30,4 +31,3 @@ sgdmf: epsilon: 2. delta: 0.5 R: 5. - theta: -1 diff --git a/federatedscope/mf/baseline/hfl_fedavg_standalone_on_movielens1m.yaml b/federatedscope/mf/baseline/hfl_fedavg_standalone_on_movielens1m.yaml index 5b528eb9a..f6f2637f8 100644 --- a/federatedscope/mf/baseline/hfl_fedavg_standalone_on_movielens1m.yaml +++ b/federatedscope/mf/baseline/hfl_fedavg_standalone_on_movielens1m.yaml @@ -9,8 +9,8 @@ federate: data: root: data/ type: HFLMovieLens1M - batch_size: 64 - num_workers: 0 +dataloader: + type: mf model: type: HMFNet hidden: 20 diff --git a/federatedscope/mf/baseline/hfl_fedavg_standalone_on_netflix.yaml b/federatedscope/mf/baseline/hfl_fedavg_standalone_on_netflix.yaml index 37d645380..e764eba44 100644 --- a/federatedscope/mf/baseline/hfl_fedavg_standalone_on_netflix.yaml +++ b/federatedscope/mf/baseline/hfl_fedavg_standalone_on_netflix.yaml @@ -11,8 +11,9 @@ federate: data: root: data/ type: HFLNetflix +dataloader: batch_size: 32 - num_workers: 0 + type: mf model: type: HMFNet hidden: 10 diff --git a/federatedscope/mf/baseline/vfl-sgdmf_fedavg_standalone_on_movielens1m.yaml b/federatedscope/mf/baseline/vfl-sgdmf_fedavg_standalone_on_movielens1m.yaml index 20b40ec53..e01d598e4 100644 --- a/federatedscope/mf/baseline/vfl-sgdmf_fedavg_standalone_on_movielens1m.yaml +++ b/federatedscope/mf/baseline/vfl-sgdmf_fedavg_standalone_on_movielens1m.yaml @@ -9,8 +9,10 @@ federate: data: root: data/ type: VFLMovieLens1M +dataloader: + type: mf + theta: -1 batch_size: 8 - num_workers: 0 model: type: VMFNet hidden: 20 @@ -30,4 +32,3 @@ sgdmf: epsilon: 4. delta: 0.75 R: 5. - theta: -1 diff --git a/federatedscope/mf/baseline/vfl_fedavg_standalone_on_movielens1m.yaml b/federatedscope/mf/baseline/vfl_fedavg_standalone_on_movielens1m.yaml index 7b21dbf22..3f2acaf64 100644 --- a/federatedscope/mf/baseline/vfl_fedavg_standalone_on_movielens1m.yaml +++ b/federatedscope/mf/baseline/vfl_fedavg_standalone_on_movielens1m.yaml @@ -9,8 +9,8 @@ federate: data: root: data/ type: VFLMovieLens1M - batch_size: 64 - num_workers: 0 +dataloader: + type: mf model: type: VMFNet hidden: 20 diff --git a/federatedscope/mf/dataloader/dataloader.py b/federatedscope/mf/dataloader/dataloader.py index c65f18603..ebd4df252 100644 --- a/federatedscope/mf/dataloader/dataloader.py +++ b/federatedscope/mf/dataloader/dataloader.py @@ -17,7 +17,7 @@ } -def load_mf_dataset(config=None): +def load_mf_dataset(config=None, client_cfgs=None): """Return the dataset of matrix factorization Format: @@ -46,26 +46,15 @@ def load_mf_dataset(config=None): raise NotImplementedError("Dataset {} is not implemented.".format( config.data.type)) - data_local_dict = collections.defaultdict(dict) - for id_client, data in dataset.data.items(): - data_local_dict[id_client]["train"] = MFDataLoader( - data["train"], - shuffle=config.data.shuffle, - batch_size=config.data.batch_size, - drop_last=config.data.drop_last, - theta=config.sgdmf.theta) - data_local_dict[id_client]["test"] = MFDataLoader( - data["test"], - shuffle=False, - batch_size=config.data.batch_size, - drop_last=config.data.drop_last, - theta=config.sgdmf.theta) + data_dict = collections.defaultdict(dict) + for client_idx, data in dataset.data.items(): + data_dict[client_idx] = data # Modify config config.merge_from_list(['model.num_user', dataset.n_user]) config.merge_from_list(['model.num_item', dataset.n_item]) - return data_local_dict, config + return data_dict, config class MFDataLoader(object): diff --git a/federatedscope/mf/trainer/trainer_sgdmf.py b/federatedscope/mf/trainer/trainer_sgdmf.py index 7e5dddacc..653eeb555 100644 --- a/federatedscope/mf/trainer/trainer_sgdmf.py +++ b/federatedscope/mf/trainer/trainer_sgdmf.py @@ -37,13 +37,13 @@ def init_sgdmf_ctx(base_trainer): ctx = base_trainer.ctx cfg = base_trainer.cfg - sample_ratio = float(cfg.data.batch_size) / cfg.model.num_user + sample_ratio = float(cfg.dataloader.batch_size) / cfg.model.num_user # Noise multiplier tmp = cfg.sgdmf.constant * np.power(sample_ratio, 2) * ( cfg.federate.total_round_num * ctx.num_total_train_batch) * np.log( 1. / cfg.sgdmf.delta) noise_multipler = np.sqrt(tmp / np.power(cfg.sgdmf.epsilon, 2)) - ctx.scale = max(cfg.sgdmf.theta, 1.) * noise_multipler * np.power( + ctx.scale = max(cfg.dataloader.theta, 1.) * noise_multipler * np.power( cfg.sgdmf.R, 1.5) logger.info("Inject noise: (loc=0, scale={})".format(ctx.scale)) ctx.sgdmf_R = cfg.sgdmf.R diff --git a/federatedscope/nlp/baseline/fedavg_bert_on_sst2.yaml b/federatedscope/nlp/baseline/fedavg_bert_on_sst2.yaml index b85d0febf..898ecd514 100644 --- a/federatedscope/nlp/baseline/fedavg_bert_on_sst2.yaml +++ b/federatedscope/nlp/baseline/fedavg_bert_on_sst2.yaml @@ -11,10 +11,10 @@ data: root: 'glue' type: 'sst2@huggingface_datasets' args: [{'max_len': 512}] - batch_size: 128 splitter: 'lda' splitter_args: [{'alpha': 0.5}] - num_workers: 0 +dataloader: + batch_size: 128 model: type: 'google/bert_uncased_L-2_H-128_A-2@transformers' task: 'SequenceClassification' diff --git a/federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml b/federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml index 8aa9b8dc2..0aeb0a318 100644 --- a/federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml +++ b/federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml @@ -10,7 +10,6 @@ federate: data: root: data/ type: synthetic - batch_size: 64 subsample: 1.0 model: type: lr diff --git a/federatedscope/nlp/baseline/fedavg_lr_on_twitter.yaml b/federatedscope/nlp/baseline/fedavg_lr_on_twitter.yaml index b510ed79f..eef8963de 100644 --- a/federatedscope/nlp/baseline/fedavg_lr_on_twitter.yaml +++ b/federatedscope/nlp/baseline/fedavg_lr_on_twitter.yaml @@ -11,9 +11,9 @@ federate: data: root: data/ type: twitter - batch_size: 5 subsample: 0.005 - num_workers: 0 +dataloader: + batch_size: 5 model: type: lr out_channels: 2 diff --git a/federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml b/federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml index 86d5aca27..3603d3de9 100644 --- a/federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml +++ b/federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml @@ -9,9 +9,7 @@ federate: data: root: data/ type: shakespeare - batch_size: 64 subsample: 0.2 - num_workers: 0 splits: [0.6,0.2,0.2] model: type: lstm diff --git a/federatedscope/nlp/baseline/fedavg_lstm_on_subreddit.yaml b/federatedscope/nlp/baseline/fedavg_lstm_on_subreddit.yaml index 1080bb591..6ee95a43c 100644 --- a/federatedscope/nlp/baseline/fedavg_lstm_on_subreddit.yaml +++ b/federatedscope/nlp/baseline/fedavg_lstm_on_subreddit.yaml @@ -9,8 +9,9 @@ federate: data: root: data/ type: subreddit - batch_size: 5 subsample: 1.0 +dataloader: + batch_size: 5 model: type: lstm in_channels: 10000 diff --git a/federatedscope/nlp/baseline/fedavg_transformer_on_cola.yaml b/federatedscope/nlp/baseline/fedavg_transformer_on_cola.yaml index 7eb686495..9dd6e243c 100644 --- a/federatedscope/nlp/baseline/fedavg_transformer_on_cola.yaml +++ b/federatedscope/nlp/baseline/fedavg_transformer_on_cola.yaml @@ -20,10 +20,8 @@ data: args: [{'load_disk_dir': 'huggingface/datasets/glue/cola', 'hg_cache_dir': 'huggingface', 'max_len': 128, 'val_as_dummy_test': True, 'part_train_dummy_val': 0.2} ] - batch_size: 64 splitter: 'lda' splitter_args: [ { 'alpha': 0.4, 'min_size': 1} ] - num_workers: 0 model: type: 'google/bert_uncased_L-2_H-128_A-2@transformers' task: 'SequenceClassification' diff --git a/federatedscope/nlp/baseline/fedavg_transformer_on_imdb.yaml b/federatedscope/nlp/baseline/fedavg_transformer_on_imdb.yaml index a9e818aa1..0d2924cb4 100644 --- a/federatedscope/nlp/baseline/fedavg_transformer_on_imdb.yaml +++ b/federatedscope/nlp/baseline/fedavg_transformer_on_imdb.yaml @@ -12,10 +12,10 @@ data: type: 'IMDB@torchtext' args: [{'max_len': 512}] splits: [0.8, 0.2, 0.0] # test is fixed - batch_size: 128 splitter: 'lda' splitter_args: [{'alpha': 0.5}] - num_workers: 0 +dataloader: + batch_size: 128 model: type: 'google/bert_uncased_L-2_H-128_A-2@transformers' task: 'SequenceClassification' diff --git a/federatedscope/nlp/dataloader/dataloader.py b/federatedscope/nlp/dataloader/dataloader.py index c33a75ec7..d950a2a63 100644 --- a/federatedscope/nlp/dataloader/dataloader.py +++ b/federatedscope/nlp/dataloader/dataloader.py @@ -1,5 +1,3 @@ -from torch.utils.data import DataLoader - from federatedscope.nlp.dataset.leaf_nlp import LEAF_NLP from federatedscope.nlp.dataset.leaf_twitter import LEAF_TWITTER from federatedscope.nlp.dataset.leaf_synthetic import LEAF_SYNTHETIC @@ -20,7 +18,6 @@ def load_nlp_dataset(config=None): path = config.data.root name = config.data.type.lower() - batch_size = config.data.batch_size transforms_funcs = get_transform(config, 'torchtext') if name in ['shakespeare', 'subreddit']: @@ -49,26 +46,8 @@ def load_nlp_dataset(config=None): config.merge_from_list(['federate.client_num', client_num]) # get local dataset - data_local_dict = dict() - for client_idx in range(client_num): - dataloader = { - 'train': DataLoader(dataset[client_idx]['train'], - batch_size, - shuffle=config.data.shuffle, - num_workers=config.data.num_workers) - } - if 'test' in dataset[client_idx]: - dataloader['test'] = DataLoader( - dataset[client_idx]['test'], - batch_size, - shuffle=False, - num_workers=config.data.num_workers) - if 'val' in dataset[client_idx]: - dataloader['val'] = DataLoader(dataset[client_idx]['val'], - batch_size, - shuffle=False, - num_workers=config.data.num_workers) - - data_local_dict[client_idx + 1] = dataloader + data_dict = dict() + for client_idx in range(1, client_num + 1): + data_dict[client_idx] = dataset[client_idx - 1] - return data_local_dict, config + return data_dict, config diff --git a/federatedscope/tabular/dataloader/quadratic.py b/federatedscope/tabular/dataloader/quadratic.py index 37b73829d..335984672 100644 --- a/federatedscope/tabular/dataloader/quadratic.py +++ b/federatedscope/tabular/dataloader/quadratic.py @@ -1,10 +1,8 @@ import numpy as np -from torch.utils.data import DataLoader - def load_quadratic_dataset(config): - dataset = dict() + data_dict = dict() d = config.data.quadratic.dim base = np.exp( np.log(config.data.quadratic.max_curv / config.data.quadratic.min_curv) @@ -13,9 +11,9 @@ def load_quadratic_dataset(config): # TODO: enable sphere a = 0.02 * base**(i - 1) * np.identity(d) # TODO: enable non-zero minimizer, i.e., provide a shift - client_data = dict() - client_data['train'] = DataLoader([(a.astype(np.float32), .0)]) - client_data['val'] = DataLoader([(a.astype(np.float32), .0)]) - client_data['test'] = DataLoader([(a.astype(np.float32), .0)]) - dataset[i] = client_data - return dataset, config + data_dict[i] = { + 'train': [(a.astype(np.float32), .0)], + 'val': [(a.astype(np.float32), .0)], + 'test': [(a.astype(np.float32), .0)] + } + return data_dict, config diff --git a/federatedscope/tabular/dataloader/toy.py b/federatedscope/tabular/dataloader/toy.py new file mode 100644 index 000000000..a5a4cb0aa --- /dev/null +++ b/federatedscope/tabular/dataloader/toy.py @@ -0,0 +1,120 @@ +import pickle + +import numpy as np + +from federatedscope.core.data.wrap_dataset import WrapDataset + + +def load_toy_data(config=None): + generate = config.federate.mode.lower() == 'standalone' + + def _generate_data(client_num=5, + instance_num=1000, + feature_num=5, + save_data=False): + """ + Generate data in FedRunner format + Args: + client_num: + instance_num: + feature_num: + save_data: + + Returns: + { + '{client_id}': { + 'train': { + 'x': ..., + 'y': ... + }, + 'test': { + 'x': ..., + 'y': ... + }, + 'val': { + 'x': ..., + 'y': ... + } + } + } + + """ + weights = np.random.normal(loc=0.0, scale=1.0, size=feature_num) + bias = np.random.normal(loc=0.0, scale=1.0) + data = dict() + for each_client in range(1, client_num + 1): + data[each_client] = dict() + client_x = np.random.normal(loc=0.0, + scale=0.5 * each_client, + size=(instance_num, feature_num)) + client_y = np.sum(client_x * weights, axis=-1) + bias + client_y = np.expand_dims(client_y, -1) + client_data = {'x': client_x, 'y': client_y} + data[each_client]['train'] = client_data + + # test data + test_x = np.random.normal(loc=0.0, + scale=1.0, + size=(instance_num, feature_num)) + test_y = np.sum(test_x * weights, axis=-1) + bias + test_y = np.expand_dims(test_y, -1) + test_data = {'x': test_x, 'y': test_y} + for each_client in range(1, client_num + 1): + data[each_client]['test'] = test_data + + # val data + val_x = np.random.normal(loc=0.0, + scale=1.0, + size=(instance_num, feature_num)) + val_y = np.sum(val_x * weights, axis=-1) + bias + val_y = np.expand_dims(val_y, -1) + val_data = {'x': val_x, 'y': val_y} + for each_client in range(1, client_num + 1): + data[each_client]['val'] = val_data + + # server_data + data[0] = dict() + # data[0]['train'] = None + data[0]['val'] = val_data + data[0]['test'] = test_data + + if save_data: + # server_data = dict() + save_client_data = dict() + + for client_idx in range(0, client_num + 1): + if client_idx == 0: + filename = 'data/server_data' + else: + filename = 'data/client_{:d}_data'.format(client_idx) + with open(filename, 'wb') as f: + save_client_data['train'] = { + k: v.tolist() + for k, v in data[client_idx]['train'].items() + } + save_client_data['val'] = { + k: v.tolist() + for k, v in data[client_idx]['val'].items() + } + save_client_data['test'] = { + k: v.tolist() + for k, v in data[client_idx]['test'].items() + } + pickle.dump(save_client_data, f) + + return data + + if generate: + data = _generate_data(client_num=config.federate.client_num, + save_data=config.data.save_data) + else: + with open(config.distribute.data_file, 'rb') as f: + data = pickle.load(f) + data = {config.distribute.data_idx: data} + for client_id in data.keys(): + data[client_id] = { + k: WrapDataset(v) + for k, v in data[client_id].items() + } if data[client_id] is not None else None + + return data, config diff --git a/federatedscope/vertical_fl/worker/vertical_client.py b/federatedscope/vertical_fl/worker/vertical_client.py index b463ea82e..09fcb9c18 100644 --- a/federatedscope/vertical_fl/worker/vertical_client.py +++ b/federatedscope/vertical_fl/worker/vertical_client.py @@ -37,7 +37,7 @@ def __init__(self, self.batch_index = None self.own_label = ('y' in self.data['train']) self.dataloader = batch_iter(self.data['train'], - self._cfg.data.batch_size, + self._cfg.dataloader.batch_size, shuffled=True) self.register_handlers('public_keys', diff --git a/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml b/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml index 8b0436c3c..2d78ba93e 100644 --- a/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml +++ b/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml @@ -16,11 +16,11 @@ data: type: femnist # form: dataloader splits: [0.6,0.2,0.2] - batch_size: 32 subsample: 0.05 - num_workers: 0 # transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - transform: [['ToTensor']] + transform: [ [ 'ToTensor' ] ] +dataloader: + batch_size: 32 model: type: convnet2 hidden: 2048 diff --git a/scripts/attack_exp_scripts/privacy_attack/CRA_fedavg_convnet2_on_femnist.yaml b/scripts/attack_exp_scripts/privacy_attack/CRA_fedavg_convnet2_on_femnist.yaml index c7659b192..43d1120dc 100644 --- a/scripts/attack_exp_scripts/privacy_attack/CRA_fedavg_convnet2_on_femnist.yaml +++ b/scripts/attack_exp_scripts/privacy_attack/CRA_fedavg_convnet2_on_femnist.yaml @@ -12,10 +12,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 10 subsample: 0.0001 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 10 model: type: convnet2 hidden: 2048 diff --git a/scripts/attack_exp_scripts/privacy_attack/PIA_toy.yaml b/scripts/attack_exp_scripts/privacy_attack/PIA_toy.yaml index fb36f4b63..19a816d68 100644 --- a/scripts/attack_exp_scripts/privacy_attack/PIA_toy.yaml +++ b/scripts/attack_exp_scripts/privacy_attack/PIA_toy.yaml @@ -10,9 +10,9 @@ data: root: data/ type: toy splits: [0.6,0.2,0.2] - batch_size: 1 subsample: 0.0001 - num_workers: 0 +dataloader: + batch_size: 1 model: type: lr hidden: 2048 diff --git a/scripts/attack_exp_scripts/privacy_attack/gradient_ascent_MIA_on_femnist.yaml b/scripts/attack_exp_scripts/privacy_attack/gradient_ascent_MIA_on_femnist.yaml index f60e64dff..619e2b972 100644 --- a/scripts/attack_exp_scripts/privacy_attack/gradient_ascent_MIA_on_femnist.yaml +++ b/scripts/attack_exp_scripts/privacy_attack/gradient_ascent_MIA_on_femnist.yaml @@ -12,10 +12,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 1 subsample: 0.0001 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 1 model: type: convnet2 hidden: 2048 diff --git a/scripts/attack_exp_scripts/privacy_attack/gradient_ascent_MIA_on_femnist_simu_in.yaml b/scripts/attack_exp_scripts/privacy_attack/gradient_ascent_MIA_on_femnist_simu_in.yaml index b99423d1b..c752fdd08 100644 --- a/scripts/attack_exp_scripts/privacy_attack/gradient_ascent_MIA_on_femnist_simu_in.yaml +++ b/scripts/attack_exp_scripts/privacy_attack/gradient_ascent_MIA_on_femnist_simu_in.yaml @@ -12,10 +12,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 1 subsample: 0.0001 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 1 model: type: convnet2 hidden: 2048 diff --git a/scripts/attack_exp_scripts/privacy_attack/reconstruct_IG_fedavg_opt_on_femnist.yaml b/scripts/attack_exp_scripts/privacy_attack/reconstruct_IG_fedavg_opt_on_femnist.yaml index a8a6da4a4..2a0a75be9 100644 --- a/scripts/attack_exp_scripts/privacy_attack/reconstruct_IG_fedavg_opt_on_femnist.yaml +++ b/scripts/attack_exp_scripts/privacy_attack/reconstruct_IG_fedavg_opt_on_femnist.yaml @@ -11,10 +11,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 1 subsample: 0.0001 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 1 model: type: convnet2 hidden: 2048 diff --git a/scripts/attack_exp_scripts/privacy_attack/reconstruct_fedavg_opt_on_femnist.yaml b/scripts/attack_exp_scripts/privacy_attack/reconstruct_fedavg_opt_on_femnist.yaml index 582e44159..40094a34b 100644 --- a/scripts/attack_exp_scripts/privacy_attack/reconstruct_fedavg_opt_on_femnist.yaml +++ b/scripts/attack_exp_scripts/privacy_attack/reconstruct_fedavg_opt_on_femnist.yaml @@ -12,10 +12,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 1 subsample: 0.0001 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 1 model: type: convnet2 hidden: 2048 diff --git a/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_1.yaml b/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_1.yaml index 7ea86b85d..eb3d54c34 100644 --- a/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_1.yaml +++ b/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_1.yaml @@ -21,10 +21,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 10 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 10 model: type: convnet2 hidden: 128 diff --git a/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_2.yaml b/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_2.yaml index 6d5374c8c..4fa8738dc 100644 --- a/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_2.yaml +++ b/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_2.yaml @@ -21,10 +21,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 10 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 10 model: type: convnet2 hidden: 128 diff --git a/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_3.yaml b/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_3.yaml index 83abc02b0..b08e14b68 100644 --- a/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_3.yaml +++ b/scripts/distributed_scripts/distributed_configs/distributed_femnist_client_3.yaml @@ -21,10 +21,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 10 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 10 model: type: convnet2 hidden: 128 diff --git a/scripts/distributed_scripts/distributed_configs/distributed_femnist_server.yaml b/scripts/distributed_scripts/distributed_configs/distributed_femnist_server.yaml index 05b4fb124..599a08a69 100644 --- a/scripts/distributed_scripts/distributed_configs/distributed_femnist_server.yaml +++ b/scripts/distributed_scripts/distributed_configs/distributed_femnist_server.yaml @@ -19,10 +19,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 10 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 10 model: type: convnet2 hidden: 128 diff --git a/scripts/example_configs/asyn_cifar10.yaml b/scripts/example_configs/asyn_cifar10.yaml index 01d5150de..ffbf7030b 100644 --- a/scripts/example_configs/asyn_cifar10.yaml +++ b/scripts/example_configs/asyn_cifar10.yaml @@ -17,12 +17,12 @@ data: type: CIFAR10@torchvision args: [{'download': True}] splits: [0.8,0.2,0.2] - batch_size: 10 subsample: 0.2 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] splitter: 'lda' splitter_args: [{'alpha': 0.2}] +dataloader: + batch_size: 10 model: type: convnet2 hidden: 512 diff --git a/scripts/example_configs/cora/sha.yaml b/scripts/example_configs/cora/sha.yaml index b2eff0477..00444abb6 100644 --- a/scripts/example_configs/cora/sha.yaml +++ b/scripts/example_configs/cora/sha.yaml @@ -15,6 +15,7 @@ data: root: data/ type: cora splitter: 'louvain' +dataloader: batch_size: 1 model: type: gcn diff --git a/scripts/example_configs/cora/sha_wrap_fedex.yaml b/scripts/example_configs/cora/sha_wrap_fedex.yaml index 822bbc9da..a5c729c14 100644 --- a/scripts/example_configs/cora/sha_wrap_fedex.yaml +++ b/scripts/example_configs/cora/sha_wrap_fedex.yaml @@ -15,6 +15,7 @@ data: root: data/ type: cora splitter: 'louvain' +dataloader: batch_size: 1 model: type: gcn diff --git a/scripts/example_configs/cora/sha_wrap_fedex_arm.yaml b/scripts/example_configs/cora/sha_wrap_fedex_arm.yaml index 2e9b726ec..6566791d5 100644 --- a/scripts/example_configs/cora/sha_wrap_fedex_arm.yaml +++ b/scripts/example_configs/cora/sha_wrap_fedex_arm.yaml @@ -15,6 +15,7 @@ data: root: data/ type: cora splitter: 'louvain' +dataloader: batch_size: 1 model: type: gcn diff --git a/scripts/example_configs/fed_node_cls.yaml b/scripts/example_configs/fed_node_cls.yaml index adb65211e..1f84b165d 100644 --- a/scripts/example_configs/fed_node_cls.yaml +++ b/scripts/example_configs/fed_node_cls.yaml @@ -12,6 +12,7 @@ data: root: data/ type: cora splitter: 'louvain' +dataloader: batch_size: 1 model: type: gcn diff --git a/scripts/example_configs/femnist.yaml b/scripts/example_configs/femnist.yaml index 5e9f39ce1..766c3e4b0 100644 --- a/scripts/example_configs/femnist.yaml +++ b/scripts/example_configs/femnist.yaml @@ -11,10 +11,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 10 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 10 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/bo_gp.yaml b/scripts/example_configs/femnist/avg/bo_gp.yaml index a9730acab..35915c31f 100644 --- a/scripts/example_configs/femnist/avg/bo_gp.yaml +++ b/scripts/example_configs/femnist/avg/bo_gp.yaml @@ -14,10 +14,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml b/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml index 89cdd15aa..815325682 100644 --- a/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml +++ b/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml @@ -15,10 +15,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/bo_kde.yaml b/scripts/example_configs/femnist/avg/bo_kde.yaml index 9d7550f23..e8096feeb 100644 --- a/scripts/example_configs/femnist/avg/bo_kde.yaml +++ b/scripts/example_configs/femnist/avg/bo_kde.yaml @@ -14,10 +14,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/bo_kde_wrap.yaml b/scripts/example_configs/femnist/avg/bo_kde_wrap.yaml index e19f71b7e..1ad4f0003 100644 --- a/scripts/example_configs/femnist/avg/bo_kde_wrap.yaml +++ b/scripts/example_configs/femnist/avg/bo_kde_wrap.yaml @@ -15,10 +15,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/bo_rf.yaml b/scripts/example_configs/femnist/avg/bo_rf.yaml index 7eead0983..1e8bca28e 100644 --- a/scripts/example_configs/femnist/avg/bo_rf.yaml +++ b/scripts/example_configs/femnist/avg/bo_rf.yaml @@ -14,10 +14,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml b/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml index 36c52e4e8..f2977a2eb 100644 --- a/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml +++ b/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml @@ -15,10 +15,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/bohb.yaml b/scripts/example_configs/femnist/avg/bohb.yaml index 5273252a0..b60970464 100644 --- a/scripts/example_configs/femnist/avg/bohb.yaml +++ b/scripts/example_configs/femnist/avg/bohb.yaml @@ -14,10 +14,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/bohb_wrap.yaml b/scripts/example_configs/femnist/avg/bohb_wrap.yaml index 77ed33363..e3ca1f733 100644 --- a/scripts/example_configs/femnist/avg/bohb_wrap.yaml +++ b/scripts/example_configs/femnist/avg/bohb_wrap.yaml @@ -15,10 +15,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/hb.yaml b/scripts/example_configs/femnist/avg/hb.yaml index 7e2c3dcca..f48d9de93 100644 --- a/scripts/example_configs/femnist/avg/hb.yaml +++ b/scripts/example_configs/femnist/avg/hb.yaml @@ -14,10 +14,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/hb_wrap.yaml b/scripts/example_configs/femnist/avg/hb_wrap.yaml index c9e8b6c84..1cfdeeca6 100644 --- a/scripts/example_configs/femnist/avg/hb_wrap.yaml +++ b/scripts/example_configs/femnist/avg/hb_wrap.yaml @@ -15,10 +15,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/rs.yaml b/scripts/example_configs/femnist/avg/rs.yaml index 41eed9bb7..d6dd868a0 100644 --- a/scripts/example_configs/femnist/avg/rs.yaml +++ b/scripts/example_configs/femnist/avg/rs.yaml @@ -14,10 +14,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/rs_wrap.yaml b/scripts/example_configs/femnist/avg/rs_wrap.yaml index 40acdb5cc..9d4680524 100644 --- a/scripts/example_configs/femnist/avg/rs_wrap.yaml +++ b/scripts/example_configs/femnist/avg/rs_wrap.yaml @@ -15,10 +15,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/sha.yaml b/scripts/example_configs/femnist/avg/sha.yaml index c07af3842..0c9350fe0 100644 --- a/scripts/example_configs/femnist/avg/sha.yaml +++ b/scripts/example_configs/femnist/avg/sha.yaml @@ -14,10 +14,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/sha_wrap.yaml b/scripts/example_configs/femnist/avg/sha_wrap.yaml index 481c1d5a6..cf476e100 100644 --- a/scripts/example_configs/femnist/avg/sha_wrap.yaml +++ b/scripts/example_configs/femnist/avg/sha_wrap.yaml @@ -15,10 +15,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/avg/ss.yaml b/scripts/example_configs/femnist/avg/ss.yaml index 199a1c1d2..e7d57b0ae 100644 --- a/scripts/example_configs/femnist/avg/ss.yaml +++ b/scripts/example_configs/femnist/avg/ss.yaml @@ -14,6 +14,6 @@ train.local_update_steps: type: int lower: 1 upper: 4 -data.batch_size: +dataloader.batch_size: type: cate choices: [16, 32, 64] diff --git a/scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml b/scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml index 6f1e7ef33..ea438a1ad 100644 --- a/scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml +++ b/scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml @@ -10,6 +10,6 @@ model.dropout: train.local_update_steps: type: cate choices: [1, 2, 3, 4] -data.batch_size: +dataloader.batch_size: type: cate choices: [16, 32, 64] \ No newline at end of file diff --git a/scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml b/scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml index c7e9138b0..774265c55 100644 --- a/scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml +++ b/scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml @@ -2,4 +2,4 @@ train.optimizer.lr: [0.01, 0.01668, 0.02783, 0.04642, 0.07743, 0.12915, 0.21544, train.optimizer.weight_decay: [0.0, 0.001, 0.01, 0.1] model.dropout: [0.0, 0.5] train.local_update_steps: [1, 2, 3, 4] -data.batch_size: [16, 32, 64] \ No newline at end of file +dataloader.batch_size: [16, 32, 64] \ No newline at end of file diff --git a/scripts/example_configs/femnist/hpo_ss_sha.yaml b/scripts/example_configs/femnist/hpo_ss_sha.yaml index 200f8624d..4a65f2cef 100644 --- a/scripts/example_configs/femnist/hpo_ss_sha.yaml +++ b/scripts/example_configs/femnist/hpo_ss_sha.yaml @@ -12,6 +12,6 @@ model.dropout: train.local_update_steps: type: cate choices: [1, 2, 3, 4] -data.batch_size: +dataloader.batch_size: type: cate choices: [16, 32, 64] \ No newline at end of file diff --git a/scripts/example_configs/femnist/sha.yaml b/scripts/example_configs/femnist/sha.yaml index 2207777a9..2343d43d6 100644 --- a/scripts/example_configs/femnist/sha.yaml +++ b/scripts/example_configs/femnist/sha.yaml @@ -14,10 +14,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist/sha_wrap_fedex.yaml b/scripts/example_configs/femnist/sha_wrap_fedex.yaml index 56d3602a3..58309fe31 100644 --- a/scripts/example_configs/femnist/sha_wrap_fedex.yaml +++ b/scripts/example_configs/femnist/sha_wrap_fedex.yaml @@ -14,10 +14,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 16 subsample: 0.05 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] - num_workers: 0 +dataloader: + batch_size: 16 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/femnist_global_train.yaml b/scripts/example_configs/femnist_global_train.yaml index 54d776c12..15871047a 100644 --- a/scripts/example_configs/femnist_global_train.yaml +++ b/scripts/example_configs/femnist_global_train.yaml @@ -14,10 +14,10 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 10 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 10 model: type: convnet2 hidden: 2048 diff --git a/scripts/example_configs/quadratic.yaml b/scripts/example_configs/quadratic.yaml index 21e6b645b..99ad58096 100644 --- a/scripts/example_configs/quadratic.yaml +++ b/scripts/example_configs/quadratic.yaml @@ -16,5 +16,6 @@ model: type: 'quadratic' criterion: type: 'L1Loss' -optimizer: - lr: 0.01 +train: + optimizer: + lr: 0.01 diff --git a/scripts/mf_exp_scripts/run_movielens1m_hfl_standalone.sh b/scripts/mf_exp_scripts/run_movielens1m_hfl_standalone.sh index 3afd0c519..e68245dc8 100644 --- a/scripts/mf_exp_scripts/run_movielens1m_hfl_standalone.sh +++ b/scripts/mf_exp_scripts/run_movielens1m_hfl_standalone.sh @@ -9,4 +9,4 @@ python federatedscope/main.py --cfg federatedscope/mf/baseline/hfl_fedavg_standa train.optimizer.lr 0.8 \ train.local_update_steps 20 \ federate.total_round_num 50 \ - data.batch_size 32 + dataloader.batch_size 32 diff --git a/scripts/mf_exp_scripts/run_movielens1m_hflsgdmf_standalone.sh b/scripts/mf_exp_scripts/run_movielens1m_hflsgdmf_standalone.sh index f063beb66..ef24de8f1 100644 --- a/scripts/mf_exp_scripts/run_movielens1m_hflsgdmf_standalone.sh +++ b/scripts/mf_exp_scripts/run_movielens1m_hflsgdmf_standalone.sh @@ -11,4 +11,4 @@ python federatedscope/main.py --cfg federatedscope/mf/baseline/hfl-sgdmf_fedavg_ train.optimizer.lr 0.1 \ train.local_update_steps 20 \ federate.total_round_num 50 \ - data.batch_size 64 + dataloader.batch_size 64 diff --git a/scripts/mf_exp_scripts/run_movielens1m_vfl_standalone.sh b/scripts/mf_exp_scripts/run_movielens1m_vfl_standalone.sh index 35f253438..6951676ae 100644 --- a/scripts/mf_exp_scripts/run_movielens1m_vfl_standalone.sh +++ b/scripts/mf_exp_scripts/run_movielens1m_vfl_standalone.sh @@ -9,4 +9,4 @@ python federatedscope/main.py --cfg federatedscope/mf/baseline/vfl_fedavg_standa train.optimizer.lr 0.8 \ train.local_update_steps 20 \ federate.total_round_num 50 \ - data.batch_size 32 \ No newline at end of file + dataloader.batch_size 32 \ No newline at end of file diff --git a/scripts/mf_exp_scripts/run_movielens1m_vflsgdmf_standalone.sh b/scripts/mf_exp_scripts/run_movielens1m_vflsgdmf_standalone.sh index 0000a0260..98628a467 100644 --- a/scripts/mf_exp_scripts/run_movielens1m_vflsgdmf_standalone.sh +++ b/scripts/mf_exp_scripts/run_movielens1m_vflsgdmf_standalone.sh @@ -11,4 +11,4 @@ python federatedscope/main.py --cfg federatedscope/mf/baseline/vfl-sgdmf_fedavg_ train.optimizer.lr 0.1 \ train.local_update_steps 20 \ federate.total_round_num 50 \ - data.batch_size 64 + dataloader.batch_size 64 diff --git a/scripts/personalization_exp_scripts/ditto/ditto_convnet2_on_femnist.yaml b/scripts/personalization_exp_scripts/ditto/ditto_convnet2_on_femnist.yaml index 51ade8fcb..b9b0a4268 100644 --- a/scripts/personalization_exp_scripts/ditto/ditto_convnet2_on_femnist.yaml +++ b/scripts/personalization_exp_scripts/ditto/ditto_convnet2_on_femnist.yaml @@ -11,9 +11,7 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 64 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] model: type: convnet2 diff --git a/scripts/personalization_exp_scripts/ditto/ditto_lr_on_synthetic.yaml b/scripts/personalization_exp_scripts/ditto/ditto_lr_on_synthetic.yaml index 4b3e52a9f..824fedcd4 100644 --- a/scripts/personalization_exp_scripts/ditto/ditto_lr_on_synthetic.yaml +++ b/scripts/personalization_exp_scripts/ditto/ditto_lr_on_synthetic.yaml @@ -11,7 +11,6 @@ federate: data: root: data/ type: synthetic - batch_size: 64 subsample: 1.0 personalization: local_update_steps: 30 diff --git a/scripts/personalization_exp_scripts/ditto/ditto_lstm_on_shakespeare.yaml b/scripts/personalization_exp_scripts/ditto/ditto_lstm_on_shakespeare.yaml index fab240443..077ca0548 100644 --- a/scripts/personalization_exp_scripts/ditto/ditto_lstm_on_shakespeare.yaml +++ b/scripts/personalization_exp_scripts/ditto/ditto_lstm_on_shakespeare.yaml @@ -10,9 +10,7 @@ federate: data: root: data/ type: shakespeare - batch_size: 64 subsample: 0.2 - num_workers: 0 splits: [0.6,0.2,0.2] model: type: lstm diff --git a/scripts/personalization_exp_scripts/fedbn/fedbn_convnet2_on_femnist.yaml b/scripts/personalization_exp_scripts/fedbn/fedbn_convnet2_on_femnist.yaml index 3dda571cb..0ff8b2464 100644 --- a/scripts/personalization_exp_scripts/fedbn/fedbn_convnet2_on_femnist.yaml +++ b/scripts/personalization_exp_scripts/fedbn/fedbn_convnet2_on_femnist.yaml @@ -11,9 +11,7 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 64 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] model: type: convnet2 diff --git a/scripts/personalization_exp_scripts/fedem/fedem_convnet2_on_femnist.yaml b/scripts/personalization_exp_scripts/fedem/fedem_convnet2_on_femnist.yaml index 4b6237b04..ad066ad4c 100644 --- a/scripts/personalization_exp_scripts/fedem/fedem_convnet2_on_femnist.yaml +++ b/scripts/personalization_exp_scripts/fedem/fedem_convnet2_on_femnist.yaml @@ -11,9 +11,7 @@ data: root: data/ type: femnist splits: [0.6,0.2,0.2] - batch_size: 64 subsample: 0.05 - num_workers: 0 transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] model: model_num_per_trainer: 3 diff --git a/scripts/personalization_exp_scripts/fedem/fedem_lr_on_synthetic.yaml b/scripts/personalization_exp_scripts/fedem/fedem_lr_on_synthetic.yaml index c50068373..69d6c0161 100644 --- a/scripts/personalization_exp_scripts/fedem/fedem_lr_on_synthetic.yaml +++ b/scripts/personalization_exp_scripts/fedem/fedem_lr_on_synthetic.yaml @@ -11,7 +11,6 @@ federate: data: root: data/ type: synthetic - batch_size: 64 subsample: 1.0 personalization: local_update_steps: 30 diff --git a/scripts/personalization_exp_scripts/fedem/fedem_lstm_on_shakespeare.yaml b/scripts/personalization_exp_scripts/fedem/fedem_lstm_on_shakespeare.yaml index 6bce97cb4..52ecc5234 100644 --- a/scripts/personalization_exp_scripts/fedem/fedem_lstm_on_shakespeare.yaml +++ b/scripts/personalization_exp_scripts/fedem/fedem_lstm_on_shakespeare.yaml @@ -8,9 +8,7 @@ federate: data: root: data/ type: shakespeare - batch_size: 64 subsample: 0.2 - num_workers: 0 splits: [0.6,0.2,0.2] model: model_num_per_trainer: 3 diff --git a/scripts/personalization_exp_scripts/run_femnist_ditto.sh b/scripts/personalization_exp_scripts/run_femnist_ditto.sh index 7cc61cc1f..dd26cfcf7 100755 --- a/scripts/personalization_exp_scripts/run_femnist_ditto.sh +++ b/scripts/personalization_exp_scripts/run_femnist_ditto.sh @@ -23,7 +23,7 @@ do do for k in {1..3} do - python federatedscope/main.py --cfg federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml federate.method ${method} personalization.regular_weight ${personalization_regular_weight} data.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml federate.method ${method} personalization.regular_weight ${personalization_regular_weight} dataloader.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_femnist_fedavg.sh b/scripts/personalization_exp_scripts/run_femnist_fedavg.sh index f920032df..17238cc65 100755 --- a/scripts/personalization_exp_scripts/run_femnist_fedavg.sh +++ b/scripts/personalization_exp_scripts/run_femnist_fedavg.sh @@ -22,7 +22,7 @@ do do for k in {1..3} do - python federatedscope/main.py --cfg federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml federate.method ${method} data.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml federate.method ${method} dataloader.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_femnist_fedbn.sh b/scripts/personalization_exp_scripts/run_femnist_fedbn.sh index 8d3cb3cae..0d808a78f 100755 --- a/scripts/personalization_exp_scripts/run_femnist_fedbn.sh +++ b/scripts/personalization_exp_scripts/run_femnist_fedbn.sh @@ -22,7 +22,7 @@ do do for k in {1..3} do - python federatedscope/main.py --cfg federatedscope/cv/baseline/fedbn_convnet2_on_femnist.yaml data.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/cv/baseline/fedbn_convnet2_on_femnist.yaml dataloader.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_femnist_fedem.sh b/scripts/personalization_exp_scripts/run_femnist_fedem.sh index 1c067d2fd..0dba67f77 100755 --- a/scripts/personalization_exp_scripts/run_femnist_fedem.sh +++ b/scripts/personalization_exp_scripts/run_femnist_fedem.sh @@ -23,7 +23,7 @@ do do for (( g=0; g<${#models[@]}; g++ )) do - python federatedscope/main.py --cfg federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml federate.method ${method} model.model_num_per_trainer ${model_num_per_trainer} data.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml federate.method ${method} model.model_num_per_trainer ${model_num_per_trainer} dataloader.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_femnist_pfedme.sh b/scripts/personalization_exp_scripts/run_femnist_pfedme.sh index 77f55126e..e23ee0d7d 100755 --- a/scripts/personalization_exp_scripts/run_femnist_pfedme.sh +++ b/scripts/personalization_exp_scripts/run_femnist_pfedme.sh @@ -25,7 +25,7 @@ do do for k in {1..3} do - python federatedscope/main.py --cfg federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml federate.method ${method} data.batch_size ${bs} personalization.K ${personalization_K} personalization.lr ${personalization_lr} personalization.regular_weight ${personalization_beta} device ${cudaid} optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/cv/baseline/fedavg_convnet2_on_femnist.yaml federate.method ${method} dataloader.batch_size ${bs} personalization.K ${personalization_K} personalization.lr ${personalization_lr} personalization.regular_weight ${personalization_beta} device ${cudaid} optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_shakespeare_ditto.sh b/scripts/personalization_exp_scripts/run_shakespeare_ditto.sh index 0a0f38edf..ce53722cf 100755 --- a/scripts/personalization_exp_scripts/run_shakespeare_ditto.sh +++ b/scripts/personalization_exp_scripts/run_shakespeare_ditto.sh @@ -23,7 +23,7 @@ do do for (( g=0; g<${#models[@]}; g++ )) do - python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml federate.method ${method} data.batch_size ${bs} personalization.regular_weight ${personalization_regular_weight} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml federate.method ${method} dataloader.batch_size ${bs} personalization.regular_weight ${personalization_regular_weight} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_shakespeare_fedavg.sh b/scripts/personalization_exp_scripts/run_shakespeare_fedavg.sh index cc6d988c3..ecb7c57e1 100755 --- a/scripts/personalization_exp_scripts/run_shakespeare_fedavg.sh +++ b/scripts/personalization_exp_scripts/run_shakespeare_fedavg.sh @@ -22,7 +22,7 @@ do do for (( g=0; g<${#models[@]}; g++ )) do - python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml federate.method ${method} data.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml federate.method ${method} dataloader.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_shakespeare_fedem.sh b/scripts/personalization_exp_scripts/run_shakespeare_fedem.sh index fe4de1fdd..83160d69b 100755 --- a/scripts/personalization_exp_scripts/run_shakespeare_fedem.sh +++ b/scripts/personalization_exp_scripts/run_shakespeare_fedem.sh @@ -28,7 +28,7 @@ do do for (( g=0; g<${#models[@]}; g++ )) do - python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml federate.method ${method} data.batch_size ${bs} model.model_num_per_trainer ${model_num_per_trainer} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml federate.method ${method} dataloader.batch_size ${bs} model.model_num_per_trainer ${model_num_per_trainer} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_shakespeare_pfedme.sh b/scripts/personalization_exp_scripts/run_shakespeare_pfedme.sh index 40757595e..45d821721 100755 --- a/scripts/personalization_exp_scripts/run_shakespeare_pfedme.sh +++ b/scripts/personalization_exp_scripts/run_shakespeare_pfedme.sh @@ -25,7 +25,7 @@ do do for (( g=0; g<${#models[@]}; g++ )) do - python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml federate.method ${method} data.batch_size ${bs} personalization.K ${personalization_K} personalization.lr ${personalization_lr} personalization.regular_weight ${personalization_beta} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lstm_on_shakespeare.yaml federate.method ${method} dataloader.batch_size ${bs} personalization.K ${personalization_K} personalization.lr ${personalization_lr} personalization.regular_weight ${personalization_beta} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_synthetic_ditto.sh b/scripts/personalization_exp_scripts/run_synthetic_ditto.sh index 58d126261..d802d8b2a 100755 --- a/scripts/personalization_exp_scripts/run_synthetic_ditto.sh +++ b/scripts/personalization_exp_scripts/run_synthetic_ditto.sh @@ -23,7 +23,7 @@ do do for k in {1..3} do - python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml federate.method ${method} data.batch_size ${bs} personalization.regular_weight ${personalization_regular_weight} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml federate.method ${method} dataloader.batch_size ${bs} personalization.regular_weight ${personalization_regular_weight} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_synthetic_fedavg.sh b/scripts/personalization_exp_scripts/run_synthetic_fedavg.sh index 74e928a1a..6d6e28780 100755 --- a/scripts/personalization_exp_scripts/run_synthetic_fedavg.sh +++ b/scripts/personalization_exp_scripts/run_synthetic_fedavg.sh @@ -22,7 +22,7 @@ do do for k in {1..3} do - python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml federate.method ${method} data.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml federate.method ${method} dataloader.batch_size ${bs} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_synthetic_fedem.sh b/scripts/personalization_exp_scripts/run_synthetic_fedem.sh index 67a0b3bb2..b73934851 100755 --- a/scripts/personalization_exp_scripts/run_synthetic_fedem.sh +++ b/scripts/personalization_exp_scripts/run_synthetic_fedem.sh @@ -28,7 +28,7 @@ do do for k in {1..3} do - python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml federate.method ${method} data.batch_size ${bs} model.model_num_per_trainer ${model_num_per_trainer} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml federate.method ${method} dataloader.batch_size ${bs} model.model_num_per_trainer ${model_num_per_trainer} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/scripts/personalization_exp_scripts/run_synthetic_pfedme.sh b/scripts/personalization_exp_scripts/run_synthetic_pfedme.sh index 2679f24f1..0a34d2c54 100755 --- a/scripts/personalization_exp_scripts/run_synthetic_pfedme.sh +++ b/scripts/personalization_exp_scripts/run_synthetic_pfedme.sh @@ -25,7 +25,7 @@ do do for k in {1..3} do - python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml federate.method ${method} data.batch_size ${bs} personalization.K ${personalization_K} personalization.lr ${personalization_lr} personalization.regular_weight ${personalization_beta} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} + python federatedscope/main.py --cfg federatedscope/nlp/baseline/fedavg_lr_on_synthetic.yaml federate.method ${method} dataloader.batch_size ${bs} personalization.K ${personalization_K} personalization.lr ${personalization_lr} personalization.regular_weight ${personalization_beta} device ${cudaid} train.optimizer.lr ${lrs[$i]} train.local_update_steps ${local_updates[$j]} model.type ${models[$g]} seed $k outdir ${outdir}/${models[$g]}_${lrs[$i]}_${local_updates[$j]}_bs${bs}_on_${dataset} done done done diff --git a/setup.py b/setup.py index 4fb9a7407..8a402f766 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import setuptools __name__ = 'federatedscope' -__version__ = '0.2.0' +__version__ = '0.2.1' URL = 'https://github.com/alibaba/FederatedScope' minimal_requires = [ diff --git a/tests/test_fedsageplus.py b/tests/test_fedsageplus.py index 44fa2c680..ee9706219 100644 --- a/tests/test_fedsageplus.py +++ b/tests/test_fedsageplus.py @@ -29,7 +29,9 @@ def set_config_fedsageplus(self, cfg): cfg.data.root = 'test_data/' cfg.data.type = 'cora' cfg.data.splitter = 'louvain' - cfg.data.batch_size = 1 + + cfg.dataloader.type = 'pyg' + cfg.dataloader.batch_size = 1 cfg.model.type = 'sage' cfg.model.hidden = 64 diff --git a/tests/test_graph_node_trainer.py b/tests/test_graph_node_trainer.py index 8c1618c03..a6e441bd5 100644 --- a/tests/test_graph_node_trainer.py +++ b/tests/test_graph_node_trainer.py @@ -27,9 +27,11 @@ def set_config_node(self, cfg): cfg.data.root = 'test_data/' cfg.data.type = 'cora' - cfg.data.batch_size = 1 # full batch train cfg.data.splitter = 'louvain' + cfg.dataloader.type = 'pyg' + cfg.dataloader.batch_size = 1 # full batch train + cfg.model.type = 'gcn' cfg.model.hidden = 64 cfg.model.dropout = 0.5 diff --git a/tests/test_mf.py b/tests/test_mf.py index 1714d65f4..c602d8621 100644 --- a/tests/test_mf.py +++ b/tests/test_mf.py @@ -30,7 +30,9 @@ def set_config_movielens1m(self, cfg): cfg.data.root = 'test_data/' cfg.data.type = 'vflmovielens1m' - cfg.data.batch_size = 32 + + cfg.dataloader.type = 'mf' + cfg.dataloader.batch_size = 32 cfg.model.type = 'VMFNet' cfg.model.hidden = 20 diff --git a/tests/test_vertical_fl.py b/tests/test_vertical_fl.py index dad09038c..95613c466 100644 --- a/tests/test_vertical_fl.py +++ b/tests/test_vertical_fl.py @@ -31,6 +31,8 @@ def set_config(self, cfg): cfg.data.type = 'vertical_fl_data' cfg.data.size = 50 + cfg.dataloader.type = 'raw' + cfg.vertical.use = True cfg.vertical.key_size = 256