Dataserver synthetic data

devops/scripts/requirements.txt

@@ -1,7 +1,6 @@
 protobuf
 grpcio
 grpcio-tools
-loguru
 dill
 multiprocess
 nvidia-ml-py3

python/examples/cross_silo/mqtt_s3_fedavg_cifar10_lr_example/torch_client.py

@@ -3,59 +3,60 @@
 import fedml
 from fedml import FedMLRunner
 from fedml.data.MNIST.data_loader import download_mnist, load_partition_data_mnist
-from .trainer.classification_aggregator import ClassificationAggregator
-from .trainer.classification_trainer import ClassificationTrainer
+from trainer.classification_aggregator import ClassificationAggregator
+from trainer.classification_trainer import ClassificationTrainer
+from fedml.data.data_loader import load
 
 
-def load_data(args):
-    download_mnist(args.data_cache_dir)
-    fedml.logging.info("load_data. dataset_name = %s" % args.dataset)
-
-    """
-    Please read through the data loader at to see how to customize the dataset for FedML framework.
-    """
-    (
-        client_num,
-        train_data_num,
-        test_data_num,
-        train_data_global,
-        test_data_global,
-        train_data_local_num_dict,
-        train_data_local_dict,
-        test_data_local_dict,
-        class_num,
-    ) = load_partition_data_mnist(
-        args,
-        args.batch_size,
-        train_path=args.data_cache_dir + "/MNIST/train",
-        test_path=args.data_cache_dir + "/MNIST/test",
-    )
-    """
-    For shallow NN or linear models, 
-    we uniformly sample a fraction of clients each round (as the original FedAvg paper)
-    """
-    args.client_num_in_total = client_num
-    dataset = [
-        train_data_num,
-        test_data_num,
-        train_data_global,
-        test_data_global,
-        train_data_local_num_dict,
-        train_data_local_dict,
-        test_data_local_dict,
-        class_num,
-    ]
-    return dataset, class_num
-
-
-class LogisticRegression(torch.nn.Module):
-    def __init__(self, input_dim, output_dim):
-        super(LogisticRegression, self).__init__()
-        self.linear = torch.nn.Linear(input_dim, output_dim)
-
-    def forward(self, x):
-        outputs = torch.sigmoid(self.linear(x))
-        return outputs
+# def load_data(args):
+#     download_mnist(args.data_cache_dir)
+#     fedml.logging.info("load_data. dataset_name = %s" % args.dataset)
+#
+#     """
+#     Please read through the data loader at to see how to customize the dataset for FedML framework.
+#     """
+#     (
+#         client_num,
+#         train_data_num,
+#         test_data_num,
+#         train_data_global,
+#         test_data_global,
+#         train_data_local_num_dict,
+#         train_data_local_dict,
+#         test_data_local_dict,
+#         class_num,
+#     ) = load_partition_data_mnist(
+#         args,
+#         args.batch_size,
+#         train_path=args.data_cache_dir + "/MNIST/train",
+#         test_path=args.data_cache_dir + "/MNIST/test",
+#     )
+#     """
+#     For shallow NN or linear models,
+#     we uniformly sample a fraction of clients each round (as the original FedAvg paper)
+#     """
+#     args.client_num_in_total = client_num
+#     dataset = [
+#         train_data_num,
+#         test_data_num,
+#         train_data_global,
+#         test_data_global,
+#         train_data_local_num_dict,
+#         train_data_local_dict,
+#         test_data_local_dict,
+#         class_num,
+#     ]
+#     return dataset, class_num
+#
+#
+# class LogisticRegression(torch.nn.Module):
+#     def __init__(self, input_dim, output_dim):
+#         super(LogisticRegression, self).__init__()
+#         self.linear = torch.nn.Linear(input_dim, output_dim)
+#
+#     def forward(self, x):
+#         outputs = torch.sigmoid(self.linear(x))
+#         return outputs
 
 
 if __name__ == "__main__":
@@ -66,7 +67,7 @@ def forward(self, x):
     device = fedml.device.get_device(args)
 
     # load data
-    dataset, class_num = load_data(args)
+    dataset, class_num = load(args)
 
     # create model and trainer
     model = fedml.model.create(args, output_dim=class_num)

python/examples/cross_silo/mqtt_s3_fedavg_cifar10_lr_example/torch_server.py

@@ -2,50 +2,51 @@
 
 import fedml
 from fedml import FedMLRunner
-from fedml.data.MNIST.data_loader import download_mnist, load_partition_data_mnist
-from .trainer.classification_aggregator import ClassificationAggregator
-from .trainer.classification_trainer import ClassificationTrainer
+# from fedml.data.MNIST.data_loader import download_mnist, load_partition_data_mnist
+from trainer.classification_aggregator import ClassificationAggregator
+from trainer.classification_trainer import ClassificationTrainer
+from fedml.data.data_loader import load
 
 
-def load_data(args):
-    download_mnist(args.data_cache_dir)
-    fedml.logging.info("load_data. dataset_name = %s" % args.dataset)
-
-    """
-    Please read through the data loader at to see how to customize the dataset for FedML framework.
-    """
-    (
-        client_num,
-        train_data_num,
-        test_data_num,
-        train_data_global,
-        test_data_global,
-        train_data_local_num_dict,
-        train_data_local_dict,
-        test_data_local_dict,
-        class_num,
-    ) = load_partition_data_mnist(
-        args,
-        args.batch_size,
-        train_path=args.data_cache_dir + "/MNIST/train",
-        test_path=args.data_cache_dir + "/MNIST/test",
-    )
-    """
-    For shallow NN or linear models, 
-    we uniformly sample a fraction of clients each round (as the original FedAvg paper)
-    """
-    args.client_num_in_total = client_num
-    dataset = [
-        train_data_num,
-        test_data_num,
-        train_data_global,
-        test_data_global,
-        train_data_local_num_dict,
-        train_data_local_dict,
-        test_data_local_dict,
-        class_num,
-    ]
-    return dataset, class_num
+# def load_data(args):
+#     download_mnist(args.data_cache_dir)
+#     fedml.logging.info("load_data. dataset_name = %s" % args.dataset)
+#
+#     """
+#     Please read through the data loader at to see how to customize the dataset for FedML framework.
+#     """
+#     (
+#         client_num,
+#         train_data_num,
+#         test_data_num,
+#         train_data_global,
+#         test_data_global,
+#         train_data_local_num_dict,
+#         train_data_local_dict,
+#         test_data_local_dict,
+#         class_num,
+#     ) = load_partition_data_mnist(
+#         args,
+#         args.batch_size,
+#         train_path=args.data_cache_dir + "/MNIST/train",
+#         test_path=args.data_cache_dir + "/MNIST/test",
+#     )
+#     """
+#     For shallow NN or linear models,
+#     we uniformly sample a fraction of clients each round (as the original FedAvg paper)
+#     """
+#     args.client_num_in_total = client_num
+#     dataset = [
+#         train_data_num,
+#         test_data_num,
+#         train_data_global,
+#         test_data_global,
+#         train_data_local_num_dict,
+#         train_data_local_dict,
+#         test_data_local_dict,
+#         class_num,
+#     ]
+#     return dataset, class_num
 
 
 class LogisticRegression(torch.nn.Module):
@@ -66,7 +67,7 @@ def forward(self, x):
     device = fedml.device.get_device(args)
 
     # load data
-    dataset, class_num = load_data(args)
+    dataset, class_num = load(args)
 
     # create model and trainer
     model = fedml.model.create(args, output_dim=class_num)

python/fedml/core/security/common/attack_defense_data_loader.py

@@ -50,7 +50,6 @@ def load_data_loader_from_file(cls, filename):
         """
         Loads DataLoader object from a file if available.
 
-        :param logger: loguru.Logger
         :param filename: string
         """
         print("Loading data loader from file: {}".format(filename))

python/fedml/data/cifar10/efficient_loader.py

@@ -94,26 +94,30 @@ def _data_transforms_cifar10():
     return train_transform, valid_transform
 
 
-def load_cifar10_data(datadir, resize=32, augmentation=True, data_efficient_load=False):
+def load_cifar10_data(datadir, process_id, synthetic_data_url, private_local_data, resize=32, augmentation=True, data_efficient_load=False):
     train_transform, test_transform = _data_transforms_cifar10()
 
+    is_download = True;
+    if process_id != 0:
+        is_download = False if (len(synthetic_data_url) != 0 or len(private_local_data) != 0) else True;
+
     if data_efficient_load:
         cifar10_train_ds = CIFAR10(datadir, train=True, download=True, transform=train_transform)
         cifar10_test_ds = CIFAR10(datadir, train=False, download=True, transform=test_transform)
     else:
-        cifar10_train_ds = CIFAR10_truncated(datadir, train=True, download=True, transform=train_transform)
-        cifar10_test_ds = CIFAR10_truncated(datadir, train=False, download=True, transform=test_transform)
+        cifar10_train_ds = CIFAR10_truncated(datadir, train=True, download=is_download, transform=train_transform)
+        cifar10_test_ds = CIFAR10_truncated(datadir, train=False, download=is_download, transform=test_transform)
 
     X_train, y_train = cifar10_train_ds.data, cifar10_train_ds.targets
     X_test, y_test = cifar10_test_ds.data, cifar10_test_ds.targets
 
     return (X_train, y_train, X_test, y_test, cifar10_train_ds, cifar10_test_ds)
 
 
-def partition_data(dataset, datadir, partition, n_nets, alpha):
+def partition_data(dataset, datadir, partition, n_nets, alpha, process_id, synthetic_data_url, private_local_data):
     np.random.seed(10)
     logging.info("*********partition data***************")
-    X_train, y_train, X_test, y_test, cifar10_train_ds, cifar10_test_ds = load_cifar10_data(datadir)
+    X_train, y_train, X_test, y_test, cifar10_train_ds, cifar10_test_ds = load_cifar10_data(datadir, process_id, synthetic_data_url, private_local_data)
     n_train = X_train.shape[0]
     # n_test = X_test.shape[0]
 
@@ -311,6 +315,8 @@ def efficient_load_partition_data_cifar10(
     client_number,
     batch_size,
     process_id,
+    synthetic_data_url="",
+    private_local_data="",
     n_proc_in_silo=0,
     data_efficient_load=True,
 ):
@@ -323,7 +329,7 @@ def efficient_load_partition_data_cifar10(
         traindata_cls_counts,
         cifar10_train_ds,
         cifar10_test_ds,
-    ) = partition_data(dataset, data_dir, partition_method, client_number, partition_alpha, process_id)
+    ) = partition_data(dataset, data_dir, partition_method, client_number, partition_alpha, process_id, synthetic_data_url, private_local_data)
     class_num = len(np.unique(y_train))
     logging.info("traindata_cls_counts = " + str(traindata_cls_counts))
     train_data_num = sum([len(net_dataidx_map[r]) for r in range(client_number)])

python/fedml/data/cifar100/datasets.py

@@ -45,14 +45,8 @@ def __build_truncated_dataset__(self):
 
         cifar_dataobj = CIFAR100(self.root, self.train, self.transform, self.target_transform, self.download)
 
-        if self.train:
-            # print("train member of the class: {}".format(self.train))
-            # data = cifar_dataobj.train_data
-            data = cifar_dataobj.data
-            target = np.array(cifar_dataobj.targets)
-        else:
-            data = cifar_dataobj.data
-            target = np.array(cifar_dataobj.targets)
+        data = cifar_dataobj.data
+        target = np.array(cifar_dataobj.targets)
 
         if self.dataidxs is not None:
             data = data[self.dataidxs]