diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..5c74391b8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-yaml + exclude: | + (?x)^( + meta.yaml + )$ + - repo: https://github.com/pre-commit/mirrors-yapf + rev: v0.32.0 + hooks: + - id: yapf + - repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + - repo: https://github.com/regebro/pyroma + rev: "4.0" + hooks: + - id: pyroma + args: [--min=9, .] \ No newline at end of file diff --git a/README.md b/README.md index 6a6f2db0d..cf7746d59 100644 --- a/README.md +++ b/README.md @@ -26,21 +26,16 @@ We provide an end-to-end example for users to start running a standard FL course ### Step 1. Installation -First of all, users need to clone the source code and install the required packages (we suggest python version >= 3.9). +First of all, users need to clone the source code and install the required packages (we suggest python version >= 3.9). You can choose between the following two installation methods (via docker or conda) to install FederatedScope. ```bash git clone https://github.com/alibaba/FederatedScope.git cd FederatedScope ``` -You can install the dependencies from the requirement file: -``` -# For minimal version -conda install --file enviroment/requirements-torch1.10.txt -c pytorch -c conda-forge -c nvidia +#### Use Docker + +You can build docker image and run with docker env (cuda 11 and torch 1.10): -# For application version -conda install --file enviroment/requirements-torch1.10-application.txt -c pytorch -c conda-forge -c nvidia -c pyg -``` -or build docker image and run with docker env (cuda 11 and torch 1.10): ``` docker build -f enviroment/docker_files/federatedscope-torch1.10.Dockerfile -t alibaba/federatedscope:base-env-torch1.10 . docker run --gpus device=all --rm -it --name "fedscope" -w $(pwd) alibaba/federatedscope:base-env-torch1.10 /bin/bash @@ -56,12 +51,55 @@ enviroment/docker_files/federatedscope-torch1.10-application.Dockerfile Note: You can choose to use cuda 10 and torch 1.8 via changing `torch1.10` to `torch1.8`. The docker images are based on the nvidia-docker. Please pre-install the NVIDIA drivers and `nvidia-docker2` in the host machine. See more details [here](https://github.com/alibaba/FederatedScope/tree/master/enviroment/docker_files). -Finally, after all the dependencies are installed, run: +#### Use Conda + +We recommend using a new virtual environment to install FederatedScope: + +```bash +conda create -n fs python=3.9 +conda activate fs +``` + +If your backend is torch, please install torch in advance ([torch-get-started](https://pytorch.org/get-started/locally/)). For example, if your cuda version is 11.3 please execute the following command: + +```bash +conda install -y pytorch=1.10.1 torchvision=0.11.2 torchaudio=0.10.1 cudatoolkit=11.3 -c pytorch -c conda-forge +``` + +For users with Apple M1 chips: +```bash +conda install pytorch torchvision torchaudio -c pytorch +# Downgrade torchvision to avoid segmentation fault +python -m pip install torchvision==0.11.3 +``` + +Finally, after the backend is installed, you can install FederatedScope from `source`, `conda` or `pip`: + +##### From source + ```bash python setup.py install # Or (for dev mode) -pip install -e . +pip install -e .[dev] +``` + +##### From conda + +```bash +conda install -c federatedscope federatedscope +``` + +##### From pip + +```bash +python -m pip install federatedscope +``` + +Now, you have successfully installed the minimal version of FederatedScope. (**Optinal**) For application version including graph, nlp and speech, run: + +```bash +bash enviroment/extra_dependencies_torch1.10-application.sh ``` ### Step 2. Prepare datasets diff --git a/enviroment/extra_dependencies_torch1.10-application.sh b/enviroment/extra_dependencies_torch1.10-application.sh new file mode 100644 index 000000000..acc298e3b --- /dev/null +++ b/enviroment/extra_dependencies_torch1.10-application.sh @@ -0,0 +1,13 @@ +set -e + +# Graph +conda install -y pyg==2.0.4 -c pyg +conda install -y rdkit=2021.09.4=py39hccf6a74_0 -c conda-forge +conda install -y nltk + +# Speech and NLP +conda install -y sentencepiece textgrid typeguard -c conda-forge +conda install -y transformers==4.16.2 tokenizers==0.10.3 datasets -c huggingface -c conda-forge +conda install -y torchtext -c pytorch + +conda clean -a -y \ No newline at end of file diff --git a/federatedscope/__init__.py b/federatedscope/__init__.py index 7a805d435..270b04d74 100644 --- a/federatedscope/__init__.py +++ b/federatedscope/__init__.py @@ -1,14 +1,13 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division +from __future__ import absolute_import, division, print_function -__version__ = '0.1.0' +__version__ = '0.1.9' def _setup_logger(): import logging - logging_fmt = "%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s" + logging_fmt = "%(asctime)s (%(module)s:%(lineno)d)" \ + "%(levelname)s: %(message)s" logger = logging.getLogger("federatedscope") handler = logging.StreamHandler() handler.setFormatter(logging.Formatter(logging_fmt)) diff --git a/meta.yaml b/meta.yaml new file mode 100644 index 000000000..6d980a57c --- /dev/null +++ b/meta.yaml @@ -0,0 +1,54 @@ +{% set name = "federatedscope" %} +{% set version = "0.1.9" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/federatedscope-{{ version }}.tar.gz + sha256: f7802afa41b7edc34adb3128fbadd18ceb2a92521384e57b9d7cbb9aae894ea6 + +build: + noarch: python + script: {{ PYTHON }} -m pip install . -vv + number: 0 + +requirements: + host: + - pip + - python >=3.9 + run: + - fvcore + - grpcio >=1.45.0 + - grpcio-tools + - iopath + - numpy <1.23.0 + - pandas + - protobuf ==3.19.4 + - pympler + - python >=3.9 + - pyyaml >=5.1 + - scikit-learn >=1.0.2 + - scipy ==1.7.3 + - tensorboard + - tensorboardx + - wandb + +test: + imports: + - federatedscope + commands: + - pip check + requires: + - pip + +about: + home: https://github.com/alibaba/FederatedScope + summary: Federated learning package + license: Apache-2.0 + license_file: LICENSE + +extra: + recipe-maintainers: + - rayrayraykk diff --git a/setup.py b/setup.py index 74dff1fe0..c3250e484 100644 --- a/setup.py +++ b/setup.py @@ -1,37 +1,62 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division +from __future__ import absolute_import, division, print_function import setuptools +__name__ = 'federatedscope' +__version__ = '0.1.9' +URL = 'https://github.com/alibaba/FederatedScope' + +minimal_requires = [ + 'numpy<1.23.0', 'scikit-learn==1.0.2', 'scipy==1.7.3', 'pandas', + 'grpcio>=1.45.0', 'grpcio-tools', 'pyyaml>=5.1', 'fvcore', 'iopath', + 'wandb', 'tensorboard', 'tensorboardX', 'pympler', 'protobuf==3.19.4' +] + +test_requires = [] + +dev_requires = test_requires + ['pre-commit'] + +benchmark_hpo_requires = [ + 'configspace==0.5.0', 'hpbandster==0.7.4', 'smac==1.3.3', 'optuna==2.10.0' +] + +# TODO: add requirements for pfl +benchmark_pfl_requires = [] + +# TODO: add requirements for htl +benchmark_htl_requires = [] + with open("README.md", "r") as fh: long_description = fh.read() setuptools.setup( - name="federatedscope", - version="0.1.0", + name=__name__, + version=__version__, author="Alibaba Damo Academy", - author_email="", + author_email="jones.wz@alibaba-inc.com", description="Federated learning package", long_description=long_description, long_description_content_type="text/markdown", - url="", + url=URL, + download_url=f'{URL}/archive/{__version__}.tar.gz', + keywords=['deep-learning', 'federated-learning', 'benchmark'], packages=[ package for package in setuptools.find_packages() - if package.startswith('federatedscope') - ], - install_requires=[ - 'torch', 'networkx', 'numpy', 'grpcio>=1.45.0', 'grpcio-tools' + if package.startswith(__name__) ], - setup_requires=[], - extras_require={'yaml': ['yaml>=5.1']}, + install_requires=minimal_requires, + extras_require={ + 'test': test_requires, + 'dev': dev_requires, + 'benchmark_hpo': benchmark_hpo_requires, + 'benchmark_pfl': benchmark_pfl_requires, + 'benchmark_htl': benchmark_htl_requires, + }, license="Apache License 2.0", classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ], - test_suite='nose.collector', - test_require=['nose'], python_requires='>=3.9', )