Merge branch 'alibaba:master' into test_jks

alibaba · Nov 15, 2023 · 661f185 · 661f185
2 parents b1de3ae + 480b67d
commit 661f185
Show file tree

Hide file tree

Showing 562 changed files with 29,794 additions and 3,518 deletions.
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -0,0 +1,41 @@
+name: CodeQL (Code Scanning)
+
+on:
+  pull_request:
+    branches: [ "master" ]
+    types: [opened, synchronize, edited]
+  schedule:
+    - cron: '0 8 * * *'
+
+jobs:
+  analyze:
+    name: Analyze
+    if: false == contains(github.event.pull_request.title, 'WIP')
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
@@ -0,0 +1,28 @@
+name: Pre-commit (Required)
+
+on: [push, pull_request]
+
+jobs:
+  run:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: True
+      matrix:
+        os: [ubuntu-latest]
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: '3.9'
+    steps:
+    - uses: actions/checkout@master
+    - name: Setup Python
+      uses: actions/setup-python@master
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        pip install pre-commit
+        pre-commit install
+    - name: Pre-commit starts
+      run: |
+        pre-commit run --all-files
+        [ $? -eq 1 ] && exit 1 || echo "Passed"
diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml
@@ -0,0 +1,28 @@
+name: "API Reference"
+on:
+  pull_request:
+    types: [opened, synchronize, edited]
+
+jobs:
+  docs:
+    if: true == contains(github.event.pull_request.title, 'DOC')
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: '3.9'
+    steps:
+    - uses: actions/checkout@master
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@master
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Generate Documentation
+      uses: ammaraskar/sphinx-action@master
+      with:
+        docs-folder: "doc/"
+    - name: Upload Documentation
+      uses: actions/upload-artifact@v3
+      with:
+        name: APIReference
+        path: doc/build/html/
diff --git a/.github/workflows/test_atc.yml b/.github/workflows/test_atc.yml
@@ -0,0 +1,101 @@
+name: UnitTests for ATC
+
+on:
+  schedule:
+    - cron: '0 8 * * 0'
+
+
+jobs:
+  run:
+    if: (false == contains(github.event.pull_request.title, 'WIP') && github.repository == 'alibaba/FederatedScope')
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9']
+        torch-version: ['1.10.1']
+        torchvision-version: ['0.11.2']
+        torchaudio-version: ['0.10.1']
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: '3.9'
+    steps:
+    - uses: actions/checkout@master
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@master
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install PyTorch ${{ matrix.torch-version }}+cpu
+      run: |
+        pip install numpy typing-extensions dataclasses
+        pip install torch==${{ matrix.torch-version}}+cpu torchvision==${{matrix.torchvision-version}}+cpu torchaudio==${{matrix.torchaudio-version}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+    - name: Install FS
+      run: |
+        pip install -e .[test]
+    - name: Install NLTK
+      run: |
+        pip install nltk
+    - name: Install Transformers
+      run: |
+        pip install transformers==4.21.0
+    - name: Install ROUGE
+      run: |
+        git clone https://github.com/bheinzerling/pyrouge
+        cd pyrouge
+        pip install -e .
+        git clone https://github.com/andersjo/pyrouge.git rouge
+        pyrouge_set_rouge_path $(realpath rouge/tools/ROUGE-1.5.5/)
+        sudo apt-get install libxml-parser-perl --fix-missing
+        cd rouge/tools/ROUGE-1.5.5/data
+        rm WordNet-2.0.exc.db
+        ./WordNet-2.0-Exceptions/buildExeptionDB.pl ./WordNet-2.0-Exceptions ./smart_common_words.txt ./WordNet-2.0.exc.db
+        python -m pyrouge.test
+    - name: Download METEOR packages
+      run: |
+        wget -c http://www.cs.cmu.edu/~alavie/METEOR/download/meteor-1.5.tar.gz
+        tar -zxvf meteor-1.5.tar.gz
+        mkdir federatedscope/nlp/metric/meteor/data/
+        mv meteor-1.5/data/paraphrase-en.gz federatedscope/nlp/metric/meteor/data/
+        mv meteor-1.5/meteor-1.5.jar federatedscope/nlp/metric/meteor/
+    - name: Test ATC
+      run: |
+        python federatedscope/main.py \
+          --cfg federatedscope/nlp/hetero_tasks/baseline/config_isolated.yaml \
+          --client_cfg federatedscope/nlp/hetero_tasks/baseline/config_client_isolated.yaml \
+          outdir exp/isolated/ \
+          use_gpu False \
+          data.is_debug True \
+          data.root test_data/ \
+
+        [ $? -eq 1 ] && exit 1
+
+        python federatedscope/main.py \
+          --cfg federatedscope/nlp/hetero_tasks/baseline/config_fedavg.yaml \
+          --client_cfg federatedscope/nlp/hetero_tasks/baseline/config_client_fedavg.yaml \
+          outdir exp/fedavg/ \
+          use_gpu False \
+          data.is_debug True \
+          data.root test_data/ \
+
+        [ $? -eq 1 ] && exit 1
+
+        python federatedscope/main.py \
+          --cfg federatedscope/nlp/hetero_tasks/baseline/config_pretrain.yaml \
+          outdir exp/atc/pretrain/ \
+          use_gpu False \
+          data.is_debug True \
+          data.root test_data/ \
+
+        [ $? -eq 1 ] && exit 1
+
+        python federatedscope/main.py \
+          --cfg federatedscope/nlp/hetero_tasks/baseline/config_atc.yaml \
+          --client_cfg federatedscope/nlp/hetero_tasks/baseline/config_client_atc.yaml \
+          outdir exp/atc/train/ \
+          use_gpu False \
+          data.is_debug True \
+          data.root test_data/ \
+
+        [ $? -eq 1 ] && exit 1 || echo "Passed"
+
diff --git a/.github/workflows/test_autotune.yml b/.github/workflows/test_autotune.yml
@@ -0,0 +1,38 @@
+name: UnitTests for Autotune Module
+
+on:
+  pull_request:
+    types: [opened, synchronize, edited]
+
+jobs:
+  run:
+    if: false == contains(github.event.pull_request.title, 'WIP')
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9']
+        torch-version: ['1.10.1']
+        torchvision-version: ['0.11.2']
+        torchaudio-version: ['0.10.1']
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: '3.9'
+    steps:
+    - uses: actions/checkout@master
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@master
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install PyTorch ${{ matrix.torch-version }}+cpu
+      run: |
+        pip install numpy typing-extensions dataclasses
+        pip install torch==${{ matrix.torch-version}}+cpu torchvision==${{matrix.torchvision-version}}+cpu torchaudio==${{matrix.torchaudio-version}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+    - name: Install FS
+      run: |
+        pip install -e .[test,hpo]
+    - name: Test Autotune
+      run: |
+        python federatedscope/hpo.py --cfg federatedscope/autotune/baseline/fedhpo_vfl.yaml
+        [ $? -eq 1 ] && exit 1 || echo "Passed"
diff --git a/.github/workflows/test_distribute.yml b/.github/workflows/test_distribute.yml
@@ -0,0 +1,54 @@
+name: UnitTests for Distributed Mode
+
+on:
+  pull_request:
+    types: [opened, synchronize, edited]
+
+jobs:
+  run:
+    if: false == contains(github.event.pull_request.title, 'WIP')
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9']
+        torch-version: ['1.10.1']
+        torchvision-version: ['0.11.2']
+        torchaudio-version: ['0.10.1']
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: '3.9'
+    steps:
+    - uses: actions/checkout@master
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@master
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install PyTorch ${{ matrix.torch-version }}+cpu
+      run: |
+        pip install numpy typing-extensions dataclasses
+        pip install torch==${{ matrix.torch-version}}+cpu torchvision==${{matrix.torchvision-version}}+cpu torchaudio==${{matrix.torchaudio-version}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+    - name: Install FS
+      run: |
+        pip install -e .[test]
+    - name: Test Distributed (LR on toy with a unified files)
+      run: |
+        python scripts/distributed_scripts/gen_data.py
+        python federatedscope/main.py --cfg scripts/distributed_scripts/distributed_configs/distributed_server_no_data.yaml distribute.grpc_compression gzip &
+        sleep 2
+        python federatedscope/main.py --cfg scripts/distributed_scripts/distributed_configs/distributed_client_1.yaml distribute.grpc_compression gzip &
+        sleep 2
+        python federatedscope/main.py --cfg scripts/distributed_scripts/distributed_configs/distributed_client_2.yaml distribute.grpc_compression gzip &
+        sleep 2
+        python federatedscope/main.py --cfg scripts/distributed_scripts/distributed_configs/distributed_client_3.yaml distribute.grpc_compression gzip
+        [ $? -eq 1 ] && exit 1 || echo "Passed"
+    - name: Test Distributed (LR on toy with multiple files)
+      run: |
+        python federatedscope/main.py --cfg scripts/distributed_scripts/distributed_configs/distributed_server.yaml data.file_path 'toy_data/server_data' distribute.data_idx -1 &
+        sleep 2
+        python federatedscope/main.py --cfg scripts/distributed_scripts/distributed_configs/distributed_client_1.yaml data.file_path 'toy_data/client_1_data' distribute.data_idx -1 &
+        sleep 2
+        python federatedscope/main.py --cfg scripts/distributed_scripts/distributed_configs/distributed_client_2.yaml data.file_path 'toy_data/client_2_data' distribute.data_idx -1 &
+        sleep 2
+        python federatedscope/main.py --cfg scripts/distributed_scripts/distributed_configs/distributed_client_3.yaml data.file_path 'toy_data/client_3_data' distribute.data_idx -1