1474 refactor the concept of public and private test set

.github/workflows/black_checker.yml

@@ -0,0 +1,47 @@
+name: black-format-check
+
+on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
+  push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
+  pull_request:
+    branches:
+      - master
+      - development
+
+env:
+  #If STRICT is set to true, it will fail on black check fail
+  STRICT: false
+
+jobs:
+
+  black-format-check:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Setup Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.7"
+
+    - name: Install black
+      run: |
+        pip install black
+
+    - name: Run Black Check
+      run: |
+        black --check --diff --line-length 100 ./autosklearn || ! $STRICT
+        black --check --diff --line-length 100 ./test || ! $STRICT
+        black --check --diff --line-length 100 ./examples|| ! $STRICT

.github/workflows/dist.yml

@@ -1,31 +1,58 @@
 name: dist-check
 
-on: [push, pull_request]
+on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
+  push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
+  pull_request:
+    branches:
+      - master
+      - development
 
 jobs:
   dist:
     runs-on: ubuntu-latest
+
     steps:
-    - uses: actions/checkout@v2
+    - name: Check out the repo
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
+
     - name: Build dist
       run: |
         python setup.py sdist
+
     - name: Twine check
       run: |
         pip install twine
         last_dist=$(ls -t dist/auto-sklearn-*.tar.gz | head -n 1)
-        twine_output=`twine check "$last_dist"`
-        if [[ "$twine_output" != "Checking $last_dist: PASSED" ]]; then echo $twine_output && exit 1;fi
+        twine check "$last_dist" --strict
+
+
     - name: Install dist
       run: |
         last_dist=$(ls -t dist/auto-sklearn-*.tar.gz | head -n 1)
         pip install $last_dist
+
     - name: PEP 561 Compliance
       run: |
         pip install mypy
+
         cd ..  # required to use the installed version of autosklearn
-        if ! python -c "import autosklearn"; then exit 1; fi
+
+        # Note this doesnt perform mypy checks, only
+        # that the types are exported
+        if ! mypy -c "import autosklearn"; then exit 1; fi

.github/workflows/docker-publish.yml

@@ -1,24 +1,31 @@
 #https://help.github.com/en/actions/language-and-framework-guides/publishing-docker-images#publishing-images-to-github-packages
 name: Publish Docker image
+
 on:
+
   push:
-    # Push  to `master` or `development`
     branches:
       - master
       - development
       - docker_workflow
 
 jobs:
+
   push_to_registry:
     name: Push Docker image to GitHub Packages
     runs-on: ubuntu-latest
+
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
       - name: Extract branch name
         shell: bash
         run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
         id: extract_branch
+
       - name: Push to GitHub Packages
         uses: docker/build-push-action@v1
         with:
@@ -28,26 +35,32 @@ jobs:
           repository: automl/auto-sklearn/auto-sklearn
           tag_with_ref: true
           tags: ${{ steps.extract_branch.outputs.branch }}
+
       - name: Push to Docker Hub
         uses: docker/build-push-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: mfeurer/auto-sklearn
           tags: ${{ steps.extract_branch.outputs.branch }}
+
       - name: Docker Login
         run: docker login docker.pkg.github.com -u $GITHUB_ACTOR -p $GITHUB_TOKEN
         env:
             GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
       - name: Pull Docker image
         run: docker pull docker.pkg.github.com/$GITHUB_REPOSITORY/auto-sklearn:$BRANCH
         env:
             BRANCH: ${{ steps.extract_branch.outputs.branch }}
+
       - name: Run image
         run: docker run -i -d --name unittester -v $GITHUB_WORKSPACE:/workspace -w /workspace docker.pkg.github.com/$GITHUB_REPOSITORY/auto-sklearn:$BRANCH
         env:
             BRANCH: ${{ steps.extract_branch.outputs.branch }}
+
       - name: Auto-Sklearn loaded
         run: docker exec  -i unittester python3 -c 'import autosklearn; print(f"Auto-sklearn imported from {autosklearn.__file__}")'
+
       - name: Run unit testing
         run: docker exec  -i unittester python3 -m pytest -v test

.github/workflows/docs.yml

@@ -1,38 +1,65 @@
 name: Docs
-on: [pull_request, push]
+
+on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
+  push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
+  pull_request:
+    branches:
+      - master
+      - development
 
 jobs:
+
   build-and-deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
+
     - name: Install dependencies
       run: |
-        pip install -e .[docs,examples,examples_unix]
+        pip install -e .[docs,examples]
+
     - name: Make docs
       run: |
         cd doc
         make html
+
     - name: Check links
       run: |
         cd doc
         make linkcheck
+
     - name: Pull latest gh-pages
       if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
       run: |
         cd ..
         git clone https://github.com/automl/auto-sklearn.git --branch gh-pages --single-branch gh-pages
+
     - name: Copy new doc into gh-pages
       if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
       run: |
         branch_name=${GITHUB_REF##*/}
         cd ../gh-pages
         rm -rf $branch_name
         cp -r ../auto-sklearn/doc/build/html $branch_name
+
     - name: Push to gh-pages
       if: (contains(github.ref, 'develop') || contains(github.ref, 'master')) && github.event_name == 'push'
       run: |

.github/workflows/isort_checker.yml

@@ -0,0 +1,45 @@
+name: isort-check
+
+on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
+  push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
+  pull_request:
+    branches:
+      - master
+      - development
+
+env:
+  #If STRICT is set to true, it will fail on isort check fail
+  STRICT: false
+
+jobs:
+
+  isort-format-check:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Setup Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.7"
+
+    - name: Install isort
+      run: |
+        pip install isort
+
+    - name: Run isort Check
+      run: |
+        isort --check-only autosklearn || ! $STRICT

.github/workflows/pre-commit.yaml

@@ -1,20 +1,39 @@
 name: pre-commit
 
-on: [push, pull_request]
+on:
+  # Manually triggerable in github
+  workflow_dispatch:
+
+  # When a push occurs on either of these branches
+  push:
+    branches:
+      - master
+      - development
+
+  # When a push occurs on a PR that targets these branches
+  pull_request:
+    branches:
+      - master
+      - development
 
 jobs:
   run-all-files:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
     - name: Setup Python 3.7
       uses: actions/setup-python@v2
       with:
         python-version: 3.7
+
     - name: Install pre-commit
       run: |
         pip install pre-commit
         pre-commit install
+
     - name: Run pre-commit
       run: |
         pre-commit run --all-files