pyopenms-wheels.yml

# This builds python wheels on manylinux, macOS and Windows and uploads them
# to public pypi for release and for nightlies also to pypi.cs.uni-tuebingen.de

name: pyopenms-wheels-and-packages

# Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the master branch
on:
  workflow_dispatch:
  push:
    tags:
      - 'Release*'
    branches: [ nightly, py_nightly_test ]

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  build-win:
    runs-on: windows-latest
    steps:
    - uses: actions/checkout@v3
      with:
        path: OpenMS
      
    - name: Setup cmake
      uses: jwlawson/actions-setup-cmake@v1.13
      with:
        cmake-version: '3.23.x'

    - name: Reduce PATH
      run: echo "PATH=C:\hostedtoolcache\windows\Python\3.7.9\x64\Scripts;C:\hostedtoolcache\windows\Python\3.7.9\x64;C:\hostedtoolcache\windows\Ruby\2.5.8\x64\bin;C:\Program Files\Java\jdk8u265-b01\bin;C:\ProgramData\kind;C:\vcpkg;C:\cf-cli;C:\Program Files (x86)\NSIS;C:\windows\system32;C:\windows;C:\windows\System32\Wbem;C:\windows\System32\WindowsPowerShell\v1.0;C:\windows\System32\OpenSSH;C:\ProgramData\Chocolatey\bin;C:\Program Files\Docker;C:\Program Files\PowerShell\7;C:\Program Files\OpenSSL\bin;C:\Strawberry\c\bin;C:\Strawberry\perl\site\bin;C:\Strawberry\perl\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin;C:\Program Files\Git\usr\bin;C:\Program Files (x86)\Microsoft Visual Studio\2019\\Enterprise\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

    - name: Install Qt
      uses: jurplel/install-qt-action@v3
      with:
        version: '5.12.6' # 5.12.7 is broken https://bugreports.qt.io/browse/QTBUG-81715
        host: 'windows' # default: win64_msvc2017_64
        target: 'desktop'
        install-deps: 'true'
        archives: 'qtsvg qtbase'
        cache: 'false'

    - name: Install Miniconda
      shell: bash # uses git bash on windows
      run: |
            MINICONDA_FILENAME=Miniconda3-latest-Windows-x86_64.exe
            curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
            ./Miniconda3-latest-Windows-x86_64.exe //InstallationType=JustMe //RegisterPython=0 //S //D=$HOME/miniconda3

    # https://github.com/marketplace/actions/visual-studio-shell
    - name: Set up Visual Studio shell
      uses: egor-tensin/vs-shell@v2
      with:
        arch: x64

    - name: Load contrib build
      run: |
          mkdir contribbld
          cd contribbld
          curl -o contribbld.tar.gz https://abibuilder.cs.uni-tuebingen.de/archive/openms/contrib/windows/x64/msvc-14.2/contrib_build.tar.gz
          tar -xzf contribbld.tar.gz
          rm contribbld.tar.gz

    - name: Setup conda paths
      shell: bash
      run: |
        echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
        echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH

    - name: Build
      shell: bash
      run: |
        PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
        mkdir bld
        pushd bld
        # TODO: set generator via variable, then we can share this step
        cmake --version
        cmake -G "Visual Studio 17 2022" -A x64 -DOPENMS_CONTRIB_LIBS="$GITHUB_WORKSPACE/contribbld" -DCMAKE_PREFIX_PATH="$(echo $Qt5_Dir)/lib/cmake;${Qt5_Dir}" ../OpenMS
        # Note: multiple --targets only supported by CMake 3.15+
        cmake --build . --config Release --target OpenMS
      
        mkdir pyopenms_whls

        # The growing PATH leads to a failure in the last iteration of the pyopenms build process.
        # We will store the PATH variable in the beginning and reset it with every iteration.
        RESET_PATH=$PATH

        for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
          # reset the path
          PATH=$RESET_PATH

          # create and activate conda environment
          py=$(echo "$py" | tr -d " \n")
          pynodot=$(echo "$py" | tr -d ".")
          conda create -n pyoms-bld-"${pynodot}" python="${py}"
          source activate pyoms-bld-"${pynodot}"

          # set current python executable
          CURRENT_PYTHON_EXECUTABLE=$(which python)

          ${CURRENT_PYTHON_EXECUTABLE} -m pip install --upgrade pip
          # pip install all the stuff
          pip install -U setuptools
          pip install -U pip
          pip install -U autowrap
          pip install -U pytest
          pip install -U numpy
          pip install -U wheel

          echo $CURRENT_PYTHON_EXECUTABLE

          # build pyopenms distribution
          cmake --version
          cmake -DPYTHON_EXECUTABLE:FILEPATH=$CURRENT_PYTHON_EXECUTABLE -DPYOPENMS=ON .
          cmake --build . --config Release --target pyopenms

          # copy to directory
          cp pyOpenMS/dist/*.whl pyopenms_whls/
          # clean up
          conda deactivate
          conda remove --name "pyoms-bld-${pynodot}" --all

          # clean previous pyopenms libs
          find . -name "pyopenms*.so" -exec rm -rf {} \;

        done

    - uses: actions/upload-artifact@v3
      name: Upload artifacts
      with:
        name: ${{ runner.os }}-wheels
        path: bld/pyopenms_whls

  build-macos:
    runs-on: macos-latest
    steps:
    - uses: actions/checkout@v3
      with:
        path: OpenMS

    - name: Setup cmake
      uses: jwlawson/actions-setup-cmake@v1.13
      with:
        cmake-version: '3.23.x'

    - name: Install Qt
      uses: jurplel/install-qt-action@v3
      with:
        version: '5.12.7'
        host: 'mac'
        target: 'desktop'
        install-deps: 'true'
        cache: 'false'
        setup-python: 'false' # https://github.com/jurplel/install-qt-action/issues/130

    - name: Install Miniconda
      shell: bash
      run: |
            MINICONDA_FILENAME=Miniconda3-latest-MacOSX-x86_64.sh
            curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
            bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3

    - name: Setup conda paths
      shell: bash
      run: |
        echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
        echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH

    - name: Install build tools
      run: brew install autoconf automake libtool

    - name: Load contrib build
      run: |
          mkdir contribbld
          cd contribbld
          curl -o contribbld.tar.gz https://abibuilder.cs.uni-tuebingen.de/archive/openms/contrib/macOS/10.15.5/x64/appleclang-11.0.0/contrib_build.tar.gz
          tar -xzf contribbld.tar.gz
          rm contribbld.tar.gz
          cd .. 

    - name: Load contrib and build OpenMP
      run: |
          git clone https://github.com/OpenMS/contrib.git
          cd contribbld
          cmake -DBUILD_TYPE=OPENMP -DNUMBER_OF_JOBS=4 -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -DCMAKE_OSX_DEPLOYMENT_TARGET=10.9 ../contrib

    - name: Fix contrib install names
      shell: bash
      run: |
          for lib in $GITHUB_WORKSPACE/contribbld/lib/*.dylib
          do
            install_name_tool -id $lib $lib
          done

    - name: Build
      run: |
        PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
        export CC=clang
        export CXX=clang++
        export MACOSX_DEPLOYMENT_TARGET=10.9
        # Unfortunately, on macOS due to the inofficial way of enabling OpenMP on AppleClang, passing the following
        # options to setup.py extra_compile_args does not work. See also https://gist.github.com/andyfaff/084005bee32aee83d6b59e843278ab3e
        export CFLAGS="-Xpreprocessor -fopenmp $CFLAGS"
        export CXXFLAGS="-Xpreprocessor -fopenmp $CXXFLAGS"

        # create symlinks like brew https://github.com/Homebrew/homebrew-core/blob/master/Formula/qt.rb#L61
        pushd $Qt5_Dir/include
          for f in $Qt5_Dir/lib/*.framework/Headers
          do
            lnknm=$(basename "${f%/*}" ".framework")
            ln -s $f $lnknm
          done
          for f in $Qt5_Dir/lib/*.framework/Versions/5/Qt*
          do
            install_name_tool -id $Qt5_Dir/lib/$(basename $f).framework/Versions/Current/$(basename $f) $f
          done
          for f in $Qt5_Dir/lib/*.framework/Versions/Current/Qt*
          do
            install_name_tool -id $f $f
          done
        popd
        mkdir bld
        pushd bld
        # Use -DCMAKE_FIND_DEBUG_MODE=ON for debug
        cmake -DCMAKE_BUILD_TYPE="Release" -DOPENMS_CONTRIB_LIBS="$GITHUB_WORKSPACE/contribbld" -DCMAKE_PREFIX_PATH="$(echo $Qt5_Dir)/lib/cmake;${Qt5_Dir};$GITHUB_WORKSPACE/contribbld/lib/" -DCMAKE_OSX_DEPLOYMENT_TARGET=10.9 -DBOOST_USE_STATIC=ON ../OpenMS
        make -j4 OpenMS

        mkdir pyopenms_whls

        for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
          py=$(echo "$py" | tr -d " \n")
          pynodot=$(echo "$py" | tr -d ".")
          conda create -n pyoms-bld-"${pynodot}" python="${py}"
          source activate pyoms-bld-"${pynodot}"

          # set current python executable
          CURRENT_PYTHON_EXECUTABLE=$(which python)

          # pip install all the stuff
          pip install -U setuptools
          pip install -U pip
          pip install -U autowrap
          pip install -U pytest
          pip install -U numpy
          pip install -U wheel
          pip install -U pandas

          #conda deactivate

          echo $CURRENT_PYTHON_EXECUTABLE

          # build pyopenms distribution (macOS)
          cmake -DPYTHON_EXECUTABLE:FILEPATH=$CURRENT_PYTHON_EXECUTABLE -DPYOPENMS=ON .
          make -j4 pyopenms

          # copy to directory
          cp pyOpenMS/dist/*.whl pyopenms_whls/

          # clean up
          conda deactivate
          conda remove --name pyoms-bld-"${pynodot}" --all

          # clean previous pyopenms libs
          find . -name "pyopenms*.so" -exec rm -rf {} \;

        done

    - uses: actions/upload-artifact@v3
      name: Upload artifacts
      if: always()
      with:
        name: ${{ runner.os }}-buildfiles
        path: |
          bld/CMakeCache.txt
          bld/pyOpenMS/env.py
        
    - uses: actions/upload-artifact@v3
      name: Upload wheels
      with:
        name: ${{ runner.os }}-wheels
        path: bld/pyopenms_whls

  build-lnx:
    runs-on: ubuntu-latest
    container: ghcr.io/openms/contrib_manylinux2014:latest

    steps:
    # Cancels older builds if still running
    - uses: rokroskar/workflow-run-cleanup-action@master
      env:
        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
      if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'"

    - uses: actions/checkout@v3
      name: Checkout sources
      with:
        path: OpenMS

    - name: Setup cmake
      uses: jwlawson/actions-setup-cmake@v1.13
      with:
        cmake-version: '3.23.x'

    - name: Build on manylinux2014
      shell: bash
      run: |
        PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
        yum install -y jq
        cmake --version

        mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/
        mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/before_fix/

        LD_OLD_LIBRARY_PATH=$LD_LIBRARY_PATH

        mkdir openms-build
        cd openms-build
        cmake -DCMAKE_BUILD_TYPE="Release" -DOPENMS_CONTRIB_LIBS="/contrib-build/" -DCMAKE_PREFIX_PATH="/contrib-build/" $GITHUB_WORKSPACE/OpenMS
        make -j4 OpenMS

        # compile and configure OpenMS
        for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
          py=$(echo "$py" | tr -d " \n")
          pynodot=$(echo "$py" | tr -d ".")
          PYBIN=$(echo /opt/python/cp*${pynodot})
          "$PYBIN/bin/pip" install -U pip
          "$PYBIN/bin/pip" install -U Cython
          "$PYBIN/bin/pip" install -U setuptools
          "$PYBIN/bin/pip" install -U wheel==0.31.1
          "$PYBIN/bin/pip" install -U numpy
          "$PYBIN/bin/pip" install -U pytest
          "$PYBIN/bin/pip" install -U autowrap

          # configure (don't copy deps since we use auditwheel)
          cmake -DNO_DEPENDENCIES=ON -DOPENMS_CONTRIB_LIBS="/contrib-build/" -DCMAKE_PREFIX_PATH="/contrib-build/" -DPYOPENMS=On -DPython_ROOT_DIR=$PYBIN -DPython_FIND_STRATEGY="LOCATION" $GITHUB_WORKSPACE/OpenMS
          make -j4 pyopenms
          
          # ensure auditwheel can find the libraries
          export LD_LIBRARY_PATH=$LD_OLD_LIBRARY_PATH:`pwd`/lib

          # strip the libraries before repairing
          strip --strip-all lib/libOpenMS.so
          strip --strip-all lib/libOpenSwathAlgo.so

          pushd pyOpenMS
            # Bundle stripped plus external shared libraries into the wheels
            for whl in dist/pyopenms*.whl; do
              auditwheel repair "$whl" -w wheelhouse/
            done

            mv wheelhouse/* $GITHUB_WORKSPACE/data/wheelhouse/
          popd
          export LD_LIBRARY_PATH=$LD_OLD_LIBRARY_PATH
        done

    - uses: actions/upload-artifact@v3
      name: Upload artifacts
      with:
        name: ${{ runner.os }}-wheels
        path: data/wheelhouse/

# Tests the build workflows
  test:
    needs: [build-win, build-macos, build-lnx]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
    steps:
    - uses: actions/checkout@v3
      name: Checkout sources
      with:
        path: OpenMS

    - name: Install Miniconda
      shell: bash
      run: |
          if [[ "${{ runner.os }}" == "Linux" ]]; then
            MINICONDA_FILENAME=Miniconda3-latest-Linux-x86_64.sh
            curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
            bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
          elif [[ "${{ runner.os }}" == "macOS" ]]; then
            MINICONDA_FILENAME=Miniconda3-latest-MacOSX-x86_64.sh
            curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
            bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
          elif [[ "${{ runner.os }}" == "Windows" ]]; then
            MINICONDA_FILENAME=Miniconda3-latest-Windows-x86_64.exe
            curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
            ./Miniconda3-latest-Windows-x86_64.exe //InstallationType=JustMe //RegisterPython=0 //S
          fi

    - name: Setup conda paths
      shell: bash
      run: |
        echo "$HOME/miniconda3/bin" >> $GITHUB_PATH
        echo "$HOME/miniconda3/Scripts" >> $GITHUB_PATH

    - name: Download artifacts
      uses: actions/download-artifact@v3
      with:
          name: ${{ runner.os }}-wheels

    - name: Test
      shell: bash 
      run: |
        PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
        echo $PYTHON_VERSIONS

        for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
          py=$(echo "$py" | tr -d " \n")
          pynodot=$(echo "$py" | tr -d ".")
          conda create -n pyoms-bld-$pynodot python=$py
          source activate pyoms-bld-$pynodot

          # Find .whl for current python version
          CURRENT_VERSION=$(echo $py | sed "s/\.//g") # remove "." for find
          echo "Testing python version $CURRENT_VERSION"
          CURRENT_WHL=$(find $GITHUB_WORKSPACE -name "*$CURRENT_VERSION*.whl") # Probably $GITHUB_WORKSPACE
          if [[ "$CURRENT_WHL" == '' ]]; then

            continue

          fi
          echo "Found wheel $CURRENT_WHL : Testing.."

          # pip install all the stuff
          python -m pip install --upgrade pip
          python -m pip install $CURRENT_WHL

          # check if package was installed
          python -c "import importlib.util; p_name = 'pyopenms'; package = importlib.util.find_spec(p_name); print(p_name + ' was sucessfully installed! Nice one!') if package is not None else print('Something seems to be wrong!')"
          # run test
          python -c "exec(\"import pyopenms\nprint(pyopenms.EmpiricalFormula(\'C6H12O6\').getMonoWeight())\")"

          conda deactivate

          # clean up
          conda remove --name pyoms-bld-$pynodot --all

        done

  # Upload the wheels (the action only runs on ubuntu slaves)
  publish:
    needs: [test] # doesn't really NEED it but we want the tests to pass first
    runs-on: ubuntu-latest
    steps:
    # haha github is just ridiculous.. not getting their actions to do basic stuff for years
    - name: Download artifacts
      uses: actions/download-artifact@v3
      with:
          name: Linux-wheels
          path: /home/runner/work/OpenMS/OpenMS/Linux-wheels
    - name: Download artifacts
      uses: actions/download-artifact@v3
      with:
          name: Windows-wheels
          path: /home/runner/work/OpenMS/OpenMS/Windows-wheels
    - name: Download artifacts
      uses: actions/download-artifact@v3
      with:
          name: macOS-wheels
          path: /home/runner/work/OpenMS/OpenMS/macOS-wheels

    - name: Merge all artifacts
      run: |
            mkdir wheels
            cp -lr *-wheels/* wheels/
            pwd
            ls -la wheels/

    - name: Publish package to nightly PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        repository_url: https://pypi.cs.uni-tuebingen.de/
        user: openms
        password: ${{ secrets.abiservices_pypi_pw }}
        packages_dir: ${{ github.workspace }}/wheels

    - name: Publish package to PyPI
      if: "contains(github.ref, 'Release')"
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.pypi_api_token_release }}
        packages_dir: ${{ github.workspace }}/wheels