azure-pipelines.yml

# https://aka.ms/yaml

trigger:
  - master

variables:
    regression: 1

jobs:
  - job: build_ubuntu_userguide
    displayName: Ubuntu Userguide
    pool:
      vmImage: 'Ubuntu-20.04'
    steps:
      - checkout: "none"
      - task: Cache@2
        inputs:
          key: '"tectonic" | "$(Agent.OS)"'
          restoreKeys: |
            "tectonic" | "$(Agent.OS)"
            "tectonic"
          path: $(Pipeline.Workspace)/tectonic-cache/
        displayName: Cache Tectonic
      - script: |
          wget -qO- https://github.com/tectonic-typesetting/tectonic/releases/download/tectonic%400.8.0/tectonic-0.8.0-x86_64-unknown-linux-gnu.tar.gz | tar xzvf - tectonic
          wget -qO- https://github.com/jgm/pandoc/releases/download/2.11.3.2/pandoc-2.11.3.2-linux-amd64.tar.gz | tar --strip-components=2 -xzvf - pandoc-2.11.3.2/bin/pandoc
          sudo mv -f pandoc tectonic /usr/local/bin
        displayName: Install Dependencies
      - script: |
          cd ${SYSTEM_DEFAULTWORKINGDIRECTORY}
          git clone https://github.com/soedinglab/MMseqs2.wiki.git .
          export XDG_CACHE_HOME=${PIPELINE_WORKSPACE}/tectonic-cache/
          .pandoc/make-pdf.sh
        displayName: Build Userguide
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(System.DefaultWorkingDirectory)/userguide.pdf
          artifactName: userguide

  - job: build_macos
    displayName: macOS
    pool:
      vmImage: 'macos-12'
    steps:
      - checkout: self
        submodules: true
      - script: |
          cd ${BUILD_SOURCESDIRECTORY}
          ./util/build_osx.sh . build
        displayName: Build MMseqs2
      - script: |
          ${BUILD_SOURCESDIRECTORY}/util/regression/run_regression.sh ${BUILD_SOURCESDIRECTORY}/build/mmseqs ${BUILD_SOURCESDIRECTORY}/regression
        displayName: Run Regression Suite
        condition: eq(variables['regression'], 1)
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(Build.SourcesDirectory)/build/mmseqs
          artifactName: mmseqs-darwin-universal

  - job: build_ubuntu
    displayName: Ubuntu MMseqs2
    pool:
      vmImage: 'Ubuntu-20.04'
    timeoutInMinutes: 120
    strategy:
      matrix:
        avx2:
          SIMD: 'AVX2'
          STATIC: 1
          MPI: 0
          BUILD_TYPE: RelWithDebInfo
        sse41:
          SIMD: 'SSE4_1'
          STATIC: 1
          MPI: 0
          BUILD_TYPE: RelWithDebInfo
        sse2:
          SIMD: 'SSE2'
          STATIC: 1
          MPI: 0
          BUILD_TYPE: RelWithDebInfo
        avx2_mpi:
          SIMD: 'AVX2'
          STATIC: 0
          MPI: 1
          BUILD_TYPE: RelWithDebInfo
        asan:
          SIMD: 'AVX2'
          STATIC: 0
          MPI: 0
          BUILD_TYPE: ASanOpt

    steps:
      - checkout: self
        submodules: true
      - script: |
          #sudo add-apt-repository ppa:ubuntu-toolchain-r/test
          #sudo apt-get update
          sudo apt-get install -y g++-10
        condition: eq(variables['BUILD_TYPE'], 'ASanOpt')
        displayName: Install newer G++
      - script: |
          sudo apt-get update
          sudo apt-get -y install mpi-default-dev mpi-default-bin
        displayName: Install Dependencies
        condition: eq(variables['MPI'], 1)
      - script: |
          mkdir build && cd build
          if [ "${BUILD_TYPE}" = "ASanOpt" ]; then
              export CC=gcc-10 ; export CXX=g++-10
          fi
          if [ "${STATIC}" -eq "1" ]; then
              cmake -DHAVE_SANITIZER=1 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DHAVE_TESTS=1 \
                -DBUILD_SHARED_LIBS=OFF \
                -DCMAKE_EXE_LINKER_FLAGS="-static -static-libgcc \
                -static-libstdc++" -DCMAKE_FIND_LIBRARY_SUFFIXES=".a" \
                -DENABLE_WERROR=1 -DHAVE_${SIMD}=1 -DHAVE_MPI=${MPI} ..
          else
              cmake -DHAVE_SANITIZER=1 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DHAVE_TESTS=1 \
                -DENABLE_WERROR=1 -DHAVE_${SIMD}=1 -DHAVE_MPI=${MPI} ..
          fi

          make -j $(nproc --all)
        displayName: Build MMseqs2
      - script: |
          export TTY=0
          if [ "${BUILD_TYPE}" = "ASan" ]; then
            echo "leak:libgomp1" > ${BUILD_SOURCESDIRECTORY}/ASan.supp
            export ASAN_OPTIONS=suppressions=${BUILD_SOURCESDIRECTORY}/ASan.supp
          fi
          ${BUILD_SOURCESDIRECTORY}/util/regression/run_regression.sh ${BUILD_SOURCESDIRECTORY}/build/src/mmseqs ${BUILD_SOURCESDIRECTORY}/regression
        displayName: Run Regression Suite
        condition: eq(variables['regression'], 1)
      - task: PublishPipelineArtifact@0
        condition: eq(variables['STATIC'], 1)
        inputs:
          targetPath: $(Build.SourcesDirectory)/build/src/mmseqs
          artifactName: mmseqs-linux-$(SIMD)

  - job: build_ubuntu_gpu
    displayName: Ubuntu MMseqs2 GPU
    pool:
      vmImage: 'Ubuntu-20.04'
    timeoutInMinutes: 120
    steps:
      - checkout: self
        submodules: false
      - script: |
          wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb
          sudo dpkg -i cuda-keyring_1.1-1_all.deb
          sudo apt-get update
          sudo apt-get install -y gcc-10 g++-10 cuda-nvcc-12-6 build-essential libgcc-10-dev ninja-build
          sudo apt-get purge -y cmake
          wget https://github.com/Kitware/CMake/releases/download/v3.31.0/cmake-3.31.0-linux-x86_64.sh
          chmod +x cmake-3.31.0-linux-x86_64.sh
          sudo ./cmake-3.31.0-linux-x86_64.sh --skip-license --prefix=/usr/local
        displayName: Install newer G++
      - script: |
          mkdir build && cd build
          export CC=gcc-10 ; export CXX=g++-10; export CUDAHOSTCXX=$CXX; export CUDACXX=/usr/local/cuda/bin/nvcc;
          /usr/local/bin/cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=1 -DHAVE_AVX2=1 \
            -DPREFER_STATIC=1 -DCMAKE_LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/10 -DBUILD_SHARED_LIBS=OFF \
            -DCMAKE_EXE_LINKER_FLAGS="-static-libgcc -static-libstdc++" \
            -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" ..
          cmake --build . -j$(nproc --all)
          # fail if GLIBC_PRIVATE or too new GLIBC is present
          if readelf -Ws mmseqs | grep -q GLIBC_PRIVATE; then
            exit 1
          fi
          LIBC_V=$(readelf -V mmseqs | awk '$3 ~ /^GLIBC_/ { print $3 }' | sort -V | tail -n1)
          if [[ "$LIBC_V" > "GLIBC_2.29" ]]; then
              exit 1
          fi
        displayName: Build MMseqs2
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(Build.SourcesDirectory)/build/src/mmseqs
          artifactName: mmseqs-linux-gpu

  - job: build_ubuntu_cross
    displayName: Ubuntu Cross-Compile
    pool:
      vmImage: 'Ubuntu-20.04'
    timeoutInMinutes: 120
    strategy:
      matrix:
        power8:
          SIMD: POWER8
          ARCH: ppc64el
          CPREF: powerpc64le
        power9:
          SIMD: POWER9
          ARCH: ppc64el
          CPREF: powerpc64le
        arm64:
          SIMD: ARM8
          ARCH: arm64
          CPREF: aarch64

    steps:
      - script: |
          sudo dpkg --add-architecture $ARCH
          cat << HEREDOC | sudo tee /etc/apt/sources.list
          deb [arch=amd64,i386] http://archive.ubuntu.com/ubuntu/ focal main universe multiverse
          deb [arch=amd64,i386] http://archive.ubuntu.com/ubuntu/ focal-updates main universe multiverse
          deb [arch=amd64,i386] http://archive.ubuntu.com/ubuntu/ focal-backports main universe multiverse
          deb [arch=amd64,i386] http://security.ubuntu.com/ubuntu/ focal-security main universe multiverse
          deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports/ focal main  universe multiverse
          deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports/ focal-updates main universe multiverse
          deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports/ focal-backports main universe multiverse
          deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports/ focal-security main universe multiverse
          HEREDOC
          sudo apt-get update
          sudo apt-get -y install -o APT::Immediate-Configure=false crossbuild-essential-$ARCH zlib1g-dev:$ARCH libbz2-dev:$ARCH
        displayName: Install Toolchain
      - script: |
          mkdir build && cd build
          CC=${CPREF}-linux-gnu-gcc CXX=${CPREF}-linux-gnu-g++ \
           cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DHAVE_TESTS=1 \
            -DBUILD_SHARED_LIBS=OFF \
            -DCMAKE_EXE_LINKER_FLAGS="-static -static-libgcc \
            -static-libstdc++" -DCMAKE_FIND_LIBRARY_SUFFIXES=".a" \
            -DENABLE_WERROR=1 -DHAVE_${SIMD}=1 ..
          make -j $(nproc --all)
        displayName: Build MMseqs2
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(Build.SourcesDirectory)/build/src/mmseqs
          artifactName: mmseqs-linux-$(SIMD)

  - job: build_windows
    displayName: Windows
    pool:
      vmImage: 'windows-2019'
    variables:
      CYGWIN_ROOT: $(System.Workfolder)\cygwin
      CYGWIN_MIRROR: http://cygwin.mirror.constant.com
    timeoutInMinutes: 120
    steps:
      - powershell: git config --global core.autocrlf false
        displayName: core.autocrlf false
      - checkout: self
        submodules: true
      - script: |
          rmdir /s /q C:\Strawberry
        displayName: Remove Strawberry Perl (Conflict with Cygwin)
      - script: |
          choco install cygwin --params="/InstallDir:%CYGWIN_ROOT%"
        displayName: Install Cygwin
      - script: |
          %CYGWIN_ROOT%\cygwinsetup.exe -qnNdO -R "%CYGWIN_ROOT%" -s "%CYGWIN_MIRROR%" -g -P ^
          bash,^
          xxd,^
          cmake,^
          make,^
          gcc-core,^
          gcc-g++,^
          zlib-devel,^
          libbz2-devel,^
          busybox-standalone,^
          git,^
          binutils,^
          wget
        displayName: Install Dependencies
      - script: |
          %CYGWIN_ROOT%\bin\bash.exe -cl "${BUILD_SOURCESDIRECTORY}/util/build_windows.sh $(cygpath ${BUILD_SOURCESDIRECTORY}) $(cygpath ${BUILD_SOURCESDIRECTORY}/build)"
        displayName: Build MMseqs2
      - task: "ArchiveFiles@2"
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)\build\mmseqs
          archiveFile: $(Build.SourcesDirectory)\mmseqs-win64.zip
          includeRootFolder: true
          archiveType: zip
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(Build.SourcesDirectory)\mmseqs-win64.zip
          artifactName: mmseqs-win64
      # - script: |
      #     %BUILD_SOURCESDIRECTORY%\build\mmseqs\mmseqs.bat version
      #   displayName: Setup Busybox
      - script: |
          %CYGWIN_ROOT%\bin\bash.exe -cl "${BUILD_SOURCESDIRECTORY}/util/regression/run_regression.sh $(cygpath ${BUILD_SOURCESDIRECTORY}/build/build_sse41/src/mmseqs.exe) $(cygpath ${BUILD_SOURCESDIRECTORY}/regression)"
        displayName: Run Regression Suite
        condition: eq(variables['regression'], 1)

  - job: upload_artifacts
    displayName: Upload Artifacts
    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
    pool:
      vmImage: 'ubuntu-latest'
    dependsOn:
      - build_ubuntu_userguide
      - build_macos
      - build_ubuntu
      - build_ubuntu_gpu
      - build_ubuntu_cross
      - build_windows
    steps:
      - script: |
          cd "${BUILD_SOURCESDIRECTORY}"
          mkdir mmseqs
          cp -f README.md LICENSE.md mmseqs
          cp -r examples mmseqs
          mkdir mmseqs/matrices
          cp -f data/*.out mmseqs/matrices
          mkdir mmseqs/util
          cp -f util/bash-completion.sh mmseqs/util
          mkdir mmseqs/bin
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: userguide
          targetPath: $(Build.SourcesDirectory)/mmseqs
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: mmseqs-darwin-universal
          targetPath: $(Build.SourcesDirectory)/mmseqs/bin
      - script:
          chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
          archiveFile: $(Build.SourcesDirectory)/mmseqs-osx-universal.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: mmseqs-linux-SSE4_1
          targetPath: $(Build.SourcesDirectory)/mmseqs/bin
      - script:
          chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
          archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-sse41.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: mmseqs-linux-AVX2
          targetPath: $(Build.SourcesDirectory)/mmseqs/bin
      - script:
          chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
          archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-avx2.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: mmseqs-linux-SSE2
          targetPath: $(Build.SourcesDirectory)/mmseqs/bin
      - script:
          chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
          archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-sse2.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: mmseqs-linux-gpu
          targetPath: $(Build.SourcesDirectory)/mmseqs/bin
      - script:
          chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
          archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-gpu.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: mmseqs-linux-POWER9
          targetPath: $(Build.SourcesDirectory)/mmseqs/bin
      - script:
          chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
          archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-ppc64le-power9.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: mmseqs-linux-POWER8
          targetPath: $(Build.SourcesDirectory)/mmseqs/bin
      - script:
          chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
          archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-ppc64le-power8.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: mmseqs-linux-ARM8
          targetPath: $(Build.SourcesDirectory)/mmseqs/bin
      - script:
          chmod +x "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
          archiveFile: $(Build.SourcesDirectory)/mmseqs-linux-arm64.tar.gz
          includeRootFolder: true
          archiveType: tar
      - script:
          rm "${BUILD_SOURCESDIRECTORY}/mmseqs/bin/mmseqs"
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: mmseqs-win64
          targetPath: $(Build.SourcesDirectory)
      - script: |
          unzip "${BUILD_SOURCESDIRECTORY}/mmseqs-win64.zip"
          chmod +x mmseqs/mmseqs.bat mmseqs/bin/*
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/mmseqs
          archiveFile: $(Build.SourcesDirectory)/mmseqs-win64.zip
          includeRootFolder: true
          archiveType: zip
      - task: DownloadSecureFile@1
        inputs:
          secureFile: secretKeyPleaseDontSteal
        displayName: Get Deployment Key
      - script: |
          mkdir ~/.ssh && mv $DOWNLOADSECUREFILE_SECUREFILEPATH ~/.ssh/id_rsa
          chmod 700 ~/.ssh && chmod 600 ~/.ssh/id_rsa
          ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub
          cd "${BUILD_SOURCESDIRECTORY}"
          cp mmseqs/userguide.pdf userguide.pdf
          ssh-keygen -Y sign -f ~/.ssh/id_rsa -n file \
            userguide.pdf \
            mmseqs-osx-universal.tar.gz \
            mmseqs-linux-sse2.tar.gz \
            mmseqs-linux-sse41.tar.gz \
            mmseqs-linux-avx2.tar.gz \
            mmseqs-linux-gpu.tar.gz \
            mmseqs-linux-arm64.tar.gz \
            mmseqs-linux-ppc64le-power8.tar.gz \
            mmseqs-linux-ppc64le-power9.tar.gz \
            mmseqs-win64.zip
          curl --retry 5 --retry-all-errors -X POST \
            -F file[]=@userguide.pdf -F signature[]=@userguide.pdf.sig \
            -F file[]=@mmseqs-osx-universal.tar.gz -F signature[]=@mmseqs-osx-universal.tar.gz.sig \
            -F file[]=@mmseqs-linux-sse2.tar.gz -F signature[]=@mmseqs-linux-sse2.tar.gz.sig \
            -F file[]=@mmseqs-linux-sse41.tar.gz -F signature[]=@mmseqs-linux-sse41.tar.gz.sig \
            -F file[]=@mmseqs-linux-avx2.tar.gz -F signature[]=@mmseqs-linux-avx2.tar.gz.sig \
            -F file[]=@mmseqs-linux-gpu.tar.gz -F signature[]=@mmseqs-linux-gpu.tar.gz.sig \
            -F file[]=@mmseqs-linux-arm64.tar.gz -F signature[]=@mmseqs-linux-arm64.tar.gz.sig \
            -F file[]=@mmseqs-linux-ppc64le-power8.tar.gz -F signature[]=@mmseqs-linux-ppc64le-power8.tar.gz.sig \
            -F file[]=@mmseqs-linux-ppc64le-power9.tar.gz -F signature[]=@mmseqs-linux-ppc64le-power9.tar.gz.sig \
            -F file[]=@mmseqs-win64.zip  -F signature[]=@mmseqs-win64.zip.sig \
            -F identifier="mmseqs" -F directory="${BUILD_SOURCEVERSION}" \
              https://mmseqs.com/upload