release.yml

# This workflow builds packages with Marian executables and then attaches them
# to an existing GitHub release or drafts a new release if needed.
# There are two ways of utilizing the workflow:
#
# 1. Create a new tag locally and push it to the remote, for example:
#
#     git tag 1.2.3
#     git push origin 1.2.3
#
# The workflow, if built successfully, will open a new draft release (not seen
# for the public, available from under the /releases subpage on GitHub), use
# the last unreleased changes from CHANGELOG as the release description, and
# attach built packages as assets. Until the draft release is not published,
# the tag will be separated from the release, but they get merged into a single
# item with all assets.  An admin user need to manually publish the draft
# release then. (Note: this behavior can be easily changed if needed).
#
# 2. Publish a new (non-draft) release with a tag via GitHub web interface. The
# workflow, if built successfully, will only attach built packages as
# additional assets. It is the responsibility of an admin user to add
# description to the release if it was not provided before..
#
name: Release

# The job is triggered by pushing a tag that follows the given pattern name
on:
  push:
    tags:
      - "[0-9]+.[0-9]+.[0-9]+*"

env:
  cuda_version: "11.2"
  gcc_version: 8

jobs:
  ######################################################################
  build-release-ubuntu:
    strategy:
      matrix:
        include:
          # No builds with FBGEMM because they are not portable enough
          # requiring same CPU architecture at compilation and runtime
          - name: "Build Ubuntu CPU"
            suffix: cpu
            fbgemm: false
            cuda: false
          - name: "Build Ubuntu CPU+CUDA"
            suffix: cuda11.2
            fbgemm: false
            cuda: true

    runs-on: ubuntu-18.04
    name: ${{ matrix.name }}

    steps:
    # Set env.github_tag_name and env.archive_name for subsequent steps
    - name: Archive name
      run: |
        # Get the tag name only to use it in the archive name. The variable github.ref can not be used because it starts with refs/tags/
        TAG_NAME=$(echo ${{ github.ref }} | cut -d/ -f3-)
        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
        echo "github_tag_name=${TAG_NAME}" >> $GITHUB_ENV
        echo "archive_name=${{ github.event.repository.name }}-${TAG_NAME}_linux-x64-static_${{ matrix.suffix }}" >> $GITHUB_ENV
      shell: bash

    - name: Checkout
      uses: actions/checkout@v2
      with:
        submodules: recursive

    # The following packages are already installed on GitHub-hosted runners: build-essential openssl libssl-dev
    # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
    # Boost is no longer pre-installed on GitHub-hosted runners
    - name: Install dependencies
      run: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler libboost-system-dev gcc-${{ env.gcc_version }} g++-${{ env.gcc_version }}

    # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
    - name: Install MKL
      run: |
        wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
        sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
        sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
        sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
    # The script simplifies installation of different versions of CUDA
    - name: Install CUDA
      run: ./scripts/ci/install_cuda_ubuntu.sh ${{ env.cuda_version }}
      if: matrix.cuda == true

    - name: Configure CMake
      run: |
        mkdir -p build
        cd build
        CC=/usr/bin/gcc-${{ env.gcc_version }} CXX=/usr/bin/g++-${{ env.gcc_version }} CUDAHOSTCXX=/usr/bin/g++-${{ env.gcc_version }} \
        cmake .. \
          -DBoost_ARCHITECTURE=-x64 \
          -DCMAKE_BUILD_TYPE=Slim \
          -DCOMPILE_CPU=on \
          -DCOMPILE_CUDA=${{ matrix.cuda }} \
          -DCOMPILE_KEPLER=${{ matrix.cuda }} \
          -DCOMPILE_MAXWELL=${{ matrix.cuda }} \
          -DCOMPILE_SERVER=on \
          -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ env.cuda_version }} \
          -DUSE_FBGEMM=${{ matrix.fbgemm }} \
          -DUSE_SENTENCEPIECE=on \
          -DUSE_STATIC_LIBS=on \
    - name: Compile
      working-directory: build
      run: make -j2

    - name: Create archive
      working-directory: build
      run: tar zcvf ../${{ env.archive_name }}.tar.gz marian* ../README.md

    # For testing only
    #- name: Test archive
      #run: tar zcvf ${{ env.archive_name }}.tar.gz README.md

    - name: Upload archive
      uses: actions/upload-artifact@v2
      with:
        name: ${{ env.archive_name }}.tar.gz
        path: ${{ env.archive_name }}.tar.gz

  ######################################################################
  build-release-windows:
    strategy:
      matrix:
        include:
          # No builds with FBGEMM because they are not portable enough
          # requiring same CPU architecture at compilation and runtime
          - name: "Build Windows CPU"
            suffix: cpu
            fbgemm: false
            cuda: false
          - name: "Build Windows CPU+CUDA"
            suffix: cuda11.2
            fbgemm: false
            cuda: true

    runs-on: windows-2019
    name: ${{ matrix.name }}

    steps:
    # Set env.github_tag_name and env.archive_name for subsequent steps
    - name: Archive name
      run: |
        # Get the tag name only to use it in the archive name. The variable github.ref can not be used because it starts with refs/tags/
        TAG_NAME=$(echo ${{ github.ref }} | cut -d/ -f3-)
        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
        echo "github_tag_name=${TAG_NAME}" >> $GITHUB_ENV
        echo "archive_name=${{ github.event.repository.name }}-${TAG_NAME}_windows-x64_${{ matrix.suffix }}" >> $GITHUB_ENV
      shell: bash

    - name: Checkout
      uses: actions/checkout@v2
      with:
        submodules: recursive

    - name: Download MKL
      run: |
        C:\msys64\usr\bin\wget.exe -nv https://data.statmt.org/romang/marian-regression-tests/ci/mkl-2020.1-windows-static.zip -O mkl.zip
        Expand-Archive -Force mkl.zip ${{ github.workspace }}\mkl
        # Set the MKLROOT environment variable so that CMake can find MKL.
        # GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners
        echo "MKLROOT=$env:GITHUB_WORKSPACE/mkl" | Out-File -FilePath $env:GITHUB_ENV -Append
      shell: pwsh

    - name: Install CUDA
      run: |
        .\scripts\ci\install_cuda_windows.ps1 '${{ env.cuda_version }}'
        # Set path to CUDA for subsequent steps so that CMake can find it
        echo "CUDA_PATH=$env:CUDA_PATH" | Out-File -FilePath $env:GITHUB_ENV -Append
        echo "$env:CUDA_PATH/bin" | Out-File -FilePath $env:GITHUB_PATH -Append
      shell: pwsh
      if: matrix.cuda == true

    - name: Prepare vcpkg
      uses: lukka/run-vcpkg@v4
      with:
        vcpkgArguments: protobuf
        vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
        vcpkgDirectory: ${{ github.workspace }}/vcpkg/
        vcpkgTriplet: x64-windows-static

    # Build with a simplified CMake settings JSON file
    - name: Run CMake
      uses: lukka/run-cmake@v3
      with:
        buildDirectory: ${{ github.workspace }}/build/
        cmakeAppendedArgs: '-G Ninja
          -DCMAKE_BUILD_TYPE="Release"
          -DOPENSSL_USE_STATIC_LIBS="TRUE"
          -DOPENSSL_MSVC_STATIC_RT="TRUE"
          -DCOMPILE_CPU="TRUE"
          -DCOMPILE_CUDA="${{ matrix.cuda }}"
          -DCOMPILE_SERVER="FALSE"
          -DUSE_FBGEMM="${{ matrix.fbgemm }}"
          -DUSE_MPI="FALSE"
          -DUSE_NCCL="FALSE"
          -DUSE_SENTENCEPIECE="TRUE"
          -DUSE_STATIC_LIBS="TRUE"'
        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
        cmakeListsTxtPath: ${{ github.workspace }}/CMakeLists.txt
        useVcpkgToolchainFile: true

    - name: Create archive
      run: |
        cp ../README.md .
        Compress-Archive -Path marian*.exe,README.md -DestinationPath ../${{ env.archive_name }}.zip
      shell: pwsh
      working-directory: build

    # For testing only
    #- name: Test archive
      #run: |
        #Compress-Archive -Path README.md -DestinationPath ${{ env.archive_name }}.zip
      #shell: pwsh

    - name: Upload archive
      uses: actions/upload-artifact@v2
      with:
        name: ${{ env.archive_name }}.zip
        path: ${{ env.archive_name }}.zip

  ######################################################################
  release:
    needs: [build-release-ubuntu, build-release-windows]

    runs-on: ubuntu-18.04
    name: Release

    steps:
    - name: Checkout
      uses: actions/checkout@v2
      with:
        submodules: false  # the repo is needed only for CHANGELOG, so submodules are not needed

    # Extract entire markdown text from the [Unreleased] section to be used as the release description
    - name: Extract changelog
      run: |
        cat CHANGELOG.md | sed -n "/^## \[Unreleased\]/,/^## \[[0-9]/p;/^## \[[0-9]/q" | sed '1d;$d' > RELEASE.md
        cat RELEASE.md
    # Downloads all artifacts to the current working directory
    - name: Download archives
      uses: actions/download-artifact@v2

    # Artifacts are downloaded into directories with the same names as actual files.
    # They are moved to a common directory ./assets/
    - name: Prepare archives
      run: |
        mkdir -p assets
        mv ${{ github.event.repository.name }}-*/* assets/
        rmdir ${{ github.event.repository.name }}-*
        ls -lh assets/
    - name: Release
      uses: softprops/action-gh-release@v1
      if: startsWith(github.ref, 'refs/tags/')
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
        body_path: RELEASE.md
        draft: true
        files: |
          assets/*.zip
          assets/*.tar.gz
        prerelease: false