[CI] Fix the problem with building thrift #178
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a single version of Python | |
| # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python | |
| name: Python | |
| on: | |
| push: | |
| pull_request: | |
| permissions: | |
| contents: read | |
| jobs: | |
| test: | |
| if: github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| working-directory: ./python | |
| shell: bash | |
| run: | | |
| sudo apt-get install -y libthrift-dev libboost-all-dev | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install flake8 pytest cibuildwheel build pytest | |
| - name: Lint with flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Test with pytest | |
| working-directory: ./python | |
| run: | | |
| pip install --editable . | |
| pytest | |
| build: | |
| needs: test | |
| if: github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| working-directory: ./python | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install flake8 pytest cibuildwheel build | |
| # Compute vcpkg triplet and root | |
| - name: Compute vcpkg triplet and root | |
| id: vcpkg-info | |
| run: | | |
| triplet="x64-" | |
| case ${{ runner.os }} in | |
| Linux) | |
| triplet+="linux" | |
| ;; | |
| macOS) | |
| triplet+="osx" | |
| ;; | |
| Windows) | |
| triplet+="windows" | |
| ;; | |
| esac | |
| echo "triplet=$triplet" >> $GITHUB_OUTPUT | |
| echo "root=$VCPKG_INSTALLATION_ROOT" >> $GITHUB_OUTPUT | |
| echo "cmake_version=$(cmake --version | head -n1 | awk '{print $3}')" >> $GITHUB_OUTPUT | |
| echo "runner_info=$ImageOS-$ImageVersion" >> $GITHUB_OUTPUT | |
| shell: bash | |
| # Check for cached vcpkg dependencies (use these if we can). | |
| - name: Get cached vcpkg dependencies | |
| id: get-cached-vcpkg | |
| uses: actions/cache@v3 | |
| with: | |
| path: cache/vcpkg | |
| # https://vcpkg.readthedocs.io/en/stable/users/binarycaching/ | |
| # Binary caching relies on hashing everything that contributes to a particular package build. This includes: | |
| # - The triplet file and name | |
| # - The C and C++ compilers executable | |
| # - The version of CMake used | |
| # - Every file in the port directory | |
| # - & more... (subject to change without notice) | |
| # | |
| # We use Vcpkg and C/C++ compilers which are preinstalled on the runner, so we include the runner's image identity as part of the hash key. | |
| key: vcpkg-${{ steps.vcpkg-info.outputs.triplet }}-cmake:${{ steps.vcpkg-info.outputs.cmake_version }}-vcpkg_json:${{ hashFiles('**/vcpkg*.json') }}-runner:${{ steps.vcpkg-info.outputs.runner_info }} | |
| restore-keys: | | |
| vcpkg-${{ steps.vcpkg-info.outputs.triplet }}-cmake:${{ steps.vcpkg-info.outputs.cmake_version }}-vcpkg_json:${{ hashFiles('**/vcpkg*.json') }} | |
| vcpkg-${{ steps.vcpkg-info.outputs.triplet }}-cmake:${{ steps.vcpkg-info.outputs.cmake_version }} | |
| vcpkg-${{ steps.vcpkg-info.outputs.triplet }} | |
| # Install vcpkg dependencies | |
| - name: Install vcpkg build dependencies (macOS) | |
| if: runner.os == 'macOS' | |
| run: brew install bison | |
| - name: Install thrift | |
| working-directory: ./python | |
| run: | | |
| vcpkg install --triplet ${{ steps.vcpkg-info.outputs.triplet }} --x-manifest-root . --feature-flags=versions | |
| env: | |
| VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/cache/vcpkg,readwrite | |
| VCPKG_TARGET_TRIPLET: ${{ steps.vcpkg-info.outputs.triplet }} | |
| - name: Build sdist (Linux) | |
| if: runner.os == 'linux' | |
| working-directory: ./python | |
| run: | | |
| python -m build --sdist | |
| - name: Build wheels | |
| working-directory: ./python | |
| run: python -m cibuildwheel --output-dir dist | |
| # to supply options, put them in 'env', like: | |
| env: | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude libarrow.so.1601 --exclude libparquet.so.1601 -w {dest_dir} {wheel} | |
| CIBW_ENVIRONMENT: VCPKG_TARGET_TRIPLET="${{ steps.vcpkg-info.outputs.triplet }}" | |
| CIBW_BUILD_VERBOSITY: 1 | |
| # We use manylinux_2_28 for ABI compatibility with pyarrow | |
| # With the default image we were getting "undefined symbol: _ZNK5arrow6Status8ToStringEv" error (e.g https://github.com/ray-project/ray/issues/24566) | |
| CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 | |
| # Disable unsupported builds | |
| CIBW_SKIP: "pp* *_i686 *-musllinux_* *win32" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: dist-${{ matrix.os }} | |
| path: ./python/dist/* | |
| test-binary: | |
| needs: build | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: dist-* | |
| path: dist | |
| merge-multiple: true | |
| - name: Test with pytest | |
| run: | | |
| # Keep in mind that if the local and remote versions are the same, the remote version will be installed | |
| pip install PalletJack --pre --find-links ./dist --break-system-packages --only-binary=:all: | |
| # So now ensure that the local version is installed | |
| pip install PalletJack --pre --find-links ./dist --break-system-packages --only-binary=:all: --force-reinstall --no-index --no-deps | |
| python3 python/test/test_palletjack.py | |
| # Virtual job that can be configured as a required check before a PR can be merged. | |
| # As GitHub considers a check as successful if it is skipped, we need to check its status in | |
| # another workflow (check-required.yml) and create a check there. | |
| all-required-checks-done: | |
| name: All required checks done | |
| needs: | |
| - test | |
| - build | |
| - test-binary | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "All required checks done" | |
| benchmarks: | |
| needs: build | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: dist-* | |
| path: dist | |
| merge-multiple: true | |
| - name: Run benchmarks | |
| run: | | |
| # Keep in mind that if the local and remote versions are the same, the remote version will be installed | |
| pip install PalletJack --pre --find-links ./dist --break-system-packages --only-binary=:all: | |
| # So now ensure that the local version is installed | |
| pip install PalletJack --pre --find-links ./dist --break-system-packages --only-binary=:all: --force-reinstall --no-index --no-deps | |
| python3 ./benchmarks/benchmark_palletjack_metadata.py | |
| publish: | |
| if: ${{ !github.event.repository.fork && startsWith(github.ref, 'refs/tags/v') }} | |
| needs: [all-required-checks-done] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: dist-* | |
| path: dist | |
| merge-multiple: true | |
| - name: Display structure of downloaded files | |
| run: ls -R dist | |
| - name: Validate tag | |
| shell: pwsh | |
| run: | | |
| $tag = "${{ github.ref }}".SubString(11) | |
| $expectedFile = "dist/palletjack-$tag.tar.gz" | |
| # Check whether the tag and the package version match together | |
| if (-not (Test-Path -Path $expectedFile)) { | |
| echo "::error ::Expected file $expectedFile doesn't exist" | |
| Get-ChildItem -Path dist | |
| exit 1 | |
| } | |
| - name: Publish to PyPi | |
| uses: pypa/gh-action-pypi-publish@c12cc61414480c03e10ea76e2a0a1a17d6c764e2 | |
| with: | |
| password: ${{ secrets.PYPI_API_TOKEN }} | |
| packages-dir: dist | |