python.yml

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

name: Python

on:
  push:
  pull_request:
  schedule:
    # Run daily at 00:00 so we get notified if CI is broken before a pull request is submitted. 
    # It also notifies us about new Arrow releases for which we need to release a corresponding version of PalletJack.
    - cron:  '0 0 * * *'

permissions:
  contents: read

jobs:
        
  test:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python 3.10
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"
        
    - name: Install dependencies
      working-directory: ./python
      shell: bash
      run: |
        sudo apt-get install -y libthrift-dev libboost-all-dev
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        pip install flake8 pytest

    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

    - name: Test with pytest
      working-directory: ./python
      run: |
        pip install --editable . 
        pytest

  build:
    needs: test
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    runs-on: ${{ matrix.runner }}
    strategy:
      matrix:
        include:
          - runner: ubuntu-latest
            arch: x64
          - runner: windows-latest
            arch: x64
          - runner: ubuntu-22.04-arm64
            arch: arm64
      fail-fast: false

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python 3.10
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"
        
    - name: Install dependencies
      working-directory: ./python
      shell: bash
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        pip install flake8 pytest cibuildwheel build

    # Compute vcpkg triplet and root
    - name: Compute vcpkg triplet and root
      id: vcpkg-info
      run: |
        triplet="${{ matrix.arch }}-"
        case ${{ runner.os }} in
          Linux)
            triplet+="linux"
            ;;
          macOS)
            triplet+="osx"
            ;;
          Windows)
            triplet+="windows"
            ;;
        esac
        echo "triplet=$triplet" >> $GITHUB_OUTPUT
        echo "root=$VCPKG_INSTALLATION_ROOT" >> $GITHUB_OUTPUT
        echo "cmake_version=$(cmake --version | head -n1 | awk '{print $3}')" >> $GITHUB_OUTPUT
        echo "runner_info=$ImageOS-$ImageVersion" >> $GITHUB_OUTPUT
      shell: bash
      
    - uses: actions/github-script@v7
      with:
        script: |
          core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
          core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
      
    # Install vcpkg dependencies
    - name: Install vcpkg build dependencies (macOS)
      if: runner.os == 'macOS'
      run: brew install bison

    - name: Install thrift
      working-directory: ./python
      run: |
        vcpkg install --triplet ${{ steps.vcpkg-info.outputs.triplet }} --x-manifest-root . --feature-flags=versions
      env:
        VCPKG_BINARY_SOURCES: clear;x-gha,readwrite

    - name: Upload vcpkg arrow logs
      if: success() || failure()
      uses: actions/upload-artifact@v4
      with:
        name: ${{ steps.vcpkg-info.outputs.triplet }}-vcpkg-arrow-logs
        path: ${{ steps.vcpkg-info.outputs.root }}/buildtrees/arrow/*.log

    - name: Build sdist (Linux)
      if: runner.os == 'linux'
      working-directory: ./python
      run: |
        python -m build --sdist         
        
    - name: Build wheels
      working-directory: ./python
      run: python -m cibuildwheel --output-dir dist
      # to supply options, put them in 'env', like:
      env:
        CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude libarrow.so.1801 --exclude libparquet.so.1801 -w {dest_dir} {wheel}
        CIBW_ENVIRONMENT: VCPKG_TARGET_TRIPLET="${{ steps.vcpkg-info.outputs.triplet }}"
        CIBW_BUILD_VERBOSITY: 1
        # We use manylinux_2_28 for ABI compatibility with pyarrow
        # With the default image we were getting "undefined symbol: _ZNK5arrow6Status8ToStringEv" error (e.g https://github.com/ray-project/ray/issues/24566) 
        CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 
        CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
        # Disable unsupported builds
        CIBW_SKIP: "pp* *_i686 *-musllinux_* *win32 cp313-*"

    - uses: actions/upload-artifact@v4
      with:
        name: dist-${{ matrix.runner }}
        path: ./python/dist/*

  test-binary:
    needs: build
    runs-on: ${{ matrix.runner }}
    strategy:
      matrix:
        include:
          - runner: ubuntu-latest
            arch: x64
          - runner: windows-latest
            arch: x64
          - runner: ubuntu-22.04-arm64
            arch: arm64
      fail-fast: false

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python 3.10
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"

    - name: Download artifacts
      uses: actions/download-artifact@v4
      with:
        pattern: dist-*
        path: dist
        merge-multiple: true
          
    - name: Test with pytest
      run: |
        pip install -r python/requirements.txt
        # Keep in mind that if the local and remote versions are the same, the remote version will be installed
        pip install PalletJack --pre --find-links ./dist --break-system-packages --only-binary=:all:
        # So now ensure that the local version is installed 
        pip install PalletJack --pre --find-links ./dist --break-system-packages --only-binary=:all: --force-reinstall --no-index --no-deps
        python3 python/test/test_palletjack.py

  # Virtual job that can be configured as a required check before a PR can be merged.
  # As GitHub considers a check as successful if it is skipped, we need to check its status in
  # another workflow (check-required.yml) and create a check there.
  all-required-checks-done:
    name: All required checks done
    needs:
      - test
      - build
      - test-binary
    runs-on: ubuntu-latest
    steps:
      - run: echo "All required checks done"

  benchmarks:
    needs: build
    runs-on: ${{ matrix.runner }}
    strategy:
      matrix:
        include:
          - runner: ubuntu-latest
            arch: x64
          - runner: windows-latest
            arch: x64
          - runner: ubuntu-22.04-arm64
            arch: arm64
      fail-fast: false

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python 3.10
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"

    - name: Download artifacts
      uses: actions/download-artifact@v4
      with:
        pattern: dist-*
        path: dist
        merge-multiple: true

    - name: Run benchmarks
      run: |
        pip install -r python/requirements.txt
        # Keep in mind that if the local and remote versions are the same, the remote version will be installed
        pip install PalletJack --pre --find-links ./dist --break-system-packages --only-binary=:all:
        # So now ensure that the local version is installed 
        pip install PalletJack --pre --find-links ./dist --break-system-packages --only-binary=:all: --force-reinstall --no-index --no-deps
        python3 ./benchmarks/benchmark_palletjack_metadata.py

  publish:
    if: ${{ !github.event.repository.fork && startsWith(github.ref, 'refs/tags/v') }}
    needs: [all-required-checks-done]
    runs-on: ubuntu-latest
    environment: release
    permissions:
      id-token: write

    steps:

    - name: Download artifacts
      uses: actions/download-artifact@v4
      with:
        pattern: dist-*
        path: dist
        merge-multiple: true
    
    - name: Display structure of downloaded files
      run: ls -R dist
    
    - name: Validate tag
      shell: pwsh
      run: |
        $tag = "${{ github.ref }}".SubString(11)
        $expectedFile = "dist/palletjack-$tag.tar.gz"
    
        # Check whether the tag and the package version match together
        if (-not (Test-Path -Path $expectedFile)) {
            echo "::error ::Expected file $expectedFile doesn't exist"
            Get-ChildItem -Path dist
            exit 1
        }
    
    - name: Publish to PyPi
      uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70
      with:
        packages-dir: dist