Enable Linux ARM builds; skip tests for Linux ARM; Use external image… #20
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Wheels | |
| on: | |
| push: | |
| tags: | |
| - '*' | |
| branches: | |
| - develop | |
| - ci/* | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| branches: | |
| - develop | |
| env: | |
| PYTHON_VERSION: "3.12" | |
| jobs: | |
| build-wheels: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, windows-latest, macOS-latest ] | |
| env: | |
| CIBW_SKIP: "*-musllinux*" | |
| CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/jonded94/resiliparse-manylinux_2_28_x86_64 # replace with ${{ github.repository_owner }} | |
| CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/jonded94/resiliparse-manylinux_2_28_aarch64 | |
| CIBW_ARCHS_MACOS: "x86_64 arm64" | |
| CIBW_ARCHS_LINUX: "x86_64 aarch64" | |
| CIBW_TEST_SKIP: "*-macosx_arm64 *-manylinux_aarch64" # Apple Silicon wheels cannot be tested | |
| CIBW_REPAIR_WHEEL_COMMAND_MACOS: >- | |
| DYLD_LIBRARY_PATH=$LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} | |
| CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel" | |
| CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >- | |
| delvewheel repair --add-path C:\vcpkg\installed\x64-windows\bin -w {dest_dir} {wheel} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Cache Vcpkg | |
| uses: actions/cache@v3 | |
| id: cache-vcpkg | |
| if: runner.os == 'macOS' || runner.os == 'Windows' | |
| with: | |
| path: | | |
| /usr/local/share/vcpkg/installed | |
| C:\vcpkg\installed | |
| key: ${{ runner.os }}-vcpkg-14 # INCREMENT ME!! | |
| - name: Install Vcpkg Dependencies | |
| if: (runner.os == 'macOS' || runner.os == 'Windows') && steps.cache-vcpkg.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| set -e | |
| VCPKG_CMD="vcpkg --overlay-ports .vcpkg/ports --overlay-triplets .vcpkg/triplets install" | |
| PKG_LIST="lexbor lz4 re2 uchardet zlib" | |
| if [ -d /usr/local/share/vcpkg/ports ]; then | |
| # MacOS | |
| $VCPKG_CMD --triplet=x64-osx $PKG_LIST | |
| $VCPKG_CMD --triplet=arm64-osx $PKG_LIST | |
| elif [ -d /c/vcpkg ]; then | |
| # Windows | |
| $VCPKG_CMD --triplet=x64-windows $PKG_LIST | |
| else | |
| echo "Unsupported platform." >&2 | |
| exit 1 | |
| fi | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Set up QEMU | |
| if: runner.os == 'Linux' | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: all | |
| - name: Install cibuildwheel | |
| run: python -m pip install cibuildwheel | |
| - name: Build FastWARC | |
| run: python -m cibuildwheel --output-dir wheelhouse fastwarc | |
| env: | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc | |
| - name: Build Resiliparse | |
| run: python -m cibuildwheel --output-dir wheelhouse resiliparse | |
| env: | |
| CIBW_BEFORE_TEST: >- | |
| python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" && | |
| python -m pip install -r fastwarc.txt | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse | |
| - name: Upload Wheels | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: ./wheelhouse/*.whl | |
| build-sdist: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Build Module | |
| run: python -m pip install build | |
| - name: Build FastWARC Source Dist | |
| run: python -m build --sdist --outdir dist fastwarc | |
| - name: Build Resiliparse Source Dist | |
| run: python -m build --sdist --outdir dist resiliparse | |
| - name: Upload Source Dists | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sdist | |
| path: ./dist/*.tar.gz | |
| build-asan: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux2014_x86_64 | |
| env: | |
| DEBUG: "1" | |
| ASAN: "1" | |
| ASAN_OPTIONS: leak_check_at_exit=0 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 # TODO: Upgrade to @v4 causes GLIBC failures | |
| with: | |
| fetch-depth: 0 | |
| - name: Build Extensions | |
| run: | | |
| set -e | |
| python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]" | |
| python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]" | |
| - name: Run Tests | |
| run: | | |
| export LD_PRELOAD="$(ldconfig -p | grep libasan | head -n1 | awk '{print $4}')" | |
| python${PYTHON_VERSION} -m pytest --capture=sys --verbose tests/ | |
| build-coverage: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/jonded94/resiliparse-manylinux_2_28_x86_64 | |
| needs: [ build-wheels, build-asan, build-sdist ] | |
| env: | |
| TRACE: "1" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 # TODO: Upgrade to @v4 causes GLIBC failures | |
| with: | |
| fetch-depth: 0 | |
| - name: Build Extensions | |
| run: | | |
| set -e | |
| python${PYTHON_VERSION} -m pip install cython | |
| python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]" | |
| python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]" | |
| - name: Run Tests | |
| run: python${PYTHON_VERSION} -m pytest --cov=resiliparse/ --cov=fastwarc/ --cov=resiliparse_common/ --cov-report xml tests/ | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage | |
| path: coverage.xml | |
| - name: Upload to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| fail_ci_if_error: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| build-documentation: | |
| runs-on: ubuntu-latest | |
| needs: build-wheels | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| path: wheelhouse | |
| - name: Build Documentation | |
| run: | | |
| set -e | |
| python -m pip install twine | |
| grep -vE "fastwarc|resiliparse" docs/requirements.txt | xargs python -m pip install | |
| PYTHON_ABI="cp${PYTHON_VERSION/./}" | |
| find wheelhouse -name "FastWARC-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]" | |
| find wheelhouse -name "Resiliparse-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]" | |
| cd docs | |
| make html | |
| - name: Trigger Readthedocs Build | |
| if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v') | |
| uses: dfm/rtds-action@v1 | |
| with: | |
| webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }} | |
| webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }} | |
| commit_ref: ${{ github.ref }} | |
| publish-wheels: | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| needs: [ build-wheels, build-sdist, build-documentation ] | |
| steps: | |
| - name: Download Wheels | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| - name: Download Source Dist | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: sdist | |
| - name: Publish to PyPi | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD1: ${{ secrets.PYPI_FASTWARC_API_TOKEN }} | |
| TWINE_PASSWORD2: ${{ secrets.PYPI_RESILIPARSE_API_TOKEN }} | |
| run: | | |
| set -e | |
| python3 -m pip install twine | |
| TWINE_PASSWORD="$TWINE_PASSWORD1" python3 -m twine upload FastWARC-*.whl FastWARC-*.tar.gz | |
| TWINE_PASSWORD="$TWINE_PASSWORD2" python3 -m twine upload Resiliparse-*.whl Resiliparse-*.tar.gz | |
| - name: Wait | |
| run: sleep 30 | |
| - name: Trigger Readthedocs Build | |
| uses: dfm/rtds-action@v1 | |
| with: | |
| webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }} | |
| webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }} | |
| commit_ref: ${{ github.ref }} |