Build wheels using vcpkg for Linux/MacOS

.github/workflows/build_wheel_windows.yml

@@ -3,10 +3,10 @@ name: Build wheels for Windows
 on:
   push:
     branches:
-      - main  # just build the sdist & wheel, skip release
+      - main # just build the sdist & wheel, skip release
     tags:
       - "*"
-  pull_request:  # also build on PRs touching files that affect wheels
+  pull_request: # also build on PRs touching files that affect wheels
     paths:
       - ".github/workflows/build_wheel.yml"
       - "MANIFEST.in"
@@ -46,13 +46,13 @@ jobs:
           path: ./wheelhouse/*.whl
 
   test_wheels:
-    name: windows py${{ matrix.python-version }} tests
+    name: windows python ${{ matrix.python-version }} tests
     runs-on: windows-latest
     needs: build_wheels
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ["3.8", "3.9", "3.10"]
 
     steps:
       - name: Checkout
@@ -64,8 +64,8 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: 'ci/requirements-wheel-test.txt'
+          cache: "pip"
+          cache-dependency-path: "ci/requirements-wheel-test.txt"
 
       - name: Download wheels from artifacts
         uses: actions/download-artifact@v2
@@ -89,4 +89,4 @@ jobs:
         shell: bash
         run: |
           cd ..
-          python -m pytest --pyargs pyogrio
+          python -m pytest --pyargs pyogrio.tests -v

.github/workflows/build_wheels_linux.yml

@@ -0,0 +1,114 @@
+name: Build Linux wheels
+
+on:
+  push:
+    branches:
+      - main # just build the sdist & wheel, skip release
+    tags:
+      - "*"
+  pull_request: # also build on PRs touching this file
+    paths:
+      - ".github/workflows/build_wheels_linux.yml"
+      - "ci/*"
+      - "MANIFEST.in"
+      - "pyproject.toml"
+      - "setup.py"
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }} ${{ matrix.platform }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-20.04"]
+        platform: ["x86_64"]
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+
+      # - name: Build Docker image with vcpkg and gdal
+      #   run: |
+      #     docker build -t manylinux-vcpkg-gdal -f ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile .
+
+      - name: Build Docker image with vcpkg and gdal
+        # using build-push-action (without push) to make use of cache arguments
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile
+          tags: manylinux-vcpkg-gdal:latest
+          push: false
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - run: |
+          docker images
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl
+
+  test-wheel:
+    name: Test wheels on ${{ matrix.os }} ${{ matrix.platform }}
+    needs: build_wheels
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-20.04"]
+        platform: ["x86_64"]
+        python-version: ["3.8", "3.9", "3.10"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Download wheels from artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: wheelhouse
+
+      - name: Set version string
+        shell: bash
+        run: echo "py_version_str=cp$(echo ${{ matrix.python-version }} | tr -d -c 0-9)" >> $GITHUB_ENV
+
+      - name: Install package from wheel
+        shell: bash
+        run: |
+          ls -R wheelhouse
+          python -m pip install --upgrade pip
+          python -m pip install wheelhouse/artifact/pyogrio-*${{ env.py_version_str }}*.whl
+
+      - name: Run tests
+        # TODO temporary set env variable for curl certificate
+        env:
+          CURL_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
+        run: |
+          python -m pip install pytest pandas pyproj shapely packaging
+          python -m pip install --no-deps geopandas
+          python -m pip list
+          pwd
+          ls
+          cd ..
+          python -m pytest --pyargs pyogrio.tests

.github/workflows/build_wheels_macos.yml

@@ -0,0 +1,114 @@
+name: Build MacOS wheels
+
+on:
+  push:
+    branches:
+      - main # just build the sdist & wheel, skip release
+    tags:
+      - "*"
+  pull_request: # also build on PRs touching this file
+    paths:
+      - ".github/workflows/build_wheels_macos.yml"
+      - "ci/*"
+      - "MANIFEST.in"
+      - "pyproject.toml"
+      - "setup.py"
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }} ${{ matrix.platform }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["macos-10.15"]
+        platform: ["x86_64"]
+
+    env:
+      MACOSX_DEPLOYMENT_TARGET: "10.15"
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Cache vcpkg
+        uses: actions/cache@v1
+        id: vcpkgcache
+        with:
+          path: "/Users/runner/.cache/vcpkg/archives"
+          key: ${{ matrix.os }}-${{ matrix.platform }}-vcpkg-gdal3.4.1-dynamic
+
+      # MacOS build requires aclocal, which is part of automake, but appears
+      # to be missing in default image
+      - name: Reinstall automake
+        if: ${{ startsWith(matrix.os, 'macos') }}
+        run: |
+          brew reinstall automake
+          echo $(which aclocal)
+
+      - name: Install GDAL
+        run: |
+          vcpkg install gdal[core]:x64-osx-dynamic --overlay-triplets=./ci/custom-triplets
+
+      - name: Upload build logs
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          path: /usr/local/share/vcpkg/buildtrees/gdal/*.log
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          PYOGRIO_PACKAGE_DATA: 1
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl
+
+  test-wheel:
+    name: Test wheels on ${{ matrix.os }} ${{ matrix.platform }}
+    needs: build_wheels
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["macos-latest", "macos-10.15"]
+        platform: ["x86_64"]
+        python-version: ["3.8", "3.9"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Download wheels from artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: wheelhouse
+
+      - name: Set version string
+        shell: bash
+        run: echo "py_version_str=cp$(echo ${{ matrix.python-version }} | tr -d -c 0-9)" >> $GITHUB_ENV
+
+      - name: Install package from wheel
+        shell: bash
+        run: |
+          ls -R wheelhouse
+          python -m pip install --upgrade pip
+          python -m pip install wheelhouse/artifact/pyogrio-*${{ env.py_version_str }}*.whl
+
+      - name: Run tests
+        run: |
+          python -m pip install pytest pandas pyproj shapely packaging
+          python -m pip install --no-deps geopandas
+          python -m pip list
+          cd ..
+          python -m pytest --pyargs pyogrio.tests -v

MANIFEST.in

@@ -2,4 +2,7 @@ include versioneer.py
 include pyogrio/_version.py
 include pyogrio/*.pyx pyogrio/*.pxd
 exclude pyogrio/*.c
+include LICENSE
+include pyproject.toml
+exclude MANIFEST.in
 recursive-include pyogrio/tests/fixtures *

ci/custom-triplets/x64-linux-dynamic.cmake

@@ -0,0 +1,5 @@
+set(VCPKG_TARGET_ARCHITECTURE x64)
+set(VCPKG_CRT_LINKAGE dynamic)
+set(VCPKG_LIBRARY_LINKAGE dynamic)
+set(VCPKG_CMAKE_SYSTEM_NAME Linux)
+set(VCPKG_BUILD_TYPE release)

ci/custom-triplets/x64-osx-dynamic.cmake

@@ -0,0 +1,5 @@
+set(VCPKG_TARGET_ARCHITECTURE x64)
+set(VCPKG_CRT_LINKAGE dynamic)
+set(VCPKG_LIBRARY_LINKAGE dynamic)
+set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
+set(VCPKG_BUILD_TYPE release)

ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile

@@ -0,0 +1,17 @@
+FROM quay.io/pypa/manylinux2014_x86_64
+
+RUN yum install -y curl unzip zip tar python3
+
+RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg
+
+ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg"
+ENV PATH="${PATH}:/opt/vcpkg"
+
+RUN bootstrap-vcpkg.sh && \
+    vcpkg integrate install && \
+    vcpkg integrate bash
+
+COPY ci/custom-triplets/x64-linux-dynamic.cmake opt/vcpkg/custom-triplets/x64-linux-dynamic.cmake
+
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/x64-linux-dynamic/lib"
+RUN vcpkg install gdal[core]:x64-linux-dynamic --overlay-triplets=opt/vcpkg/custom-triplets

pyogrio/tests/conftest.py

@@ -18,7 +18,7 @@ def pytest_report_header(config):
     )
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def data_dir():
     return _data_dir
 
@@ -28,7 +28,7 @@ def naturalearth_lowres():
     return _data_dir / Path("naturalearth_lowres/naturalearth_lowres.shp")
 
 
-@pytest.fixture
+@pytest.fixture(scope="function")
 def naturalearth_lowres_vsi(tmp_path, naturalearth_lowres):
     """Wrap naturalearth_lowres as a zip file for vsi tests"""
 

pyproject.toml

@@ -8,7 +8,35 @@ requires = [
 ]
 
 [tool.cibuildwheel]
-skip = ["cp36-*", "cp37-*", "pp*"]
+skip = ["cp36-*", "cp37-*", "pp*", "*musllinux*"]
+archs = ["auto64"]
+manylinux-x86_64-image = "manylinux-vcpkg-gdal:latest"
+build-verbosity = 3
+
+[tool.cibuildwheel.linux]
+# explicitly mention 2_27 for now because it fails for manylinux2014 (2_17)
+repair-wheel-command = "auditwheel repair --plat manylinux_2_27_x86_64 -w {dest_dir} {wheel}"
+
+[tool.cibuildwheel.linux.environment]
+VCPKG_INSTALL = "$VCPKG_INSTALLATION_ROOT/installed/x64-linux-dynamic"
+GDAL_INCLUDE_PATH = "$VCPKG_INSTALL/include"
+GDAL_LIBRARY_PATH = "$VCPKG_INSTALL/lib"
+PYOGRIO_PACKAGE_DATA = 1
+GDAL_DATA = "$VCPKG_INSTALL/share/gdal/gdal"
+PROJ_LIB = "$VCPKG_INSTALL/share/proj/data"
+
+[tool.cibuildwheel.macos]
+repair-wheel-command = [
+    "DYLD_LIBRARY_PATH=$GDAL_LIBRARY_PATH delocate-listdeps {wheel}",
+    "DYLD_LIBRARY_PATH=$GDAL_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}"
+]
+
+[tool.cibuildwheel.macos.environment]
+VCPKG_INSTALL = "$VCPKG_INSTALLATION_ROOT/installed/x64-osx-dynamic"
+GDAL_INCLUDE_PATH = "$VCPKG_INSTALL/include"
+GDAL_LIBRARY_PATH = "$VCPKG_INSTALL/lib"
+GDAL_DATA = "$VCPKG_INSTALL/share/gdal/gdal"
+PROJ_LIB = "$VCPKG_INSTALL/share/proj4"
 
 [tool.cibuildwheel.windows]
 archs = ["auto64"]
@@ -23,3 +51,6 @@ GDAL_LIBRARY_PATH = "$VCPKG_INSTALL/lib"
 PYOGRIO_PACKAGE_DATA = 1
 GDAL_DATA = "$VCPKG_INSTALL/share/gdal"
 PROJ_LIB = "$VCPKG_INSTALL/share/proj4"
+
+[tool.pytest.ini_options]
+testpaths = ["pyogrio/tests"]

setup.py

@@ -78,9 +78,7 @@ def get_gdal_paths():
         if not gdal_version >= MIN_GDAL_VERSION:
             sys.exit("GDAL must be >= 2.4.x")
 
-        include_dirs = [
-            entry[2:] for entry in config["cflags"].split(" ")
-        ]
+        include_dirs = [entry[2:] for entry in config["cflags"].split(" ")]
         library_dirs = []
         libraries = []
         extra_link_args = []
@@ -92,7 +90,7 @@ def get_gdal_paths():
                 libraries.append(entry[2:])
             else:
                 extra_link_args.append(entry)
-        
+
         return {
             "include_dirs": include_dirs,
             "library_dirs": library_dirs,