Refactor cibuildwheel setup

.github/workflows/build.yml

@@ -131,18 +131,6 @@ jobs:
   wheels:
     name: Build wheel on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    env:
-      CIBW_SKIP: "pp* *-win32 cp312-*"  # remove once numpy supports Python 3.12
-      CIBW_TEST_REQUIRES: pytest numpy
-      CIBW_TEST_COMMAND: "pytest -v {project}/tests"
-      # we are copying the shared libraries ourselves so skip magical copy
-      CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
-      MACOSX_DEPLOYMENT_TARGET: 10.9
-      CIBW_BUILD_VERBOSITY_MACOS: 3
-      CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-musllinux* *i686"
-      CIBW_ARCHS_MACOS: "x86_64 arm64"
-      CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: ""
-      CIBW_BEFORE_ALL_LINUX: "pip install cmake; bash {project}/ci/install_libspatialindex.bash"
     strategy:
       matrix:
         os: [windows-latest, ubuntu-latest, macos-latest]
@@ -173,39 +161,11 @@ jobs:
         name: ${{ matrix.os }}-whl
         path: wheelhouse/*.whl
 
-  wheels_aarch64:
-    name: Build Linux wheels on aarch64
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      name: Install Python
-      with:
-        python-version: '3.11'
-    - uses: docker/setup-qemu-action@v2
-      name: Set up QEMU
-      with:
-        platforms: arm64
-    - name: Build wheels
-      uses: pypa/[email protected]
-      env:
-        CIBW_SKIP: pp*
-        CIBW_ARCHS_LINUX: aarch64
-        CIBW_TEST_REQUIRES: pytest numpy
-        CIBW_TEST_COMMAND: "pytest -v {project}/tests"
-        CIBW_TEST_SKIP: "*-musllinux*"
-        CIBW_BEFORE_ALL_LINUX: "pip install cmake; bash {project}/ci/install_libspatialindex.bash"
-    - uses: actions/upload-artifact@v3
-      with:
-        name: aarch64-whl
-        path: wheelhouse/*.whl
-
   collect-artifacts:
     name: Package and push release
 
     #needs: [windows-wheel, linux-wheel, osx-wheel, conda, ubuntu]
-    needs: [conda, ubuntu, wheels, wheels_aarch64]
+    needs: [conda, ubuntu, wheels]
 
     runs-on: 'ubuntu-latest'
     strategy:

.gitignore

@@ -8,3 +8,5 @@ dist/
 include
 lib
 .coverage
+.tox
+wheelhouse

pyproject.toml

@@ -50,6 +50,33 @@ target-version = ["py38", "py39", "py310", "py311"]
 color = true
 skip_magic_trailing_comma = true
 
+[tool.cibuildwheel]
+build = "cp39-*"
+build-verbosity = "3"
+repair-wheel-command = "python scripts/repair_wheel.py -w {dest_dir} {wheel}"
+test-requires = "tox"
+test-command = "tox --conf {project} --installpkg {wheel}"
+
+[tool.cibuildwheel.linux]
+archs = ["auto", "aarch64"]
+# test-skip = "*-musllinux* *i686"
+before-all = [
+    "yum install -y cmake libffi-devel",
+    "bash {project}/ci/install_libspatialindex.bash",
+]
+
+[[tool.cibuildwheel.overrides]]
+select = "*-musllinux*"
+before-all = [
+    "apk add cmake libffi-dev",
+    "bash {project}/ci/install_libspatialindex.bash",
+]
+
+[tool.cibuildwheel.macos]
+archs = ["x86_64", "arm64"]
+test-skip = "*-macosx_arm64 *-macosx_universal2:arm64"
+environment = { MACOSX_DEPLOYMENT_TARGET="10.9" }
+
 [tool.coverage.report]
 # Ignore warnings for overloads
 # https://github.com/nedbat/coveragepy/issues/970#issuecomment-612602180

rtree/finder.py

@@ -73,9 +73,21 @@ def load() -> ctypes.CDLL:
             # macos shared libraries are `.dylib`
             lib_name = "libspatialindex_c.dylib"
         else:
+            import importlib.metadata
+
             # linux shared libraries are `.so`
             lib_name = "libspatialindex_c.so"
 
+            # add path for binary wheel prepared with cibuildwheel/auditwheel
+            for file in importlib.metadata.files("rtree"):
+                if (
+                    file.parent.name == "Rtree.libs"
+                    and file.stem.startswith("libspatialindex")
+                    and ".so" in file.suffixes
+                ):
+                    _candidates.insert(1, os.path.join(str(file.locate())))
+                    break
+
         # get the starting working directory
         cwd = os.getcwd()
         for cand in _candidates:

scripts/repair_wheel.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+
+import argparse
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def main():
+    if sys.platform.startswith("linux"):
+        os_ = "linux"
+    elif sys.platform.startswith("darwin"):
+        os_ = "macos"
+    elif sys.platform.startswith("win32"):
+        os_ = "windows"
+    else:
+        raise NotImplementedError(
+            "sys.platform '{}' is not supported yet.".format(sys.platform)
+        )
+
+    p = argparse.ArgumentParser(
+        description="Convert wheel to be independent of python implementation and ABI"
+    )
+    p.set_defaults(prog=Path(sys.argv[0]).name)
+    p.add_argument("WHEEL_FILE", help="Path to wheel file.")
+    p.add_argument(
+        "-w",
+        "--wheel-dir",
+        dest="WHEEL_DIR",
+        help=('Directory to store delocated wheels (default: "wheelhouse/")'),
+        default="wheelhouse/",
+    )
+
+    args = p.parse_args()
+
+    file = Path(args.WHEEL_FILE).resolve(strict=True)
+    wheelhouse = Path(args.WHEEL_DIR).resolve()
+    wheelhouse.mkdir(parents=True, exist_ok=True)
+
+    with tempfile.TemporaryDirectory() as tmpdir_:
+        tmpdir = Path(tmpdir_)
+        # use the platform specific repair tool first
+        if os_ == "linux":
+            subprocess.run(
+                ["auditwheel", "repair", "-w", str(tmpdir), str(file)], check=True
+            )
+        elif os_ == "macos":
+            subprocess.run(
+                [
+                    "delocate-wheel",
+                    "--require-archs",
+                    "arm64,x86_64",
+                    "-w",
+                    str(tmpdir),
+                    str(file),
+                ],
+                check=True,
+            )
+        elif os_ == "windows":
+            # no specific tool, just copy
+            shutil.copyfile(file, tmpdir / file.name)
+        (file,) = tmpdir.glob("*.whl")
+
+        # we need to handle macOS universal2 & arm64 here for now, let's use platform_tag_args for this.
+        platform_tag_args = []
+        if os_ == "macos":
+            additional_platforms = []
+
+            # first, get the target macOS deployment target from the wheel
+            match = re.match(r"^.*-macosx_(\d+)_(\d+)_x86_64\.whl$", file.name)
+            assert match is not None
+            target = tuple(map(int, match.groups()))
+
+            # let's add universal2 platform for this wheel.
+            additional_platforms = ["macosx_{}_{}_universal2".format(*target)]
+
+            # given pip support for universal2 was added after arm64 introduction
+            # let's also add arm64 platform.
+            arm64_target = target
+            if arm64_target < (11, 0):
+                arm64_target = (11, 0)
+            additional_platforms.append("macosx_{}_{}_arm64".format(*arm64_target))
+
+            if target < (11, 0):
+                # They're were also issues with pip not picking up some universal2 wheels, tag twice
+                additional_platforms.append("macosx_11_0_universal2")
+
+            platform_tag_args = [f"--platform-tag=+{'.'.join(additional_platforms)}"]
+
+        # make this a py3 wheel
+        subprocess.run(
+            [
+                "wheel",
+                "tags",
+                "--python-tag",
+                "py3",
+                "--abi-tag",
+                "none",
+                *platform_tag_args,
+                "--remove",
+                str(file),
+            ],
+            check=True,
+        )
+        (file,) = tmpdir.glob("*.whl")
+        # unpack
+        proc = subprocess.run(["wheel", "unpack", file.name], cwd=tmpdir, check=True)
+        for unpackdir in tmpdir.iterdir():
+            if unpackdir.is_dir():
+                break
+        else:
+            raise RuntimeError("subdirectory not found")
+        if os_ == "linux":
+            # remove duplicated dir
+            assert len(list((unpackdir / "Rtree.libs").glob("*.so*"))) >= 1
+            lib_dir = unpackdir / "rtree" / "lib"
+            shutil.rmtree(lib_dir)
+        # re-pack
+        subprocess.run(["wheel", "pack", str(unpackdir.name)], cwd=tmpdir, check=True)
+        files = list(tmpdir.glob("*.whl"))
+        assert len(files) == 1, files
+        file = files[0]
+        file.rename(wheelhouse / file.name)
+
+
+if __name__ == "__main__":
+    main()

tox.ini

@@ -0,0 +1,12 @@
+[tox]
+requires =
+    tox>=4
+env_list = type, py{38,39,310,311}
+
+[testenv]
+description = run unit tests
+deps =
+    pytest>=7
+    numpy
+commands =
+    pytest {posargs:tests}