Add Cython build infra + initial extension module

.github/workflows/package.yml

@@ -6,11 +6,14 @@ on:
       - main
   release:
     types: [published]
+  pull_request:
 
 jobs:
-  dist:
-    name: Build Packages
+  build_sdist:
+    name: Build Source Package
     runs-on: ubuntu-latest
+    outputs:
+      filename: ${{steps.build.outputs.filename}}
 
     steps:
       - uses: actions/checkout@v4
@@ -28,22 +31,52 @@ jobs:
       - name: Install Python deps
         run: pip install -U build
 
-      - name: Build distribution
-        run: python -m build
+      - name: Build source distribution
+        id: build
+        run: |
+          python -m build -s
+          basename dist/*.tar.gz |sed -e 's/^/filename=/' >> "$GITHUB_OUTPUT"
+          cat "$GITHUB_OUTPUT"
 
       - name: Save archive
         uses: actions/upload-artifact@v4
         with:
-          name: pypi-pkgs
+          name: dist-src
           path: dist
 
       - name: List dist dir
         run: ls -R dist
 
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs: [build_sdist]
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, windows-2019, macos-11]
+
+    steps:
+      - name: Fetch compiled package distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: dist-src
+          path: dist
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        with:
+          package-dir: dist/${{needs.build_sdist.outputs.filename}}
+          output-dir: dist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist-wheels-${{ matrix.os }}
+          path: ./dist/*.whl
+
   pypi-publish:
     name: Publish to PyPI
     runs-on: ubuntu-latest
-    needs: [dist]
+    needs: [build_sdist, build_wheels]
     if: github.event_name == 'release'
 
     environment: release
@@ -54,8 +87,9 @@ jobs:
       - name: Fetch compiled package distributions
         uses: actions/download-artifact@v4
         with:
-          name: pypi-pkgs
+          pattern: dist-*
           path: dist
+          merge-multiple: true
 
       - name: Publish package distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/test.yml

@@ -49,6 +49,10 @@ jobs:
           python -V
           numba -s
 
+      - name: Compile extension modules
+        run: |
+          just build-inplace
+
       - name: Test LKPY
         run: |
           python -m pytest --cov=lenskit --verbose --log-file=test.log
@@ -80,15 +84,11 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Create Conda environment file
-        run: |
-          pipx run pyproject2conda yaml -p 3.10 -e test -e demo -e dev -o environment.yml
-
       - name: 👢 Set up Conda environment
         uses: mamba-org/setup-micromamba@v1
         id: setup
         with:
-          environment-file: environment.yml
+          environment-file: envs/lenskit-py3.11-dev.yaml
           environment-name: lkpy
           cache-environment: true
           init-shell: bash
@@ -108,6 +108,10 @@ jobs:
             !data/*.zip
           key: test-mldata-000
 
+      - name: Compile extension modules
+        run: |
+          just build-inplace
+
       - name: Download ML data
         run: |
           python -m lenskit.datasets.fetch ml-100k
@@ -137,6 +141,40 @@ jobs:
         with:
           artifact-name: test-check-docs
 
+  cython-cover:
+    name: Measure Cython Coverage
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: 👢 Set up Conda environment
+        uses: mamba-org/setup-micromamba@v1
+        id: setup
+        with:
+          environment-file: envs/lenskit-py3.11-dev.yaml
+          environment-name: lkpy
+          cache-environment: true
+          init-shell: bash
+
+      - name: Compile extension modules
+        run: |
+          just build-inplace
+        env:
+          BUILD_FOR_COVER: 1
+
+      - name: Run Eval Tests
+        run: |
+          python -m pytest --cov=lenskit -m 'not slow' --log-file test-cython-cover.log
+
+      - name: Process test results
+        uses: lenskit/lkbuild/actions/save-test-results@main
+        with:
+          artifact-name: test-cython-cover
+
   vanilla:
     name: Vanilla Python ${{matrix.python}} on ${{matrix.platform}}
     runs-on: ${{matrix.platform}}
@@ -237,6 +275,7 @@ jobs:
     - vanilla
     - check-docs
     - mindep
+    - cython-cover
 
     steps:
       - name: Check out source

envs/lenskit-py3.10-ci.yaml

@@ -12,10 +12,12 @@ channels:
 dependencies:
   - python=3.10
   - binpickle>=0.3.2
+  - c-compiler
   - cffi>=1.15.0
   - copier==9.*
   - coverage>=5
   - csr>=0.5
+  - cython==3.*
   - docopt>=0.6
   - hypothesis>=6
   - ipython>=7
@@ -24,6 +26,7 @@ dependencies:
   - numpy>=1.23
   - pandas<3,>=1.5
   - pyproject2conda~=0.11
+  - pytest-benchmark==4.*
   - pytest-cov>=2.12
   - pytest-doctestplus==1.*
   - pytest==7.*

envs/lenskit-py3.10-dev.yaml

@@ -12,10 +12,12 @@ channels:
 dependencies:
   - python=3.10
   - binpickle>=0.3.2
+  - c-compiler
   - cffi>=1.15.0
   - copier==9.*
   - coverage>=5
   - csr>=0.5
+  - cython==3.*
   - docopt>=0.6
   - hypothesis>=6
   - ipython>=7
@@ -29,6 +31,7 @@ dependencies:
   - numpy>=1.23
   - pandas<3,>=1.5
   - pyproject2conda~=0.11
+  - pytest-benchmark==4.*
   - pytest-cov>=2.12
   - pytest-doctestplus==1.*
   - pytest==7.*
@@ -44,7 +47,6 @@ dependencies:
   - sphinx_rtd_theme>=0.5
   - sphinxcontrib-bibtex>=2.0
   - sphinxext-opengraph>=0.5
-  - tbb
   - threadpoolctl>=3.0
   - tqdm>=4
   - pip

envs/lenskit-py3.10-test.yaml

@@ -15,10 +15,12 @@ dependencies:
   - cffi>=1.15.0
   - coverage>=5
   - csr>=0.5
+  - cython==3.*
   - hypothesis>=6
   - numba<0.59,>=0.56
   - numpy>=1.23
   - pandas<3,>=1.5
+  - pytest-benchmark==4.*
   - pytest-cov>=2.12
   - pytest-doctestplus==1.*
   - pytest==7.*

envs/lenskit-py3.11-ci.yaml

@@ -12,10 +12,12 @@ channels:
 dependencies:
   - python=3.11
   - binpickle>=0.3.2
+  - c-compiler
   - cffi>=1.15.0
   - copier==9.*
   - coverage>=5
   - csr>=0.5
+  - cython==3.*
   - docopt>=0.6
   - hypothesis>=6
   - ipython>=7
@@ -24,6 +26,7 @@ dependencies:
   - numpy>=1.23
   - pandas<3,>=1.5
   - pyproject2conda~=0.11
+  - pytest-benchmark==4.*
   - pytest-cov>=2.12
   - pytest-doctestplus==1.*
   - pytest==7.*

envs/lenskit-py3.11-dev.yaml

@@ -12,10 +12,12 @@ channels:
 dependencies:
   - python=3.11
   - binpickle>=0.3.2
+  - c-compiler
   - cffi>=1.15.0
   - copier==9.*
   - coverage>=5
   - csr>=0.5
+  - cython==3.*
   - docopt>=0.6
   - hypothesis>=6
   - ipython>=7
@@ -29,6 +31,7 @@ dependencies:
   - numpy>=1.23
   - pandas<3,>=1.5
   - pyproject2conda~=0.11
+  - pytest-benchmark==4.*
   - pytest-cov>=2.12
   - pytest-doctestplus==1.*
   - pytest==7.*
@@ -44,7 +47,6 @@ dependencies:
   - sphinx_rtd_theme>=0.5
   - sphinxcontrib-bibtex>=2.0
   - sphinxext-opengraph>=0.5
-  - tbb
   - threadpoolctl>=3.0
   - tqdm>=4
   - pip

envs/lenskit-py3.11-test.yaml

@@ -15,10 +15,12 @@ dependencies:
   - cffi>=1.15.0
   - coverage>=5
   - csr>=0.5
+  - cython==3.*
   - hypothesis>=6
   - numba<0.59,>=0.56
   - numpy>=1.23
   - pandas<3,>=1.5
+  - pytest-benchmark==4.*
   - pytest-cov>=2.12
   - pytest-doctestplus==1.*
   - pytest==7.*

justfile

@@ -11,6 +11,10 @@ clean:
 build:
     python -m build -n
 
+# build the extension modules in-place for testing
+build-inplace:
+    python setup.py build_ext --inplace
+
 # install the package
 [confirm("this installs package from a wheel, continue [y/N]?")]
 install:
@@ -29,15 +33,15 @@ setup-conda-env version="3.11" env="dev":
     conda env create -n lkpy -f envs/lenskit-py{{version}}-{{env}}.yaml
 
 # run tests with default configuration
-test:
+test: build-inplace
     python -m pytest
 
 # run fast tests
-test-fast:
+test-fast: build-inplace
     python -m pytest -m 'not slow'
 
 # run tests matching a keyword query
-test-matching query:
+test-matching query: build-inplace
     python -m pytest -k {{query}}
 
 # build documentation

lenskit/util/csmatrix.pyi

@@ -0,0 +1,24 @@
+import numpy as np
+import numpy.typing as npt
+from scipy.sparse import csr_array
+
+class CSMatrix:
+    nrows: int
+    ncols: int
+    nnz: int
+
+    rowptr: npt.NDArray[np.int32]
+    colind: npt.NDArray[np.int32]
+    values: npt.NDArray[np.float64]
+
+    def __init__(
+        self,
+        nr: int,
+        nc: int,
+        rps: npt.NDArray[np.int32],
+        cis: npt.NDArray[np.int32],
+        vs: npt.NDArray[np.float64],
+    ): ...
+    def row_ep(self, row: int) -> tuple[int, int]: ...
+    @staticmethod
+    def from_scipy(matrix: csr_array) -> CSMatrix: ...

lenskit/util/csmatrix.pyx

@@ -0,0 +1,27 @@
+# cython: language_level=3str
+
+cdef class CSMatrix:
+    cdef readonly int nrows, ncols, nnz
+    cdef readonly int[:] rowptr
+    cdef readonly int[:] colind
+    cdef readonly double[:] values
+
+    def __cinit__(self, int nr, int nc, int[:] rps, int[:] cis, double[:] vs):
+        self.nrows = nr
+        self.ncols = nc
+        self.rowptr = rps
+        self.colind = cis
+        self.values = vs
+        self.nnz = self.rowptr[nr]
+
+    @staticmethod
+    def from_scipy(m):
+        nr, nc = m.shape
+
+        return CSMatrix(nr, nc, m.indptr, m.indices, m.data)
+
+    cpdef (int,int) row_ep(self, row):
+        if row < 0 or row >= self.nrows:
+            raise IndexError(f"invalid row {row} for {self.nrows}x{self.ncols} matrix")
+
+        return self.rowptr[row], self.rowptr[row+1]