Merge pull request #184 from thomasmarwitz/port-to-pixi

Port project to pixi and QuantCo OSS copier template

.copier-answers.yml

@@ -0,0 +1,13 @@
+# This file is managed by Copier; DO NOT EDIT OR REMOVE.
+_commit: v0.2.1
+_src_path: https://github.com/quantco/copier-template-python-open-source
+add_autobump_workflow: true
+author_email: [email protected]
+author_name: Data Engineering Collective
+github_url: https://github.com/data-engineering-collective/plateau
+github_user: xhochy
+minimal_python_version: py311
+project_short_description: A Python library to manage (create, read, update, delete)
+    large amounts of tabular data in a blob store.
+project_slug: plateau
+use_devcontainer: false

.gitattributes

@@ -0,0 +1 @@
+pixi.lock linguist-language=YAML linguist-generated=true

.github/workflows/benchmarks.yml

@@ -0,0 +1,25 @@
+name: Benchmarks
+
+on:
+  schedule:
+    - cron: "0 0 * * 0" # weekly on Sunday
+
+jobs:
+  benchmarks:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
+        with:
+          environments: benchmark
+      - name: Install package
+        run: pixi run -e benchmark postinstall
+      - name: Running benchmarks
+        run: |
+          pixi run -e benchmark asv --config ./asv_bench/asv.conf.json machine --machine github --os unknown --arch unknown --cpu unknown --ram unknown
+          pixi run -e benchmark asv --config ./asv_bench/asv.conf.json run -E existing:same | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log
+          if grep "failed" benchmarks.log > /dev/null ; then
+              exit 1
+          fi

.github/workflows/build.yml

@@ -0,0 +1,52 @@
+name: Build
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
+        with:
+          environments: build
+      - name: Build project
+        run: pixi run -e build build-wheel
+      - name: Check package
+        run: pixi run -e build check-wheel
+      - name: Upload package
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: artifact
+          path: dist/*
+
+  release:
+    name: Publish package
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: [build]
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    environment: pypi
+    steps:
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: artifact
+          path: dist
+      - name: Publish package on TestPyPi
+        uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # v1.10.3
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          user: __token__
+          password: ${{ secrets.PYPI_PASSWORD }}
+      - name: Publish package on PyPi
+        uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # v1.10.3
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_PASSWORD }}

.github/workflows/ci.yml

@@ -6,6 +6,7 @@ on:
     branches:
       - "main"
       - "pre-commit-autoupdate"
+      - "update-pixi"
     tags:
       - "*"
   schedule:
@@ -17,167 +18,56 @@ on:
     - cron: "0 0 * * *"
 
 jobs:
-  build:
+  pre-commit:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
+        with:
+          environments: default lint
+      - name: pre-commit
+        run: pixi run pre-commit-run --color=always --show-diff-on-failure
+
+  tests:
     strategy:
       fail-fast: false
       matrix:
-        numfocus_nightly: [false]
-        os: ["ubuntu-latest"]
-        pandas: [""]
-        pyarrow: ["4.0.1", "nightly"]
-        python: ["3.10"]
-        include:
-          - numfocus_nightly: true
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "nightly"
-            python: "3.10"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: "1.5.3"
-            pyarrow: "13.0.0"
-            python: "3.11"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "7.0.0"
-            python: "3.10"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "8.0.1"
-            python: "3.10"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "9.0.0"
-            python: "3.10"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "10.0.1"
-            python: "3.11"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "11.0.0"
-            python: "3.11"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "12.0.0"
-            python: "3.11"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "13.0.0"
-            python: "3.11"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "13.0.0"
-            python: "3.12"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "14.0.1"
-            python: "3.12"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "15.0.0"
-            python: "3.12"
-          - numfocus_nightly: false
-            os: "ubuntu-latest"
-            pandas: ""
-            pyarrow: "18.1.0"
-            python: "3.12"
-    continue-on-error: ${{ matrix.numfocus_nightly || matrix.pyarrow == 'nightly' }}
-
-    runs-on: ${{ matrix.os }}
-
-    defaults:
-      run:
-        # see https://github.com/conda-incubator/setup-miniconda/#important
-        shell: bash -l {0}
+        environment:
+          [
+            "py310-pyarrow15-0-2",
+            "py310-pyarrow16-1",
+            "py310-pyarrow17-0",
+            "py310-pyarrow18-1",
+            "py311-pyarrow15-0-2",
+            "py311-pyarrow16-1",
+            "py311-pyarrow17-0",
+            "py311-pyarrow18-1",
+            "py312-pyarrow15-0-2",
+            "py312-pyarrow16-1",
+            "py312-pyarrow17-0",
+            "py312-pyarrow18-1",
+          ]
 
-    env:
-      IS_MASTER_BUILD: ${{ !matrix.numfocus_nightly && matrix.os == 'ubuntu-latest' && matrix.pyarrow == '4.0.1' && matrix.python == '3.8' }}
-      IS_TAG: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')}}
+    runs-on: "ubuntu-latest"
 
     steps:
-      # CI setup
-      - name: Is Master Build?
-        run: echo $IS_MASTER_BUILD
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Set up Conda env
-        uses: mamba-org/setup-micromamba@068f1ab4b37ed9b3d9f73da7db90a0cda0a48d29
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1
         with:
-          environment-file: environment.yml
-          cache-environment: true
-          create-args: >-
-            python=${{ matrix.python }}
-      - name: Install Pyarrow (non-nightly)
-        run: micromamba install -y pyarrow==${{ matrix.pyarrow }}
-        if: matrix.pyarrow != 'nightly' && matrix.pandas == ''
-      - name: Install Pyarrow (nightly)
-        # Install both arrow-cpp and pyarrow to make sure that we have the
-        # latest nightly of both packages. It is sadly not guaranteed that the
-        # nightlies and the latest release would otherwise work together.
-        run: micromamba update -c arrow-nightlies -c conda-forge arrow-cpp pyarrow
-        if: matrix.pyarrow == 'nightly'
-      - name: Install Pyarrow (downgrade pandas)
-        run: micromamba install -y --no-py-pin pyarrow==${{ matrix.pyarrow }} pandas==${{ matrix.pandas }}
-        if: matrix.pyarrow != 'nightly' && matrix.pandas != ''
-      - name: Pip Install NumFOCUS nightly
-        # NumFOCUS nightly wheels, contains numpy and pandas
-        # TODO(gh-45): Re-add numpy
-        run: python -m pip install --pre --upgrade --timeout=60 --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ pandas
-        if: matrix.numfocus_nightly
+          environments: ${{ matrix.environment }} # References the environments defined in pixi.toml
       - name: Install repository
-        run: python -m pip install --no-build-isolation --no-deps --disable-pip-version-check -e .
-      - name: Test import
-        run: |
-          python -c "import plateau"
-          python -c "import plateau.api"
-          python -c "import plateau.api.dataset"
-          python -c "import plateau.api.serialization"
-          python -c "import plateau.core"
-          python -c "import plateau.io"
-          python -c "import plateau.io_components"
-          python -c "import plateau.serialization"
-          python -c "import plateau.utils"
+        run: pixi run postinstall
 
-      # Tests
       - name: Pytest
-        # FIXME: Add `-n auto` again to allow for parallelism here.
-        run: pytest --cov --cov-report xml
-
-      - name: Running benchmarks
-        run: |
-          asv --config ./asv_bench/asv.conf.json machine --machine github --os unknown --arch unknown --cpu unknown --ram unknown
-          asv --config ./asv_bench/asv.conf.json run -E existing:same | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log
-          if grep "failed" benchmarks.log > /dev/null ; then
-              exit 1
-          fi
-        if: env.IS_MASTER_BUILD == 'true'
-
-      # Builds
-      - name: Build Wheel and sdist
-        run: python -m build --no-isolation
+        run: pixi run test-coverage -n auto
 
       - name: Codecov
         uses: codecov/[email protected]
         with:
           file: ./coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}
-          name: pytest-numfocus_nightly_${{ matrix.numfocus_nightly }}-os_${{ matrix.os }}-pyarrow_${{ matrix.pyarrow }}-python_${{ matrix.python }}
-
-      # Release
-      - name: Publish to PyPI
-        if: env.IS_MASTER_BUILD == 'true' && env.IS_TAG == 'true'
-        uses: pypa/gh-action-pypi-publish@master
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_PASSWORD }}
+          name: pytest-${{ matrix.environment }}