CI: Build wheel from sdist (#53087)

* CI: Build wheel from sdist * Save sdist name to pass to ciwheelbuild * Fix typo * fix typo * Check dist * Another typo * one more typo? * Remove missing files in Manifest, checkout project * try a different variable * Try in the prior directory up * check project folder * Pull path * Exclude some data files * Replace test command * Remove .data as well * Install locale on musl * Try delvewheel * Run wheel unpack everywhere * Skip in script * Wheel unpack separately * Invalid toml * Run wheel unpack outside * dont create dir * Debug mac unpack, try alternative test * Align * for loop validate, try to fix -v * for loop validate, try to fix -v * Fix path * Try pwd * To root? * Use another bash? * try MSYS_NO_PATHCONV * Add backslash * Remove a slash * Use powershell * Use variable * Typo * Add other deps * Final cleanups * Remove unecessary select * Try no * Make multiline comment * Put pack repo owner * Try single * clean step title * Update to setup micromaba * Use create-args * Use create args * Set PANDAS_CI * Try running all on powershell * Use ; * Try this command * Add double? * try escpaing quotes * Escape brackets? * remove comma * Single quotes * Other quotes * single? * Backslash? * All the escapes * No marks * Fix quotes * test don't repair windows wheel * Uncomment delvewheel
pandas-dev · Jun 1, 2023 · c6700d5 · c6700d5
1 parent aec4daf
commit c6700d5
Show file tree

Hide file tree

Showing 7 changed files with 105 additions and 267 deletions.
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -4,7 +4,7 @@
 # In an attempt to save CI resources, wheel builds do
 # not run on each push but only weekly and for releases.
 # Wheel builds can be triggered from the Actions page
-# (if you have the perms) on a commit to master.
+# (if you have the permissions) on a commit to main.
 #
 # Alternatively, you can add labels to the pull request in order to trigger wheel
 # builds.
@@ -14,13 +14,8 @@ name: Wheel builder
 
 on:
   schedule:
-  #        ┌───────────── minute (0 - 59)
-  #        │  ┌───────────── hour (0 - 23)
-  #        │  │ ┌───────────── day of the month (1 - 31)
-  #        │  │ │ ┌───────────── month (1 - 12 or JAN-DEC)
-  #        │  │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
-  #        │  │ │ │ │
-  - cron: "27 3 */1 * *"
+  # 3:27 UTC every day
+  - cron: "27 3 * * *"
   push:
   pull_request:
     types: [labeled, opened, synchronize, reopened]
@@ -37,170 +32,136 @@ permissions:
   contents: read
 
 jobs:
-  build_wheels:
-    name: Build wheel for ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+  build_sdist:
+    name: Build sdist
     if: >-
       (github.event_name == 'schedule' && github.repository_owner == 'pandas-dev') ||
       github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'pull_request' &&
       contains(github.event.pull_request.labels.*.name, 'Build')) ||
       (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0')))
-    runs-on: ${{ matrix.buildplat[0] }}
-    strategy:
-      # Ensure that a wheel builder finishes even if another fails
-      fail-fast: false
-      matrix:
-        # GitHub Actions doesn't support pairing matrix values together, let's improvise
-        # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026
-        buildplat:
-        - [ubuntu-20.04, manylinux_x86_64]
-        - [macos-11, macosx_*]
-        - [windows-2019, win_amd64]
-        # TODO: support PyPy?
-        python: [["cp39", "3.9"], ["cp310", "3.10"],  ["cp311", "3.11"]]#  "pp39"]
+    runs-on: ubuntu-22.04
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
+    outputs:
+      sdist_file: ${{ steps.save-path.outputs.sdist_name }}
     steps:
       - name: Checkout pandas
         uses: actions/checkout@v3
         with:
-          submodules: true
-          # versioneer.py requires the latest tag to be reachable. Here we
-          # fetch the complete history to get access to the tags.
-          # A shallow clone can work when the following issue is resolved:
-          # https://github.com/actions/checkout/issues/338
           fetch-depth: 0
 
-      - name: Build wheels
-        uses: pypa/[email protected]
-        env:
-          CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
-
-      # Used to test(Windows-only) and push the built wheels
-      # You might need to use setup-python separately
-      # if the new Python-dev version
-      # is unavailable on conda-forge.
-      - uses: conda-incubator/setup-miniconda@v2
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          auto-update-conda: true
-          python-version: ${{ matrix.python[1] }}
-          activate-environment: test
-          channels: conda-forge, anaconda
-          channel-priority: true
-          # mamba fails to solve, also we really don't need this since we're just installing python
-          # mamba-version: "*"
-
-      - name: Test wheels (Windows 64-bit only)
-        if: ${{ matrix.buildplat[1] == 'win_amd64' }}
-        shell: cmd /C CALL {0}
+          python-version: '3.11'
+
+      - name: Build sdist
         run: |
-          python ci/test_wheels.py wheelhouse
+          python -m pip install build
+          python -m build --sdist
 
       - uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
-          path: ./wheelhouse/*.whl
-
+          name: sdist
+          path: ./dist/*
 
-      - name: Install anaconda client
-        if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }}
+      - name: Output sdist name
+        id: save-path
         shell: bash -el {0}
-        run: conda install -q -y anaconda-client
+        run: echo "sdist_name=$(ls ./dist)" >> "$GITHUB_OUTPUT"
 
-
-      - name: Upload wheels
-        if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }}
-        shell: bash -el {0}
-        env:
-          PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }}
-          PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }}
-        run: |
-          source ci/upload_wheels.sh
-          set_upload_vars
-          # trigger an upload to
-          # https://anaconda.org/scipy-wheels-nightly/pandas
-          # for cron jobs or "Run workflow" (restricted to main branch).
-          # Tags will upload to
-          # https://anaconda.org/multibuild-wheels-staging/pandas
-          # The tokens were originally generated at anaconda.org
-          upload_wheels
-  build_sdist:
-    name: Build sdist
+  build_wheels:
+    needs: build_sdist
+    name: Build wheel for ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
     if: >-
-      github.event_name == 'schedule' ||
+      (github.event_name == 'schedule' && github.repository_owner == 'pandas-dev') ||
       github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'pull_request' &&
       contains(github.event.pull_request.labels.*.name, 'Build')) ||
       (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0')))
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.buildplat[0] }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # GitHub Actions doesn't support pairing matrix values together, let's improvise
+        # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026
+        buildplat:
+        - [ubuntu-22.04, manylinux_x86_64]
+        - [ubuntu-22.04, musllinux_x86_64]
+        - [macos-12, macosx_*]
+        - [windows-2022, win_amd64]
+        # TODO: support PyPy?
+        python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"]]
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout pandas
         uses: actions/checkout@v3
         with:
-          submodules: true
-          # versioneer.py requires the latest tag to be reachable. Here we
-          # fetch the complete history to get access to the tags.
-          # A shallow clone can work when the following issue is resolved:
-          # https://github.com/actions/checkout/issues/338
           fetch-depth: 0
 
-      # Used to push the built sdist
-      - uses: conda-incubator/setup-miniconda@v2
+      - name: Download sdist
+        uses: actions/download-artifact@v3
         with:
-          auto-update-conda: true
-          # Really doesn't matter what version we upload with
-          # just the version we test with
-          python-version: '3.10'
-          channels: conda-forge
-          channel-priority: true
-          # mamba fails to solve, also we really don't need this since we're just installing python
-          # mamba-version: "*"
+          name: sdist
+          path: ./dist
 
-      - name: Build sdist
-        run: |
-          pip install build
-          python -m build --sdist
-      - name: Test the sdist
-        shell: bash -el {0}
-        run: |
-          # TODO: Don't run test suite, and instead build wheels from sdist
-          # by splitting the wheel builders into a two stage job
-          # (1. Generate sdist 2. Build wheels from sdist)
-          # This tests the sdists, and saves some build time
-          python -m pip install dist/*.gz
-          pip install hypothesis>=6.46.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17
-          cd .. # Not a good idea to test within the src tree
-          python -c "import pandas; print(pandas.__version__);
-          pandas.test(extra_args=['-m not clipboard and not single_cpu and not slow and not network and not db', '-n 2']);
-          pandas.test(extra_args=['-m not clipboard and single_cpu and not slow and not network and not db'])"
-      - uses: actions/upload-artifact@v3
+      - name: Build wheels
+        uses: pypa/[email protected]
         with:
-          name: sdist
-          path: ./dist/*
+          package-dir: ./dist/${{ needs.build_sdist.outputs.sdist_file }}
+        env:
+          CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
-      - name: Install anaconda client
-        if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }}
+      - name: Set up Python
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-name: wheel-env
+          create-args: >-
+            python=${{ matrix.python[1] }}
+            anaconda-client
+            wheel
+          cache-downloads: true
+          cache-environment: true
+
+      - name: Validate wheel RECORD
         shell: bash -el {0}
+        run: for whl in $(ls wheelhouse); do wheel unpack wheelhouse/$whl -d /tmp; done
+
+      # Testing on windowsservercore instead of GHA runner to fail on missing DLLs
+      - name: Test Windows Wheels
+        if: ${{ matrix.buildplat[1] == 'win_amd64' }}
+        shell: pwsh
         run: |
-          conda install -q -y anaconda-client
+          $TST_CMD = @"
+          python -m pip install pytz six numpy python-dateutil tzdata>=2022.1 hypothesis>=6.46.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17;
+          python -m pip install --find-links=pandas\wheelhouse --no-index pandas;
+          python -c `'import pandas as pd; pd.test()`';
+          "@
+          docker pull python:${{ matrix.python[1] }}-windowsservercore
+          docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] }}-windowsservercore powershell -Command $TST_CMD
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
+          path: ./wheelhouse/*.whl
 
-      - name: Upload sdist
+      - name: Upload wheels & sdist
         if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }}
         shell: bash -el {0}
         env:
           PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }}
           PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }}
+        # trigger an upload to
+        # https://anaconda.org/scipy-wheels-nightly/pandas
+        # for cron jobs or "Run workflow" (restricted to main branch).
+        # Tags will upload to
+        # https://anaconda.org/multibuild-wheels-staging/pandas
+        # The tokens were originally generated at anaconda.org
         run: |
           source ci/upload_wheels.sh
           set_upload_vars
-          # trigger an upload to
-          # https://anaconda.org/scipy-wheels-nightly/pandas
-          # for cron jobs or "Run workflow" (restricted to main branch).
-          # Tags will upload to
-          # https://anaconda.org/multibuild-wheels-staging/pandas
-          # The tokens were originally generated at anaconda.org
           upload_wheels
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,6 +1,3 @@
-include RELEASE.md
-include versioneer.py
-
 graft doc
 prune doc/build
 
@@ -10,6 +7,7 @@ graft pandas
 
 global-exclude *.bz2
 global-exclude *.csv
+global-exclude *.data
 global-exclude *.dta
 global-exclude *.feather
 global-exclude *.tar
@@ -18,9 +16,12 @@ global-exclude *.h5
 global-exclude *.html
 global-exclude *.json
 global-exclude *.jsonl
+global-exclude *.kml
 global-exclude *.msgpack
 global-exclude *.pdf
+global-exclude *.parquet
 global-exclude *.pickle
+global-exclude *.pkl
 global-exclude *.png
 global-exclude *.pptx
 global-exclude *.ods
@@ -29,12 +30,15 @@ global-exclude *.orc
 global-exclude *.sas7bdat
 global-exclude *.sav
 global-exclude *.so
+global-exclude *.txt
 global-exclude *.xls
 global-exclude *.xlsb
 global-exclude *.xlsm
 global-exclude *.xlsx
 global-exclude *.xpt
 global-exclude *.cpt
+global-exclude *.xml
+global-exclude *.xsl
 global-exclude *.xz
 global-exclude *.zip
 global-exclude *.zst

diff --git a/ci/fix_wheels.py b/ci/fix_wheels.py