From c6700d596a0a9563f1495eb65491aa48c37aa4e5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 1 Jun 2023 16:12:22 -0700 Subject: [PATCH] CI: Build wheel from sdist (#53087) * CI: Build wheel from sdist * Save sdist name to pass to ciwheelbuild * Fix typo * fix typo * Check dist * Another typo * one more typo? * Remove missing files in Manifest, checkout project * try a different variable * Try in the prior directory up * check project folder * Pull path * Exclude some data files * Replace test command * Remove .data as well * Install locale on musl * Try delvewheel * Run wheel unpack everywhere * Skip in script * Wheel unpack separately * Invalid toml * Run wheel unpack outside * dont create dir * Debug mac unpack, try alternative test * Align * for loop validate, try to fix -v * for loop validate, try to fix -v * Fix path * Try pwd * To root? * Use another bash? * try MSYS_NO_PATHCONV * Add backslash * Remove a slash * Use powershell * Use variable * Typo * Add other deps * Final cleanups * Remove unecessary select * Try no * Make multiline comment * Put pack repo owner * Try single * clean step title * Update to setup micromaba * Use create-args * Use create args * Set PANDAS_CI * Try running all on powershell * Use ; * Try this command * Add double? * try escpaing quotes * Escape brackets? * remove comma * Single quotes * Other quotes * single? * Backslash? * All the escapes * No marks * Fix quotes * test don't repair windows wheel * Uncomment delvewheel --- .github/workflows/wheels.yml | 205 ++++++++++++++--------------------- MANIFEST.in | 10 +- ci/fix_wheels.py | 61 ----------- ci/test_wheels.py | 62 ----------- ci/test_wheels_windows.bat | 9 -- pandas/util/_tester.py | 2 +- pyproject.toml | 23 ++-- 7 files changed, 105 insertions(+), 267 deletions(-) delete mode 100644 ci/fix_wheels.py delete mode 100644 ci/test_wheels.py delete mode 100644 ci/test_wheels_windows.bat diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 0ebede6501f5f..0a508a8b1701f 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -4,7 +4,7 @@ # In an attempt to save CI resources, wheel builds do # not run on each push but only weekly and for releases. # Wheel builds can be triggered from the Actions page -# (if you have the perms) on a commit to master. +# (if you have the permissions) on a commit to main. # # Alternatively, you can add labels to the pull request in order to trigger wheel # builds. @@ -14,13 +14,8 @@ name: Wheel builder on: schedule: - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - # │ │ │ │ │ - - cron: "27 3 */1 * *" + # 3:27 UTC every day + - cron: "27 3 * * *" push: pull_request: types: [labeled, opened, synchronize, reopened] @@ -37,103 +32,68 @@ permissions: contents: read jobs: - build_wheels: - name: Build wheel for ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} + build_sdist: + name: Build sdist if: >- (github.event_name == 'schedule' && github.repository_owner == 'pandas-dev') || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'Build')) || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0'))) - runs-on: ${{ matrix.buildplat[0] }} - strategy: - # Ensure that a wheel builder finishes even if another fails - fail-fast: false - matrix: - # GitHub Actions doesn't support pairing matrix values together, let's improvise - # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 - buildplat: - - [ubuntu-20.04, manylinux_x86_64] - - [macos-11, macosx_*] - - [windows-2019, win_amd64] - # TODO: support PyPy? - python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"]]# "pp39"] + runs-on: ubuntu-22.04 env: IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + outputs: + sdist_file: ${{ steps.save-path.outputs.sdist_name }} steps: - name: Checkout pandas uses: actions/checkout@v3 with: - submodules: true - # versioneer.py requires the latest tag to be reachable. Here we - # fetch the complete history to get access to the tags. - # A shallow clone can work when the following issue is resolved: - # https://github.com/actions/checkout/issues/338 fetch-depth: 0 - - name: Build wheels - uses: pypa/cibuildwheel@v2.13.0 - env: - CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} - - # Used to test(Windows-only) and push the built wheels - # You might need to use setup-python separately - # if the new Python-dev version - # is unavailable on conda-forge. - - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Python + uses: actions/setup-python@v4 with: - auto-update-conda: true - python-version: ${{ matrix.python[1] }} - activate-environment: test - channels: conda-forge, anaconda - channel-priority: true - # mamba fails to solve, also we really don't need this since we're just installing python - # mamba-version: "*" - - - name: Test wheels (Windows 64-bit only) - if: ${{ matrix.buildplat[1] == 'win_amd64' }} - shell: cmd /C CALL {0} + python-version: '3.11' + + - name: Build sdist run: | - python ci/test_wheels.py wheelhouse + python -m pip install build + python -m build --sdist - uses: actions/upload-artifact@v3 with: - name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }} - path: ./wheelhouse/*.whl - + name: sdist + path: ./dist/* - - name: Install anaconda client - if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }} + - name: Output sdist name + id: save-path shell: bash -el {0} - run: conda install -q -y anaconda-client + run: echo "sdist_name=$(ls ./dist)" >> "$GITHUB_OUTPUT" - - - name: Upload wheels - if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }} - shell: bash -el {0} - env: - PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }} - PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }} - run: | - source ci/upload_wheels.sh - set_upload_vars - # trigger an upload to - # https://anaconda.org/scipy-wheels-nightly/pandas - # for cron jobs or "Run workflow" (restricted to main branch). - # Tags will upload to - # https://anaconda.org/multibuild-wheels-staging/pandas - # The tokens were originally generated at anaconda.org - upload_wheels - build_sdist: - name: Build sdist + build_wheels: + needs: build_sdist + name: Build wheel for ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} if: >- - github.event_name == 'schedule' || + (github.event_name == 'schedule' && github.repository_owner == 'pandas-dev') || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'Build')) || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0'))) - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.buildplat[0] }} + strategy: + fail-fast: false + matrix: + # GitHub Actions doesn't support pairing matrix values together, let's improvise + # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 + buildplat: + - [ubuntu-22.04, manylinux_x86_64] + - [ubuntu-22.04, musllinux_x86_64] + - [macos-12, macosx_*] + - [windows-2022, win_amd64] + # TODO: support PyPy? + python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"]] env: IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} @@ -141,66 +101,67 @@ jobs: - name: Checkout pandas uses: actions/checkout@v3 with: - submodules: true - # versioneer.py requires the latest tag to be reachable. Here we - # fetch the complete history to get access to the tags. - # A shallow clone can work when the following issue is resolved: - # https://github.com/actions/checkout/issues/338 fetch-depth: 0 - # Used to push the built sdist - - uses: conda-incubator/setup-miniconda@v2 + - name: Download sdist + uses: actions/download-artifact@v3 with: - auto-update-conda: true - # Really doesn't matter what version we upload with - # just the version we test with - python-version: '3.10' - channels: conda-forge - channel-priority: true - # mamba fails to solve, also we really don't need this since we're just installing python - # mamba-version: "*" + name: sdist + path: ./dist - - name: Build sdist - run: | - pip install build - python -m build --sdist - - name: Test the sdist - shell: bash -el {0} - run: | - # TODO: Don't run test suite, and instead build wheels from sdist - # by splitting the wheel builders into a two stage job - # (1. Generate sdist 2. Build wheels from sdist) - # This tests the sdists, and saves some build time - python -m pip install dist/*.gz - pip install hypothesis>=6.46.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 - cd .. # Not a good idea to test within the src tree - python -c "import pandas; print(pandas.__version__); - pandas.test(extra_args=['-m not clipboard and not single_cpu and not slow and not network and not db', '-n 2']); - pandas.test(extra_args=['-m not clipboard and single_cpu and not slow and not network and not db'])" - - uses: actions/upload-artifact@v3 + - name: Build wheels + uses: pypa/cibuildwheel@v2.13.0 with: - name: sdist - path: ./dist/* + package-dir: ./dist/${{ needs.build_sdist.outputs.sdist_file }} + env: + CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} - - name: Install anaconda client - if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }} + - name: Set up Python + uses: mamba-org/setup-micromamba@v1 + with: + environment-name: wheel-env + create-args: >- + python=${{ matrix.python[1] }} + anaconda-client + wheel + cache-downloads: true + cache-environment: true + + - name: Validate wheel RECORD shell: bash -el {0} + run: for whl in $(ls wheelhouse); do wheel unpack wheelhouse/$whl -d /tmp; done + + # Testing on windowsservercore instead of GHA runner to fail on missing DLLs + - name: Test Windows Wheels + if: ${{ matrix.buildplat[1] == 'win_amd64' }} + shell: pwsh run: | - conda install -q -y anaconda-client + $TST_CMD = @" + python -m pip install pytz six numpy python-dateutil tzdata>=2022.1 hypothesis>=6.46.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17; + python -m pip install --find-links=pandas\wheelhouse --no-index pandas; + python -c `'import pandas as pd; pd.test()`'; + "@ + docker pull python:${{ matrix.python[1] }}-windowsservercore + docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] }}-windowsservercore powershell -Command $TST_CMD + + - uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }} + path: ./wheelhouse/*.whl - - name: Upload sdist + - name: Upload wheels & sdist if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }} shell: bash -el {0} env: PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }} PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }} + # trigger an upload to + # https://anaconda.org/scipy-wheels-nightly/pandas + # for cron jobs or "Run workflow" (restricted to main branch). + # Tags will upload to + # https://anaconda.org/multibuild-wheels-staging/pandas + # The tokens were originally generated at anaconda.org run: | source ci/upload_wheels.sh set_upload_vars - # trigger an upload to - # https://anaconda.org/scipy-wheels-nightly/pandas - # for cron jobs or "Run workflow" (restricted to main branch). - # Tags will upload to - # https://anaconda.org/multibuild-wheels-staging/pandas - # The tokens were originally generated at anaconda.org upload_wheels diff --git a/MANIFEST.in b/MANIFEST.in index 781a72fdb5481..9894381ed6252 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,3 @@ -include RELEASE.md -include versioneer.py - graft doc prune doc/build @@ -10,6 +7,7 @@ graft pandas global-exclude *.bz2 global-exclude *.csv +global-exclude *.data global-exclude *.dta global-exclude *.feather global-exclude *.tar @@ -18,9 +16,12 @@ global-exclude *.h5 global-exclude *.html global-exclude *.json global-exclude *.jsonl +global-exclude *.kml global-exclude *.msgpack global-exclude *.pdf +global-exclude *.parquet global-exclude *.pickle +global-exclude *.pkl global-exclude *.png global-exclude *.pptx global-exclude *.ods @@ -29,12 +30,15 @@ global-exclude *.orc global-exclude *.sas7bdat global-exclude *.sav global-exclude *.so +global-exclude *.txt global-exclude *.xls global-exclude *.xlsb global-exclude *.xlsm global-exclude *.xlsx global-exclude *.xpt global-exclude *.cpt +global-exclude *.xml +global-exclude *.xsl global-exclude *.xz global-exclude *.zip global-exclude *.zst diff --git a/ci/fix_wheels.py b/ci/fix_wheels.py deleted file mode 100644 index 76b70fdde9ea0..0000000000000 --- a/ci/fix_wheels.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -This file "repairs" our Windows wheels by copying the necessary DLLs for pandas to run -on a barebones Windows installation() into the wheel. - -NOTE: The paths for the DLLs are hard-coded to the location of the Visual Studio -redistributables -""" -import os -import shutil -import subprocess -from subprocess import CalledProcessError -import sys -import zipfile - -try: - if len(sys.argv) != 3: - raise ValueError( - "User must pass the path to the wheel and the destination directory." - ) - wheel_path = sys.argv[1] - dest_dir = sys.argv[2] - # Figure out whether we are building on 32 or 64 bit python - is_32 = sys.maxsize <= 2**32 - PYTHON_ARCH = "x86" if is_32 else "x64" -except ValueError: - # Too many/little values to unpack - raise ValueError( - "User must pass the path to the wheel and the destination directory." - ) -if not os.path.isdir(dest_dir): - print(f"Created directory {dest_dir}") - os.mkdir(dest_dir) - -wheel_name = os.path.basename(wheel_path) -success = True - -try: - # Use the wheel CLI for zipping up the wheel since the CLI will - # take care of rebuilding the hashes found in the record file - tmp_dir = os.path.join(dest_dir, "tmp") - with zipfile.ZipFile(wheel_path, "r") as f: - # Extracting all the members of the zip - # into a specific location. - f.extractall(path=tmp_dir) - base_redist_dir = ( - f"C:/Program Files (x86)/Microsoft Visual Studio/2019/" - f"Enterprise/VC/Redist/MSVC/14.29.30133/{PYTHON_ARCH}/" - f"Microsoft.VC142.CRT/" - ) - required_dlls = ["msvcp140.dll", "concrt140.dll"] - if not is_32: - required_dlls += ["vcruntime140_1.dll"] - dest_dll_dir = os.path.join(tmp_dir, "pandas/_libs/window") - for dll in required_dlls: - src = os.path.join(base_redist_dir, dll) - shutil.copy(src, dest_dll_dir) - subprocess.run(["wheel", "pack", tmp_dir, "-d", dest_dir], check=True) -except CalledProcessError: - print("Failed to add DLLS to wheel.") - sys.exit(1) -print("Successfully repaired wheel") diff --git a/ci/test_wheels.py b/ci/test_wheels.py deleted file mode 100644 index 75675d7e4ffc3..0000000000000 --- a/ci/test_wheels.py +++ /dev/null @@ -1,62 +0,0 @@ -import glob -import os -import shutil -import subprocess -from subprocess import CalledProcessError -import sys - -if os.name == "nt": - py_ver = f"{sys.version_info.major}.{sys.version_info.minor}" - is_32_bit = os.getenv("IS_32_BIT") == "true" - try: - wheel_dir = sys.argv[1] - wheel_path = glob.glob(f"{wheel_dir}/*.whl")[0] - except IndexError: - # Not passed - wheel_path = None - print(f"IS_32_BIT is {is_32_bit}") - print(f"Path to built wheel is {wheel_path}") - - print("Verifying file hashes in wheel RECORD file") - try: - tmp_dir = "tmp" - subprocess.run(["wheel", "unpack", wheel_path, "-d", tmp_dir], check=True) - except CalledProcessError: - print("wheel RECORD file hash verification failed.") - sys.exit(1) - finally: - shutil.rmtree(tmp_dir) - - if is_32_bit: - sys.exit(0) # No way to test Windows 32-bit(no docker image) - if wheel_path is None: - raise ValueError("Wheel path must be passed in if on 64-bit Windows") - print(f"Pulling docker image to test Windows 64-bit Python {py_ver}") - subprocess.run(f"docker pull python:{py_ver}-windowsservercore", check=True) - pandas_base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) - print(f"pandas project dir is {pandas_base_dir}") - dist_dir = os.path.join(pandas_base_dir, "dist") - print(f"Copying wheel into pandas_base_dir/dist ({dist_dir})") - os.mkdir(dist_dir) - shutil.copy(wheel_path, dist_dir) - print(os.listdir(dist_dir)) - subprocess.run( - rf"docker run -v %cd%:c:\pandas " - f"python:{py_ver}-windowsservercore /pandas/ci/test_wheels_windows.bat", - check=True, - shell=True, - cwd=pandas_base_dir, - ) -else: - import pandas as pd - - multi_args = [ - "-m not clipboard and not single_cpu and not slow and not network and not db", - "-n 2", - ] - pd.test(extra_args=multi_args) - pd.test( - extra_args=[ - "-m not clipboard and single_cpu and not slow and not network and not db", - ] - ) diff --git a/ci/test_wheels_windows.bat b/ci/test_wheels_windows.bat deleted file mode 100644 index 9864446d71137..0000000000000 --- a/ci/test_wheels_windows.bat +++ /dev/null @@ -1,9 +0,0 @@ -set test_command=import pandas as pd; print(pd.__version__); ^ -pd.test(extra_args=['-m not clipboard and not single_cpu and not slow and not network and not db', '-n 2']); ^ -pd.test(extra_args=['-m not clipboard and single_cpu and not slow and not network and not db']) - -python --version -pip install pytz six numpy python-dateutil tzdata>=2022.1 -pip install hypothesis>=6.46.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 -pip install --find-links=pandas/dist --no-index pandas -python -c "%test_command%" diff --git a/pandas/util/_tester.py b/pandas/util/_tester.py index e9f516bac6ad2..6a2a8189ad896 100644 --- a/pandas/util/_tester.py +++ b/pandas/util/_tester.py @@ -28,7 +28,7 @@ def test(extra_args: list[str] | None = None, run_doctests: bool = False) -> Non """ pytest = import_optional_dependency("pytest") import_optional_dependency("hypothesis") - cmd = ['-m "not slow and not network and not db"'] + cmd = ["-m not slow and not network and not db"] if extra_args: if not isinstance(extra_args, list): extra_args = [extra_args] diff --git a/pyproject.toml b/pyproject.toml index 9da2951272541..4440c65cfb777 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -143,23 +143,32 @@ parentdir_prefix = "pandas-" setup = ['--vsenv'] # For Windows [tool.cibuildwheel] -skip = "cp36-* cp37-* pp37-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux*" +skip = "cp36-* cp37-* cp38-* pp* *_i686 *_ppc64le *_s390x *-musllinux_aarch64" build-verbosity = "3" -environment = {LDFLAGS="-Wl,--strip-all" } +environment = {LDFLAGS="-Wl,--strip-all"} test-requires = "hypothesis>=6.46.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17" -test-command = "python {project}/ci/test_wheels.py" +test-command = """ + PANDAS_CI='1' python -c 'import pandas as pd; \ + pd.test(extra_args=["-m not clipboard and not single_cpu and not slow and not network and not db", "-n 2"]); \ + pd.test(extra_args=["-m not clipboard and single_cpu and not slow and not network and not db"]);' \ + """ [tool.cibuildwheel.macos] archs = "x86_64 arm64" test-skip = "*_arm64" [tool.cibuildwheel.windows] -repair-wheel-command = "python ci/fix_wheels.py {wheel} {dest_dir}" +before-build = "pip install delvewheel" +repair-wheel-command = "delvewheel repair -w {dest_dir} {wheel}" + +[[tool.cibuildwheel.overrides]] +select = "*-musllinux*" +before-test = "apk update && apk add musl-locales" [[tool.cibuildwheel.overrides]] select = "*-win*" # We test separately for Windows, since we use -# the base windows docker image to check if any dlls are +# the windowsservercore docker image to check if any dlls are # missing from the wheel test-command = "" @@ -170,10 +179,6 @@ test-command = "" select = "*-macosx*" environment = {CFLAGS="-g0"} -[[tool.cibuildwheel.overrides]] -select = "*-win32" -environment = { IS_32_BIT="true" } - [tool.black] target-version = ['py39', 'py310'] required-version = '23.3.0'