From 5896ca5ec5856638324b3c77c65c1ef708dc5a1a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 6 Dec 2023 12:43:32 +0000 Subject: [PATCH] Working on Python 3.12: numpy pandas pywavelets kiwisolver contourpy matplotlib --- server/pypi/build-wheel.py | 2 + server/pypi/env/lib/python/sitecustomize.py | 47 +++++++++------- server/pypi/packages/contourpy/meta.yaml | 6 +-- server/pypi/packages/matplotlib/meta.yaml | 4 +- .../matplotlib/patches/chaquopy.patch | 11 ---- server/pypi/packages/numpy/meta.yaml | 8 ++- .../packages/numpy/patches/chaquopy.patch | 53 ++++++++++--------- server/pypi/packages/pandas/meta.yaml | 15 ++++-- .../packages/pandas/patches/chaquopy.patch | 38 +++++++++++++ server/pypi/packages/pywavelets/meta.yaml | 2 +- .../pywavelets/patches/chaquopy.patch | 14 +++++ 11 files changed, 133 insertions(+), 67 deletions(-) diff --git a/server/pypi/build-wheel.py b/server/pypi/build-wheel.py index 91eeb42dd6..265bf4f149 100755 --- a/server/pypi/build-wheel.py +++ b/server/pypi/build-wheel.py @@ -468,6 +468,7 @@ def build_with_script(self, build_script): return self.package_wheel(prefix_dir, self.src_dir) def build_with_pep517(self): + log("Building with PEP 517") try: return self.builder.build("wheel", "dist") except build.BuildBackendException as e: @@ -561,6 +562,7 @@ def env_vars(self): pypi_env = f"{PYPI_DIR}/env" env["PATH"] = os.pathsep.join([ f"{pypi_env}/bin", + f"{self.build_env}/bin", f"{self.host_env}/chaquopy/bin", # For "-config" scripts. os.environ["PATH"]]) diff --git a/server/pypi/env/lib/python/sitecustomize.py b/server/pypi/env/lib/python/sitecustomize.py index 7e6c8f0dc2..67def8c9bb 100644 --- a/server/pypi/env/lib/python/sitecustomize.py +++ b/server/pypi/env/lib/python/sitecustomize.py @@ -5,28 +5,37 @@ import shlex import sys +# Recent versions of setuptools redirect distutils to their own bundled copy, so try +# to import that first. Even more recent versions of setuptools provide a .pth file +# which makes this import unnecessary, but the package we're building might have +# pinned an older version in its pyproject.toml file. try: - from setuptools._distutils import sysconfig + import setuptools # noqa: F401 except ImportError: - from distutils import sysconfig - - -# TODO: look into using crossenv to extract this from the Android sysconfigdata. -sysconfig.get_config_vars() # Ensure _config_vars has been initialized. -sysconfig._config_vars["CFLAGS"] = \ - "-Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall" - - -# Fix distutils ignoring LDFLAGS when building executables. -customize_compiler_original = sysconfig.customize_compiler + pass -def customize_compiler_override(compiler): - customize_compiler_original(compiler) - ldflags = os.environ["LDFLAGS"] - if ldflags not in " ".join(compiler.linker_exe): - compiler.linker_exe += shlex.split(ldflags) - -sysconfig.customize_compiler = customize_compiler_override +try: + from distutils import sysconfig +except ImportError: + # distutils was removed in Python 3.12, so it will only exist if setuptools is + # in the build environment. + pass +else: + # TODO: look into using crossenv to extract this from the Android sysconfigdata. + sysconfig.get_config_vars() # Ensure _config_vars has been initialized. + sysconfig._config_vars["CFLAGS"] = \ + "-Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall" + + # Fix distutils ignoring LDFLAGS when building executables. + customize_compiler_original = sysconfig.customize_compiler + + def customize_compiler_override(compiler): + customize_compiler_original(compiler) + ldflags = os.environ["LDFLAGS"] + if ldflags not in " ".join(compiler.linker_exe): + compiler.linker_exe += shlex.split(ldflags) + + sysconfig.customize_compiler = customize_compiler_override # Call the next sitecustomize script if there is one diff --git a/server/pypi/packages/contourpy/meta.yaml b/server/pypi/packages/contourpy/meta.yaml index 05a14ca922..2e94315519 100644 --- a/server/pypi/packages/contourpy/meta.yaml +++ b/server/pypi/packages/contourpy/meta.yaml @@ -1,10 +1,6 @@ package: name: contourpy - version: "1.0.5" + version: "1.0.7" build: number: 0 - -requirements: - build: - - pybind11 2.10.0 diff --git a/server/pypi/packages/matplotlib/meta.yaml b/server/pypi/packages/matplotlib/meta.yaml index 0394360ff3..710a24e00b 100644 --- a/server/pypi/packages/matplotlib/meta.yaml +++ b/server/pypi/packages/matplotlib/meta.yaml @@ -1,12 +1,12 @@ {% if PY_VER == "3.8" %} {% set numpy_version = "1.19.5" %} {% else %} - {% set numpy_version = "1.23.3" %} + {% set numpy_version = "1.26.2" %} {% endif %} package: name: matplotlib - version: "3.6.0" + version: "3.8.2" build: number: 0 diff --git a/server/pypi/packages/matplotlib/patches/chaquopy.patch b/server/pypi/packages/matplotlib/patches/chaquopy.patch index e0d4194f99..11bb9fde5b 100644 --- a/server/pypi/packages/matplotlib/patches/chaquopy.patch +++ b/server/pypi/packages/matplotlib/patches/chaquopy.patch @@ -20,14 +20,3 @@ import numpy as np ext.include_dirs.append(np.get_include()) ext.define_macros.extend([ ---- src-original/setup.py 2022-09-15 23:26:26.000000000 +0000 -+++ src/setup.py 2022-10-04 21:05:37.058026731 +0000 -@@ -302,7 +302,7 @@ - python_requires='>={}'.format('.'.join(str(n) for n in py_min_version)), - setup_requires=[ - "certifi>=2020.06.20", -- "numpy>=1.19", -+ # Chaquopy: removed "numpy>=1.19", - "setuptools_scm>=7", - ], - install_requires=[ diff --git a/server/pypi/packages/numpy/meta.yaml b/server/pypi/packages/numpy/meta.yaml index 36af2d2e1f..5d5eb1cc64 100644 --- a/server/pypi/packages/numpy/meta.yaml +++ b/server/pypi/packages/numpy/meta.yaml @@ -1,12 +1,16 @@ package: name: numpy - version: "1.23.3" + version: "1.26.2" build: number: 0 + script_env: + - SETUPTOOLS_USE_DISTUTILS=local requirements: build: - - Cython 0.29.32 + # NumPy doesn't officially support building with setuptools on Python 3.12, but it + # still works with this version, plus a couple of patches. + - setuptools 69.0.2 host: - chaquopy-openblas 0.2.20 diff --git a/server/pypi/packages/numpy/patches/chaquopy.patch b/server/pypi/packages/numpy/patches/chaquopy.patch index 4a3d2e20c4..802e83df57 100644 --- a/server/pypi/packages/numpy/patches/chaquopy.patch +++ b/server/pypi/packages/numpy/patches/chaquopy.patch @@ -137,31 +137,36 @@ diff -ur src-original/numpy/__init__.py src/numpy/__init__.py else: try: from numpy.__config__ import show as show_config -diff -ur src-original/setup.py src/setup.py ---- src-original/setup.py 2018-04-15 20:25:55.769497865 +0000 -+++ src/setup.py 2018-04-15 19:32:59.070017179 +0000 -@@ -39,6 +39,10 @@ - import __builtin__ as builtins +--- src-original/setup.py 1970-01-01 00:00:00.000000000 +0000 ++++ src/setup.py 2023-12-06 09:28:00.475265227 +0000 +@@ -28,6 +28,9 @@ + # a lot more robust than what was previously being used. + builtins.__NUMPY_SETUP__ = True - -+# Chaquopy added ++# Chaquopy +os.environ["OPENBLAS"] = os.path.abspath("../requirements/chaquopy/lib") + -+ - CLASSIFIERS = """\ - Development Status :: 5 - Production/Stable - Intended Audience :: Science/Research ---- src-original/pyproject.toml 2022-09-09 13:36:30.619459000 +0000 -+++ src/pyproject.toml 2023-08-31 11:38:08.304445745 +0000 -@@ -2,7 +2,10 @@ - # Minimum requirements for the build system to execute. + # Needed for backwards code compatibility below and in some CI scripts. + # The version components are changed from ints to strings, but only VERSION + # seems to matter outside of this module and it was already a str. +@@ -82,7 +85,9 @@ + except KeyError: + os.environ['SETUPTOOLS_USE_DISTUTILS'] = "stdlib" + else: +- if setuptools_use_distutils != "stdlib": ++ # Chaquopy: SETUPTOOLS_USE_DISTUTILS=stdlib doesn't work on Python 3.12, ++ # but the setuptools version in meta.yaml still works for now. ++ if False and setuptools_use_distutils != "stdlib": + raise RuntimeError("setuptools versions >= '60.0.0' require " + "SETUPTOOLS_USE_DISTUTILS=stdlib in the environment") + +--- src-original/pyproject.toml 1970-01-01 00:00:00.000000000 +0000 ++++ src/pyproject.toml 2023-12-06 08:34:11.926771744 +0000 +@@ -1,6 +1,5 @@ + [build-system] +-build-backend = "mesonpy" +-backend-path = ['./vendored-meson/meson-python'] ++# Chaquopy: disabled meson, since the setuptools build still works for now requires = [ - "packaging==20.5; platform_machine=='arm64'", # macos M1 -- "setuptools==59.2.0", -+ -+ # Chaquopy: was 59.2.0, which failed to detect modf and frexp for some reason. -+ "setuptools==63.0.0", -+ - "wheel==0.37.0", - "Cython>=0.29.30,<3.0", - ] + "Cython>=0.29.34,<3.1", + # All dependencies of the vendored meson-python (except for meson, because diff --git a/server/pypi/packages/pandas/meta.yaml b/server/pypi/packages/pandas/meta.yaml index 226b0c3b98..2c3af32e34 100644 --- a/server/pypi/packages/pandas/meta.yaml +++ b/server/pypi/packages/pandas/meta.yaml @@ -1,16 +1,25 @@ {% if PY_VER == "3.8" %} {% set numpy_version = "1.19.5" %} {% else %} - {% set numpy_version = "1.23.3" %} + {% set numpy_version = "1.26.2" %} {% endif %} +{% set version = "2.1.3" %} + package: name: pandas - version: "1.5.0" + version: "{{ version }}" + +# The setuptools build still works for now, but the sdist on PyPI doesn't include a +# setup.py, and neither does the GitHub "source code" archive. +source: + git_url: https://github.com/pandas-dev/pandas + git_rev: v{{ version }} build: - number: 1 + number: 0 requirements: host: + - python - numpy {{ numpy_version }} diff --git a/server/pypi/packages/pandas/patches/chaquopy.patch b/server/pypi/packages/pandas/patches/chaquopy.patch index fd76b0ae37..529c9dce67 100644 --- a/server/pypi/packages/pandas/patches/chaquopy.patch +++ b/server/pypi/packages/pandas/patches/chaquopy.patch @@ -11,3 +11,41 @@ import numpy from pkg_resources import parse_version from setuptools import ( +--- src-original/pyproject.toml 2023-12-06 12:10:08.000000000 +0000 ++++ src/pyproject.toml 2023-12-06 12:33:14.841679887 +0000 +@@ -2,8 +2,9 @@ + # Minimum requirements for the build system to execute. + # See https://github.com/scipy/scipy/pull/12940 for the AIX issue. + requires = [ +- "meson-python==0.13.1", +- "meson==1.2.1", ++ # Chaquopy: disabled meson, since the setuptools build still works for now ++ "setuptools==69.0.2", ++ + "wheel", + "Cython>=0.29.33,<3", # Note: sync with setup.py, environment.yml and asv.conf.json + # Note: numpy 1.25 has a backwards compatible C API by default +@@ -14,7 +15,8 @@ + "versioneer[toml]" + ] + +-build-backend = "mesonpy" ++# Chaquopy ++# build-backend = "mesonpy" + + [project] + name = 'pandas' +@@ -137,7 +139,12 @@ + # resulting files. + [tool.versioneer] + VCS = "git" +-style = "pep440" ++ ++# Chaquopy: `-pre` prevents versioneer from returning a "dirty" version when we're ++# building from a git checkout. This can be reverted once we go back to building from an ++# sdist. ++style = "pep440-pre" ++ + versionfile_source = "pandas/_version.py" + versionfile_build = "pandas/_version.py" + tag_prefix = "v" diff --git a/server/pypi/packages/pywavelets/meta.yaml b/server/pypi/packages/pywavelets/meta.yaml index ce7fcb1a4c..a05363a0a4 100644 --- a/server/pypi/packages/pywavelets/meta.yaml +++ b/server/pypi/packages/pywavelets/meta.yaml @@ -7,4 +7,4 @@ build: requirements: host: - - numpy 1.23.3 + - numpy 1.26.2 diff --git a/server/pypi/packages/pywavelets/patches/chaquopy.patch b/server/pypi/packages/pywavelets/patches/chaquopy.patch index dd96f18882..5ff749e12a 100644 --- a/server/pypi/packages/pywavelets/patches/chaquopy.patch +++ b/server/pypi/packages/pywavelets/patches/chaquopy.patch @@ -11,3 +11,17 @@ from numpy import get_include as get_numpy_include cython_modules, cython_sources = get_cython_sources(use_cython) ext_modules = [ +--- src-original/pyproject.toml 2022-09-16 14:25:42.000000000 +0000 ++++ src/pyproject.toml 2023-12-06 10:36:40.960979478 +0000 +@@ -7,7 +7,10 @@ + [build-system] + requires = [ + "wheel", +- "setuptools<65", ++ ++ # Chaquopy: updated for Python 3.12 ++ "setuptools==69.0.2", ++ + "Cython>=0.29.24,<3.0", + + # NumPy dependencies - to update these, sync from