ARROW-17487: [Python][Packaging][CI] Add support for Python 3.11

ci/docker/python-wheel-windows-test-vs2017.dockerfile

@@ -37,7 +37,8 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 ARG python=3.8
 RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
     (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
-    (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
-    (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
+    (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+    (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \
+    (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts")
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
 RUN python -m pip install -U pip setuptools

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -80,8 +80,9 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
 ARG python=3.8
 RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
     (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
-    (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
-    (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
+    (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+    (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \
+    (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts")
 RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
 RUN python -m pip install -U pip setuptools
 

ci/scripts/install_gcs_testbench.sh

@@ -34,18 +34,9 @@ case "$(uname -m)" in
     ;;
 esac
 
-case "$(uname -s)-$(uname -m)" in
-  Darwin-arm64)
-    # Workaround for https://github.com/grpc/grpc/issues/28387 .
-    # Build grpcio instead of using wheel.
-    # storage-testbench 0.27.0 pins grpcio to 1.46.1.
-    ${PYTHON:-python3} -m pip install --no-binary :all: "grpcio==1.46.1"
-    ;;
-esac
-
 version=$1
 if [[ "${version}" -eq "default" ]]; then
-  version="v0.27.0"
+  version="v0.32.0"
 fi
 
 ${PYTHON:-python3} -m pip install \

ci/scripts/install_python.sh

@@ -27,8 +27,9 @@ platforms=([windows]=Windows
 declare -A versions
 versions=([3.7]=3.7.9
           [3.8]=3.8.10
-          [3.9]=3.9.9
-          [3.10]=3.10.1)
+          [3.9]=3.9.13
+          [3.10]=3.10.8
+          [3.11]=3.11.0)
 
 if [ "$#" -ne 2 ]; then
   echo "Usage: $0 <platform> <version>"
@@ -45,7 +46,7 @@ full_version=${versions[$2]}
 if [ $platform = "MacOSX" ]; then
     echo "Downloading Python installer..."
 
-    if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ]; then
+    if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then
         fname="python-${full_version}-macos11.pkg"
     else
         fname="python-${full_version}-macosx10.9.pkg"

dev/release/verify-release-candidate.sh

@@ -1019,7 +1019,7 @@ test_linux_wheels() {
     local arch="x86_64"
   fi
 
-  local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10}"
+  local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10 3.11}"
   local platform_tags="manylinux_2_17_${arch}.manylinux2014_${arch}"
 
   for python in ${python_versions}; do
@@ -1041,11 +1041,11 @@ test_macos_wheels() {
 
   # apple silicon processor
   if [ "$(uname -m)" = "arm64" ]; then
-    local python_versions="3.8 3.9 3.10"
+    local python_versions="3.8 3.9 3.10 3.11"
     local platform_tags="macosx_11_0_arm64"
     local check_flight=OFF
   else
-    local python_versions="3.7m 3.8 3.9 3.10"
+    local python_versions="3.7m 3.8 3.9 3.10 3.11"
     local platform_tags="macosx_10_14_x86_64"
   fi
 
@@ -1071,6 +1071,8 @@ test_macos_wheels() {
   # verify arm64 and universal2 wheels using an universal2 python binary
   # the interpreter should be installed from python.org:
   #   https://www.python.org/ftp/python/3.9.6/python-3.9.6-macosx10.9.pkg
+  # From Python 3.11 onwards we are not providing universal2 wheels.
+  # Only architecture specific.
   if [ "$(uname -m)" = "arm64" ]; then
     for pyver in 3.9 3.10; do
       local python="/Library/Frameworks/Python.framework/Versions/${pyver}/bin/python${pyver}"

dev/tasks/tasks.yml

@@ -427,7 +427,8 @@ tasks:
 {% for python_version, python_tag, abi_tag in [("3.7", "cp37", "cp37m"),
                                                ("3.8", "cp38", "cp38"),
                                                ("3.9", "cp39", "cp39"),
-                                               ("3.10", "cp310", "cp310")] %}
+                                               ("3.10", "cp310", "cp310"),
+                                               ("3.11", "cp311", "cp311")] %}
 
 {############################## Wheel Linux ##################################}
 
@@ -488,7 +489,7 @@ tasks:
     artifacts:
       - pyarrow-{no_rc_version}-cp38-cp38-macosx_11_0_arm64.whl
 
-{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310")] %}
+{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311")] %}
   wheel-macos-big-sur-{{ python_tag }}-arm64:
     ci: github
     template: python-wheels/github.osx.arm64.yml
@@ -498,7 +499,9 @@ tasks:
       macos_deployment_target: "11.0"
     artifacts:
       - pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_11_0_arm64.whl
+{% endfor %}
 
+{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310")] %}
   wheel-macos-big-sur-{{ python_tag }}-universal2:
     ci: github
     template: python-wheels/github.osx.arm64.yml
@@ -1193,7 +1196,7 @@ tasks:
         UBUNTU: 20.04
       image: ubuntu-cpp-thread-sanitizer
 
-{% for python_version in ["3.7", "3.8", "3.9", "3.10"] %}
+{% for python_version in ["3.7", "3.8", "3.9", "3.10", "3.11"] %}
   test-conda-python-{{ python_version }}:
     ci: github
     template: docker-tests/github.linux.yml

docker-compose.yml

@@ -953,7 +953,7 @@ services:
       args:
         arch: ${ARCH}
         arch_short: ${ARCH_SHORT}
-        base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2021-10-11-14ac00e
+        base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2022-10-25-fbea779
         vcpkg: ${VCPKG}
         python: ${PYTHON}
         manylinux: 2014

python/pyarrow/_fs.pyx

@@ -78,6 +78,12 @@ cdef CFileType _unwrap_file_type(FileType ty) except *:
     assert 0
 
 
+def _file_type_to_string(ty):
+    # Python 3.11 changed str(IntEnum) to return the string representation
+    # of the integer value: https://github.com/python/cpython/issues/94763
+    return f"{ty.__class__.__name__}.{ty._name_}"
+
+
 cdef class FileInfo(_Weakrefable):
     """
     FileSystem entry info.
@@ -185,9 +191,10 @@ cdef class FileInfo(_Weakrefable):
             except ValueError:
                 return ''
 
-        s = '<FileInfo for {!r}: type={}'.format(self.path, str(self.type))
+        s = (f'<FileInfo for {self.path!r}: '
+             f'type={_file_type_to_string(self.type)}')
         if self.is_file:
-            s += ', size={}'.format(self.size)
+            s += f', size={self.size}'
         s += '>'
         return s
 

python/requirements-wheel-test.txt

@@ -8,14 +8,19 @@ pytz
 tzdata; sys_platform == 'win32'
 
 numpy==1.19.5; platform_system == "Linux"   and platform_machine == "aarch64" and python_version <  "3.7"
-numpy==1.21.3; platform_system == "Linux"   and platform_machine == "aarch64" and python_version >= "3.7"
+numpy==1.21.3; platform_system == "Linux"   and platform_machine == "aarch64" and python_version >= "3.7" and python_version < "3.11"
+numpy==1.23.4; platform_system == "Linux"   and platform_machine == "aarch64" and python_version >= "3.11"
 numpy==1.19.5; platform_system == "Linux"   and platform_machine != "aarch64" and python_version <  "3.9"
-numpy==1.21.3; platform_system == "Linux"   and platform_machine != "aarch64" and python_version >= "3.9"
-numpy==1.21.3; platform_system == "Darwin"  and platform_machine == "arm64"
+numpy==1.21.3; platform_system == "Linux"   and platform_machine != "aarch64" and python_version >= "3.9" and python_version < "3.11"
+numpy==1.23.4; platform_system == "Linux"   and platform_machine != "aarch64" and python_version >= "3.11"
+numpy==1.21.3; platform_system == "Darwin"  and platform_machine == "arm64"   and python_version <  "3.11"
+numpy==1.23.4; platform_system == "Darwin"  and platform_machine == "arm64"   and python_version >= "3.11"
 numpy==1.19.5; platform_system == "Darwin"  and platform_machine != "arm64"   and python_version <  "3.9"
-numpy==1.21.3; platform_system == "Darwin"  and platform_machine != "arm64"   and python_version >= "3.9"
+numpy==1.21.3; platform_system == "Darwin"  and platform_machine != "arm64"   and python_version >= "3.9" and python_version < "3.11"
+numpy==1.23.4; platform_system == "Darwin"  and platform_machine != "arm64"   and python_version >= "3.11"
 numpy==1.19.5; platform_system == "Windows"                                   and python_version <  "3.9"
-numpy==1.21.3; platform_system == "Windows"                                   and python_version >= "3.9"
+numpy==1.21.3; platform_system == "Windows"                                   and python_version >= "3.9" and python_version < "3.11"
+numpy==1.23.4; platform_system == "Windows"                                   and python_version >= "3.11"
 
 pandas<1.1.0;  platform_system == "Linux"   and platform_machine != "aarch64" and python_version <  "3.8"
 pandas;        platform_system == "Linux"   and platform_machine != "aarch64" and python_version >= "3.8"

python/setup.py

@@ -764,6 +764,7 @@ def has_ext_modules(foo):
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
     ],
     license='Apache License, Version 2.0',
     maintainer='Apache Arrow Developers',