Merge branch 'python:main' into re_improvements

.github/CODEOWNERS

@@ -17,7 +17,7 @@ Makefile.pre.in               @erlend-aasland
 Modules/Setup*                @erlend-aasland
 
 # asyncio
-**/*asyncio*                  @1st1 @asvetlov @gvanrossum @kumaraditya303 @willingc
+**/*asyncio*                  @1st1 @asvetlov @kumaraditya303 @willingc
 
 # Core
 **/*context*                  @1st1
@@ -281,4 +281,4 @@ Lib/test/test_configparser.py @jaraco
 # Doc sections
 Doc/reference/                @willingc
 
-**/*weakref*                  @kumaraditya303
+**/*weakref*                  @kumaraditya303

.github/workflows/build.yml

@@ -46,7 +46,7 @@ jobs:
     # reproducible: to get the same tools versions (autoconf, aclocal, ...)
     runs-on: ubuntu-24.04
     container:
-      image: ghcr.io/python/autoconf:2024.10.11.11293396815
+      image: ghcr.io/python/autoconf:2024.10.16.11360930377
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
@@ -536,6 +536,7 @@ jobs:
     needs:
     - check_source  # Transitive dependency, needed to access `run_tests` value
     - check-docs
+    - check_autoconf_regen
     - check_generated_files
     - build_macos
     - build_ubuntu
@@ -571,6 +572,7 @@ jobs:
           ${{
             needs.check_source.outputs.run_tests != 'true'
             && '
+            check_autoconf_regen,
             check_generated_files,
             build_macos,
             build_ubuntu,

.github/workflows/posix-deps-apt.sh

@@ -1,11 +1,9 @@
 #!/bin/sh
 apt-get update
 
-# autoconf-archive is needed by autoreconf (check_generated_files job)
 apt-get -yq install \
     build-essential \
     pkg-config \
-    autoconf-archive \
     ccache \
     gdb \
     lcov \

Android/android-env.sh

@@ -3,7 +3,7 @@
 : ${HOST:?}  # GNU target triplet
 
 # You may also override the following:
-: ${api_level:=21}  # Minimum Android API level the build will run on
+: ${api_level:=24}  # Minimum Android API level the build will run on
 : ${PREFIX:-}  # Path in which to find required libraries
 
 
@@ -24,7 +24,7 @@ fail() {
 # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md
 #   where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.:
 #   https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md
-ndk_version=26.2.11394342
+ndk_version=27.1.12297006
 
 ndk=$ANDROID_HOME/ndk/$ndk_version
 if ! [ -e $ndk ]; then
@@ -58,8 +58,8 @@ for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP";
     fi
 done
 
-export CFLAGS=""
-export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment"
+export CFLAGS="-D__BIONIC_NO_PAGE_SIZE_MACRO"
+export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment -Wl,-z,max-page-size=16384"
 
 # Unlike Linux, Android does not implicitly use a dlopened library to resolve
 # relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used
@@ -85,6 +85,10 @@ if [ -n "${PREFIX:-}" ]; then
     export PKG_CONFIG_LIBDIR="$abs_prefix/lib/pkgconfig"
 fi
 
+# When compiling C++, some build systems will combine CFLAGS and CXXFLAGS, and some will
+# use CXXFLAGS alone.
+export CXXFLAGS=$CFLAGS
+
 # Use the same variable name as conda-build
 if [ $(uname) = "Darwin" ]; then
     export CPU_COUNT=$(sysctl -n hw.ncpu)

Android/android.py

@@ -138,8 +138,8 @@ def make_build_python(context):
 
 def unpack_deps(host):
     deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
-    for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.15-0",
-                     "sqlite-3.45.1-0", "xz-5.4.6-0"]:
+    for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4",
+                     "sqlite-3.45.3-3", "xz-5.4.6-1"]:
         filename = f"{name_ver}-{host}.tar.gz"
         download(f"{deps_url}/{name_ver}/{filename}")
         run(["tar", "-xf", filename])
@@ -189,12 +189,13 @@ def configure_host_python(context):
 
 def make_host_python(context):
     # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so
-    # delete any previously-installed Python libs and include files to prevent
-    # them being used during the build.
+    # delete any previous Python installation to prevent it being used during
+    # the build.
     host_dir = subdir(context.host)
     prefix_dir = host_dir / "prefix"
     delete_glob(f"{prefix_dir}/include/python*")
     delete_glob(f"{prefix_dir}/lib/libpython*")
+    delete_glob(f"{prefix_dir}/lib/python*")
 
     os.chdir(host_dir / "build")
     run(["make", "-j", str(os.cpu_count())], host=context.host)

Android/testbed/app/build.gradle.kts

@@ -30,36 +30,47 @@ val PYTHON_VERSION = file("$PYTHON_DIR/Include/patchlevel.h").useLines {
     throw GradleException("Failed to find Python version")
 }
 
-android.ndkVersion = file("../../android-env.sh").useLines {
-    for (line in it) {
-        val match = """ndk_version=(\S+)""".toRegex().find(line)
-        if (match != null) {
-            return@useLines match.groupValues[1]
-        }
-    }
-    throw GradleException("Failed to find NDK version")
-}
-
 
 android {
+    val androidEnvFile = file("../../android-env.sh").absoluteFile
+
     namespace = "org.python.testbed"
     compileSdk = 34
 
     defaultConfig {
         applicationId = "org.python.testbed"
-        minSdk = 21
+
+        minSdk = androidEnvFile.useLines {
+            for (line in it) {
+                """api_level:=(\d+)""".toRegex().find(line)?.let {
+                    return@useLines it.groupValues[1].toInt()
+                }
+            }
+            throw GradleException("Failed to find API level in $androidEnvFile")
+        }
         targetSdk = 34
+
         versionCode = 1
         versionName = "1.0"
 
         ndk.abiFilters.addAll(ABIS.keys)
         externalNativeBuild.cmake.arguments(
             "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR",
-            "-DPYTHON_VERSION=$PYTHON_VERSION")
+            "-DPYTHON_VERSION=$PYTHON_VERSION",
+            "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON",
+        )
 
         testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
     }
 
+    ndkVersion = androidEnvFile.useLines {
+        for (line in it) {
+            """ndk_version=(\S+)""".toRegex().find(line)?.let {
+                return@useLines it.groupValues[1]
+            }
+        }
+        throw GradleException("Failed to find NDK version in $androidEnvFile")
+    }
     externalNativeBuild.cmake {
         path("src/main/c/CMakeLists.txt")
     }

Android/testbed/app/src/main/c/main_activity.c

@@ -34,9 +34,12 @@ typedef struct {
     int pipe[2];
 } StreamInfo;
 
+// The FILE member can't be initialized here because stdout and stderr are not
+// compile-time constants. Instead, it's initialized immediately before the
+// redirection.
 static StreamInfo STREAMS[] = {
-    {stdout, STDOUT_FILENO, ANDROID_LOG_INFO, "native.stdout", {-1, -1}},
-    {stderr, STDERR_FILENO, ANDROID_LOG_WARN, "native.stderr", {-1, -1}},
+    {NULL, STDOUT_FILENO, ANDROID_LOG_INFO, "native.stdout", {-1, -1}},
+    {NULL, STDERR_FILENO, ANDROID_LOG_WARN, "native.stderr", {-1, -1}},
     {NULL, -1, ANDROID_LOG_UNKNOWN, NULL, {-1, -1}},
 };
 
@@ -87,6 +90,8 @@ static char *redirect_stream(StreamInfo *si) {
 JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToLogcat(
     JNIEnv *env, jobject obj
 ) {
+    STREAMS[0].file = stdout;
+    STREAMS[1].file = stderr;
     for (StreamInfo *si = STREAMS; si->file; si++) {
         char *error_prefix;
         if ((error_prefix = redirect_stream(si))) {

Android/testbed/build.gradle.kts

@@ -1,5 +1,5 @@
 // Top-level build file where you can add configuration options common to all sub-projects/modules.
 plugins {
-    id("com.android.application") version "8.4.2" apply false
+    id("com.android.application") version "8.6.1" apply false
     id("org.jetbrains.kotlin.android") version "1.9.22" apply false
 }

Android/testbed/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 #Mon Feb 19 20:29:06 GMT 2024
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

Doc/c-api/init.rst

@@ -625,7 +625,7 @@ Process-wide parameters
    returned string points into static storage; the caller should not modify its
    value.  This corresponds to the :makevar:`prefix` variable in the top-level
    :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure`
-   script at build time.  The value is available to Python code as ``sys.prefix``.
+   script at build time.  The value is available to Python code as ``sys.base_prefix``.
    It is only useful on Unix.  See also the next function.
 
    This function should not be called before :c:func:`Py_Initialize`, otherwise
@@ -635,7 +635,8 @@ Process-wide parameters
       It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
 
    .. deprecated-removed:: 3.13 3.15
-      Get :data:`sys.prefix` instead.
+      Get :data:`sys.base_prefix` instead, or :data:`sys.prefix` if
+      :ref:`virtual environments <venv-def>` need to be handled.
 
 
 .. c:function:: wchar_t* Py_GetExecPrefix()
@@ -648,7 +649,8 @@ Process-wide parameters
    should not modify its value.  This corresponds to the :makevar:`exec_prefix`
    variable in the top-level :file:`Makefile` and the ``--exec-prefix``
    argument to the :program:`configure` script at build  time.  The value is
-   available to Python code as ``sys.exec_prefix``.  It is only useful on Unix.
+   available to Python code as ``sys.base_exec_prefix``.  It is only useful on
+   Unix.
 
    Background: The exec-prefix differs from the prefix when platform dependent
    files (such as executables and shared libraries) are installed in a different
@@ -679,7 +681,8 @@ Process-wide parameters
       It now returns ``NULL`` if called before :c:func:`Py_Initialize`.
 
    .. deprecated-removed:: 3.13 3.15
-      Get :data:`sys.exec_prefix` instead.
+      Get :data:`sys.base_exec_prefix` instead, or :data:`sys.exec_prefix` if
+      :ref:`virtual environments <venv-def>` need to be handled.
 
 
 .. c:function:: wchar_t* Py_GetProgramFullPath()
@@ -2418,7 +2421,7 @@ Example usage::
 
 In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which
 can call arbitrary code through an object's deallocation function.  The critical
-section API avoids potentital deadlocks due to reentrancy and lock ordering
+section API avoids potential deadlocks due to reentrancy and lock ordering
 by allowing the runtime to temporarily suspend the critical section if the
 code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`.
 

Doc/c-api/long.rst

@@ -511,7 +511,7 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
       free(bignum);
 
    *flags* is either ``-1`` (``Py_ASNATIVEBYTES_DEFAULTS``) to select defaults
-   that behave most like a C cast, or a combintation of the other flags in
+   that behave most like a C cast, or a combination of the other flags in
    the table below.
    Note that ``-1`` cannot be combined with other flags.
 

Doc/c-api/monitoring.rst

@@ -147,7 +147,7 @@ would typically correspond to a python function.
 
    The ``version`` argument is a pointer to a value which should be allocated
    by the user together with ``state_array`` and initialized to 0,
-   and then set only by :c:func:`!PyMonitoring_EnterScope` itelf. It allows this
+   and then set only by :c:func:`!PyMonitoring_EnterScope` itself. It allows this
    function to determine whether event states have changed since the previous call,
    and to return quickly if they have not.
 

Doc/c-api/type.rst

@@ -413,6 +413,20 @@ The following functions and structs are used to create
       Creating classes whose metaclass overrides
       :c:member:`~PyTypeObject.tp_new` is no longer allowed.
 
+.. c:function:: int PyType_Freeze(PyTypeObject *type)
+
+   Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag.
+
+   All base classes of *type* must be immutable.
+
+   On success, return ``0``.
+   On error, set an exception and return ``-1``.
+
+   The type must not be used before it's made immutable. For example, type
+   instances must not be created before the type is made immutable.
+
+   .. versionadded:: 3.14
+
 .. raw:: html
 
    <!-- Keep old URL fragments working (see gh-97908) -->

Doc/c-api/typeobj.rst

@@ -682,6 +682,19 @@ and :c:data:`PyType_Type` effectively act as defaults.)
          Py_DECREF(tp);
      }
 
+   .. warning::
+
+      In a garbage collected Python, :c:member:`!tp_dealloc` may be called from
+      any Python thread, not just the thread which created the object (if the
+      object becomes part of a refcount cycle, that cycle might be collected by
+      a garbage collection on any thread).  This is not a problem for Python
+      API calls, since the thread on which :c:member:`!tp_dealloc` is called
+      will own the Global Interpreter Lock (GIL).  However, if the object being
+      destroyed in turn destroys objects from some other C or C++ library, care
+      should be taken to ensure that destroying those objects on the thread
+      which called :c:member:`!tp_dealloc` will not violate any assumptions of
+      the library.
+
 
    **Inheritance:**
 
@@ -2109,17 +2122,6 @@ and :c:data:`PyType_Type` effectively act as defaults.)
           PyErr_Restore(error_type, error_value, error_traceback);
       }
 
-   Also, note that, in a garbage collected Python,
-   :c:member:`~PyTypeObject.tp_dealloc` may be called from
-   any Python thread, not just the thread which created the object (if the object
-   becomes part of a refcount cycle, that cycle might be collected by a garbage
-   collection on any thread).  This is not a problem for Python API calls, since
-   the thread on which tp_dealloc is called will own the Global Interpreter Lock
-   (GIL). However, if the object being destroyed in turn destroys objects from some
-   other C or C++ library, care should be taken to ensure that destroying those
-   objects on the thread which called tp_dealloc will not violate any assumptions
-   of the library.
-
    **Inheritance:**
 
    This field is inherited by subtypes.

Doc/conf.py

@@ -94,7 +94,8 @@
 
 # Create table of contents entries for domain objects (e.g. functions, classes,
 # attributes, etc.). Default is True.
-toc_object_entries = False
+toc_object_entries = True
+toc_object_entries_show_parents = 'hide'
 
 # Ignore any .rst files in the includes/ directory;
 # they're embedded in pages but not rendered individually.

Doc/deprecations/c-api-pending-removal-in-3.15.rst

@@ -13,11 +13,11 @@ Pending removal in Python 3.15
   * :c:func:`PySys_ResetWarnOptions`:
     Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead.
   * :c:func:`Py_GetExecPrefix`:
-    Get :data:`sys.exec_prefix` instead.
+    Get :data:`sys.base_exec_prefix` and :data:`sys.exec_prefix` instead.
   * :c:func:`Py_GetPath`:
     Get :data:`sys.path` instead.
   * :c:func:`Py_GetPrefix`:
-    Get :data:`sys.prefix` instead.
+    Get :data:`sys.base_prefix` and :data:`sys.prefix` instead.
   * :c:func:`Py_GetProgramFullPath`:
     Get :data:`sys.executable` instead.
   * :c:func:`Py_GetProgramName`: