Merge branch 'fix-issue-127567' of https://github.com/Agent-Hellboy/c…

…python into fix-issue-127567

InternalDocs/garbage_collector.md

@@ -199,22 +199,22 @@ unreachable:
 
 ```pycon
 >>> import gc
->>> 
+>>>
 >>> class Link:
 ...    def __init__(self, next_link=None):
 ...        self.next_link = next_link
-...  
+...
 >>> link_3 = Link()
 >>> link_2 = Link(link_3)
 >>> link_1 = Link(link_2)
 >>> link_3.next_link = link_1
 >>> A = link_1
 >>> del link_1, link_2, link_3
->>> 
+>>>
 >>> link_4 = Link()
 >>> link_4.next_link = link_4
 >>> del link_4
->>> 
+>>>
 >>> # Collect the unreachable Link object (and its .__dict__ dict).
 >>> gc.collect()
 2
@@ -459,11 +459,11 @@ specifically in a generation by calling `gc.collect(generation=NUM)`.
 >>> # Create a reference cycle.
 >>> x = MyObj()
 >>> x.self = x
->>> 
+>>>
 >>> # Initially the object is in the young generation.
 >>> gc.get_objects(generation=0)
 [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...]
->>> 
+>>>
 >>> # After a collection of the youngest generation the object
 >>> # moves to the old generation.
 >>> gc.collect(generation=0)
@@ -515,6 +515,44 @@ increment. All objects directly referred to from those stack frames are
 added to the working set.
 Then the above algorithm is repeated, starting from step 2.
 
+Determining how much work to do
+-------------------------------
+
+We need to do a certain amount of work to enusre that garbage is collected,
+but doing too much work slows down execution.
+
+To work out how much work we need to do, consider a heap with `L` live objects
+and `G0` garbage objects at the start of a full scavenge and `G1` garbage objects
+at the end of the scavenge. We don't want the amount of garbage to grow, `G1 ≤ G0`, and
+we don't want too much garbage (say 1/3 of the heap maximum), `G0 ≤ L/2`.
+For each full scavenge we must visit all objects, `T == L + G0 + G1`, during which
+`G1` garbage objects are created.
+
+The number of new objects created `N` must be at least the new garbage created, `N ≥ G1`,
+assuming that the number of live objects remains roughly constant.
+If we set `T == 4*N` we get `T > 4*G1` and `T = L + G0 + G1` => `L + G0 > 3G1`
+For a steady state heap (`G0 == G1`) we get `L > 2G0` and the desired garbage ratio.
+
+In other words, to keep the garbage fraction to 1/3 or less we need to visit
+4 times as many objects as are newly created.
+
+We can do better than this though. Not all new objects will be garbage.
+Consider the heap at the end of the scavenge with `L1` live objects and `G1`
+garbage. Also, note that `T == M + I` where `M` is the number of objects marked
+as reachable and `I` is the number of objects visited in increments.
+Everything in `M` is live, so `I ≥ G0` and in practice `I` is closer to `G0 + G1`.
+
+If we choose the amount of work done such that `2*M + I == 6N` then we can do
+less work in most cases, but are still guaranteed to keep up.
+Since `I ≳ G0 + G1` (not strictly true, but close enough)
+`T == M + I == (6N + I)/2` and `(6N + I)/2 ≳ 4G`, so we can keep up.
+
+The reason that this improves performance is that `M` is usually much larger
+than `I`. If `M == 10I`, then `T ≅ 3N`.
+
+Finally, instead of using a fixed multiple of 8, we gradually increase it as the
+heap grows. This avoids wasting work for small heaps and during startup.
+
 
 Optimization: reusing fields to save memory
 ===========================================

Lib/asyncio/selector_events.py

@@ -1175,6 +1175,7 @@ def writelines(self, list_of_data):
         # If the entire buffer couldn't be written, register a write handler
         if self._buffer:
             self._loop._add_writer(self._sock_fd, self._write_ready)
+            self._maybe_pause_protocol()
 
     def can_write_eof(self):
         return True

Lib/test/test_asyncio/test_selector_events.py

@@ -805,6 +805,18 @@ def test_writelines_send_partial(self):
         self.assertTrue(self.sock.send.called)
         self.assertTrue(self.loop.writers)
 
+    def test_writelines_pauses_protocol(self):
+        data = memoryview(b'data')
+        self.sock.send.return_value = 2
+        self.sock.send.fileno.return_value = 7
+
+        transport = self.socket_transport()
+        transport._high_water = 1
+        transport.writelines([data])
+        self.assertTrue(self.protocol.pause_writing.called)
+        self.assertTrue(self.sock.send.called)
+        self.assertTrue(self.loop.writers)
+
     @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg')
     def test_write_sendmsg_full(self):
         data = memoryview(b'data')

Lib/test/test_complex.py

@@ -299,6 +299,22 @@ def test_mul(self):
         self.assertRaises(TypeError, operator.mul, 1j, None)
         self.assertRaises(TypeError, operator.mul, None, 1j)
 
+        for z, w, r in [(1e300+1j, complex(INF, INF), complex(NAN, INF)),
+                        (1e300+1j, complex(NAN, INF), complex(-INF, INF)),
+                        (1e300+1j, complex(INF, NAN), complex(INF, INF)),
+                        (complex(INF, 1), complex(NAN, INF), complex(NAN, INF)),
+                        (complex(INF, 1), complex(INF, NAN), complex(INF, NAN)),
+                        (complex(NAN, 1), complex(1, INF), complex(-INF, NAN)),
+                        (complex(1, NAN), complex(1, INF), complex(NAN, INF)),
+                        (complex(1e200, NAN), complex(1e200, NAN), complex(INF, NAN)),
+                        (complex(1e200, NAN), complex(NAN, 1e200), complex(NAN, INF)),
+                        (complex(NAN, 1e200), complex(1e200, NAN), complex(NAN, INF)),
+                        (complex(NAN, 1e200), complex(NAN, 1e200), complex(-INF, NAN)),
+                        (complex(NAN, NAN), complex(NAN, NAN), complex(NAN, NAN))]:
+            with self.subTest(z=z, w=w, r=r):
+                self.assertComplexesAreIdentical(z * w, r)
+                self.assertComplexesAreIdentical(w * z, r)
+
     def test_mod(self):
         # % is no longer supported on complex numbers
         with self.assertRaises(TypeError):
@@ -340,6 +356,7 @@ def test_pow(self):
         self.assertAlmostEqual(pow(1j, 200), 1)
         self.assertRaises(ValueError, pow, 1+1j, 1+1j, 1+1j)
         self.assertRaises(OverflowError, pow, 1e200+1j, 1e200+1j)
+        self.assertRaises(OverflowError, pow, 1e200+1j, 5)
         self.assertRaises(TypeError, pow, 1j, None)
         self.assertRaises(TypeError, pow, None, 1j)
         self.assertAlmostEqual(pow(1j, 0.5), 0.7071067811865476+0.7071067811865475j)

Lib/test/test_gc.py

@@ -1161,27 +1161,19 @@ def make_ll(depth):
             return head
 
         head = make_ll(1000)
-        count = 1000
-
-        # There will be some objects we aren't counting,
-        # e.g. the gc stats dicts. This test checks
-        # that the counts don't grow, so we try to
-        # correct for the uncounted objects
-        # This is just an estimate.
-        CORRECTION = 20
 
         enabled = gc.isenabled()
         gc.enable()
         olds = []
         initial_heap_size = _testinternalcapi.get_tracked_heap_size()
-        for i in range(20_000):
+        iterations = max(20_000, initial_heap_size)
+        for i in range(iterations):
             newhead = make_ll(20)
-            count += 20
             newhead.surprise = head
             olds.append(newhead)
             if len(olds) == 20:
                 new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size
-                self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations")
+                self.assertLess(new_objects, initial_heap_size/2, f"Heap growing. Reached limit after {i} iterations")
                 del olds[:]
         if not enabled:
             gc.disable()

Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst

@@ -0,0 +1,2 @@
+Correct invalid corner cases which resulted in ``(nan+nanj)`` output in complex
+multiplication, e.g., ``(1e300+1j)*(nan+infj)``.  Patch by Sergey B Kirpichev.

Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst

@@ -0,0 +1,2 @@
+Added ``posix._emscripten_debugger()`` to help with debugging the test suite on
+the Emscripten target.

Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst

@@ -0,0 +1 @@
+Fixed the :class:`!asyncio.selector_events._SelectorSocketTransport` transport not pausing writes for the protocol when the buffer reaches the high water mark when using :meth:`asyncio.WriteTransport.writelines`.

Modules/posixmodule.c

@@ -84,6 +84,9 @@
    extern char * _getpty(int *, int, mode_t, int);
 #endif
 
+#ifdef __EMSCRIPTEN__
+#include "emscripten.h" // emscripten_debugger()
+#endif
 
 /*
  * A number of APIs are available on macOS from a certain macOS version.
@@ -16845,8 +16848,24 @@ os__create_environ_impl(PyObject *module)
 }
 
 
-static PyMethodDef posix_methods[] = {
+#ifdef __EMSCRIPTEN__
+/*[clinic input]
+os._emscripten_debugger
+
+Create a breakpoint for the JavaScript debugger. Emscripten only.
+[clinic start generated code]*/
+
+static PyObject *
+os__emscripten_debugger_impl(PyObject *module)
+/*[clinic end generated code: output=ad47dc3bf0661343 input=d814b1877fb6083a]*/
+{
+    emscripten_debugger();
+    Py_RETURN_NONE;
+}
+#endif /* __EMSCRIPTEN__ */
+
 
+static PyMethodDef posix_methods[] = {
     OS_STAT_METHODDEF
     OS_ACCESS_METHODDEF
     OS_TTYNAME_METHODDEF
@@ -17060,6 +17079,7 @@ static PyMethodDef posix_methods[] = {
     OS__INPUTHOOK_METHODDEF
     OS__IS_INPUTHOOK_INSTALLED_METHODDEF
     OS__CREATE_ENVIRON_METHODDEF
+    OS__EMSCRIPTEN_DEBUGGER_METHODDEF
     {NULL,              NULL}            /* Sentinel */
 };
 

Objects/complexobject.c

@@ -85,11 +85,63 @@ _Py_c_neg(Py_complex a)
 }
 
 Py_complex
-_Py_c_prod(Py_complex a, Py_complex b)
+_Py_c_prod(Py_complex z, Py_complex w)
 {
-    Py_complex r;
-    r.real = a.real*b.real - a.imag*b.imag;
-    r.imag = a.real*b.imag + a.imag*b.real;
+    double a = z.real, b = z.imag, c = w.real, d = w.imag;
+    double ac = a*c, bd = b*d, ad = a*d, bc = b*c;
+    Py_complex r = {ac - bd, ad + bc};
+
+    /* Recover infinities that computed as nan+nanj.  See e.g. the C11,
+       Annex G.5.1, routine _Cmultd(). */
+    if (isnan(r.real) && isnan(r.imag)) {
+        int recalc = 0;
+
+        if (isinf(a) || isinf(b)) {  /* z is infinite */
+            /* "Box" the infinity and change nans in the other factor to 0 */
+            a = copysign(isinf(a) ? 1.0 : 0.0, a);
+            b = copysign(isinf(b) ? 1.0 : 0.0, b);
+            if (isnan(c)) {
+                c = copysign(0.0, c);
+            }
+            if (isnan(d)) {
+                d = copysign(0.0, d);
+            }
+            recalc = 1;
+        }
+        if (isinf(c) || isinf(d)) {  /* w is infinite */
+            /* "Box" the infinity and change nans in the other factor to 0 */
+            c = copysign(isinf(c) ? 1.0 : 0.0, c);
+            d = copysign(isinf(d) ? 1.0 : 0.0, d);
+            if (isnan(a)) {
+                a = copysign(0.0, a);
+            }
+            if (isnan(b)) {
+                b = copysign(0.0, b);
+            }
+            recalc = 1;
+        }
+        if (!recalc && (isinf(ac) || isinf(bd) || isinf(ad) || isinf(bc))) {
+            /* Recover infinities from overflow by changing nans to 0 */
+            if (isnan(a)) {
+                a = copysign(0.0, a);
+            }
+            if (isnan(b)) {
+                b = copysign(0.0, b);
+            }
+            if (isnan(c)) {
+                c = copysign(0.0, c);
+            }
+            if (isnan(d)) {
+                d = copysign(0.0, d);
+            }
+            recalc = 1;
+        }
+        if (recalc) {
+            r.real = Py_INFINITY*(a*c - b*d);
+            r.imag = Py_INFINITY*(a*d + b*c);
+        }
+    }
+
     return r;
 }
 

Objects/dictobject.c

@@ -7064,9 +7064,7 @@ int
 PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
 {
     PyTypeObject *tp = Py_TYPE(obj);
-    if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
-        return 0;
-    }
+    assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
     if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
         PyDictValues *values = _PyObject_InlineValues(obj);
         if (values->valid) {