From b1ca34d4d5e463b8108eea20090f12292390f0cf Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 2 May 2023 01:55:41 +0200 Subject: [PATCH 01/14] gh-104016: Skip test for deeply neste f-strings on wasi (#104071) --- Lib/test/test_fstring.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 5c5176dc54a6d9..be71fde5aaba54 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -561,11 +561,12 @@ def test_mismatched_parens(self): ]) self.assertRaises(SyntaxError, eval, "f'{" + "("*500 + "}'") + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_fstring_nested_too_deeply(self): self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", ['f"{1+2:{1+2:{1+1:{1}}}}"']) - + def create_nested_fstring(n): if n == 0: return "1+1" @@ -575,13 +576,13 @@ def create_nested_fstring(n): self.assertAllRaise(SyntaxError, "too many nested f-strings", [create_nested_fstring(160)]) - + def test_syntax_error_in_nested_fstring(self): # See gh-104016 for more information on this crash self.assertAllRaise(SyntaxError, "invalid syntax", ['f"{1 1:' + ('{f"1:' * 199)]) - + def test_double_braces(self): self.assertEqual(f'{{', '{') self.assertEqual(f'a{{', 'a{') From fdd878650d325297cd801305bc2d1b0e903e42b4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 1 May 2023 19:36:00 -0600 Subject: [PATCH 02/14] gh-94673: Properly Initialize and Finalize Static Builtin Types for Each Interpreter (gh-104072) Until now, we haven't been initializing nor finalizing the per-interpreter state properly. --- Include/internal/pycore_object.h | 3 +- Include/internal/pycore_pylifecycle.h | 2 +- Include/internal/pycore_structseq.h | 10 ++- Include/internal/pycore_typeobject.h | 8 +- Modules/_io/_iomodule.c | 8 +- Objects/exceptions.c | 8 +- Objects/floatobject.c | 9 +- Objects/longobject.c | 11 +-- Objects/object.c | 8 +- Objects/structseq.c | 21 +++-- Objects/typeobject.c | 117 ++++++++++++++++---------- Objects/unicodeobject.c | 16 ++-- Objects/weakrefobject.c | 4 +- Python/errors.c | 11 +-- Python/pylifecycle.c | 6 +- Python/sysmodule.c | 28 +++--- Python/thread.c | 9 +- 17 files changed, 145 insertions(+), 134 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 2ca047846e0935..2ee0180c0554d2 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -272,8 +272,9 @@ _PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) { if (PyType_Check(op) && ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); static_builtin_state *state = _PyStaticType_GetState( - (PyTypeObject *)op); + interp, (PyTypeObject *)op); return _PyStaticType_GET_WEAKREFS_LISTPTR(state); } // Essentially _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(): diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index f96261a650dac7..7f8cc643ec0c96 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -39,7 +39,7 @@ extern PyStatus _PySys_Create( extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options); extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config); extern int _PySys_UpdateConfig(PyThreadState *tstate); -extern void _PySys_Fini(PyInterpreterState *interp); +extern void _PySys_FiniTypes(PyInterpreterState *interp); extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); diff --git a/Include/internal/pycore_structseq.h b/Include/internal/pycore_structseq.h index bd1e85c6883f01..6f5dfc12707cf8 100644 --- a/Include/internal/pycore_structseq.h +++ b/Include/internal/pycore_structseq.h @@ -16,18 +16,22 @@ PyAPI_FUNC(PyTypeObject *) _PyStructSequence_NewType( unsigned long tp_flags); extern int _PyStructSequence_InitBuiltinWithFlags( + PyInterpreterState *interp, PyTypeObject *type, PyStructSequence_Desc *desc, unsigned long tp_flags); static inline int -_PyStructSequence_InitBuiltin(PyTypeObject *type, +_PyStructSequence_InitBuiltin(PyInterpreterState *interp, + PyTypeObject *type, PyStructSequence_Desc *desc) { - return _PyStructSequence_InitBuiltinWithFlags(type, desc, 0); + return _PyStructSequence_InitBuiltinWithFlags(interp, type, desc, 0); } -extern void _PyStructSequence_FiniBuiltin(PyTypeObject *type); +extern void _PyStructSequence_FiniBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); #ifdef __cplusplus } diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 76253fd5fd864c..5bd04736c01d2d 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -104,10 +104,10 @@ _PyType_GetModuleState(PyTypeObject *type) } -extern int _PyStaticType_InitBuiltin(PyTypeObject *type); -extern static_builtin_state * _PyStaticType_GetState(PyTypeObject *); -extern void _PyStaticType_ClearWeakRefs(PyTypeObject *type); -extern void _PyStaticType_Dealloc(PyTypeObject *type); +extern int _PyStaticType_InitBuiltin(PyInterpreterState *, PyTypeObject *type); +extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTypeObject *); +extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); +extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); PyObject * _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index a3bfbc9ac5a1b1..8ec3a6081c98d9 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -680,7 +680,7 @@ _PyIO_InitTypes(PyInterpreterState *interp) for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; - if (_PyStaticType_InitBuiltin(type) < 0) { + if (_PyStaticType_InitBuiltin(interp, type) < 0) { return _PyStatus_ERR("Can't initialize builtin type"); } } @@ -691,15 +691,11 @@ _PyIO_InitTypes(PyInterpreterState *interp) void _PyIO_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - // Deallocate types in the reverse order to deallocate subclasses before // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types) - 1; i >= 0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(type); + _PyStaticType_Dealloc(interp, type); } } diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 6c9dfbd9b415cf..ba5ee291f08b0c 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3598,7 +3598,7 @@ _PyExc_InitTypes(PyInterpreterState *interp) { for (size_t i=0; i < Py_ARRAY_LENGTH(static_exceptions); i++) { PyTypeObject *exc = static_exceptions[i].exc; - if (_PyStaticType_InitBuiltin(exc) < 0) { + if (_PyStaticType_InitBuiltin(interp, exc) < 0) { return -1; } } @@ -3609,13 +3609,9 @@ _PyExc_InitTypes(PyInterpreterState *interp) static void _PyExc_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - for (Py_ssize_t i=Py_ARRAY_LENGTH(static_exceptions) - 1; i >= 0; i--) { PyTypeObject *exc = static_exceptions[i].exc; - _PyStaticType_Dealloc(exc); + _PyStaticType_Dealloc(interp, exc); } } diff --git a/Objects/floatobject.c b/Objects/floatobject.c index a694ddcd019ee8..d257857d9c619c 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1991,8 +1991,9 @@ PyStatus _PyFloat_InitTypes(PyInterpreterState *interp) { /* Init float info */ - if (_PyStructSequence_InitBuiltin(&FloatInfoType, - &floatinfo_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &FloatInfoType, + &floatinfo_desc) < 0) + { return _PyStatus_ERR("can't init float info type"); } @@ -2028,9 +2029,7 @@ _PyFloat_Fini(PyInterpreterState *interp) void _PyFloat_FiniType(PyInterpreterState *interp) { - if (_Py_IsMainInterpreter(interp)) { - _PyStructSequence_FiniBuiltin(&FloatInfoType); - } + _PyStructSequence_FiniBuiltin(interp, &FloatInfoType); } /* Print summary info about the state of the optimized allocator */ diff --git a/Objects/longobject.c b/Objects/longobject.c index de043488d7a173..853e934e2107ea 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -7,7 +7,6 @@ #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() -#include "pycore_pystate.h" // _Py_IsMainInterpreter() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() @@ -6352,7 +6351,9 @@ PyStatus _PyLong_InitTypes(PyInterpreterState *interp) { /* initialize int_info */ - if (_PyStructSequence_InitBuiltin(&Int_InfoType, &int_info_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &Int_InfoType, + &int_info_desc) < 0) + { return _PyStatus_ERR("can't init int info type"); } @@ -6363,9 +6364,5 @@ _PyLong_InitTypes(PyInterpreterState *interp) void _PyLong_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStructSequence_FiniBuiltin(&Int_InfoType); + _PyStructSequence_FiniBuiltin(interp, &Int_InfoType); } diff --git a/Objects/object.c b/Objects/object.c index 4ce10cf1192d3f..ee8690101d3cc0 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2105,7 +2105,7 @@ _PyTypes_InitTypes(PyInterpreterState *interp) // All other static types (unless initialized elsewhere) for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; - if (_PyStaticType_InitBuiltin(type) < 0) { + if (_PyStaticType_InitBuiltin(interp, type) < 0) { return _PyStatus_ERR("Can't initialize builtin type"); } if (type == &PyType_Type) { @@ -2128,15 +2128,11 @@ _PyTypes_InitTypes(PyInterpreterState *interp) void _PyTypes_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - // Deallocate types in the reverse order to deallocate subclasses before // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types)-1; i>=0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(type); + _PyStaticType_Dealloc(interp, type); } } diff --git a/Objects/structseq.c b/Objects/structseq.c index d8f55dc1eae5ed..ea476bf7a6a954 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -502,7 +502,8 @@ initialize_static_type(PyTypeObject *type, PyStructSequence_Desc *desc, } int -_PyStructSequence_InitBuiltinWithFlags(PyTypeObject *type, +_PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, + PyTypeObject *type, PyStructSequence_Desc *desc, unsigned long tp_flags) { @@ -536,7 +537,7 @@ _PyStructSequence_InitBuiltinWithFlags(PyTypeObject *type, } #endif - if (_PyStaticType_InitBuiltin(type) < 0) { + if (_PyStaticType_InitBuiltin(interp, type) < 0) { PyErr_Format(PyExc_RuntimeError, "Can't initialize builtin type %s", desc->name); @@ -606,7 +607,7 @@ PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) initialized via _PyStructSequence_InitBuiltinWithFlags(). */ void -_PyStructSequence_FiniBuiltin(PyTypeObject *type) +_PyStructSequence_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) { // Ensure that the type is initialized assert(type->tp_name != NULL); @@ -620,13 +621,15 @@ _PyStructSequence_FiniBuiltin(PyTypeObject *type) return; } - _PyStaticType_Dealloc(type); + _PyStaticType_Dealloc(interp, type); - // Undo _PyStructSequence_InitBuiltinWithFlags(). - type->tp_name = NULL; - PyMem_Free(type->tp_members); - type->tp_members = NULL; - type->tp_base = NULL; + if (_Py_IsMainInterpreter(interp)) { + // Undo _PyStructSequence_InitBuiltinWithFlags(). + type->tp_name = NULL; + PyMem_Free(type->tp_members); + type->tp_members = NULL; + type->tp_base = NULL; + } } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 060d14e254ab2d..2ed806fb01554f 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -69,13 +69,11 @@ static inline PyTypeObject * subclass_from_ref(PyObject *ref); /* helpers for for static builtin types */ -#ifndef NDEBUG static inline int static_builtin_index_is_set(PyTypeObject *self) { return self->tp_subclasses != NULL; } -#endif static inline size_t static_builtin_index_get(PyTypeObject *self) @@ -107,43 +105,46 @@ static_builtin_state_get(PyInterpreterState *interp, PyTypeObject *self) /* For static types we store some state in an array on each interpreter. */ static_builtin_state * -_PyStaticType_GetState(PyTypeObject *self) +_PyStaticType_GetState(PyInterpreterState *interp, PyTypeObject *self) { assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - PyInterpreterState *interp = _PyInterpreterState_GET(); return static_builtin_state_get(interp, self); } +/* Set the type's per-interpreter state. */ static void -static_builtin_state_init(PyTypeObject *self) +static_builtin_state_init(PyInterpreterState *interp, PyTypeObject *self) { - /* Set the type's per-interpreter state. */ - PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!static_builtin_index_is_set(self)) { + static_builtin_index_set(self, interp->types.num_builtins_initialized); + } + static_builtin_state *state = static_builtin_state_get(interp, self); /* It should only be called once for each builtin type. */ - assert(!static_builtin_index_is_set(self)); - - static_builtin_index_set(self, interp->types.num_builtins_initialized); - interp->types.num_builtins_initialized++; - - static_builtin_state *state = static_builtin_state_get(interp, self); + assert(state->type == NULL); state->type = self; + /* state->tp_subclasses is left NULL until init_subclasses() sets it. */ /* state->tp_weaklist is left NULL until insert_head() or insert_after() (in weakrefobject.c) sets it. */ + + interp->types.num_builtins_initialized++; } +/* Reset the type's per-interpreter state. + This basically undoes what static_builtin_state_init() did. */ static void -static_builtin_state_clear(PyTypeObject *self) +static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) { - /* Reset the type's per-interpreter state. - This basically undoes what static_builtin_state_init() did. */ - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, self); + + assert(state->type != NULL); state->type = NULL; assert(state->tp_weaklist == NULL); // It was already cleared out. - static_builtin_index_clear(self); + + if (_Py_IsMainInterpreter(interp)) { + static_builtin_index_clear(self); + } assert(interp->types.num_builtins_initialized > 0); interp->types.num_builtins_initialized--; @@ -4491,33 +4492,37 @@ clear_static_tp_subclasses(PyTypeObject *type) clear_subclasses(type); } +static void +clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) +{ + if (_Py_IsMainInterpreter(interp)) { + Py_CLEAR(type->tp_dict); + Py_CLEAR(type->tp_bases); + Py_CLEAR(type->tp_mro); + Py_CLEAR(type->tp_cache); + } + clear_static_tp_subclasses(type); +} + void -_PyStaticType_Dealloc(PyTypeObject *type) +_PyStaticType_Dealloc(PyInterpreterState *interp, PyTypeObject *type) { - assert(!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)); + assert(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); + assert(_Py_IsImmortal((PyObject *)type)); type_dealloc_common(type); - Py_CLEAR(type->tp_dict); - Py_CLEAR(type->tp_bases); - Py_CLEAR(type->tp_mro); - Py_CLEAR(type->tp_cache); - clear_static_tp_subclasses(type); + clear_static_type_objects(interp, type); - // PyObject_ClearWeakRefs() raises an exception if Py_REFCNT() != 0 - if (Py_REFCNT(type) == 0) { - PyObject_ClearWeakRefs((PyObject *)type); + if (_Py_IsMainInterpreter(interp)) { + type->tp_flags &= ~Py_TPFLAGS_READY; + type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; + type->tp_version_tag = 0; } - type->tp_flags &= ~Py_TPFLAGS_READY; - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; - - if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - _PyStaticType_ClearWeakRefs(type); - static_builtin_state_clear(type); - /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ - } + _PyStaticType_ClearWeakRefs(interp, type); + static_builtin_state_clear(interp, type); + /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ } @@ -4564,7 +4569,8 @@ static PyObject * lookup_subclasses(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - static_builtin_state *state = _PyStaticType_GetState(self); + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); return state->tp_subclasses; } @@ -4574,8 +4580,9 @@ lookup_subclasses(PyTypeObject *self) int _PyType_HasSubclasses(PyTypeObject *self) { + PyInterpreterState *interp = _PyInterpreterState_GET(); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN && - _PyStaticType_GetState(self) == NULL) { + _PyStaticType_GetState(interp, self) == NULL) { return 0; } if (lookup_subclasses(self) == NULL) { @@ -6938,7 +6945,8 @@ type_ready_post_checks(PyTypeObject *type) else if (type->tp_dictoffset < (Py_ssize_t)sizeof(PyObject)) { if (type->tp_dictoffset + type->tp_basicsize <= 0) { PyErr_Format(PyExc_SystemError, - "type %s has a tp_dictoffset that is too small"); + "type %s has a tp_dictoffset that is too small", + type->tp_name); } } return 0; @@ -7029,17 +7037,32 @@ PyType_Ready(PyTypeObject *type) } int -_PyStaticType_InitBuiltin(PyTypeObject *self) +_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) { assert(_Py_IsImmortal((PyObject *)self)); assert(!(self->tp_flags & Py_TPFLAGS_HEAPTYPE)); + assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); + assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); +#ifndef NDEBUG + int ismain = _Py_IsMainInterpreter(interp); +#endif if (self->tp_flags & Py_TPFLAGS_READY) { + assert(!ismain); assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); + assert(self->tp_flags & Py_TPFLAGS_VALID_VERSION_TAG); + + static_builtin_state_init(interp, self); + + /* Per-interpreter tp_subclasses is done lazily. + Otherwise we would initialize it here. */ + assert(_PyType_CheckConsistency(self)); return 0; } + assert(ismain); + self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; @@ -7047,11 +7070,11 @@ _PyStaticType_InitBuiltin(PyTypeObject *self) self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; - static_builtin_state_init(self); + static_builtin_state_init(interp, self); int res = type_ready(self); if (res < 0) { - static_builtin_state_clear(self); + static_builtin_state_clear(interp, self); } return res; } @@ -7065,7 +7088,8 @@ init_subclasses(PyTypeObject *self) return NULL; } if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - static_builtin_state *state = _PyStaticType_GetState(self); + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); state->tp_subclasses = subclasses; return subclasses; } @@ -7080,7 +7104,8 @@ clear_subclasses(PyTypeObject *self) callers also test if tp_subclasses is NULL to check if a static type has no subclass. */ if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - static_builtin_state *state = _PyStaticType_GetState(self); + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); Py_CLEAR(state->tp_subclasses); return; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7537c12e92680c..6ae68cc20f7dee 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14573,13 +14573,13 @@ _PyUnicode_InitGlobalObjects(PyInterpreterState *interp) PyStatus _PyUnicode_InitTypes(PyInterpreterState *interp) { - if (_PyStaticType_InitBuiltin(&EncodingMapType) < 0) { + if (_PyStaticType_InitBuiltin(interp, &EncodingMapType) < 0) { goto error; } - if (_PyStaticType_InitBuiltin(&PyFieldNameIter_Type) < 0) { + if (_PyStaticType_InitBuiltin(interp, &PyFieldNameIter_Type) < 0) { goto error; } - if (_PyStaticType_InitBuiltin(&PyFormatterIter_Type) < 0) { + if (_PyStaticType_InitBuiltin(interp, &PyFormatterIter_Type) < 0) { goto error; } return _PyStatus_OK(); @@ -15158,13 +15158,9 @@ unicode_is_finalizing(void) void _PyUnicode_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStaticType_Dealloc(&EncodingMapType); - _PyStaticType_Dealloc(&PyFieldNameIter_Type); - _PyStaticType_Dealloc(&PyFormatterIter_Type); + _PyStaticType_Dealloc(interp, &EncodingMapType); + _PyStaticType_Dealloc(interp, &PyFieldNameIter_Type); + _PyStaticType_Dealloc(interp, &PyFormatterIter_Type); } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index c1afe63ecf66f6..aee79fc1410b29 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1017,9 +1017,9 @@ PyObject_ClearWeakRefs(PyObject *object) * or anything else. */ void -_PyStaticType_ClearWeakRefs(PyTypeObject *type) +_PyStaticType_ClearWeakRefs(PyInterpreterState *interp, PyTypeObject *type) { - static_builtin_state *state = _PyStaticType_GetState(type); + static_builtin_state *state = _PyStaticType_GetState(interp, type); PyObject **list = _PyStaticType_GET_WEAKREFS_LISTPTR(state); while (*list != NULL) { /* Note that clear_weakref() pops the first ref off the type's diff --git a/Python/errors.c b/Python/errors.c index ce72049b92de27..a8000ac94918db 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1342,8 +1342,9 @@ static PyStructSequence_Desc UnraisableHookArgs_desc = { PyStatus _PyErr_InitTypes(PyInterpreterState *interp) { - if (_PyStructSequence_InitBuiltin(&UnraisableHookArgsType, - &UnraisableHookArgs_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &UnraisableHookArgsType, + &UnraisableHookArgs_desc) < 0) + { return _PyStatus_ERR("failed to initialize UnraisableHookArgs type"); } return _PyStatus_OK(); @@ -1353,11 +1354,7 @@ _PyErr_InitTypes(PyInterpreterState *interp) void _PyErr_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStructSequence_FiniBuiltin(&UnraisableHookArgsType); + _PyStructSequence_FiniBuiltin(interp, &UnraisableHookArgsType); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ba248d208e425a..b8a115236900b9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1663,8 +1663,10 @@ flush_std_files(void) static void finalize_interp_types(PyInterpreterState *interp) { + _PyIO_FiniTypes(interp); + _PyUnicode_FiniTypes(interp); - _PySys_Fini(interp); + _PySys_FiniTypes(interp); _PyExc_Fini(interp); _PyAsyncGen_Fini(interp); _PyContext_Fini(interp); @@ -1706,8 +1708,6 @@ finalize_interp_clear(PyThreadState *tstate) /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); - _PyIO_FiniTypes(tstate->interp); - /* Clear all loghooks */ /* Both _PySys_Audit function and users still need PyObject, such as tuple. Call _PySys_ClearAuditHooks when PyObject available. */ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 81dabe6102f18d..781588b0df4ead 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3141,6 +3141,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) { PyObject *version_info; int res; + PyInterpreterState *interp = tstate->interp; /* stdin/stdout/stderr are set in pylifecycle.c */ @@ -3166,7 +3167,9 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("float_info", PyFloat_GetInfo()); SET_SYS("int_info", PyLong_GetInfo()); /* initialize hash_info */ - if (_PyStructSequence_InitBuiltin(&Hash_InfoType, &hash_info_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &Hash_InfoType, + &hash_info_desc) < 0) + { goto type_init_failed; } SET_SYS("hash_info", get_hash_info(tstate)); @@ -3190,7 +3193,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) #define ENSURE_INFO_TYPE(TYPE, DESC) \ do { \ if (_PyStructSequence_InitBuiltinWithFlags( \ - &TYPE, &DESC, Py_TPFLAGS_DISALLOW_INSTANTIATION) < 0) { \ + interp, &TYPE, &DESC, Py_TPFLAGS_DISALLOW_INSTANTIATION) < 0) { \ goto type_init_failed; \ } \ } while (0) @@ -3226,8 +3229,9 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("thread_info", PyThread_GetInfo()); /* initialize asyncgen_hooks */ - if (_PyStructSequence_InitBuiltin( - &AsyncGenHooksType, &asyncgen_hooks_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &AsyncGenHooksType, + &asyncgen_hooks_desc) < 0) + { goto type_init_failed; } @@ -3489,20 +3493,20 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) void -_PySys_Fini(PyInterpreterState *interp) +_PySys_FiniTypes(PyInterpreterState *interp) { - if (_Py_IsMainInterpreter(interp)) { - _PyStructSequence_FiniBuiltin(&VersionInfoType); - _PyStructSequence_FiniBuiltin(&FlagsType); + _PyStructSequence_FiniBuiltin(interp, &VersionInfoType); + _PyStructSequence_FiniBuiltin(interp, &FlagsType); #if defined(MS_WINDOWS) - _PyStructSequence_FiniBuiltin(&WindowsVersionType); + _PyStructSequence_FiniBuiltin(interp, &WindowsVersionType); #endif - _PyStructSequence_FiniBuiltin(&Hash_InfoType); - _PyStructSequence_FiniBuiltin(&AsyncGenHooksType); + _PyStructSequence_FiniBuiltin(interp, &Hash_InfoType); + _PyStructSequence_FiniBuiltin(interp, &AsyncGenHooksType); #ifdef __EMSCRIPTEN__ + if (_Py_IsMainInterpreter(interp)) { Py_CLEAR(EmscriptenInfoType); -#endif } +#endif } diff --git a/Python/thread.c b/Python/thread.c index f90cd34a073540..7fc53f9b61360b 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -137,7 +137,8 @@ PyThread_GetInfo(void) int len; #endif - if (_PyStructSequence_InitBuiltin(&ThreadInfoType, &threadinfo_desc) < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyStructSequence_InitBuiltin(interp, &ThreadInfoType, &threadinfo_desc) < 0) { return NULL; } @@ -191,9 +192,5 @@ PyThread_GetInfo(void) void _PyThread_FiniType(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStructSequence_FiniBuiltin(&ThreadInfoType); + _PyStructSequence_FiniBuiltin(interp, &ThreadInfoType); } From f73abf8e03fd370c86fbb2a249fe1e065f7d84b4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 1 May 2023 20:34:43 -0600 Subject: [PATCH 03/14] gh-94673: Hide Objects in PyTypeObject Behind Accessors (gh-104074) This makes it much cleaner to move more PyTypeObject fields to PyInterpreterState. --- Include/internal/pycore_object.h | 6 - Include/internal/pycore_typeobject.h | 14 + Modules/_abc.c | 7 +- Objects/structseq.c | 8 +- Objects/typeobject.c | 559 ++++++++++++++++----------- Python/context.c | 2 +- 6 files changed, 352 insertions(+), 244 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 2ee0180c0554d2..91853ad0525b55 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -331,10 +331,6 @@ extern int _Py_CheckSlotResult( const char *slot_name, int success); -// PyType_Ready() must be called if _PyType_IsReady() is false. -// See also the Py_TPFLAGS_READY flag. -#define _PyType_IsReady(type) ((type)->tp_dict != NULL) - // Test if a type supports weak references static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { return (type->tp_weaklistoffset != 0); @@ -392,8 +388,6 @@ _PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); -extern int _PyType_HasSubclasses(PyTypeObject *); -extern PyObject* _PyType_GetSubclasses(PyTypeObject *); // Access macro to the members which are floating "behind" the object static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 5bd04736c01d2d..f865e51aeba503 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -109,6 +109,20 @@ extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTyp extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); +PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); +extern PyObject * _PyType_GetBases(PyTypeObject *type); +extern PyObject * _PyType_GetMRO(PyTypeObject *type); +extern PyObject* _PyType_GetSubclasses(PyTypeObject *); +extern int _PyType_HasSubclasses(PyTypeObject *); + +// PyType_Ready() must be called if _PyType_IsReady() is false. +// See also the Py_TPFLAGS_READY flag. +static inline int +_PyType_IsReady(PyTypeObject *type) +{ + return _PyType_GetDict(type) != NULL; +} + PyObject * _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); PyObject * diff --git a/Modules/_abc.c b/Modules/_abc.c index 9d6654b4e58aad..997b618d557ab2 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -452,7 +452,8 @@ _abc__abc_init(PyObject *module, PyObject *self) * their special status w.r.t. pattern matching. */ if (PyType_Check(self)) { PyTypeObject *cls = (PyTypeObject *)self; - PyObject *flags = PyDict_GetItemWithError(cls->tp_dict, + PyObject *dict = _PyType_GetDict(cls); + PyObject *flags = PyDict_GetItemWithError(dict, &_Py_ID(__abc_tpflags__)); if (flags == NULL) { if (PyErr_Occurred()) { @@ -471,7 +472,7 @@ _abc__abc_init(PyObject *module, PyObject *self) } ((PyTypeObject *)self)->tp_flags |= (val & COLLECTION_FLAGS); } - if (PyDict_DelItem(cls->tp_dict, &_Py_ID(__abc_tpflags__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__abc_tpflags__)) < 0) { return NULL; } } @@ -742,7 +743,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, Py_DECREF(ok); /* 4. Check if it's a direct subclass. */ - PyObject *mro = ((PyTypeObject *)subclass)->tp_mro; + PyObject *mro = _PyType_GetMRO((PyTypeObject *)subclass); assert(PyTuple_Check(mro)); for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); diff --git a/Objects/structseq.c b/Objects/structseq.c index ea476bf7a6a954..f63660acb639c3 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -26,7 +26,7 @@ const char * const PyStructSequence_UnnamedField = "unnamed field"; static Py_ssize_t get_type_attr_as_size(PyTypeObject *tp, PyObject *name) { - PyObject *v = PyDict_GetItemWithError(tp->tp_dict, name); + PyObject *v = PyDict_GetItemWithError(_PyType_GetDict(tp), name); if (v == NULL && !PyErr_Occurred()) { PyErr_Format(PyExc_TypeError, "Missed attribute '%U' of type %s", @@ -493,7 +493,7 @@ initialize_static_type(PyTypeObject *type, PyStructSequence_Desc *desc, Py_INCREF(type); if (initialize_structseq_dict( - desc, type->tp_dict, n_members, n_unnamed_members) < 0) { + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { Py_DECREF(type); return -1; } @@ -549,7 +549,7 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, } if (initialize_structseq_dict( - desc, type->tp_dict, n_members, n_unnamed_members) < 0) { + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { goto error; } @@ -675,7 +675,7 @@ _PyStructSequence_NewType(PyStructSequence_Desc *desc, unsigned long tp_flags) } if (initialize_structseq_dict( - desc, type->tp_dict, n_members, n_unnamed_members) < 0) { + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { Py_DECREF(type); return NULL; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2ed806fb01554f..a9d3a69263fb40 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -64,7 +64,19 @@ lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound); static int slot_tp_setattro(PyObject *self, PyObject *name, PyObject *value); -static inline PyTypeObject * subclass_from_ref(PyObject *ref); + +static inline PyTypeObject * +type_from_ref(PyObject *ref) +{ + assert(PyWeakref_CheckRef(ref)); + PyObject *obj = PyWeakref_GET_OBJECT(ref); // borrowed ref + assert(obj != NULL); + if (obj == Py_None) { + return NULL; + } + assert(PyType_Check(obj)); + return _PyType_CAST(obj); +} /* helpers for for static builtin types */ @@ -155,6 +167,178 @@ static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) /* end static builtin helpers */ +/* accessors for objects stored on PyTypeObject */ + +static inline PyObject * +lookup_tp_dict(PyTypeObject *self) +{ + return self->tp_dict; +} + +PyObject * +_PyType_GetDict(PyTypeObject *self) +{ + return lookup_tp_dict(self); +} + +static inline void +set_tp_dict(PyTypeObject *self, PyObject *dict) +{ + self->tp_dict = dict; +} + +static inline void +clear_tp_dict(PyTypeObject *self) +{ + Py_CLEAR(self->tp_dict); +} + + +static inline PyObject * +lookup_tp_bases(PyTypeObject *self) +{ + return self->tp_bases; +} + +PyObject * +_PyType_GetBases(PyTypeObject *self) +{ + return lookup_tp_bases(self); +} + +static inline void +set_tp_bases(PyTypeObject *self, PyObject *bases) +{ + self->tp_bases = bases; +} + +static inline void +clear_tp_bases(PyTypeObject *self) +{ + Py_CLEAR(self->tp_bases); +} + + +static inline PyObject * +lookup_tp_mro(PyTypeObject *self) +{ + return self->tp_mro; +} + +PyObject * +_PyType_GetMRO(PyTypeObject *self) +{ + return lookup_tp_mro(self); +} + +static inline void +set_tp_mro(PyTypeObject *self, PyObject *mro) +{ + self->tp_mro = mro; +} + +static inline void +clear_tp_mro(PyTypeObject *self) +{ + Py_CLEAR(self->tp_mro); +} + + +static PyObject * +init_tp_subclasses(PyTypeObject *self) +{ + PyObject *subclasses = PyDict_New(); + if (subclasses == NULL) { + return NULL; + } + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + state->tp_subclasses = subclasses; + return subclasses; + } + self->tp_subclasses = (void *)subclasses; + return subclasses; +} + +static void +clear_tp_subclasses(PyTypeObject *self) +{ + /* Delete the dictionary to save memory. _PyStaticType_Dealloc() + callers also test if tp_subclasses is NULL to check if a static type + has no subclass. */ + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + Py_CLEAR(state->tp_subclasses); + return; + } + Py_CLEAR(self->tp_subclasses); +} + +static inline PyObject * +lookup_tp_subclasses(PyTypeObject *self) +{ + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + return state->tp_subclasses; + } + return (PyObject *)self->tp_subclasses; +} + +int +_PyType_HasSubclasses(PyTypeObject *self) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN + // XXX _PyStaticType_GetState() should never return NULL. + && _PyStaticType_GetState(interp, self) == NULL) + { + return 0; + } + if (lookup_tp_subclasses(self) == NULL) { + return 0; + } + return 1; +} + +PyObject* +_PyType_GetSubclasses(PyTypeObject *self) +{ + PyObject *list = PyList_New(0); + if (list == NULL) { + return NULL; + } + + PyObject *subclasses = lookup_tp_subclasses(self); // borrowed ref + if (subclasses == NULL) { + return list; + } + assert(PyDict_CheckExact(subclasses)); + // The loop cannot modify tp_subclasses, there is no need + // to hold a strong reference (use a borrowed reference). + + Py_ssize_t i = 0; + PyObject *ref; // borrowed ref + while (PyDict_Next(subclasses, &i, NULL, &ref)) { + PyTypeObject *subclass = type_from_ref(ref); // borrowed + if (subclass == NULL) { + continue; + } + + if (PyList_Append(list, _PyObject_CAST(subclass)) < 0) { + Py_DECREF(list); + return NULL; + } + } + return list; +} + +/* end accessors for objects stored on PyTypeObject */ + + /* * finds the beginning of the docstring's introspection signature. * if present, returns a pointer pointing to the first '('. @@ -225,7 +409,7 @@ _PyType_CheckConsistency(PyTypeObject *type) CHECK(PyType_Check(type)); CHECK(!(type->tp_flags & Py_TPFLAGS_READYING)); - CHECK(type->tp_dict != NULL); + CHECK(lookup_tp_dict(type) != NULL); if (type->tp_flags & Py_TPFLAGS_HAVE_GC) { // bpo-44263: tp_traverse is required if Py_TPFLAGS_HAVE_GC is set. @@ -235,7 +419,7 @@ _PyType_CheckConsistency(PyTypeObject *type) if (type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION) { CHECK(type->tp_new == NULL); - CHECK(PyDict_Contains(type->tp_dict, &_Py_ID(__new__)) == 0); + CHECK(PyDict_Contains(lookup_tp_dict(type), &_Py_ID(__new__)) == 0); } return 1; @@ -361,8 +545,6 @@ _PyTypes_Fini(PyInterpreterState *interp) } -static PyObject * lookup_subclasses(PyTypeObject *); - int PyType_AddWatcher(PyType_WatchCallback callback) { @@ -462,14 +644,14 @@ PyType_Modified(PyTypeObject *type) return; } - PyObject *subclasses = lookup_subclasses(type); + PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses != NULL) { assert(PyDict_CheckExact(subclasses)); Py_ssize_t i = 0; PyObject *ref; while (PyDict_Next(subclasses, &i, NULL, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == NULL) { continue; } @@ -598,7 +780,7 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) assert (type->tp_version_tag != 0); } - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); Py_ssize_t n = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < n; i++) { PyObject *b = PyTuple_GET_ITEM(bases, i); @@ -749,7 +931,8 @@ type_module(PyTypeObject *type, void *context) PyObject *mod; if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { - mod = PyDict_GetItemWithError(type->tp_dict, &_Py_ID(__module__)); + PyObject *dict = lookup_tp_dict(type); + mod = PyDict_GetItemWithError(dict, &_Py_ID(__module__)); if (mod == NULL) { if (!PyErr_Occurred()) { PyErr_Format(PyExc_AttributeError, "__module__"); @@ -781,7 +964,8 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyType_Modified(type); - return PyDict_SetItem(type->tp_dict, &_Py_ID(__module__), value); + PyObject *dict = lookup_tp_dict(type); + return PyDict_SetItem(dict, &_Py_ID(__module__), value); } static PyObject * @@ -790,9 +974,10 @@ type_abstractmethods(PyTypeObject *type, void *context) PyObject *mod = NULL; /* type itself has an __abstractmethods__ descriptor (this). Don't return that. */ - if (type != &PyType_Type) - mod = PyDict_GetItemWithError(type->tp_dict, - &_Py_ID(__abstractmethods__)); + if (type != &PyType_Type) { + PyObject *dict = lookup_tp_dict(type); + mod = PyDict_GetItemWithError(dict, &_Py_ID(__abstractmethods__)); + } if (!mod) { if (!PyErr_Occurred()) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); @@ -810,15 +995,16 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) special to update subclasses. */ int abstract, res; + PyObject *dict = lookup_tp_dict(type); if (value != NULL) { abstract = PyObject_IsTrue(value); if (abstract < 0) return -1; - res = PyDict_SetItem(type->tp_dict, &_Py_ID(__abstractmethods__), value); + res = PyDict_SetItem(dict, &_Py_ID(__abstractmethods__), value); } else { abstract = 0; - res = PyDict_DelItem(type->tp_dict, &_Py_ID(__abstractmethods__)); + res = PyDict_DelItem(dict, &_Py_ID(__abstractmethods__)); if (res && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); return -1; @@ -837,7 +1023,7 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) static PyObject * type_get_bases(PyTypeObject *type, void *context) { - return Py_NewRef(type->tp_bases); + return Py_NewRef(lookup_tp_bases(type)); } static PyTypeObject *best_base(PyObject *); @@ -865,7 +1051,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) /* error / reentrance */ return res; } - PyObject *new_mro = type->tp_mro; + PyObject *new_mro = lookup_tp_mro(type); PyObject *tuple; if (old_mro != NULL) { @@ -884,7 +1070,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) Py_XDECREF(tuple); if (res < 0) { - type->tp_mro = old_mro; + set_tp_mro(type, old_mro); Py_DECREF(new_mro); return -1; } @@ -963,7 +1149,8 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) below), which in turn may cause an inheritance cycle through tp_base chain. And this is definitely not what you want to ever happen. */ - (base->tp_mro != NULL && type_is_subtype_base_chain(base, type))) + (lookup_tp_mro(base) != NULL + && type_is_subtype_base_chain(base, type))) { PyErr_SetString(PyExc_TypeError, "a __bases__ item causes an inheritance cycle"); @@ -980,11 +1167,11 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) return -1; } - PyObject *old_bases = type->tp_bases; + PyObject *old_bases = lookup_tp_bases(type); assert(old_bases != NULL); PyTypeObject *old_base = type->tp_base; - type->tp_bases = Py_NewRef(new_bases); + set_tp_bases(type, Py_NewRef(new_bases)); type->tp_base = (PyTypeObject *)Py_NewRef(new_base); PyObject *temp = PyList_New(0); @@ -999,7 +1186,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) /* Take no action in case if type->tp_bases has been replaced through reentrance. */ int res; - if (type->tp_bases == new_bases) { + if (lookup_tp_bases(type) == new_bases) { /* any base that was in __bases__ but now isn't, we need to remove |type| from its tp_subclasses. conversely, any class now in __bases__ that wasn't @@ -1030,18 +1217,18 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) PyArg_UnpackTuple(PyList_GET_ITEM(temp, i), "", 2, 3, &cls, &new_mro, &old_mro); /* Do not rollback if cls has a newer version of MRO. */ - if (cls->tp_mro == new_mro) { - cls->tp_mro = Py_XNewRef(old_mro); + if (lookup_tp_mro(cls) == new_mro) { + set_tp_mro(cls, Py_XNewRef(old_mro)); Py_DECREF(new_mro); } } Py_DECREF(temp); bail: - if (type->tp_bases == new_bases) { + if (lookup_tp_bases(type) == new_bases) { assert(type->tp_base == new_base); - type->tp_bases = old_bases; + set_tp_bases(type, old_bases); type->tp_base = old_base; Py_DECREF(new_bases); @@ -1059,10 +1246,11 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) static PyObject * type_dict(PyTypeObject *type, void *context) { - if (type->tp_dict == NULL) { + PyObject *dict = lookup_tp_dict(type); + if (dict == NULL) { Py_RETURN_NONE; } - return PyDictProxy_New(type->tp_dict); + return PyDictProxy_New(dict); } static PyObject * @@ -1072,7 +1260,8 @@ type_get_doc(PyTypeObject *type, void *context) if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE) && type->tp_doc != NULL) { return _PyType_GetDocFromInternalDoc(type->tp_name, type->tp_doc); } - result = PyDict_GetItemWithError(type->tp_dict, &_Py_ID(__doc__)); + PyObject *dict = lookup_tp_dict(type); + result = PyDict_GetItemWithError(dict, &_Py_ID(__doc__)); if (result == NULL) { if (!PyErr_Occurred()) { result = Py_NewRef(Py_None); @@ -1100,7 +1289,8 @@ type_set_doc(PyTypeObject *type, PyObject *value, void *context) if (!check_set_special_type_attr(type, value, "__doc__")) return -1; PyType_Modified(type); - return PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), value); + PyObject *dict = lookup_tp_dict(type); + return PyDict_SetItem(dict, &_Py_ID(__doc__), value); } static PyObject * @@ -1113,9 +1303,9 @@ type_get_annotations(PyTypeObject *type, void *context) PyObject *annotations; /* there's no _PyDict_GetItemId without WithError, so let's LBYL. */ - if (PyDict_Contains(type->tp_dict, &_Py_ID(__annotations__))) { - annotations = PyDict_GetItemWithError( - type->tp_dict, &_Py_ID(__annotations__)); + PyObject *dict = lookup_tp_dict(type); + if (PyDict_Contains(dict, &_Py_ID(__annotations__))) { + annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); /* ** PyDict_GetItemWithError could still fail, ** for instance with a well-timed Ctrl-C or a MemoryError. @@ -1133,7 +1323,7 @@ type_get_annotations(PyTypeObject *type, void *context) annotations = PyDict_New(); if (annotations) { int result = PyDict_SetItem( - type->tp_dict, &_Py_ID(__annotations__), annotations); + dict, &_Py_ID(__annotations__), annotations); if (result) { Py_CLEAR(annotations); } else { @@ -1155,16 +1345,17 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) } int result; + PyObject *dict = lookup_tp_dict(type); if (value != NULL) { /* set */ - result = PyDict_SetItem(type->tp_dict, &_Py_ID(__annotations__), value); + result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); } else { /* delete */ - if (!PyDict_Contains(type->tp_dict, &_Py_ID(__annotations__))) { + if (!PyDict_Contains(dict, &_Py_ID(__annotations__))) { PyErr_Format(PyExc_AttributeError, "__annotations__"); return -1; } - result = PyDict_DelItem(type->tp_dict, &_Py_ID(__annotations__)); + result = PyDict_DelItem(dict, &_Py_ID(__annotations__)); } if (result == 0) { @@ -1751,7 +1942,7 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) { PyObject *mro; - mro = a->tp_mro; + mro = lookup_tp_mro(a); if (mro != NULL) { /* Deal with multiple inheritance without recursion by walking the MRO tuple */ @@ -2135,17 +2326,17 @@ mro_implementation(PyTypeObject *type) return NULL; } - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); Py_ssize_t n = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); - if (base->tp_mro == NULL) { + if (lookup_tp_mro(base) == NULL) { PyErr_Format(PyExc_TypeError, "Cannot extend an incomplete type '%.100s'", base->tp_name); return NULL; } - assert(PyTuple_Check(base->tp_mro)); + assert(PyTuple_Check(lookup_tp_mro(base))); } if (n == 1) { @@ -2153,7 +2344,8 @@ mro_implementation(PyTypeObject *type) * is trivial. */ PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, 0)); - Py_ssize_t k = PyTuple_GET_SIZE(base->tp_mro); + PyObject *base_mro = lookup_tp_mro(base); + Py_ssize_t k = PyTuple_GET_SIZE(base_mro); PyObject *result = PyTuple_New(k + 1); if (result == NULL) { return NULL; @@ -2162,7 +2354,7 @@ mro_implementation(PyTypeObject *type) ; PyTuple_SET_ITEM(result, 0, Py_NewRef(type)); for (Py_ssize_t i = 0; i < k; i++) { - PyObject *cls = PyTuple_GET_ITEM(base->tp_mro, i); + PyObject *cls = PyTuple_GET_ITEM(base_mro, i); PyTuple_SET_ITEM(result, i + 1, Py_NewRef(cls)); } return result; @@ -2189,7 +2381,7 @@ mro_implementation(PyTypeObject *type) for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); - to_merge[i] = base->tp_mro; + to_merge[i] = lookup_tp_mro(base); } to_merge[n] = bases; @@ -2344,9 +2536,9 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) /* Keep a reference to be able to do a reentrancy check below. Don't let old_mro be GC'ed and its address be reused for another object, like (suddenly!) a new tp_mro. */ - old_mro = Py_XNewRef(type->tp_mro); + old_mro = Py_XNewRef(lookup_tp_mro(type)); new_mro = mro_invoke(type); /* might cause reentrance */ - reent = (type->tp_mro != old_mro); + reent = (lookup_tp_mro(type) != old_mro); Py_XDECREF(old_mro); if (new_mro == NULL) { return -1; @@ -2357,12 +2549,12 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) return 0; } - type->tp_mro = new_mro; + set_tp_mro(type, new_mro); - type_mro_modified(type, type->tp_mro); + type_mro_modified(type, new_mro); /* corner case: the super class might have been hidden from the custom MRO */ - type_mro_modified(type, type->tp_bases); + type_mro_modified(type, lookup_tp_bases(type)); // XXX Expand this to Py_TPFLAGS_IMMUTABLETYPE? if (!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)) { @@ -2964,7 +3156,7 @@ type_new_alloc(type_new_ctx *ctx) type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; - type->tp_bases = Py_NewRef(ctx->bases); + set_tp_bases(type, Py_NewRef(ctx->bases)); type->tp_base = (PyTypeObject *)Py_NewRef(ctx->base); type->tp_dealloc = subtype_dealloc; @@ -3004,7 +3196,8 @@ type_new_set_name(const type_new_ctx *ctx, PyTypeObject *type) static int type_new_set_module(PyTypeObject *type) { - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__module__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__module__)); if (r < 0) { return -1; } @@ -3025,7 +3218,7 @@ type_new_set_module(PyTypeObject *type) return 0; } - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__module__), module) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__module__), module) < 0) { return -1; } return 0; @@ -3038,8 +3231,8 @@ static int type_new_set_ht_name(PyTypeObject *type) { PyHeapTypeObject *et = (PyHeapTypeObject *)type; - PyObject *qualname = PyDict_GetItemWithError( - type->tp_dict, &_Py_ID(__qualname__)); + PyObject *dict = lookup_tp_dict(type); + PyObject *qualname = PyDict_GetItemWithError(dict, &_Py_ID(__qualname__)); if (qualname != NULL) { if (!PyUnicode_Check(qualname)) { PyErr_Format(PyExc_TypeError, @@ -3048,7 +3241,7 @@ type_new_set_ht_name(PyTypeObject *type) return -1; } et->ht_qualname = Py_NewRef(qualname); - if (PyDict_DelItem(type->tp_dict, &_Py_ID(__qualname__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__qualname__)) < 0) { return -1; } } @@ -3068,7 +3261,8 @@ type_new_set_ht_name(PyTypeObject *type) static int type_new_set_doc(PyTypeObject *type) { - PyObject *doc = PyDict_GetItemWithError(type->tp_dict, &_Py_ID(__doc__)); + PyObject *dict = lookup_tp_dict(type); + PyObject *doc = PyDict_GetItemWithError(dict, &_Py_ID(__doc__)); if (doc == NULL) { if (PyErr_Occurred()) { return -1; @@ -3103,7 +3297,8 @@ type_new_set_doc(PyTypeObject *type) static int type_new_staticmethod(PyTypeObject *type, PyObject *attr) { - PyObject *func = PyDict_GetItemWithError(type->tp_dict, attr); + PyObject *dict = lookup_tp_dict(type); + PyObject *func = PyDict_GetItemWithError(dict, attr); if (func == NULL) { if (PyErr_Occurred()) { return -1; @@ -3118,7 +3313,7 @@ type_new_staticmethod(PyTypeObject *type, PyObject *attr) if (static_func == NULL) { return -1; } - if (PyDict_SetItem(type->tp_dict, attr, static_func) < 0) { + if (PyDict_SetItem(dict, attr, static_func) < 0) { Py_DECREF(static_func); return -1; } @@ -3130,7 +3325,8 @@ type_new_staticmethod(PyTypeObject *type, PyObject *attr) static int type_new_classmethod(PyTypeObject *type, PyObject *attr) { - PyObject *func = PyDict_GetItemWithError(type->tp_dict, attr); + PyObject *dict = lookup_tp_dict(type); + PyObject *func = PyDict_GetItemWithError(dict, attr); if (func == NULL) { if (PyErr_Occurred()) { return -1; @@ -3146,7 +3342,7 @@ type_new_classmethod(PyTypeObject *type, PyObject *attr) return -1; } - if (PyDict_SetItem(type->tp_dict, attr, method) < 0) { + if (PyDict_SetItem(dict, attr, method) < 0) { Py_DECREF(method); return -1; } @@ -3232,8 +3428,8 @@ type_new_set_slots(const type_new_ctx *ctx, PyTypeObject *type) static int type_new_set_classcell(PyTypeObject *type) { - PyObject *cell = PyDict_GetItemWithError( - type->tp_dict, &_Py_ID(__classcell__)); + PyObject *dict = lookup_tp_dict(type); + PyObject *cell = PyDict_GetItemWithError(dict, &_Py_ID(__classcell__)); if (cell == NULL) { if (PyErr_Occurred()) { return -1; @@ -3250,7 +3446,7 @@ type_new_set_classcell(PyTypeObject *type) } (void)PyCell_Set(cell, (PyObject *) type); - if (PyDict_DelItem(type->tp_dict, &_Py_ID(__classcell__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__classcell__)) < 0) { return -1; } return 0; @@ -3357,7 +3553,7 @@ type_new_init(type_new_ctx *ctx) goto error; } - type->tp_dict = dict; + set_tp_dict(type, dict); PyHeapTypeObject *et = (PyHeapTypeObject*)type; et->ht_slots = ctx->slots; @@ -3856,7 +4052,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, /* Set slots we have prepared */ type->tp_base = (PyTypeObject *)Py_NewRef(base); - type->tp_bases = bases; + set_tp_bases(type, bases); bases = NULL; // We give our reference to bases to the type type->tp_doc = tp_doc; @@ -3936,12 +4132,13 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, goto finally; } + PyObject *dict = lookup_tp_dict(type); if (type->tp_doc) { PyObject *__doc__ = PyUnicode_FromString(_PyType_DocWithoutSignature(type->tp_name, type->tp_doc)); if (!__doc__) { goto finally; } - r = PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), __doc__); + r = PyDict_SetItem(dict, &_Py_ID(__doc__), __doc__); Py_DECREF(__doc__); if (r < 0) { goto finally; @@ -3949,18 +4146,18 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, } if (weaklistoffset) { - if (PyDict_DelItem((PyObject *)type->tp_dict, &_Py_ID(__weaklistoffset__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__weaklistoffset__)) < 0) { goto finally; } } if (dictoffset) { - if (PyDict_DelItem((PyObject *)type->tp_dict, &_Py_ID(__dictoffset__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__dictoffset__)) < 0) { goto finally; } } /* Set type.__module__ */ - r = PyDict_Contains(type->tp_dict, &_Py_ID(__module__)); + r = PyDict_Contains(dict, &_Py_ID(__module__)); if (r < 0) { goto finally; } @@ -3972,7 +4169,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, if (modname == NULL) { goto finally; } - r = PyDict_SetItem(type->tp_dict, &_Py_ID(__module__), modname); + r = PyDict_SetItem(dict, &_Py_ID(__module__), modname); Py_DECREF(modname); if (r != 0) { goto finally; @@ -4094,7 +4291,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); - PyObject *mro = type->tp_mro; + PyObject *mro = lookup_tp_mro(type); // The type must be ready assert(mro != NULL); assert(PyTuple_Check(mro)); @@ -4143,14 +4340,14 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) } /* Look in tp_dict of types in MRO */ - PyObject *mro = type->tp_mro; + PyObject *mro = lookup_tp_mro(type); if (mro == NULL) { if ((type->tp_flags & Py_TPFLAGS_READYING) == 0) { if (PyType_Ready(type) < 0) { *error = -1; return NULL; } - mro = type->tp_mro; + mro = lookup_tp_mro(type); } if (mro == NULL) { *error = 1; @@ -4165,7 +4362,7 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 0; i < n; i++) { PyObject *base = PyTuple_GET_ITEM(mro, i); - PyObject *dict = _PyType_CAST(base)->tp_dict; + PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); assert(dict && PyDict_Check(dict)); res = _PyDict_GetItem_KnownHash(dict, name, hash); if (res != NULL) { @@ -4439,20 +4636,19 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys); static void type_dealloc_common(PyTypeObject *type) { - if (type->tp_bases != NULL) { + PyObject *bases = lookup_tp_bases(type); + if (bases != NULL) { PyObject *exc = PyErr_GetRaisedException(); - remove_all_subclasses(type, type->tp_bases); + remove_all_subclasses(type, bases); PyErr_SetRaisedException(exc); } } -static void clear_subclasses(PyTypeObject *self); - static void clear_static_tp_subclasses(PyTypeObject *type) { - PyObject *subclasses = lookup_subclasses(type); + PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses == NULL) { return; } @@ -4481,7 +4677,7 @@ clear_static_tp_subclasses(PyTypeObject *type) Py_ssize_t i = 0; PyObject *key, *ref; // borrowed ref while (PyDict_Next(subclasses, &i, &key, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == NULL) { continue; } @@ -4489,16 +4685,16 @@ clear_static_tp_subclasses(PyTypeObject *type) assert(!(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); } - clear_subclasses(type); + clear_tp_subclasses(type); } static void clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) { if (_Py_IsMainInterpreter(interp)) { - Py_CLEAR(type->tp_dict); - Py_CLEAR(type->tp_bases); - Py_CLEAR(type->tp_mro); + clear_tp_dict(type); + clear_tp_bases(type); + clear_tp_mro(type); Py_CLEAR(type->tp_cache); } clear_static_tp_subclasses(type); @@ -4545,7 +4741,7 @@ type_dealloc(PyTypeObject *type) Py_XDECREF(type->tp_bases); Py_XDECREF(type->tp_mro); Py_XDECREF(type->tp_cache); - clear_subclasses(type); + clear_tp_subclasses(type); /* A type's tp_doc is heap allocated, unlike the tp_doc slots * of most other objects. It's okay to cast it to char *. @@ -4565,65 +4761,6 @@ type_dealloc(PyTypeObject *type) } -static PyObject * -lookup_subclasses(PyTypeObject *self) -{ - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - assert(state != NULL); - return state->tp_subclasses; - } - return (PyObject *)self->tp_subclasses; -} - -int -_PyType_HasSubclasses(PyTypeObject *self) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN && - _PyStaticType_GetState(interp, self) == NULL) { - return 0; - } - if (lookup_subclasses(self) == NULL) { - return 0; - } - return 1; -} - -PyObject* -_PyType_GetSubclasses(PyTypeObject *self) -{ - PyObject *list = PyList_New(0); - if (list == NULL) { - return NULL; - } - - PyObject *subclasses = lookup_subclasses(self); // borrowed ref - if (subclasses == NULL) { - return list; - } - assert(PyDict_CheckExact(subclasses)); - // The loop cannot modify tp_subclasses, there is no need - // to hold a strong reference (use a borrowed reference). - - Py_ssize_t i = 0; - PyObject *ref; // borrowed ref - while (PyDict_Next(subclasses, &i, NULL, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed - if (subclass == NULL) { - continue; - } - - if (PyList_Append(list, _PyObject_CAST(subclass)) < 0) { - Py_DECREF(list); - return NULL; - } - } - return list; -} - - /*[clinic input] type.__subclasses__ @@ -4837,8 +4974,9 @@ type_clear(PyTypeObject *type) */ PyType_Modified(type); - if (type->tp_dict) { - PyDict_Clear(type->tp_dict); + PyObject *dict = lookup_tp_dict(type); + if (dict) { + PyDict_Clear(dict); } Py_CLEAR(((PyHeapTypeObject *)type)->ht_module); @@ -5387,7 +5525,8 @@ _PyType_GetSlotNames(PyTypeObject *cls) assert(PyType_Check(cls)); /* Get the slot names from the cache in the class if possible. */ - slotnames = PyDict_GetItemWithError(cls->tp_dict, &_Py_ID(__slotnames__)); + PyObject *dict = lookup_tp_dict(cls); + slotnames = PyDict_GetItemWithError(dict, &_Py_ID(__slotnames__)); if (slotnames != NULL) { if (slotnames != Py_None && !PyList_Check(slotnames)) { PyErr_Format(PyExc_TypeError, @@ -5887,8 +6026,8 @@ object___reduce_ex___impl(PyObject *self, int protocol) PyObject *reduce, *res; if (objreduce == NULL) { - objreduce = PyDict_GetItemWithError( - PyBaseObject_Type.tp_dict, &_Py_ID(__reduce__)); + PyObject *dict = lookup_tp_dict(&PyBaseObject_Type); + objreduce = PyDict_GetItemWithError(dict, &_Py_ID(__reduce__)); if (objreduce == NULL && PyErr_Occurred()) { return NULL; } @@ -6154,11 +6293,12 @@ type_add_method(PyTypeObject *type, PyMethodDef *meth) } int err; + PyObject *dict = lookup_tp_dict(type); if (!(meth->ml_flags & METH_COEXIST)) { - err = PyDict_SetDefault(type->tp_dict, name, descr) == NULL; + err = PyDict_SetDefault(dict, name, descr) == NULL; } else { - err = PyDict_SetItem(type->tp_dict, name, descr) < 0; + err = PyDict_SetItem(dict, name, descr) < 0; } if (!isdescr) { Py_DECREF(name); @@ -6197,7 +6337,7 @@ type_add_members(PyTypeObject *type) return 0; } - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); for (; memb->name != NULL; memb++) { PyObject *descr = PyDescr_NewMember(type, memb); if (descr == NULL) @@ -6221,7 +6361,7 @@ type_add_getset(PyTypeObject *type) return 0; } - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); for (; gsp->name != NULL; gsp++) { PyObject *descr = PyDescr_NewGetSet(type, gsp); if (descr == NULL) { @@ -6300,7 +6440,7 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) static int overrides_hash(PyTypeObject *type) { - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); assert(dict != NULL); int r = PyDict_Contains(dict, &_Py_ID(__eq__)); @@ -6579,7 +6719,7 @@ type_ready_set_bases(PyTypeObject *type) } /* Initialize tp_bases */ - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); if (bases == NULL) { PyTypeObject *base = type->tp_base; if (base == NULL) { @@ -6591,7 +6731,7 @@ type_ready_set_bases(PyTypeObject *type) if (bases == NULL) { return -1; } - type->tp_bases = bases; + set_tp_bases(type, bases); } return 0; } @@ -6600,7 +6740,7 @@ type_ready_set_bases(PyTypeObject *type) static int type_ready_set_dict(PyTypeObject *type) { - if (type->tp_dict != NULL) { + if (lookup_tp_dict(type) != NULL) { return 0; } @@ -6608,7 +6748,7 @@ type_ready_set_dict(PyTypeObject *type) if (dict == NULL) { return -1; } - type->tp_dict = dict; + set_tp_dict(type, dict); return 0; } @@ -6618,7 +6758,8 @@ type_ready_set_dict(PyTypeObject *type) static int type_dict_set_doc(PyTypeObject *type) { - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__doc__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__doc__)); if (r < 0) { return -1; } @@ -6634,14 +6775,14 @@ type_dict_set_doc(PyTypeObject *type) return -1; } - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), doc) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__doc__), doc) < 0) { Py_DECREF(doc); return -1; } Py_DECREF(doc); } else { - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), Py_None) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__doc__), Py_None) < 0) { return -1; } } @@ -6706,14 +6847,14 @@ type_ready_mro(PyTypeObject *type) if (mro_internal(type, NULL) < 0) { return -1; } - assert(type->tp_mro != NULL); - assert(PyTuple_Check(type->tp_mro)); + PyObject *mro = lookup_tp_mro(type); + assert(mro != NULL); + assert(PyTuple_Check(mro)); /* All bases of statically allocated type should be statically allocated, and static builtin types must have static builtin bases. */ if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { assert(type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); - PyObject *mro = type->tp_mro; Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(mro, i)); @@ -6774,8 +6915,8 @@ type_ready_inherit(PyTypeObject *type) } // Inherit slots - PyObject *mro = type->tp_mro; - Py_ssize_t n = PyTuple_GET_SIZE(type->tp_mro); + PyObject *mro = lookup_tp_mro(type); + Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 1; i < n; i++) { PyObject *b = PyTuple_GET_ITEM(mro, i); if (PyType_Check(b)) { @@ -6820,7 +6961,8 @@ type_ready_set_hash(PyTypeObject *type) return 0; } - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__hash__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__hash__)); if (r < 0) { return -1; } @@ -6828,7 +6970,7 @@ type_ready_set_hash(PyTypeObject *type) return 0; } - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__hash__), Py_None) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__hash__), Py_None) < 0) { return -1; } type->tp_hash = PyObject_HashNotImplemented; @@ -6840,7 +6982,7 @@ type_ready_set_hash(PyTypeObject *type) static int type_ready_add_subclasses(PyTypeObject *type) { - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); Py_ssize_t nbase = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < nbase; i++) { PyObject *b = PyTuple_GET_ITEM(bases, i); @@ -7080,38 +7222,6 @@ _PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) } -static PyObject * -init_subclasses(PyTypeObject *self) -{ - PyObject *subclasses = PyDict_New(); - if (subclasses == NULL) { - return NULL; - } - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - state->tp_subclasses = subclasses; - return subclasses; - } - self->tp_subclasses = (void *)subclasses; - return subclasses; -} - -static void -clear_subclasses(PyTypeObject *self) -{ - /* Delete the dictionary to save memory. _PyStaticType_Dealloc() - callers also test if tp_subclasses is NULL to check if a static type - has no subclass. */ - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - Py_CLEAR(state->tp_subclasses); - return; - } - Py_CLEAR(self->tp_subclasses); -} - static int add_subclass(PyTypeObject *base, PyTypeObject *type) { @@ -7128,9 +7238,9 @@ add_subclass(PyTypeObject *base, PyTypeObject *type) // Only get tp_subclasses after creating the key and value. // PyWeakref_NewRef() can trigger a garbage collection which can execute // arbitrary Python code and so modify base->tp_subclasses. - PyObject *subclasses = lookup_subclasses(base); + PyObject *subclasses = lookup_tp_subclasses(base); if (subclasses == NULL) { - subclasses = init_subclasses(base); + subclasses = init_tp_subclasses(base); if (subclasses == NULL) { Py_DECREF(key); Py_DECREF(ref); @@ -7161,19 +7271,6 @@ add_all_subclasses(PyTypeObject *type, PyObject *bases) return res; } -static inline PyTypeObject * -subclass_from_ref(PyObject *ref) -{ - assert(PyWeakref_CheckRef(ref)); - PyObject *obj = PyWeakref_GET_OBJECT(ref); // borrowed ref - assert(obj != NULL); - if (obj == Py_None) { - return NULL; - } - assert(PyType_Check(obj)); - return _PyType_CAST(obj); -} - static PyObject * get_subclasses_key(PyTypeObject *type, PyTypeObject *base) { @@ -7187,10 +7284,10 @@ get_subclasses_key(PyTypeObject *type, PyTypeObject *base) We fall back to manually traversing the values. */ Py_ssize_t i = 0; PyObject *ref; // borrowed ref - PyObject *subclasses = lookup_subclasses(base); + PyObject *subclasses = lookup_tp_subclasses(base); if (subclasses != NULL) { while (PyDict_Next(subclasses, &i, &key, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == type) { return Py_NewRef(key); } @@ -7203,7 +7300,7 @@ get_subclasses_key(PyTypeObject *type, PyTypeObject *base) static void remove_subclass(PyTypeObject *base, PyTypeObject *type) { - PyObject *subclasses = lookup_subclasses(base); // borrowed ref + PyObject *subclasses = lookup_tp_subclasses(base); // borrowed ref if (subclasses == NULL) { return; } @@ -7219,7 +7316,7 @@ remove_subclass(PyTypeObject *base, PyTypeObject *type) Py_XDECREF(key); if (PyDict_Size(subclasses) == 0) { - clear_subclasses(base); + clear_tp_subclasses(base); } } @@ -7517,7 +7614,7 @@ static int hackcheck(PyObject *self, setattrofunc func, const char *what) { PyTypeObject *type = Py_TYPE(self); - PyObject *mro = type->tp_mro; + PyObject *mro = lookup_tp_mro(type); if (!mro) { /* Probably ok not to check the call in this case. */ return 1; @@ -7805,7 +7902,8 @@ static struct PyMethodDef tp_new_methoddef[] = { static int add_tp_new_wrapper(PyTypeObject *type) { - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__new__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__new__)); if (r > 0) { return 0; } @@ -7817,7 +7915,7 @@ add_tp_new_wrapper(PyTypeObject *type) if (func == NULL) { return -1; } - r = PyDict_SetItem(type->tp_dict, &_Py_ID(__new__), func); + r = PyDict_SetItem(dict, &_Py_ID(__new__), func); Py_DECREF(func); return r; } @@ -9181,7 +9279,8 @@ update_all_slots(PyTypeObject* type) static int type_new_set_names(PyTypeObject *type) { - PyObject *names_to_set = PyDict_Copy(type->tp_dict); + PyObject *dict = lookup_tp_dict(type); + PyObject *names_to_set = PyDict_Copy(dict); if (names_to_set == NULL) { return -1; } @@ -9270,7 +9369,7 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, // It is safe to use a borrowed reference because update_subclasses() is // only used with update_slots_callback() which doesn't modify // tp_subclasses. - PyObject *subclasses = lookup_subclasses(type); // borrowed ref + PyObject *subclasses = lookup_tp_subclasses(type); // borrowed ref if (subclasses == NULL) { return 0; } @@ -9279,13 +9378,13 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, Py_ssize_t i = 0; PyObject *ref; while (PyDict_Next(subclasses, &i, NULL, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == NULL) { continue; } /* Avoid recursing down into unaffected classes */ - PyObject *dict = subclass->tp_dict; + PyObject *dict = lookup_tp_dict(subclass); if (dict != NULL && PyDict_Check(dict)) { int r = PyDict_Contains(dict, attr_name); if (r < 0) { @@ -9336,7 +9435,7 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, static int add_operators(PyTypeObject *type) { - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); pytype_slotdef *p; PyObject *descr; void **ptr; @@ -9432,7 +9531,7 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * PyObject *mro, *res; Py_ssize_t i, n; - mro = su_obj_type->tp_mro; + mro = lookup_tp_mro(su_obj_type); if (mro == NULL) return NULL; @@ -9453,7 +9552,7 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * Py_INCREF(mro); do { PyObject *obj = PyTuple_GET_ITEM(mro, i); - PyObject *dict = _PyType_CAST(obj)->tp_dict; + PyObject *dict = lookup_tp_dict(_PyType_CAST(obj)); assert(dict != NULL && PyDict_Check(dict)); res = PyDict_GetItemWithError(dict, name); diff --git a/Python/context.c b/Python/context.c index 5d385508405ede..1ffae9871be7b3 100644 --- a/Python/context.c +++ b/Python/context.c @@ -1309,7 +1309,7 @@ _PyContext_Init(PyInterpreterState *interp) PyObject *missing = get_token_missing(); if (PyDict_SetItemString( - PyContextToken_Type.tp_dict, "MISSING", missing)) + _PyType_GetDict(&PyContextToken_Type), "MISSING", missing)) { Py_DECREF(missing); return _PyStatus_ERR("can't init context types"); From 690df4c16ca4f0054d27a6148da9e6af809a2658 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 1 May 2023 22:53:16 -0400 Subject: [PATCH 04/14] gh-88496: IDLE - fix another test on macOS (#104075) Needed for Catalina: test_sidebar add 'idletasks' and skip assert. --- Lib/idlelib/idle_test/test_sidebar.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index 5506fd2b0e22a5..fb52b3a0179553 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -57,7 +57,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.editwin.per.close() - cls.root.update() + cls.root.update_idletasks() cls.root.destroy() del cls.text, cls.text_frame, cls.editwin, cls.root @@ -695,7 +695,8 @@ def test_mousewheel(self): delta = -1 if sys.platform == 'darwin' else 120 sidebar.canvas.event_generate('', x=0, y=0, delta=delta) yield - self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) + if sys.platform != 'darwin': # .update_idletasks() does not work. + self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) # Scroll back down using the event. sidebar.canvas.event_generate('', x=0, y=0) From 9de0cf20fa0485e327e57cc0864c7476da85cfad Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 2 May 2023 04:59:42 +0100 Subject: [PATCH 05/14] GH-103472: close response in HTTPConnection._tunnel (#103473) Avoid a potential `ResourceWarning` in `http.client.HTTPConnection` by closing the proxy / tunnel's CONNECT response explicitly. --------- Co-authored-by: Gregory P. Smith --- Lib/http/client.py | 33 ++++++++++--------- Lib/test/test_httplib.py | 23 +++++++++++++ ...-04-12-13-04-16.gh-issue-103472.C6bOHv.rst | 2 ++ 3 files changed, 43 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index 0f5cd35247ae82..74f7bcb68fb6bc 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -941,23 +941,26 @@ def _tunnel(self): del headers response = self.response_class(self.sock, method=self._method) - (version, code, message) = response._read_status() + try: + (version, code, message) = response._read_status() - if code != http.HTTPStatus.OK: - self.close() - raise OSError(f"Tunnel connection failed: {code} {message.strip()}") - while True: - line = response.fp.readline(_MAXLINE + 1) - if len(line) > _MAXLINE: - raise LineTooLong("header line") - if not line: - # for sites which EOF without sending a trailer - break - if line in (b'\r\n', b'\n', b''): - break + if code != http.HTTPStatus.OK: + self.close() + raise OSError(f"Tunnel connection failed: {code} {message.strip()}") + while True: + line = response.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("header line") + if not line: + # for sites which EOF without sending a trailer + break + if line in (b'\r\n', b'\n', b''): + break - if self.debuglevel > 0: - print('header:', line.decode()) + if self.debuglevel > 0: + print('header:', line.decode()) + finally: + response.close() def connect(self): """Connect to the host and port specified in __init__.""" diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index b4f4e2b14351a6..37f77fe0a320c7 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2390,6 +2390,29 @@ def test_tunnel_debuglog(self): lines = output.getvalue().splitlines() self.assertIn('header: {}'.format(expected_header), lines) + def test_tunnel_leak(self): + sock = None + + def _create_connection(address, timeout=None, source_address=None): + nonlocal sock + sock = FakeSocket( + 'HTTP/1.1 404 NOT FOUND\r\n\r\n', + host=address[0], + port=address[1], + ) + return sock + + self.conn._create_connection = _create_connection + self.conn.set_tunnel('destination.com') + exc = None + try: + self.conn.request('HEAD', '/', '') + except OSError as e: + # keeping a reference to exc keeps response alive in the traceback + exc = e + self.assertIsNotNone(exc) + self.assertTrue(sock.file_closed) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst b/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst new file mode 100644 index 00000000000000..01d84f024bd4a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst @@ -0,0 +1,2 @@ +Avoid a potential :exc:`ResourceWarning` in :class:`http.client.HTTPConnection` +by closing the proxy / tunnel's CONNECT response explicitly. From f0ad4567319ee4ae878d570ab7709ab63df9123e Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 2 May 2023 06:30:43 +0200 Subject: [PATCH 06/14] gh-102997: Update macOS installer to SQLite 3.41.2. (GH-102998) --- Mac/BuildScript/build-installer.py | 6 +++--- .../macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 63fa21b2b33d17..2f5937489ac03d 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.40.1", - url="https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz", - checksum="42175b1a1d23529cb133bbd2b5900afd", + name="SQLite 3.41.2", + url="https://sqlite.org/2023/sqlite-autoconf-3410200.tar.gz", + checksum="862075fd1c38324878ef809eda39edfe", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' diff --git a/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst b/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst new file mode 100644 index 00000000000000..d0b390a896b743 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst @@ -0,0 +1 @@ +Update macOS installer to SQLite 3.41.2. From 82ba6ce303d04a7b21034e38d220e23ca9f1dc0a Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 May 2023 23:05:25 -0700 Subject: [PATCH 07/14] Improve assert_type phrasing (#104081) I'd like to make the fact that this does nothing at runtime really obvious, since I suspect this is unintuitive for users who are unfamiliar with static type checking. I thought of this because of https://discuss.python.org/t/add-arg-check-type-to-types/26384 wherein I'm skeptical that the user really did want `assert_type`. --- Doc/library/typing.rst | 7 ++++--- Lib/typing.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 409a95d528b5d3..c22fc0b28a50d0 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2484,15 +2484,16 @@ Functions and decorators Ask a static type checker to confirm that *val* has an inferred type of *typ*. - When the type checker encounters a call to ``assert_type()``, it + At runtime this does nothing: it returns the first argument unchanged with no + checks or side effects, no matter the actual type of the argument. + + When a static type checker encounters a call to ``assert_type()``, it emits an error if the value is not of the specified type:: def greet(name: str) -> None: assert_type(name, str) # OK, inferred type of `name` is `str` assert_type(name, int) # type checker error - At runtime this returns the first argument unchanged with no side effects. - This function is useful for ensuring the type checker's understanding of a script is in line with the developer's intentions:: diff --git a/Lib/typing.py b/Lib/typing.py index 1a1c989dbaf37d..0dacdd9031a776 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2319,15 +2319,16 @@ def cast(typ, val): def assert_type(val, typ, /): """Ask a static type checker to confirm that the value is of the given type. - When the type checker encounters a call to assert_type(), it + At runtime this does nothing: it returns the first argument unchanged with no + checks or side effects, no matter the actual type of the argument. + + When a static type checker encounters a call to assert_type(), it emits an error if the value is not of the specified type:: def greet(name: str) -> None: assert_type(name, str) # ok assert_type(name, int) # type checker error - At runtime this returns the first argument unchanged and otherwise - does nothing. """ return val From 68ed2a2a3f1e715dc10724b0c000ec2fc498d11e Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Tue, 2 May 2023 03:34:44 -0300 Subject: [PATCH 08/14] GH-103484: Fix redirected permanently URLs (#104001) Co-authored-by: Hugo van Kemenade Co-authored-by: Oleg Iarygin --- Doc/conf.py | 24 +++++++++++++++++++++--- Doc/faq/extending.rst | 2 +- Doc/faq/general.rst | 8 ++++---- Doc/faq/programming.rst | 2 +- Doc/howto/pyporting.rst | 2 +- Doc/library/asyncio-eventloop.rst | 2 +- Doc/library/hashlib.rst | 12 ++++++------ Doc/library/http.client.rst | 2 +- Doc/library/importlib.metadata.rst | 2 +- Doc/library/os.rst | 2 +- Doc/library/plistlib.rst | 2 +- Doc/library/resource.rst | 2 +- Doc/library/select.rst | 2 +- Doc/library/ssl.rst | 2 +- Doc/library/statistics.rst | 2 +- Doc/library/struct.rst | 2 +- Doc/library/sys.rst | 2 +- Doc/library/unittest.mock-examples.rst | 2 +- Doc/library/unittest.rst | 2 +- Doc/library/xmlrpc.client.rst | 2 +- Doc/library/zipfile.rst | 2 +- Doc/reference/datamodel.rst | 2 +- Doc/reference/introduction.rst | 2 +- Doc/using/cmdline.rst | 2 +- Doc/using/mac.rst | 2 +- Doc/using/windows.rst | 2 +- Doc/whatsnew/2.0.rst | 2 +- Doc/whatsnew/2.1.rst | 2 +- Doc/whatsnew/2.2.rst | 4 ++-- Doc/whatsnew/2.3.rst | 6 +++--- Doc/whatsnew/2.4.rst | 2 +- Doc/whatsnew/2.6.rst | 4 ++-- Doc/whatsnew/3.11.rst | 8 ++++---- Doc/whatsnew/3.2.rst | 6 +++--- Doc/whatsnew/3.3.rst | 2 +- Doc/whatsnew/3.5.rst | 4 ++-- Doc/whatsnew/3.6.rst | 2 +- Doc/whatsnew/3.8.rst | 2 +- Doc/whatsnew/3.9.rst | 2 +- Misc/NEWS.d/3.7.0a1.rst | 2 +- 40 files changed, 78 insertions(+), 60 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index cef2a0e2837f6a..6a3c01cd91a77b 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -264,11 +264,29 @@ linkcheck_allowed_redirects = { # bpo-NNNN -> BPO -> GH Issues - r'https://bugs.python.org/issue\?@action=redirect&bpo=\d+': 'https://github.com/python/cpython/issues/\d+', + r'https://bugs.python.org/issue\?@action=redirect&bpo=\d+': r'https://github.com/python/cpython/issues/\d+', # GH-NNNN used to refer to pull requests - r'https://github.com/python/cpython/issues/\d+': 'https://github.com/python/cpython/pull/\d+', + r'https://github.com/python/cpython/issues/\d+': r'https://github.com/python/cpython/pull/\d+', # :source:`something` linking files in the repository - r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*' + r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*', + # Intentional HTTP use at Misc/NEWS.d/3.5.0a1.rst + r'http://www.python.org/$': 'https://www.python.org/$', + # Used in license page, keep as is + r'https://www.zope.org/': r'https://www.zope.dev/', + # Microsoft's redirects to learn.microsoft.com + r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*', + r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*', + r'https://go.microsoft.com/fwlink/\?LinkID=\d+': 'https://learn.microsoft.com/.*', + # Language redirects + r'https://toml.io': 'https://toml.io/en/', + r'https://www.redhat.com': 'https://www.redhat.com/en', + # Other redirects + r'https://www.boost.org/libs/.+': r'https://www.boost.org/doc/libs/\d_\d+_\d/.+', + r'https://support.microsoft.com/en-us/help/\d+': 'https://support.microsoft.com/en-us/topic/.+', + r'https://perf.wiki.kernel.org$': 'https://perf.wiki.kernel.org/index.php/Main_Page', + r'https://www.sqlite.org': 'https://www.sqlite.org/index.html', + r'https://mitpress.mit.edu/sicp$': 'https://mitpress.mit.edu/9780262510875/structure-and-interpretation-of-computer-programs/', + r'https://www.python.org/psf/': 'https://www.python.org/psf-landing/', } linkcheck_anchors_ignore = [ diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 07282639e4f9b4..bc3080f60ee237 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -42,7 +42,7 @@ on what you're trying to do. .. XXX make sure these all work `Cython `_ and its relative `Pyrex -`_ are compilers +`_ are compilers that accept a slightly modified form of Python and generate the corresponding C code. Cython and Pyrex make it possible to write an extension without having to learn Python's C API. diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 6256deb5797c89..a9b2622e02ef3b 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -54,8 +54,8 @@ commercial use, to sell copies of Python in source or binary form (modified or unmodified), or to sell products that incorporate Python in some form. We would still like to know about all commercial use of Python, of course. -See `the PSF license page `_ to find further -explanations and a link to the full text of the license. +See `the license page `_ to find further +explanations and the full text of the PSF License. The Python logo is trademarked, and in certain cases permission is required to use it. Consult `the Trademark Usage Policy @@ -215,7 +215,7 @@ every day, and Usenet readers are often more able to cope with this volume. Announcements of new software releases and events can be found in comp.lang.python.announce, a low-traffic moderated list that receives about five postings per day. It's available as `the python-announce mailing list -`_. +`_. More info about other mailing lists and newsgroups can be found at https://www.python.org/community/lists/. @@ -352,7 +352,7 @@ titled "Python X.Y Release Schedule", where X.Y is a version that hasn't been publicly released yet. New development is discussed on `the python-dev mailing list -`_. +`_. Is it reasonable to propose incompatible changes to Python? diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 38f9b171618b26..ab5618db84f77e 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -61,7 +61,7 @@ Yes. `Pyflakes `_ do basic checking that will help you catch bugs sooner. -Static type checkers such as `Mypy `_, +Static type checkers such as `Mypy `_, `Pyre `_, and `Pytype `_ can check type hints in Python source code. diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index add1c11be534e3..baea3e85c3b84b 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -438,7 +438,7 @@ to make sure everything functions as expected in both versions of Python. .. _Futurize: https://python-future.org/automatic_conversion.html .. _importlib2: https://pypi.org/project/importlib2 .. _Modernize: https://python-modernize.readthedocs.io/ -.. _mypy: http://mypy-lang.org/ +.. _mypy: https://mypy-lang.org/ .. _Porting to Python 3: http://python3porting.com/ .. _Pylint: https://pypi.org/project/pylint diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index e982cc166a3f2d..8d0022cc66daac 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -529,7 +529,7 @@ Opening network connections specifies requirements for algorithms that reduce this user-visible delay and provides an algorithm. - For more information: https://tools.ietf.org/html/rfc6555 + For more information: https://datatracker.ietf.org/doc/html/rfc6555 .. versionchanged:: 3.11 diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index f8d10c0c295c7a..6275f96f7d4d19 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -432,7 +432,7 @@ Constructor functions also accept the following tree hashing parameters: :alt: Explanation of tree mode parameters. See section 2.10 in `BLAKE2 specification -`_ for comprehensive review of tree +`_ for comprehensive review of tree hashing. @@ -619,7 +619,7 @@ on the hash function used in digital signatures. by the signer. (`NIST SP-800-106 "Randomized Hashing for Digital Signatures" - `_) + `_) In BLAKE2 the salt is processed as a one-time input to the hash function during initialization, rather than as an input to each compression function. @@ -628,7 +628,7 @@ initialization, rather than as an input to each compression function. *Salted hashing* (or just hashing) with BLAKE2 or any other general-purpose cryptographic hash function, such as SHA-256, is not suitable for hashing - passwords. See `BLAKE2 FAQ `_ for more + passwords. See `BLAKE2 FAQ `_ for more information. .. @@ -764,9 +764,9 @@ Domain Dedication 1.0 Universal: * *Alexandr Sokolovskiy* -.. _BLAKE2: https://blake2.net +.. _BLAKE2: https://www.blake2.net .. _HMAC: https://en.wikipedia.org/wiki/Hash-based_message_authentication_code -.. _BLAKE: https://131002.net/blake/ +.. _BLAKE: https://web.archive.org/web/20200918190133/https://131002.net/blake/ .. _SHA-3: https://en.wikipedia.org/wiki/NIST_hash_function_competition .. _ChaCha: https://cr.yp.to/chacha.html .. _pyblake2: https://pythonhosted.org/pyblake2/ @@ -782,7 +782,7 @@ Domain Dedication 1.0 Universal: Module :mod:`base64` Another way to encode binary hashes for non-binary environments. - https://blake2.net + https://www.blake2.net Official BLAKE2 website. https://csrc.nist.gov/csrc/media/publications/fips/180/2/archive/2002-08-01/documents/fips180-2.pdf diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 38821b32c91cf1..abdc6b447a8b61 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -354,7 +354,7 @@ HTTPConnection Objects the CONNECT request. As HTTP/1.1 is used for HTTP CONNECT tunnelling request, `as per the RFC - `_, a HTTP ``Host:`` + `_, a HTTP ``Host:`` header must be provided, matching the authority-form of the request target provided as the destination for the CONNECT request. If a HTTP ``Host:`` header is not provided via the headers argument, one is generated and diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index b306d5f55a714f..3097bcf47b627f 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -178,7 +178,7 @@ The "selectable" entry points were introduced in ``importlib_metadata`` no parameters and always returned a dictionary of entry points, keyed by group. With ``importlib_metadata`` 5.0 and Python 3.12, ``entry_points`` always returns an ``EntryPoints`` object. See -`backports.entry_points_selectable `_ +`backports.entry_points_selectable `_ for compatibility options. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 50e951c631fa88..76623c6305432a 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4593,7 +4593,7 @@ written in Python, such as a mail server's external command delivery program. :attr:`!children_system`, and :attr:`!elapsed` in that order. See the Unix manual page - :manpage:`times(2)` and `times(3) `_ manual page on Unix or `the GetProcessTimes MSDN + :manpage:`times(2)` and `times(3) `_ manual page on Unix or `the GetProcessTimes MSDN `_ on Windows. On Windows, only :attr:`!user` and :attr:`!system` are known; the other attributes are zero. diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 7aad15ec91a0ac..732ef3536863cc 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -46,7 +46,7 @@ or :class:`datetime.datetime` objects. .. seealso:: - `PList manual page `_ + `PList manual page `_ Apple's documentation of the file format. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index e7bf45d7d569fa..a5324c82c63484 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -244,7 +244,7 @@ platform. used by all of this user id's processes. This limit is enforced only if bit 1 of the vm.overcommit sysctl is set. Please see - `tuning(7) `__ + `tuning(7) `__ for a complete description of this sysctl. .. availability:: FreeBSD. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 2890706bab729c..b0891b0c8f584a 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -505,7 +505,7 @@ Kqueue Objects Kevent Objects -------------- -https://www.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 .. attribute:: kevent.ident diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 4b60b7c643b62c..18a6c5ab4858a4 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1719,7 +1719,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.3 .. seealso:: - `SSL/TLS & Perfect Forward Secrecy `_ + `SSL/TLS & Perfect Forward Secrecy `_ Vincent Bernat. .. method:: SSLContext.wrap_socket(sock, server_side=False, \ diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index f934b0e0319dca..395b324c860389 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -22,7 +22,7 @@ This module provides functions for calculating mathematical statistics of numeric (:class:`~numbers.Real`-valued) data. The module is not intended to be a competitor to third-party libraries such -as `NumPy `_, `SciPy `_, or +as `NumPy `_, `SciPy `_, or proprietary full-featured statistics packages aimed at professional statisticians such as Minitab, SAS and Matlab. It is aimed at the level of graphing and scientific calculators. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 9c0e32ba16bf68..78fd6e397ae635 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -602,4 +602,4 @@ The :mod:`struct` module also defines the following type: .. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_754-2008_revision -.. _IETF RFC 1700: https://tools.ietf.org/html/rfc1700 +.. _IETF RFC 1700: https://datatracker.ietf.org/doc/html/rfc1700 diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 7c0e85142e7716..95ad243bdde398 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -792,7 +792,7 @@ always available. additional garbage collector overhead if the object is managed by the garbage collector. - See `recursive sizeof recipe `_ + See `recursive sizeof recipe `_ for an example of using :func:`getsizeof` recursively to find the size of containers and all their contents. diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index f9a207bad6903f..895b9f9f07671b 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -1074,7 +1074,7 @@ subclass. Sometimes this is inconvenient. For example, `one user `_ is subclassing mock to created a `Twisted adaptor -`_. +`_. Having this applied to attributes too actually causes errors. ``Mock`` (in all its flavours) uses a method called ``_get_child_mock`` to create diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index c70153dfcd69e1..b26e6c0e6bc024 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -72,7 +72,7 @@ test runner a GUI tool for test discovery and execution. This is intended largely for ease of use for those new to unit testing. For production environments it is recommended that tests be driven by a continuous integration system such as - `Buildbot `_, `Jenkins `_, + `Buildbot `_, `Jenkins `_, `GitHub Actions `_, or `AppVeyor `_. diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index bd2c49a6edab7f..146c4fd768233b 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -161,7 +161,7 @@ between conformable Python objects and XML on the wire. .. seealso:: - `XML-RPC HOWTO `_ + `XML-RPC HOWTO `_ A good description of XML-RPC operation and client software in several languages. Contains pretty much everything an XML-RPC client developer needs to know. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 6f4826cb065c64..45f3d340bd82d3 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -128,7 +128,7 @@ The module defines the following items: Documentation on the ZIP file format by Phil Katz, the creator of the format and algorithms used. - `Info-ZIP Home Page `_ + `Info-ZIP Home Page `_ Information about the Info-ZIP project's ZIP archive programs and development libraries. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 55431f1951e50d..c35bf4016a28d2 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1564,7 +1564,7 @@ Basic customization This is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict insertion, O(n\ :sup:`2`) complexity. See - http://www.ocert.org/advisories/ocert-2011-003.html for details. + http://ocert.org/advisories/ocert-2011-003.html for details. Changing hash values affects the iteration order of sets. Python has never made guarantees about this ordering diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index 914a11556c94e6..81f0a5c5d43883 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -74,7 +74,7 @@ PyPy and a Just in Time compiler. One of the goals of the project is to encourage experimentation with the language itself by making it easier to modify the interpreter (since it is written in Python). Additional information is - available on `the PyPy project's home page `_. + available on `the PyPy project's home page `_. Each of these implementations varies in some way from the language as documented in this manual, or introduces specific information beyond what's covered in the diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b35e8454fa2a1a..9d4042ce5a7e8a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -370,7 +370,7 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict construction, O(n\ :sup:`2`) complexity. See - http://www.ocert.org/advisories/ocert-2011-003.html for details. + http://ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 9ae0270eaee7ab..69cd5c92d884d0 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -66,7 +66,7 @@ number of standard Unix command line editors, :program:`vim` and :program:`BBEdit` or :program:`TextWrangler` from Bare Bones Software (see http://www.barebones.com/products/bbedit/index.html) are good choices, as is :program:`TextMate` (see https://macromates.com/). Other editors include -:program:`Gvim` (https://macvim-dev.github.io/macvim/) and :program:`Aquamacs` +:program:`Gvim` (https://macvim.org/macvim/) and :program:`Aquamacs` (http://aquamacs.org/). To run your script from the Terminal window you must make sure that diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 380950eb507ffb..43e3c72f3e1cde 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -541,7 +541,7 @@ Besides the standard CPython distribution, there are modified packages including additional functionality. The following is a list of popular versions and their key features: -`ActivePython `_ +`ActivePython `_ Installer with multi-platform compatibility, documentation, PyWin32 `Anaconda `_ diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index 4bcb2acae1e640..0eefefd863a68f 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -933,7 +933,7 @@ using it:: parser.parse( 'hamlet.xml' ) For more information, consult the Python documentation, or the XML HOWTO at -http://pyxml.sourceforge.net/topics/howto/xml-howto.html. +https://pyxml.sourceforge.net/topics/howto/xml-howto.html. DOM Support diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index 0136de58774038..676da702b39693 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -613,7 +613,7 @@ New and Improved Modules framework based on running embedded examples in docstrings and comparing the results against the expected output. PyUnit, contributed by Steve Purcell, is a unit testing framework inspired by JUnit, which was in turn an adaptation of - Kent Beck's Smalltalk testing framework. See http://pyunit.sourceforge.net/ for + Kent Beck's Smalltalk testing framework. See https://pyunit.sourceforge.net/ for more information about PyUnit. * The :mod:`difflib` module contains a class, :class:`SequenceMatcher`, which diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 0c3bfda1933957..82aff0be1ed3b3 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -632,10 +632,10 @@ queen threatens another) and the Knight's Tour (a route that takes a knight to every square of an $NxN$ chessboard without visiting any square twice). The idea of generators comes from other programming languages, especially Icon -(https://www.cs.arizona.edu/icon/), where the idea of generators is central. In +(https://www2.cs.arizona.edu/icon/), where the idea of generators is central. In Icon, every expression and function call behaves like a generator. One example from "An Overview of the Icon Programming Language" at -https://www.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks +https://www2.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks like:: sentence := "Store it in the neighboring harbor" diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index c6e2003e92f1b3..af489d7cb45c2a 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -218,10 +218,10 @@ queen threatens another) and the Knight's Tour (a route that takes a knight to every square of an $NxN$ chessboard without visiting any square twice). The idea of generators comes from other programming languages, especially Icon -(https://www.cs.arizona.edu/icon/), where the idea of generators is central. In +(https://www2.cs.arizona.edu/icon/), where the idea of generators is central. In Icon, every expression and function call behaves like a generator. One example from "An Overview of the Icon Programming Language" at -https://www.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks +https://www2.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks like:: sentence := "Store it in the neighboring harbor" @@ -1332,7 +1332,7 @@ complete list of changes, or look through the CVS logs for all the details. (Contributed by Kevin O'Connor.) * The IDLE integrated development environment has been updated using the code - from the IDLEfork project (http://idlefork.sourceforge.net). The most notable feature is + from the IDLEfork project (https://idlefork.sourceforge.net). The most notable feature is that the code being developed is now executed in a subprocess, meaning that there's no longer any need for manual ``reload()`` operations. IDLE's core code has been incorporated into the standard library as the :mod:`idlelib` package. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 63e819876ce310..98dc83fe935d5e 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -756,7 +756,7 @@ API that perform ASCII-only conversions, ignoring the locale setting: :c:expr:`double` to an ASCII string. The code for these functions came from the GLib library -(https://developer.gnome.org/glib/stable/), whose developers kindly +(https://developer-old.gnome.org/glib/2.26/), whose developers kindly relicensed the relevant functions and donated them to the Python Software Foundation. The :mod:`locale` module can now change the numeric locale, letting extensions such as GTK+ produce the correct results. diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 4ee2aacb108a36..84bb651e68eed5 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -1433,7 +1433,7 @@ one, :func:`math.trunc`, that's been backported to Python 2.6. `Scheme's numerical tower `__, from the Guile manual. - `Scheme's number datatypes `__ from the R5RS Scheme specification. + `Scheme's number datatypes `__ from the R5RS Scheme specification. The :mod:`fractions` Module @@ -2363,7 +2363,7 @@ changes, or look through the Subversion logs for all the details. negotiation itself. (Patch contributed by Bill Fenner; :issue:`829951`.) -* The :mod:`socket` module now supports TIPC (http://tipc.sourceforge.net/), +* The :mod:`socket` module now supports TIPC (https://tipc.sourceforge.net/), a high-performance non-IP-based protocol designed for use in clustered environments. TIPC addresses are 4- or 5-tuples. (Contributed by Alberto Bertogli; :issue:`1646`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 687719a260a61c..6b591d5e184ea7 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -220,7 +220,7 @@ The copy of the :ref:`launcher` included with Python 3.11 has been significantly updated. It now supports company/tag syntax as defined in :pep:`514` using the ``-V:/`` argument instead of the limited ``-.``. This allows launching distributions other than ``PythonCore``, -the one hosted on `python.org `_. +the one hosted on `python.org `_. When using ``-V:`` selectors, either company or tag can be omitted, but all installs will be searched. For example, ``-V:OtherPython/`` will select the @@ -2481,7 +2481,7 @@ Porting to Python 3.11 #endif Or use the `pythoncapi_compat project - `__ to get these two + `__ to get these two functions on older Python versions. * Changes of the :c:type:`PyThreadState` structure members: @@ -2533,8 +2533,8 @@ Porting to Python 3.11 } #endif - Or use `the pythoncapi_compat project - `__ to get these functions + Or use `the pythoncapi-compat project + `__ to get these functions on old Python functions. * Distributors are encouraged to build Python with the optimized Blake2 diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 1b1455b72b9291..8dbe2a1d828b4e 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -785,8 +785,8 @@ functools (Contributed by Raymond Hettinger and incorporating design ideas from Jim Baker, Miki Tebeka, and Nick Coghlan; see `recipe 498245 - `_\, `recipe 577479 - `_\, :issue:`10586`, and + `_\, `recipe 577479 + `_\, :issue:`10586`, and :issue:`10593`.) * The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute @@ -2603,7 +2603,7 @@ Also, there were a number of updates to the Mac OS X build, see for details. For users running a 32/64-bit build, there is a known problem with the default Tcl/Tk on Mac OS X 10.6. Accordingly, we recommend installing an updated alternative such as -`ActiveState Tcl/Tk 8.5.9 `_\. +`ActiveState Tcl/Tk 8.5.9 `_\. See https://www.python.org/download/mac/tcltk/ for additional details. Porting to Python 3.2 diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 9e8d42469b019c..5b6c3dcd45c6f5 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1893,7 +1893,7 @@ socket * The :class:`~socket.socket` class now supports the PF_RDS protocol family (https://en.wikipedia.org/wiki/Reliable_Datagram_Sockets and - https://oss.oracle.com/projects/rds/). + `https://oss.oracle.com/projects/rds `__). * The :class:`~socket.socket` class now supports the ``PF_SYSTEM`` protocol family on OS X. (Contributed by Michael Goderbauer in :issue:`13777`.) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index f872579ef546f5..14b6425cea699e 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -425,7 +425,7 @@ are declared in the annotations:: While these annotations are available at runtime through the usual :attr:`__annotations__` attribute, *no automatic type checking happens at runtime*. Instead, it is assumed that a separate off-line type checker -(e.g. `mypy `_) will be used for on-demand +(e.g. `mypy `_) will be used for on-demand source code analysis. The type system supports unions, generic types, and a special type @@ -2212,7 +2212,7 @@ for details.) The :c:member:`PyTypeObject.tp_finalize` slot is now part of the stable ABI. Windows builds now require Microsoft Visual C++ 14.0, which -is available as part of `Visual Studio 2015 `_. +is available as part of `Visual Studio 2015 `_. Extension modules now include a platform information tag in their filename on some platforms (the tag is optional, and CPython will import extensions without diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index e4294c88b58572..c7faaebfed62b3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -238,7 +238,7 @@ and the ``__annotations__`` attribute. and Guido van Rossum. Implemented by Ivan Levkivskyi. Tools that use or will use the new syntax: - `mypy `_, + `mypy `_, `pytype `_, PyCharm, etc. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 37a6cf24e54562..85e088b64acb2d 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -2229,7 +2229,7 @@ The benchmarks were measured on an `Intel® Core™ i7-4960HQ processor `_ running the macOS 64-bit builds found at -`python.org `_. +`python.org `_. The benchmark script displays timings in nanoseconds. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index e974ee3a3f73ed..fd86db96302356 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -849,7 +849,7 @@ in nanoseconds. The benchmarks were measured on an `Intel® Core™ i7-4960HQ processor `_ running the macOS 64-bit builds found at -`python.org `_. +`python.org `_. Deprecated diff --git a/Misc/NEWS.d/3.7.0a1.rst b/Misc/NEWS.d/3.7.0a1.rst index 9bada1b76be7a8..ef93454784b77f 100644 --- a/Misc/NEWS.d/3.7.0a1.rst +++ b/Misc/NEWS.d/3.7.0a1.rst @@ -6255,7 +6255,7 @@ Fix python-gdb.py didn't support new dict implementation. .. section: Tools/Demos The pybench and pystone microbenchmark have been removed from Tools. Please -use the new Python benchmark suite https://github.com/python/performance +use the new Python benchmark suite https://github.com/python/pyperformance which is more reliable and includes a portable version of pybench working on Python 2 and Python 3. From f6314b92dcfc8ca6ff3fd150814f85448db69165 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 2 May 2023 09:37:57 +0200 Subject: [PATCH 09/14] gh-102997: Update Windows installer to SQLite 3.41.2. (#102999) --- .../next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst | 1 + PCbuild/get_externals.bat | 2 +- PCbuild/python.props | 2 +- PCbuild/readme.txt | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst diff --git a/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst b/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst new file mode 100644 index 00000000000000..c8f7259aecba6f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst @@ -0,0 +1 @@ +Update Windows installer to use SQLite 3.41.2. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 128241393f9f09..30ee873af9af24 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1t -set libraries=%libraries% sqlite-3.40.1.0 +set libraries=%libraries% sqlite-3.41.2.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tix-8.4.3.6 diff --git a/PCbuild/python.props b/PCbuild/python.props index 7994fbe7cd5e0b..29add07795f900 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -68,7 +68,7 @@ - $(ExternalsDir)sqlite-3.40.1.0\ + $(ExternalsDir)sqlite-3.41.2.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ $(ExternalsDir)libffi-3.4.4\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 4c799b64c461c1..9df56685b76a87 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -188,7 +188,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.40.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.41.2, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter From 87223f32aba872cfebde6fbe38673799eb79f248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jurica=20Bradari=C4=87?= Date: Tue, 2 May 2023 13:38:46 +0200 Subject: [PATCH 10/14] gh-103743: Add PyUnstable_Object_GC_NewWithExtraData (GH-103744) Co-authored-by: Petr Viktorin Co-authored-by: Erlend E. Aasland --- Doc/c-api/gcsupport.rst | 21 +++- Include/cpython/objimpl.h | 3 + Lib/test/test_capi/test_misc.py | 14 +++ ...-04-24-10-31-59.gh-issue-103743.2xYA1K.rst | 2 + Modules/_testcapimodule.c | 105 +++++++++++++++++- Modules/gcmodule.c | 13 +++ 6 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index cb5d64a50487fe..c3260a21bc7f8b 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -59,12 +59,31 @@ rules: Analogous to :c:func:`PyObject_New` but for container objects with the :const:`Py_TPFLAGS_HAVE_GC` flag set. - .. c:function:: TYPE* PyObject_GC_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size) Analogous to :c:func:`PyObject_NewVar` but for container objects with the :const:`Py_TPFLAGS_HAVE_GC` flag set. +.. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size) + + Analogous to :c:func:`PyObject_GC_New` but allocates *extra_size* + bytes at the end of the object (at offset + :c:member:`~PyTypeObject.tp_basicsize`). + The allocated memory is initialized to zeros, + except for the :c:type:`Python object header `. + + The extra data will be deallocated with the object, but otherwise it is + not managed by Python. + + .. warning:: + The function is marked as unstable because the final mechanism + for reserving extra data after an instance is not yet decided. + For allocating a variable number of fields, prefer using + :c:type:`PyVarObject` and :c:member:`~PyTypeObject.tp_itemsize` + instead. + + .. versionadded:: 3.12 + .. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 0b038d31080be9..5a8cdd57c7845b 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -90,3 +90,6 @@ PyAPI_FUNC(int) PyObject_IS_GC(PyObject *obj); PyAPI_FUNC(int) PyType_SUPPORTS_WEAKREFS(PyTypeObject *type); PyAPI_FUNC(PyObject **) PyObject_GET_WEAKREFS_LISTPTR(PyObject *op); + +PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *, + size_t); diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 9470cf12a7d1c4..9d5d1ca6e7dce2 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1043,6 +1043,20 @@ class dictsub(dict): ... # dict subclasses must work self.assertEqual(_testcapi.function_get_kw_defaults(some), None) self.assertEqual(some.__kwdefaults__, None) + def test_unstable_gc_new_with_extra_data(self): + class Data(_testcapi.ObjExtraData): + __slots__ = ('x', 'y') + + d = Data() + d.x = 10 + d.y = 20 + d.extra = 30 + self.assertEqual(d.x, 10) + self.assertEqual(d.y, 20) + self.assertEqual(d.extra, 30) + del d.extra + self.assertIsNone(d.extra) + class TestPendingCalls(unittest.TestCase): diff --git a/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst b/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst new file mode 100644 index 00000000000000..d074350ed3ebbe --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnstable_Object_GC_NewWithExtraData` function that can be used to +allocate additional memory after an object for data not managed by Python. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c1892f6fa0a4b8..a5d23b1b3d50ec 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3363,7 +3363,7 @@ test_gc_visit_objects_basic(PyObject *Py_UNUSED(self), } state.target = obj; state.found = 0; - + PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state); Py_DECREF(obj); if (!state.found) { @@ -3400,6 +3400,98 @@ test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self), Py_RETURN_NONE; } +typedef struct { + PyObject_HEAD +} ObjExtraData; + +static PyObject * +obj_extra_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + size_t extra_size = sizeof(PyObject *); + PyObject *obj = PyUnstable_Object_GC_NewWithExtraData(type, extra_size); + if (obj == NULL) { + return PyErr_NoMemory(); + } + PyObject_GC_Track(obj); + return obj; +} + +static PyObject ** +obj_extra_data_get_extra_storage(PyObject *self) +{ + return (PyObject **)((char *)self + Py_TYPE(self)->tp_basicsize); +} + +static PyObject * +obj_extra_data_get(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + PyObject *value = *extra_storage; + if (!value) { + Py_RETURN_NONE; + } + return Py_NewRef(value); +} + +static int +obj_extra_data_set(PyObject *self, PyObject *newval, void *Py_UNUSED(ignored)) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + Py_CLEAR(*extra_storage); + if (newval) { + *extra_storage = Py_NewRef(newval); + } + return 0; +} + +static PyGetSetDef obj_extra_data_getset[] = { + {"extra", (getter)obj_extra_data_get, (setter)obj_extra_data_set, NULL}, + {NULL} +}; + +static int +obj_extra_data_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + PyObject *value = *extra_storage; + Py_VISIT(value); + return 0; +} + +static int +obj_extra_data_clear(PyObject *self) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + Py_CLEAR(*extra_storage); + return 0; +} + +static void +obj_extra_data_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + obj_extra_data_clear(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot ObjExtraData_Slots[] = { + {Py_tp_getset, obj_extra_data_getset}, + {Py_tp_dealloc, obj_extra_data_dealloc}, + {Py_tp_traverse, obj_extra_data_traverse}, + {Py_tp_clear, obj_extra_data_clear}, + {Py_tp_new, obj_extra_data_new}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec ObjExtraData_TypeSpec = { + .name = "_testcapi.ObjExtraData", + .basicsize = sizeof(ObjExtraData), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .slots = ObjExtraData_Slots, +}; struct atexit_data { int called; @@ -4124,6 +4216,17 @@ PyInit__testcapi(void) Py_INCREF(&MethStatic_Type); PyModule_AddObject(m, "MethStatic", (PyObject *)&MethStatic_Type); + PyObject *ObjExtraData_Type = PyType_FromModuleAndSpec( + m, &ObjExtraData_TypeSpec, NULL); + if (ObjExtraData_Type == 0) { + return NULL; + } + int ret = PyModule_AddType(m, (PyTypeObject*)ObjExtraData_Type); + Py_DECREF(&ObjExtraData_Type); + if (ret < 0) { + return NULL; + } + PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX)); PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN)); PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX)); diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 3fd5f4cd70e832..8a4d1a4398281e 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2367,6 +2367,19 @@ _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) return op; } +PyObject * +PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(_PyObject_SIZE(tp) + extra_size, presize); + if (op == NULL) { + return NULL; + } + memset(op, 0, _PyObject_SIZE(tp) + extra_size); + _PyObject_Init(op, tp); + return op; +} + PyVarObject * _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) { From d81ca7ec029ba05084751c8df64292bb48f4f30f Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Wed, 3 May 2023 00:05:30 +0900 Subject: [PATCH 11/14] gh-84436: Add integration C API tests for immortal objects (gh-103962) --- Lib/test/test_capi/test_immortal.py | 16 ++++++++++ Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/immortal.c | 47 +++++++++++++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 3 ++ PCbuild/_testcapi.vcxproj | 1 + 6 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_capi/test_immortal.py create mode 100644 Modules/_testcapi/immortal.c diff --git a/Lib/test/test_capi/test_immortal.py b/Lib/test/test_capi/test_immortal.py new file mode 100644 index 00000000000000..ef5d32b7f01935 --- /dev/null +++ b/Lib/test/test_capi/test_immortal.py @@ -0,0 +1,16 @@ +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +class TestCAPI(unittest.TestCase): + def test_immortal_builtins(self): + _testcapi.test_immortal_builtins() + + def test_immortal_small_ints(self): + _testcapi.test_immortal_small_ints() + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index fe1b9f8f5380c1..a90c1e96ef0231 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c _testcapi/immortal.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c new file mode 100644 index 00000000000000..10e1733d08a9ea --- /dev/null +++ b/Modules/_testcapi/immortal.c @@ -0,0 +1,47 @@ +#include "parts.h" + +int verify_immortality(PyObject *object) +{ + assert(_Py_IsImmortal(object)); + Py_ssize_t old_count = Py_REFCNT(object); + for (int j = 0; j < 10000; j++) { + Py_DECREF(object); + } + Py_ssize_t current_count = Py_REFCNT(object); + return old_count == current_count; +} + +static PyObject * +test_immortal_builtins(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *objects[] = {Py_True, Py_False, Py_None, Py_Ellipsis}; + Py_ssize_t n = Py_ARRAY_LENGTH(objects); + for (Py_ssize_t i = 0; i < n; i++) { + assert(verify_immortality(objects[i])); + } + Py_RETURN_NONE; +} + +static PyObject * +test_immortal_small_ints(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + for (int i = -5; i <= 256; i++) { + assert(verify_immortality(PyLong_FromLong(i))); + } + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"test_immortal_builtins", test_immortal_builtins, METH_NOARGS}, + {"test_immortal_small_ints", test_immortal_small_ints, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Immortal(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 60ec81dad2ba9e..4d2d6832a827ae 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -39,6 +39,7 @@ int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_Immortal(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a5d23b1b3d50ec..30b8b6c6b3a87b 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4313,6 +4313,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_PyOS(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Immortal(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 439cd687fda61d..21941247eb9692 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -110,6 +110,7 @@ + From 8611e7bf5ceace998fefcbf26ab1c5d5bc8a0e2a Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Tue, 2 May 2023 19:08:19 +0100 Subject: [PATCH 12/14] GH-103525: Improve exception message from `pathlib.PurePath()` (GH-103526) Check that arguments are strings before calling `os.path.join()`. Also improve performance of `PurePath(PurePath(...))` while we're in the area: we now use the *unnormalized* string path of such arguments. Co-authored-by: Terry Jan Reedy --- Lib/pathlib.py | 37 ++++++++++++------- Lib/test/test_pathlib.py | 4 +- ...-04-13-19-43-15.gh-issue-103525.uY4VYg.rst | 2 + 3 files changed, 27 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f43f01ef41a97f..8eb08949fa9b43 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -300,18 +300,27 @@ def __reduce__(self): return (self.__class__, self.parts) def __init__(self, *args): - if not args: - path = '' - elif len(args) == 1: - path = os.fspath(args[0]) + paths = [] + for arg in args: + if isinstance(arg, PurePath): + path = arg._raw_path + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + if len(paths) == 0: + self._raw_path = '' + elif len(paths) == 1: + self._raw_path = paths[0] else: - path = self._flavour.join(*args) - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - self._raw_path = path + self._raw_path = self._flavour.join(*paths) @classmethod def _parse_path(cls, path): @@ -620,7 +629,7 @@ def joinpath(self, *args): paths) or a totally different path (if one of the arguments is anchored). """ - return self.__class__(self._raw_path, *args) + return self.__class__(self, *args) def __truediv__(self, key): try: @@ -630,7 +639,7 @@ def __truediv__(self, key): def __rtruediv__(self, key): try: - return type(self)(key, self._raw_path) + return type(self)(key, self) except TypeError: return NotImplemented @@ -864,7 +873,7 @@ def absolute(self): cwd = self._flavour.abspath(self.drive) else: cwd = os.getcwd() - return type(self)(cwd, self._raw_path) + return type(self)(cwd, self) def resolve(self, strict=False): """ diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 76cfadeedcea84..8b5b61a818bbbc 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -81,9 +81,9 @@ def test_bytes(self): r"where __fspath__ returns a str, not 'bytes'") with self.assertRaisesRegex(TypeError, message): P(b'a') - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, message): P(b'a', 'b') - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, message): P('a', b'b') with self.assertRaises(TypeError): P('a').joinpath(b'b') diff --git a/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst b/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst new file mode 100644 index 00000000000000..1414cb07dd9155 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst @@ -0,0 +1,2 @@ +Fix misleading exception message when mixed ``str`` and ``bytes`` arguments +are supplied to :class:`pathlib.PurePath` and :class:`~pathlib.Path`. From 587f2f018051049cf5d9de3e12ed5aa7644404dc Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 2 May 2023 11:13:47 -0700 Subject: [PATCH 13/14] gh-65022: Fix description of tuple return value in copyreg (#103892) --- Doc/library/copyreg.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 2107215c0c1967..afc3e66f0bf7ac 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -29,7 +29,7 @@ Such constructors may be factory functions or class instances. Declares that *function* should be used as a "reduction" function for objects of type *type*. *function* must return either a string or a tuple - containing two or five elements. See the :attr:`~pickle.Pickler.dispatch_table` + containing between two and six elements. See the :attr:`~pickle.Pickler.dispatch_table` for more details on the interface of *function*. The *constructor_ob* parameter is a legacy feature and is now ignored, but if From 1f5384434dce013b5dcf7e7ea3ec5312d13bba72 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Wed, 3 May 2023 01:43:31 +0530 Subject: [PATCH 14/14] gh-103822: [Calendar] change return value to enum for day and month APIs (GH-103827) --- Lib/calendar.py | 3 +-- .../Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst diff --git a/Lib/calendar.py b/Lib/calendar.py index bbd4fea3b88ca4..ea56f12ccc41d0 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -83,7 +83,6 @@ class Day(IntEnum): SUNDAY = 6 - # Number of days per month (except for February in leap years) mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] @@ -156,7 +155,7 @@ def weekday(year, month, day): """Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31).""" if not datetime.MINYEAR <= year <= datetime.MAXYEAR: year = 2000 + year % 400 - return datetime.date(year, month, day).weekday() + return Day(datetime.date(year, month, day).weekday()) def monthrange(year, month): diff --git a/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst b/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst new file mode 100644 index 00000000000000..3daf9cc093807b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst @@ -0,0 +1 @@ +Update the return type of ``weekday`` to the newly added Day attribute