From 0f128b9435fccb296714f3ea2466c3fdda77d91d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 8 Feb 2025 12:57:17 +0000 Subject: [PATCH 001/305] gh-129842: warnings.py: Remove obsolete requirement reference (GH-129845) --- Lib/warnings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/warnings.py b/Lib/warnings.py index f20b01372dd7a4..13ad6c8aacbb7f 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -473,9 +473,6 @@ def __init__(self, *, record=False, module=None, """Specify whether to record warnings and if an alternative module should be used other than sys.modules['warnings']. - For compatibility with Python 3.0, please consider all arguments to be - keyword-only. - """ self._record = record self._module = sys.modules['warnings'] if module is None else module From 421ea1291d9b8ebfe5eaa72ab041338073fb67d0 Mon Sep 17 00:00:00 2001 From: Brian Ward Date: Sat, 8 Feb 2025 08:02:36 -0500 Subject: [PATCH 002/305] gh-119349: Add ctypes.util.dllist -- list loaded shared libraries (GH-122946) Add function to list the currently loaded libraries to ctypes.util The dllist() function calls platform-specific APIs in order to list the runtime libraries loaded by Python and any imported modules. On unsupported platforms the function may be missing. Co-authored-by: Eryk Sun Co-authored-by: Peter Bierma --- Doc/library/ctypes.rst | 36 +++++ Doc/whatsnew/3.14.rst | 3 + Lib/ctypes/util.py | 131 ++++++++++++++++++ Lib/test/test_ctypes/test_dllist.py | 59 ++++++++ Misc/ACKS | 1 + ...-08-12-11-58-15.gh-issue-119349.-xTnHl.rst | 2 + 6 files changed, 232 insertions(+) create mode 100644 Lib/test/test_ctypes/test_dllist.py create mode 100644 Misc/NEWS.d/next/Library/2024-08-12-11-58-15.gh-issue-119349.-xTnHl.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 615138302e1379..cb02fd33a6e741 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1406,6 +1406,28 @@ the shared library name at development time, and hardcode that into the wrapper module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. +.. _ctypes-listing-loaded-shared-libraries: + +Listing loaded shared libraries +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When writing code that relies on code loaded from shared libraries, it can be +useful to know which shared libraries have already been loaded into the current +process. + +The :mod:`!ctypes.util` module provides the :func:`~ctypes.util.dllist` function, +which calls the different APIs provided by the various platforms to help determine +which shared libraries have already been loaded into the current process. + +The exact output of this function will be system dependent. On most platforms, +the first entry of this list represents the current process itself, which may +be an empty string. +For example, on glibc-based Linux, the return may look like:: + + >>> from ctypes.util import dllist + >>> dllist() + ['', 'linux-vdso.so.1', '/lib/x86_64-linux-gnu/libm.so.6', '/lib/x86_64-linux-gnu/libc.so.6', ... ] + .. _ctypes-loading-shared-libraries: Loading shared libraries @@ -2083,6 +2105,20 @@ Utility functions .. availability:: Windows +.. function:: dllist() + :module: ctypes.util + + Try to provide a list of paths of the shared libraries loaded into the current + process. These paths are not normalized or processed in any way. The function + can raise :exc:`OSError` if the underlying platform APIs fail. + The exact functionality is system dependent. + + On most platforms, the first element of the list represents the current + executable file. It may be an empty string. + + .. availability:: Windows, macOS, iOS, glibc, BSD libc, musl + .. versionadded:: next + .. function:: FormatError([code]) Returns a textual description of the error code *code*. If no error code is diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 9ac0e6ed2a6d40..9c4922308b7f2d 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -389,6 +389,9 @@ ctypes complex C types. (Contributed by Sergey B Kirpichev in :gh:`61103`). +* Add :func:`ctypes.util.dllist` for listing the shared libraries + loaded by the current process. + (Contributed by Brian Ward in :gh:`119349`.) datetime -------- diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 117bf06cb01013..99504911a3dbe0 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,6 +67,65 @@ def find_library(name): return fname return None + # Listing loaded DLLs on Windows relies on the following APIs: + # https://learn.microsoft.com/windows/win32/api/psapi/nf-psapi-enumprocessmodules + # https://learn.microsoft.com/windows/win32/api/libloaderapi/nf-libloaderapi-getmodulefilenamew + import ctypes + from ctypes import wintypes + + _kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + _get_current_process = _kernel32["GetCurrentProcess"] + _get_current_process.restype = wintypes.HANDLE + + _k32_get_module_file_name = _kernel32["GetModuleFileNameW"] + _k32_get_module_file_name.restype = wintypes.DWORD + _k32_get_module_file_name.argtypes = ( + wintypes.HMODULE, + wintypes.LPWSTR, + wintypes.DWORD, + ) + + _psapi = ctypes.WinDLL('psapi', use_last_error=True) + _enum_process_modules = _psapi["EnumProcessModules"] + _enum_process_modules.restype = wintypes.BOOL + _enum_process_modules.argtypes = ( + wintypes.HANDLE, + ctypes.POINTER(wintypes.HMODULE), + wintypes.DWORD, + wintypes.LPDWORD, + ) + + def _get_module_filename(module: wintypes.HMODULE): + name = (wintypes.WCHAR * 32767)() # UNICODE_STRING_MAX_CHARS + if _k32_get_module_file_name(module, name, len(name)): + return name.value + return None + + + def _get_module_handles(): + process = _get_current_process() + space_needed = wintypes.DWORD() + n = 1024 + while True: + modules = (wintypes.HMODULE * n)() + if not _enum_process_modules(process, + modules, + ctypes.sizeof(modules), + ctypes.byref(space_needed)): + err = ctypes.get_last_error() + msg = ctypes.FormatError(err).strip() + raise ctypes.WinError(err, f"EnumProcessModules failed: {msg}") + n = space_needed.value // ctypes.sizeof(wintypes.HMODULE) + if n <= len(modules): + return modules[:n] + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + modules = _get_module_handles() + libraries = [name for h in modules + if (name := _get_module_filename(h)) is not None] + return libraries + elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): @@ -80,6 +139,22 @@ def find_library(name): continue return None + # Listing loaded libraries on Apple systems relies on the following API: + # https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dyld.3.html + import ctypes + + _libc = ctypes.CDLL(find_library("c")) + _dyld_get_image_name = _libc["_dyld_get_image_name"] + _dyld_get_image_name.restype = ctypes.c_char_p + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + num_images = _libc._dyld_image_count() + libraries = [os.fsdecode(name) for i in range(num_images) + if (name := _dyld_get_image_name(i)) is not None] + + return libraries + elif sys.platform.startswith("aix"): # AIX has two styles of storing shared libraries # GNU auto_tools refer to these as svr4 and aix @@ -341,6 +416,55 @@ def find_library(name): return _findSoname_ldconfig(name) or \ _get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name)) + +# Listing loaded libraries on other systems will try to use +# functions common to Linux and a few other Unix-like systems. +# See the following for several platforms' documentation of the same API: +# https://man7.org/linux/man-pages/man3/dl_iterate_phdr.3.html +# https://man.freebsd.org/cgi/man.cgi?query=dl_iterate_phdr +# https://man.openbsd.org/dl_iterate_phdr +# https://docs.oracle.com/cd/E88353_01/html/E37843/dl-iterate-phdr-3c.html +if (os.name == "posix" and + sys.platform not in {"darwin", "ios", "tvos", "watchos"}): + import ctypes + if hasattr((_libc := ctypes.CDLL(None)), "dl_iterate_phdr"): + + class _dl_phdr_info(ctypes.Structure): + _fields_ = [ + ("dlpi_addr", ctypes.c_void_p), + ("dlpi_name", ctypes.c_char_p), + ("dlpi_phdr", ctypes.c_void_p), + ("dlpi_phnum", ctypes.c_ushort), + ] + + _dl_phdr_callback = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.POINTER(_dl_phdr_info), + ctypes.c_size_t, + ctypes.POINTER(ctypes.py_object), + ) + + @_dl_phdr_callback + def _info_callback(info, _size, data): + libraries = data.contents.value + name = os.fsdecode(info.contents.dlpi_name) + libraries.append(name) + return 0 + + _dl_iterate_phdr = _libc["dl_iterate_phdr"] + _dl_iterate_phdr.argtypes = [ + _dl_phdr_callback, + ctypes.POINTER(ctypes.py_object), + ] + _dl_iterate_phdr.restype = ctypes.c_int + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + libraries = [] + _dl_iterate_phdr(_info_callback, + ctypes.byref(ctypes.py_object(libraries))) + return libraries + ################################################################ # test code @@ -384,5 +508,12 @@ def test(): print(cdll.LoadLibrary("libcrypt.so")) print(find_library("crypt")) + try: + dllist + except NameError: + print('dllist() not available') + else: + print(dllist()) + if __name__ == "__main__": test() diff --git a/Lib/test/test_ctypes/test_dllist.py b/Lib/test/test_ctypes/test_dllist.py new file mode 100644 index 00000000000000..15603dc3d77972 --- /dev/null +++ b/Lib/test/test_ctypes/test_dllist.py @@ -0,0 +1,59 @@ +import os +import sys +import unittest +from ctypes import CDLL +import ctypes.util +from test.support import import_helper + + +WINDOWS = os.name == "nt" +APPLE = sys.platform in {"darwin", "ios", "tvos", "watchos"} + +if WINDOWS: + KNOWN_LIBRARIES = ["KERNEL32.DLL"] +elif APPLE: + KNOWN_LIBRARIES = ["libSystem.B.dylib"] +else: + # trickier than it seems, because libc may not be present + # on musl systems, and sometimes goes by different names. + # However, ctypes itself loads libffi + KNOWN_LIBRARIES = ["libc.so", "libffi.so"] + + +@unittest.skipUnless( + hasattr(ctypes.util, "dllist"), + "ctypes.util.dllist is not available on this platform", +) +class ListSharedLibraries(unittest.TestCase): + + def test_lists_system(self): + dlls = ctypes.util.dllist() + + self.assertGreater(len(dlls), 0, f"loaded={dlls}") + self.assertTrue( + any(lib in dll for dll in dlls for lib in KNOWN_LIBRARIES), f"loaded={dlls}" + ) + + def test_lists_updates(self): + dlls = ctypes.util.dllist() + + # this test relies on being able to import a library which is + # not already loaded. + # If it is (e.g. by a previous test in the same process), we skip + if any("_ctypes_test" in dll for dll in dlls): + self.skipTest("Test library is already loaded") + + _ctypes_test = import_helper.import_module("_ctypes_test") + test_module = CDLL(_ctypes_test.__file__) + dlls2 = ctypes.util.dllist() + self.assertIsNotNone(dlls2) + + dlls1 = set(dlls) + dlls2 = set(dlls2) + + self.assertGreater(dlls2, dlls1, f"newly loaded libraries: {dlls2 - dlls1}") + self.assertTrue(any("_ctypes_test" in dll for dll in dlls2), f"loaded={dlls2}") + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index 27480a1f3131bd..2a68b69f161041 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1993,6 +1993,7 @@ Edward C Wang Jiahua Wang Ke Wang Liang-Bo Wang +Brian Ward Greg Ward Tom Wardill Zachary Ware diff --git a/Misc/NEWS.d/next/Library/2024-08-12-11-58-15.gh-issue-119349.-xTnHl.rst b/Misc/NEWS.d/next/Library/2024-08-12-11-58-15.gh-issue-119349.-xTnHl.rst new file mode 100644 index 00000000000000..5dd8264a608dfa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-12-11-58-15.gh-issue-119349.-xTnHl.rst @@ -0,0 +1,2 @@ +Add the :func:`ctypes.util.dllist` function to list the loaded shared +libraries for the current process. From 1988003625409e4d6e3a1affab14b846063a4f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 8 Feb 2025 14:21:32 +0100 Subject: [PATCH 003/305] gh-111178: fix UBSan failures in `Modules/_io/*.c` (GH-129083) * fix UBSan failures for `buffered`, `rwpair`, `bytesio`, `bytesiobuf`, `iobase`, `stringio`, `nldecoder_object`, `textio`, `winconsoleio` * arg names: use 'dummy' for NOARGS method and 'args' for others --- Modules/_io/bufferedio.c | 93 +++++++++++++++---------- Modules/_io/bytesio.c | 58 ++++++++++------ Modules/_io/fileio.c | 26 +++---- Modules/_io/iobase.c | 19 +++-- Modules/_io/stringio.c | 21 +++--- Modules/_io/textio.c | 139 ++++++++++++++++++++----------------- Modules/_io/winconsoleio.c | 37 ++++++---- 7 files changed, 230 insertions(+), 163 deletions(-) diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index bc5fff54a62b6d..53c4702f673786 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -261,6 +261,8 @@ typedef struct { PyObject *weakreflist; } buffered; +#define buffered_CAST(op) ((buffered *)(op)) + /* Implementation notes: @@ -399,8 +401,9 @@ _enter_buffered_busy(buffered *self) static int -buffered_clear(buffered *self) +buffered_clear(PyObject *op) { + buffered *self = buffered_CAST(op); self->ok = 0; Py_CLEAR(self->raw); Py_CLEAR(self->dict); @@ -408,16 +411,17 @@ buffered_clear(buffered *self) } static void -buffered_dealloc(buffered *self) +buffered_dealloc(PyObject *op) { + buffered *self = buffered_CAST(op); PyTypeObject *tp = Py_TYPE(self); self->finalizing = 1; - if (_PyIOBase_finalize((PyObject *) self) < 0) + if (_PyIOBase_finalize(op) < 0) return; _PyObject_GC_UNTRACK(self); self->ok = 0; if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)self); + PyObject_ClearWeakRefs(op); if (self->buffer) { PyMem_Free(self->buffer); self->buffer = NULL; @@ -426,8 +430,8 @@ buffered_dealloc(buffered *self) PyThread_free_lock(self->lock); self->lock = NULL; } - (void)buffered_clear(self); - tp->tp_free((PyObject *)self); + (void)buffered_clear(op); + tp->tp_free(self); Py_DECREF(tp); } @@ -2227,6 +2231,8 @@ typedef struct { PyObject *weakreflist; } rwpair; +#define rwpair_CAST(op) ((rwpair *)(op)) + /*[clinic input] _io.BufferedRWPair.__init__ reader: object @@ -2276,8 +2282,9 @@ _io_BufferedRWPair___init___impl(rwpair *self, PyObject *reader, } static int -bufferedrwpair_traverse(rwpair *self, visitproc visit, void *arg) +bufferedrwpair_traverse(PyObject *op, visitproc visit, void *arg) { + rwpair *self = rwpair_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); Py_VISIT(self->reader); @@ -2286,8 +2293,9 @@ bufferedrwpair_traverse(rwpair *self, visitproc visit, void *arg) } static int -bufferedrwpair_clear(rwpair *self) +bufferedrwpair_clear(PyObject *op) { + rwpair *self = rwpair_CAST(op); Py_CLEAR(self->reader); Py_CLEAR(self->writer); Py_CLEAR(self->dict); @@ -2295,14 +2303,15 @@ bufferedrwpair_clear(rwpair *self) } static void -bufferedrwpair_dealloc(rwpair *self) +bufferedrwpair_dealloc(PyObject *op) { + rwpair *self = rwpair_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)self); - (void)bufferedrwpair_clear(self); - tp->tp_free((PyObject *) self); + PyObject_ClearWeakRefs(op); + (void)bufferedrwpair_clear(op); + tp->tp_free(self); Py_DECREF(tp); } @@ -2328,62 +2337,72 @@ _forward_call(buffered *self, PyObject *name, PyObject *args) } static PyObject * -bufferedrwpair_read(rwpair *self, PyObject *args) +bufferedrwpair_read(PyObject *op, PyObject *args) { + rwpair *self = rwpair_CAST(op); return _forward_call(self->reader, &_Py_ID(read), args); } static PyObject * -bufferedrwpair_peek(rwpair *self, PyObject *args) +bufferedrwpair_peek(PyObject *op, PyObject *args) { + rwpair *self = rwpair_CAST(op); return _forward_call(self->reader, &_Py_ID(peek), args); } static PyObject * -bufferedrwpair_read1(rwpair *self, PyObject *args) +bufferedrwpair_read1(PyObject *op, PyObject *args) { + rwpair *self = rwpair_CAST(op); return _forward_call(self->reader, &_Py_ID(read1), args); } static PyObject * -bufferedrwpair_readinto(rwpair *self, PyObject *args) +bufferedrwpair_readinto(PyObject *op, PyObject *args) { + rwpair *self = rwpair_CAST(op); return _forward_call(self->reader, &_Py_ID(readinto), args); } static PyObject * -bufferedrwpair_readinto1(rwpair *self, PyObject *args) +bufferedrwpair_readinto1(PyObject *op, PyObject *args) { + rwpair *self = rwpair_CAST(op); return _forward_call(self->reader, &_Py_ID(readinto1), args); } static PyObject * -bufferedrwpair_write(rwpair *self, PyObject *args) +bufferedrwpair_write(PyObject *op, PyObject *args) { + rwpair *self = rwpair_CAST(op); return _forward_call(self->writer, &_Py_ID(write), args); } static PyObject * -bufferedrwpair_flush(rwpair *self, PyObject *Py_UNUSED(ignored)) +bufferedrwpair_flush(PyObject *op, PyObject *Py_UNUSED(dummy)) { + rwpair *self = rwpair_CAST(op); return _forward_call(self->writer, &_Py_ID(flush), NULL); } static PyObject * -bufferedrwpair_readable(rwpair *self, PyObject *Py_UNUSED(ignored)) +bufferedrwpair_readable(PyObject *op, PyObject *Py_UNUSED(dummy)) { + rwpair *self = rwpair_CAST(op); return _forward_call(self->reader, &_Py_ID(readable), NULL); } static PyObject * -bufferedrwpair_writable(rwpair *self, PyObject *Py_UNUSED(ignored)) +bufferedrwpair_writable(PyObject *op, PyObject *Py_UNUSED(dummy)) { + rwpair *self = rwpair_CAST(op); return _forward_call(self->writer, &_Py_ID(writable), NULL); } static PyObject * -bufferedrwpair_close(rwpair *self, PyObject *Py_UNUSED(ignored)) +bufferedrwpair_close(PyObject *op, PyObject *Py_UNUSED(dummy)) { + rwpair *self = rwpair_CAST(op); PyObject *exc = NULL; PyObject *ret = _forward_call(self->writer, &_Py_ID(close), NULL); if (ret == NULL) { @@ -2401,8 +2420,9 @@ bufferedrwpair_close(rwpair *self, PyObject *Py_UNUSED(ignored)) } static PyObject * -bufferedrwpair_isatty(rwpair *self, PyObject *Py_UNUSED(ignored)) +bufferedrwpair_isatty(PyObject *op, PyObject *Py_UNUSED(dummy)) { + rwpair *self = rwpair_CAST(op); PyObject *ret = _forward_call(self->writer, &_Py_ID(isatty), NULL); if (ret != Py_False) { @@ -2415,8 +2435,9 @@ bufferedrwpair_isatty(rwpair *self, PyObject *Py_UNUSED(ignored)) } static PyObject * -bufferedrwpair_closed_get(rwpair *self, void *context) +bufferedrwpair_closed_get(PyObject *op, void *Py_UNUSED(dummy)) { + rwpair *self = rwpair_CAST(op); if (self->writer == NULL) { PyErr_SetString(PyExc_RuntimeError, "the BufferedRWPair object is being garbage-collected"); @@ -2633,20 +2654,20 @@ PyType_Spec bufferedwriter_spec = { }; static PyMethodDef bufferedrwpair_methods[] = { - {"read", (PyCFunction)bufferedrwpair_read, METH_VARARGS}, - {"peek", (PyCFunction)bufferedrwpair_peek, METH_VARARGS}, - {"read1", (PyCFunction)bufferedrwpair_read1, METH_VARARGS}, - {"readinto", (PyCFunction)bufferedrwpair_readinto, METH_VARARGS}, - {"readinto1", (PyCFunction)bufferedrwpair_readinto1, METH_VARARGS}, + {"read", bufferedrwpair_read, METH_VARARGS}, + {"peek", bufferedrwpair_peek, METH_VARARGS}, + {"read1", bufferedrwpair_read1, METH_VARARGS}, + {"readinto", bufferedrwpair_readinto, METH_VARARGS}, + {"readinto1", bufferedrwpair_readinto1, METH_VARARGS}, - {"write", (PyCFunction)bufferedrwpair_write, METH_VARARGS}, - {"flush", (PyCFunction)bufferedrwpair_flush, METH_NOARGS}, + {"write", bufferedrwpair_write, METH_VARARGS}, + {"flush", bufferedrwpair_flush, METH_NOARGS}, - {"readable", (PyCFunction)bufferedrwpair_readable, METH_NOARGS}, - {"writable", (PyCFunction)bufferedrwpair_writable, METH_NOARGS}, + {"readable", bufferedrwpair_readable, METH_NOARGS}, + {"writable", bufferedrwpair_writable, METH_NOARGS}, - {"close", (PyCFunction)bufferedrwpair_close, METH_NOARGS}, - {"isatty", (PyCFunction)bufferedrwpair_isatty, METH_NOARGS}, + {"close", bufferedrwpair_close, METH_NOARGS}, + {"isatty", bufferedrwpair_isatty, METH_NOARGS}, {NULL, NULL} }; @@ -2658,7 +2679,7 @@ static PyMemberDef bufferedrwpair_members[] = { }; static PyGetSetDef bufferedrwpair_getset[] = { - {"closed", (getter)bufferedrwpair_closed_get, NULL, NULL}, + {"closed", bufferedrwpair_closed_get, NULL, NULL}, {NULL} }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 16095333db6638..dc4e40b9f09a1d 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -21,11 +21,15 @@ typedef struct { Py_ssize_t exports; } bytesio; +#define bytesio_CAST(op) ((bytesio *)(op)) + typedef struct { PyObject_HEAD bytesio *source; } bytesiobuf; +#define bytesiobuf_CAST(op) ((bytesiobuf *)(op)) + /* The bytesio object can be in three states: * Py_REFCNT(buf) == 1, exports == 0. * Py_REFCNT(buf) > 1. exports == 0, @@ -239,8 +243,9 @@ write_bytes(bytesio *self, PyObject *b) } static PyObject * -bytesio_get_closed(bytesio *self, void *Py_UNUSED(ignored)) +bytesio_get_closed(PyObject *op, void *Py_UNUSED(closure)) { + bytesio *self = bytesio_CAST(op); if (self->buf == NULL) { Py_RETURN_TRUE; } @@ -620,9 +625,10 @@ _io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size) } static PyObject * -bytesio_iternext(bytesio *self) +bytesio_iternext(PyObject *op) { Py_ssize_t n; + bytesio *self = bytesio_CAST(op); CHECK_CLOSED(self); @@ -783,8 +789,9 @@ _io_BytesIO_close_impl(bytesio *self) */ static PyObject * -bytesio_getstate(bytesio *self, PyObject *Py_UNUSED(ignored)) +bytesio_getstate(PyObject *op, PyObject *Py_UNUSED(dummy)) { + bytesio *self = bytesio_CAST(op); PyObject *initvalue = _io_BytesIO_getvalue_impl(self); PyObject *dict; PyObject *state; @@ -808,12 +815,13 @@ bytesio_getstate(bytesio *self, PyObject *Py_UNUSED(ignored)) } static PyObject * -bytesio_setstate(bytesio *self, PyObject *state) +bytesio_setstate(PyObject *op, PyObject *state) { PyObject *result; PyObject *position_obj; PyObject *dict; Py_ssize_t pos; + bytesio *self = bytesio_CAST(op); assert(state != NULL); @@ -883,8 +891,9 @@ bytesio_setstate(bytesio *self, PyObject *state) } static void -bytesio_dealloc(bytesio *self) +bytesio_dealloc(PyObject *op) { + bytesio *self = bytesio_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); if (self->exports > 0) { @@ -895,7 +904,7 @@ bytesio_dealloc(bytesio *self) Py_CLEAR(self->buf); Py_CLEAR(self->dict); if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); + PyObject_ClearWeakRefs(op); tp->tp_free(self); Py_DECREF(tp); } @@ -961,8 +970,9 @@ _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) } static PyObject * -bytesio_sizeof(bytesio *self, void *unused) +bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) { + bytesio *self = bytesio_CAST(op); size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->buf && !SHARED_BUF(self)) { size_t s = _PySys_GetSizeOf(self->buf); @@ -975,8 +985,9 @@ bytesio_sizeof(bytesio *self, void *unused) } static int -bytesio_traverse(bytesio *self, visitproc visit, void *arg) +bytesio_traverse(PyObject *op, visitproc visit, void *arg) { + bytesio *self = bytesio_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); Py_VISIT(self->buf); @@ -984,8 +995,9 @@ bytesio_traverse(bytesio *self, visitproc visit, void *arg) } static int -bytesio_clear(bytesio *self) +bytesio_clear(PyObject *op) { + bytesio *self = bytesio_CAST(op); Py_CLEAR(self->dict); if (self->exports == 0) { Py_CLEAR(self->buf); @@ -999,7 +1011,7 @@ bytesio_clear(bytesio *self) #undef clinic_state static PyGetSetDef bytesio_getsetlist[] = { - {"closed", (getter)bytesio_get_closed, NULL, + {"closed", bytesio_get_closed, NULL, "True if the file is closed."}, {NULL}, /* sentinel */ }; @@ -1023,9 +1035,9 @@ static struct PyMethodDef bytesio_methods[] = { _IO_BYTESIO_GETVALUE_METHODDEF _IO_BYTESIO_SEEK_METHODDEF _IO_BYTESIO_TRUNCATE_METHODDEF - {"__getstate__", (PyCFunction)bytesio_getstate, METH_NOARGS, NULL}, - {"__setstate__", (PyCFunction)bytesio_setstate, METH_O, NULL}, - {"__sizeof__", (PyCFunction)bytesio_sizeof, METH_NOARGS, NULL}, + {"__getstate__", bytesio_getstate, METH_NOARGS, NULL}, + {"__setstate__", bytesio_setstate, METH_O, NULL}, + {"__sizeof__", bytesio_sizeof, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; @@ -1065,9 +1077,10 @@ PyType_Spec bytesio_spec = { */ static int -bytesiobuf_getbuffer(bytesiobuf *obj, Py_buffer *view, int flags) +bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) { - bytesio *b = (bytesio *) obj->source; + bytesiobuf *obj = bytesiobuf_CAST(op); + bytesio *b = bytesio_CAST(obj->source); if (view == NULL) { PyErr_SetString(PyExc_BufferError, @@ -1080,7 +1093,7 @@ bytesiobuf_getbuffer(bytesiobuf *obj, Py_buffer *view, int flags) } /* cannot fail if view != NULL and readonly == 0 */ - (void)PyBuffer_FillInfo(view, (PyObject*)obj, + (void)PyBuffer_FillInfo(view, op, PyBytes_AS_STRING(b->buf), b->string_size, 0, flags); b->exports++; @@ -1088,26 +1101,29 @@ bytesiobuf_getbuffer(bytesiobuf *obj, Py_buffer *view, int flags) } static void -bytesiobuf_releasebuffer(bytesiobuf *obj, Py_buffer *view) +bytesiobuf_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) { - bytesio *b = (bytesio *) obj->source; + bytesiobuf *obj = bytesiobuf_CAST(op); + bytesio *b = bytesio_CAST(obj->source); b->exports--; } static int -bytesiobuf_traverse(bytesiobuf *self, visitproc visit, void *arg) +bytesiobuf_traverse(PyObject *op, visitproc visit, void *arg) { + bytesiobuf *self = bytesiobuf_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->source); return 0; } static void -bytesiobuf_dealloc(bytesiobuf *self) +bytesiobuf_dealloc(PyObject *op) { + bytesiobuf *self = bytesiobuf_CAST(op); PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ - PyObject_GC_UnTrack(self); + PyObject_GC_UnTrack(op); Py_CLEAR(self->source); tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index f27f2ed4843271..89f1cfe6b20935 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -83,7 +83,7 @@ typedef struct { } fileio; #define PyFileIO_Check(state, op) (PyObject_TypeCheck((op), state->PyFileIO_Type)) -#define _PyFileIO_CAST(op) _Py_CAST(fileio*, (op)) +#define PyFileIO_CAST(op) ((fileio *)(op)) /* Forward declarations */ static PyObject* portable_lseek(fileio *self, PyObject *posobj, int whence, bool suppress_pipe_error); @@ -91,7 +91,7 @@ static PyObject* portable_lseek(fileio *self, PyObject *posobj, int whence, bool int _PyFileIO_closed(PyObject *self) { - return (_PyFileIO_CAST(self)->fd < 0); + return (PyFileIO_CAST(self)->fd < 0); } /* Because this can call arbitrary code, it shouldn't be called when @@ -100,7 +100,7 @@ _PyFileIO_closed(PyObject *self) static PyObject * fileio_dealloc_warn(PyObject *op, PyObject *source) { - fileio *self = _PyFileIO_CAST(op); + fileio *self = PyFileIO_CAST(op); if (self->fd >= 0 && self->closefd) { PyObject *exc = PyErr_GetRaisedException(); if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) { @@ -542,7 +542,7 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, static int fileio_traverse(PyObject *op, visitproc visit, void *arg) { - fileio *self = _PyFileIO_CAST(op); + fileio *self = PyFileIO_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; @@ -551,7 +551,7 @@ fileio_traverse(PyObject *op, visitproc visit, void *arg) static int fileio_clear(PyObject *op) { - fileio *self = _PyFileIO_CAST(op); + fileio *self = PyFileIO_CAST(op); Py_CLEAR(self->dict); return 0; } @@ -559,7 +559,7 @@ fileio_clear(PyObject *op) static void fileio_dealloc(PyObject *op) { - fileio *self = _PyFileIO_CAST(op); + fileio *self = PyFileIO_CAST(op); self->finalizing = 1; if (_PyIOBase_finalize(op) < 0) { return; @@ -1161,7 +1161,7 @@ mode_string(fileio *self) static PyObject * fileio_repr(PyObject *op) { - fileio *self = _PyFileIO_CAST(op); + fileio *self = PyFileIO_CAST(op); const char *type_name = Py_TYPE(self)->tp_name; if (self->fd < 0) { @@ -1227,9 +1227,9 @@ _io_FileIO_isatty_impl(fileio *self) context TOCTOU issues (the fd could be arbitrarily modified by surrounding code). */ static PyObject * -_io_FileIO_isatty_open_only(PyObject *op, PyObject *Py_UNUSED(ignored)) +_io_FileIO_isatty_open_only(PyObject *op, PyObject *Py_UNUSED(dummy)) { - fileio *self = _PyFileIO_CAST(op); + fileio *self = PyFileIO_CAST(op); if (self->stat_atopen != NULL && !S_ISCHR(self->stat_atopen->st_mode)) { Py_RETURN_FALSE; } @@ -1264,28 +1264,28 @@ static PyMethodDef fileio_methods[] = { static PyObject * fileio_get_closed(PyObject *op, void *closure) { - fileio *self = _PyFileIO_CAST(op); + fileio *self = PyFileIO_CAST(op); return PyBool_FromLong((long)(self->fd < 0)); } static PyObject * fileio_get_closefd(PyObject *op, void *closure) { - fileio *self = _PyFileIO_CAST(op); + fileio *self = PyFileIO_CAST(op); return PyBool_FromLong((long)(self->closefd)); } static PyObject * fileio_get_mode(PyObject *op, void *closure) { - fileio *self = _PyFileIO_CAST(op); + fileio *self = PyFileIO_CAST(op); return PyUnicode_FromString(mode_string(self)); } static PyObject * fileio_get_blksize(PyObject *op, void *closure) { - fileio *self = _PyFileIO_CAST(op); + fileio *self = PyFileIO_CAST(op); #ifdef HAVE_STRUCT_STAT_ST_BLKSIZE if (self->stat_atopen != NULL && self->stat_atopen->st_blksize > 1) { return PyLong_FromLong(self->stat_atopen->st_blksize); diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index f87043df126895..7e0822e3350eeb 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -35,6 +35,8 @@ typedef struct { PyObject *weakreflist; } iobase; +#define iobase_CAST(op) ((iobase *)(op)) + PyDoc_STRVAR(iobase_doc, "The abstract base class for all I/O classes.\n" "\n" @@ -343,16 +345,18 @@ _PyIOBase_finalize(PyObject *self) } static int -iobase_traverse(iobase *self, visitproc visit, void *arg) +iobase_traverse(PyObject *op, visitproc visit, void *arg) { + iobase *self = iobase_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } static int -iobase_clear(iobase *self) +iobase_clear(PyObject *op) { + iobase *self = iobase_CAST(op); Py_CLEAR(self->dict); return 0; } @@ -360,14 +364,15 @@ iobase_clear(iobase *self) /* Destructor */ static void -iobase_dealloc(iobase *self) +iobase_dealloc(PyObject *op) { /* NOTE: since IOBaseObject has its own dict, Python-defined attributes are still available here for close() to use. However, if the derived class declares a __slots__, those slots are already gone. */ - if (_PyIOBase_finalize((PyObject *) self) < 0) { + iobase *self = iobase_CAST(op); + if (_PyIOBase_finalize(op) < 0) { /* When called from a heap type's dealloc, the type will be decref'ed on return (see e.g. subtype_dealloc in typeobject.c). */ if (_PyType_HasFeature(Py_TYPE(self), Py_TPFLAGS_HEAPTYPE)) { @@ -378,9 +383,9 @@ iobase_dealloc(iobase *self) PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); + PyObject_ClearWeakRefs(op); Py_CLEAR(self->dict); - tp->tp_free((PyObject *)self); + tp->tp_free(self); Py_DECREF(tp); } @@ -853,7 +858,7 @@ static PyMethodDef iobase_methods[] = { static PyGetSetDef iobase_getset[] = { {"__dict__", PyObject_GenericGetDict, NULL, NULL}, - {"closed", (getter)iobase_closed_get, NULL, NULL}, + {"closed", iobase_closed_get, NULL, NULL}, {NULL} }; diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index d1a71298a9087f..9d1bfa3ea05cea 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -45,6 +45,8 @@ typedef struct { _PyIO_State *module_state; } stringio; +#define stringio_CAST(op) ((stringio *)(op)) + #define clinic_state() (find_io_state_by_def(Py_TYPE(self))) #include "clinic/stringio.c.h" #undef clinic_state @@ -402,9 +404,10 @@ _io_StringIO_readline_impl(stringio *self, Py_ssize_t size) } static PyObject * -stringio_iternext(stringio *self) +stringio_iternext(PyObject *op) { PyObject *line; + stringio *self = stringio_CAST(op); CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -416,8 +419,7 @@ stringio_iternext(stringio *self) } else { /* XXX is subclassing StringIO really supported? */ - line = PyObject_CallMethodNoArgs((PyObject *)self, - &_Py_ID(readline)); + line = PyObject_CallMethodNoArgs(op, &_Py_ID(readline)); if (line && !PyUnicode_Check(line)) { PyErr_Format(PyExc_OSError, "readline() should have returned a str object, " @@ -591,8 +593,9 @@ _io_StringIO_close_impl(stringio *self) } static int -stringio_traverse(stringio *self, visitproc visit, void *arg) +stringio_traverse(PyObject *op, visitproc visit, void *arg) { + stringio *self = stringio_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->readnl); Py_VISIT(self->writenl); @@ -602,8 +605,9 @@ stringio_traverse(stringio *self, visitproc visit, void *arg) } static int -stringio_clear(stringio *self) +stringio_clear(PyObject *op) { + stringio *self = stringio_CAST(op); Py_CLEAR(self->readnl); Py_CLEAR(self->writenl); Py_CLEAR(self->decoder); @@ -612,8 +616,9 @@ stringio_clear(stringio *self) } static void -stringio_dealloc(stringio *self) +stringio_dealloc(PyObject *op) { + stringio *self = stringio_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); self->ok = 0; @@ -622,9 +627,9 @@ stringio_dealloc(stringio *self) self->buf = NULL; } PyUnicodeWriter_Discard(self->writer); - (void)stringio_clear(self); + (void)stringio_clear(op); if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) self); + PyObject_ClearWeakRefs(op); } tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 791ee070401fe5..935aaab20a031f 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -223,6 +223,8 @@ struct nldecoder_object { unsigned int seennl: 3; }; +#define nldecoder_object_CAST(op) ((nldecoder_object *)(op)) + /*[clinic input] _io.IncrementalNewlineDecoder.__init__ decoder: object @@ -263,9 +265,9 @@ _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, } static int -incrementalnewlinedecoder_traverse(nldecoder_object *self, visitproc visit, - void *arg) +incrementalnewlinedecoder_traverse(PyObject *op, visitproc visit, void *arg) { + nldecoder_object *self = nldecoder_object_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->decoder); Py_VISIT(self->errors); @@ -273,20 +275,22 @@ incrementalnewlinedecoder_traverse(nldecoder_object *self, visitproc visit, } static int -incrementalnewlinedecoder_clear(nldecoder_object *self) +incrementalnewlinedecoder_clear(PyObject *op) { + nldecoder_object *self = nldecoder_object_CAST(op); Py_CLEAR(self->decoder); Py_CLEAR(self->errors); return 0; } static void -incrementalnewlinedecoder_dealloc(nldecoder_object *self) +incrementalnewlinedecoder_dealloc(PyObject *op) { + nldecoder_object *self = nldecoder_object_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - (void)incrementalnewlinedecoder_clear(self); - tp->tp_free((PyObject *)self); + (void)incrementalnewlinedecoder_clear(op); + tp->tp_free(self); Py_DECREF(tp); } @@ -323,7 +327,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself, { PyObject *output; Py_ssize_t output_len; - nldecoder_object *self = (nldecoder_object *) myself; + nldecoder_object *self = nldecoder_object_CAST(myself); CHECK_INITIALIZED_DECODER(self); @@ -625,8 +629,9 @@ _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self) } static PyObject * -incrementalnewlinedecoder_newlines_get(nldecoder_object *self, void *context) +incrementalnewlinedecoder_newlines_get(PyObject *op, void *Py_UNUSED(context)) { + nldecoder_object *self = nldecoder_object_CAST(op); CHECK_INITIALIZED_DECODER(self); switch (self->seennl) { @@ -652,8 +657,7 @@ incrementalnewlinedecoder_newlines_get(nldecoder_object *self, void *context) /* TextIOWrapper */ -typedef PyObject * - (*encodefunc_t)(PyObject *, PyObject *); +typedef PyObject *(*encodefunc_t)(PyObject *, PyObject *); struct textio { @@ -716,6 +720,8 @@ struct textio _PyIO_State *state; }; +#define textio_CAST(op) ((textio *)(op)) + static void textiowrapper_set_decoded_chars(textio *self, PyObject *chars); @@ -723,78 +729,81 @@ textiowrapper_set_decoded_chars(textio *self, PyObject *chars); encoding methods for the most popular encodings. */ static PyObject * -ascii_encode(textio *self, PyObject *text) +ascii_encode(PyObject *op, PyObject *text) { + textio *self = textio_CAST(op); return _PyUnicode_AsASCIIString(text, PyUnicode_AsUTF8(self->errors)); } static PyObject * -utf16be_encode(textio *self, PyObject *text) +utf16be_encode(PyObject *op, PyObject *text) { - return _PyUnicode_EncodeUTF16(text, - PyUnicode_AsUTF8(self->errors), 1); + textio *self = textio_CAST(op); + return _PyUnicode_EncodeUTF16(text, PyUnicode_AsUTF8(self->errors), 1); } static PyObject * -utf16le_encode(textio *self, PyObject *text) +utf16le_encode(PyObject *op, PyObject *text) { - return _PyUnicode_EncodeUTF16(text, - PyUnicode_AsUTF8(self->errors), -1); + textio *self = textio_CAST(op); + return _PyUnicode_EncodeUTF16(text, PyUnicode_AsUTF8(self->errors), -1); } static PyObject * -utf16_encode(textio *self, PyObject *text) +utf16_encode(PyObject *op, PyObject *text) { + textio *self = textio_CAST(op); if (!self->encoding_start_of_stream) { /* Skip the BOM and use native byte ordering */ #if PY_BIG_ENDIAN - return utf16be_encode(self, text); + return utf16be_encode(op, text); #else - return utf16le_encode(self, text); + return utf16le_encode(op, text); #endif } - return _PyUnicode_EncodeUTF16(text, - PyUnicode_AsUTF8(self->errors), 0); + return _PyUnicode_EncodeUTF16(text, PyUnicode_AsUTF8(self->errors), 0); } static PyObject * -utf32be_encode(textio *self, PyObject *text) +utf32be_encode(PyObject *op, PyObject *text) { - return _PyUnicode_EncodeUTF32(text, - PyUnicode_AsUTF8(self->errors), 1); + textio *self = textio_CAST(op); + return _PyUnicode_EncodeUTF32(text, PyUnicode_AsUTF8(self->errors), 1); } static PyObject * -utf32le_encode(textio *self, PyObject *text) +utf32le_encode(PyObject *op, PyObject *text) { - return _PyUnicode_EncodeUTF32(text, - PyUnicode_AsUTF8(self->errors), -1); + textio *self = textio_CAST(op); + return _PyUnicode_EncodeUTF32(text, PyUnicode_AsUTF8(self->errors), -1); } static PyObject * -utf32_encode(textio *self, PyObject *text) +utf32_encode(PyObject *op, PyObject *text) { + textio *self = textio_CAST(op); if (!self->encoding_start_of_stream) { /* Skip the BOM and use native byte ordering */ #if PY_BIG_ENDIAN - return utf32be_encode(self, text); + return utf32be_encode(op, text); #else - return utf32le_encode(self, text); + return utf32le_encode(op, text); #endif } - return _PyUnicode_EncodeUTF32(text, - PyUnicode_AsUTF8(self->errors), 0); + return _PyUnicode_EncodeUTF32(text, PyUnicode_AsUTF8(self->errors), 0); } static PyObject * -utf8_encode(textio *self, PyObject *text) +utf8_encode(PyObject *op, PyObject *text) { + textio *self = textio_CAST(op); return _PyUnicode_AsUTF8String(text, PyUnicode_AsUTF8(self->errors)); } static PyObject * -latin1_encode(textio *self, PyObject *text) +latin1_encode(PyObject *op, PyObject *text) { + textio *self = textio_CAST(op); return _PyUnicode_AsLatin1String(text, PyUnicode_AsUTF8(self->errors)); } @@ -802,9 +811,7 @@ latin1_encode(textio *self, PyObject *text) static inline int is_asciicompat_encoding(encodefunc_t f) { - return f == (encodefunc_t) ascii_encode - || f == (encodefunc_t) latin1_encode - || f == (encodefunc_t) utf8_encode; + return f == ascii_encode || f == latin1_encode || f == utf8_encode; } /* Map normalized encoding names onto the specialized encoding funcs */ @@ -815,15 +822,15 @@ typedef struct { } encodefuncentry; static const encodefuncentry encodefuncs[] = { - {"ascii", (encodefunc_t) ascii_encode}, - {"iso8859-1", (encodefunc_t) latin1_encode}, - {"utf-8", (encodefunc_t) utf8_encode}, - {"utf-16-be", (encodefunc_t) utf16be_encode}, - {"utf-16-le", (encodefunc_t) utf16le_encode}, - {"utf-16", (encodefunc_t) utf16_encode}, - {"utf-32-be", (encodefunc_t) utf32be_encode}, - {"utf-32-le", (encodefunc_t) utf32le_encode}, - {"utf-32", (encodefunc_t) utf32_encode}, + {"ascii", ascii_encode}, + {"iso8859-1", latin1_encode}, + {"utf-8", utf8_encode}, + {"utf-16-be", utf16be_encode}, + {"utf-16-le", utf16le_encode}, + {"utf-16", utf16_encode}, + {"utf-32-be", utf32be_encode}, + {"utf-32-le", utf32le_encode}, + {"utf-32", utf32_encode}, {NULL, NULL} }; @@ -1433,8 +1440,9 @@ _io_TextIOWrapper_reconfigure_impl(textio *self, PyObject *encoding, } static int -textiowrapper_clear(textio *self) +textiowrapper_clear(PyObject *op) { + textio *self = textio_CAST(op); self->ok = 0; Py_CLEAR(self->buffer); Py_CLEAR(self->encoding); @@ -1452,24 +1460,26 @@ textiowrapper_clear(textio *self) } static void -textiowrapper_dealloc(textio *self) +textiowrapper_dealloc(PyObject *op) { + textio *self = textio_CAST(op); PyTypeObject *tp = Py_TYPE(self); self->finalizing = 1; - if (_PyIOBase_finalize((PyObject *) self) < 0) + if (_PyIOBase_finalize(op) < 0) return; self->ok = 0; _PyObject_GC_UNTRACK(self); if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)self); - (void)textiowrapper_clear(self); - tp->tp_free((PyObject *)self); + PyObject_ClearWeakRefs(op); + (void)textiowrapper_clear(op); + tp->tp_free(self); Py_DECREF(tp); } static int -textiowrapper_traverse(textio *self, visitproc visit, void *arg) +textiowrapper_traverse(PyObject *op, visitproc visit, void *arg) { + textio *self = textio_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->buffer); Py_VISIT(self->encoding); @@ -2963,10 +2973,11 @@ _io_TextIOWrapper_truncate_impl(textio *self, PyObject *pos) } static PyObject * -textiowrapper_repr(textio *self) +textiowrapper_repr(PyObject *op) { PyObject *nameobj, *modeobj, *res, *s; int status; + textio *self = textio_CAST(op); const char *type_name = Py_TYPE(self)->tp_name; CHECK_INITIALIZED(self); @@ -2975,7 +2986,7 @@ textiowrapper_repr(textio *self) if (res == NULL) return NULL; - status = Py_ReprEnter((PyObject *)self); + status = Py_ReprEnter(op); if (status != 0) { if (status > 0) { PyErr_Format(PyExc_RuntimeError, @@ -2984,7 +2995,7 @@ textiowrapper_repr(textio *self) } goto error; } - if (PyObject_GetOptionalAttr((PyObject *) self, &_Py_ID(name), &nameobj) < 0) { + if (PyObject_GetOptionalAttr(op, &_Py_ID(name), &nameobj) < 0) { if (!PyErr_ExceptionMatches(PyExc_ValueError)) { goto error; } @@ -3000,7 +3011,7 @@ textiowrapper_repr(textio *self) if (res == NULL) goto error; } - if (PyObject_GetOptionalAttr((PyObject *) self, &_Py_ID(mode), &modeobj) < 0) { + if (PyObject_GetOptionalAttr(op, &_Py_ID(mode), &modeobj) < 0) { goto error; } if (modeobj != NULL) { @@ -3016,14 +3027,14 @@ textiowrapper_repr(textio *self) res, self->encoding); Py_DECREF(res); if (status == 0) { - Py_ReprLeave((PyObject *)self); + Py_ReprLeave(op); } return s; error: Py_XDECREF(res); if (status == 0) { - Py_ReprLeave((PyObject *)self); + Py_ReprLeave(op); } return NULL; } @@ -3163,9 +3174,10 @@ _io_TextIOWrapper_close_impl(textio *self) } static PyObject * -textiowrapper_iternext(textio *self) +textiowrapper_iternext(PyObject *op) { PyObject *line; + textio *self = textio_CAST(op); CHECK_ATTACHED(self); @@ -3175,8 +3187,7 @@ textiowrapper_iternext(textio *self) line = _textiowrapper_readline(self, -1); } else { - line = PyObject_CallMethodNoArgs((PyObject *)self, - &_Py_ID(readline)); + line = PyObject_CallMethodNoArgs(op, &_Py_ID(readline)); if (line && !PyUnicode_Check(line)) { PyErr_Format(PyExc_OSError, "readline() should have returned a str object, " @@ -3313,7 +3324,7 @@ static PyMethodDef incrementalnewlinedecoder_methods[] = { }; static PyGetSetDef incrementalnewlinedecoder_getset[] = { - {"newlines", (getter)incrementalnewlinedecoder_newlines_get, NULL, NULL}, + {"newlines", incrementalnewlinedecoder_newlines_get, NULL, NULL}, {NULL} }; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 3fa0301e337991..27c320ed073103 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -221,6 +221,8 @@ typedef struct { wchar_t wbuf; } winconsoleio; +#define winconsoleio_CAST(op) ((winconsoleio *)(op)) + int _PyWindowsConsoleIO_closed(PyObject *self) { @@ -492,32 +494,35 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, } static int -winconsoleio_traverse(winconsoleio *self, visitproc visit, void *arg) +winconsoleio_traverse(PyObject *op, visitproc visit, void *arg) { + winconsoleio *self = winconsoleio_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->dict); return 0; } static int -winconsoleio_clear(winconsoleio *self) +winconsoleio_clear(PyObject *op) { + winconsoleio *self = winconsoleio_CAST(op); Py_CLEAR(self->dict); return 0; } static void -winconsoleio_dealloc(winconsoleio *self) +winconsoleio_dealloc(PyObject *op) { + winconsoleio *self = winconsoleio_CAST(op); PyTypeObject *tp = Py_TYPE(self); self->finalizing = 1; - if (_PyIOBase_finalize((PyObject *) self) < 0) + if (_PyIOBase_finalize(op) < 0) return; _PyObject_GC_UNTRACK(self); if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); + PyObject_ClearWeakRefs(op); Py_CLEAR(self->dict); - tp->tp_free((PyObject *)self); + tp->tp_free(self); Py_DECREF(tp); } @@ -1137,9 +1142,10 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, } static PyObject * -winconsoleio_repr(winconsoleio *self) +winconsoleio_repr(PyObject *op) { - const char *type_name = (Py_TYPE((PyObject *)self)->tp_name); + winconsoleio *self = winconsoleio_CAST(op); + const char *type_name = Py_TYPE(self)->tp_name; if (self->fd == -1) { return PyUnicode_FromFormat("<%.100s [closed]>", type_name); @@ -1197,28 +1203,31 @@ static PyMethodDef winconsoleio_methods[] = { /* 'closed' and 'mode' are attributes for compatibility with FileIO. */ static PyObject * -get_closed(winconsoleio *self, void *closure) +get_closed(PyObject *op, void *Py_UNUSED(closure)) { + winconsoleio *self = winconsoleio_CAST(op); return PyBool_FromLong((long)(self->fd == -1)); } static PyObject * -get_closefd(winconsoleio *self, void *closure) +get_closefd(PyObject *op, void *Py_UNUSED(closure)) { + winconsoleio *self = winconsoleio_CAST(op); return PyBool_FromLong((long)(self->closefd)); } static PyObject * -get_mode(winconsoleio *self, void *closure) +get_mode(PyObject *op, void *Py_UNUSED(closure)) { + winconsoleio *self = winconsoleio_CAST(op); return PyUnicode_FromString(self->readable ? "rb" : "wb"); } static PyGetSetDef winconsoleio_getsetlist[] = { - {"closed", (getter)get_closed, NULL, "True if the file is closed"}, - {"closefd", (getter)get_closefd, NULL, + {"closed", get_closed, NULL, "True if the file is closed"}, + {"closefd", get_closefd, NULL, "True if the file descriptor will be closed by close()."}, - {"mode", (getter)get_mode, NULL, "String giving the file mode"}, + {"mode", get_mode, NULL, "String giving the file mode"}, {NULL}, }; From 624b93ed67332bbaf758bf963351052acc6ccf75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 8 Feb 2025 14:26:14 +0100 Subject: [PATCH 004/305] gh-111178: fix UBSan failures in `Modules/arraymodule.c` (GH-129772) * fix UBSan failures for `arrayobject`, `arrayiterobject` * suppress unused return values --- Modules/arraymodule.c | 99 ++++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 40 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index dc1729a7a3a558..5b86ec98393e48 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -79,6 +79,9 @@ get_array_state(PyObject *module) #define get_array_state_by_class(cls) \ (get_array_state(PyType_GetModule(cls))) +#define arrayobject_CAST(op) ((arrayobject *)(op)) +#define arrayiterobject_CAST(op) ((arrayiterobject *)(op)) + enum machine_format_code { UNKNOWN_FORMAT = -1, /* UNKNOWN_FORMAT is used to indicate that the machine format for an @@ -712,22 +715,25 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) /* Methods */ static int -array_tp_traverse(arrayobject *op, visitproc visit, void *arg) +array_tp_traverse(PyObject *op, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(op)); return 0; } static void -array_dealloc(arrayobject *op) +array_dealloc(PyObject *op) { PyTypeObject *tp = Py_TYPE(op); PyObject_GC_UnTrack(op); - if (op->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) op); - if (op->ob_item != NULL) - PyMem_Free(op->ob_item); + arrayobject *self = arrayobject_CAST(op); + if (self->weakreflist != NULL) { + PyObject_ClearWeakRefs(op); + } + if (self->ob_item != NULL) { + PyMem_Free(self->ob_item); + } tp->tp_free(op); Py_DECREF(tp); } @@ -843,19 +849,19 @@ array_richcompare(PyObject *v, PyObject *w, int op) } static Py_ssize_t -array_length(arrayobject *a) +array_length(PyObject *op) { - return Py_SIZE(a); + return Py_SIZE(op); } static PyObject * -array_item(arrayobject *a, Py_ssize_t i) +array_item(PyObject *op, Py_ssize_t i) { - if (i < 0 || i >= Py_SIZE(a)) { + if (i < 0 || i >= Py_SIZE(op)) { PyErr_SetString(PyExc_IndexError, "array index out of range"); return NULL; } - return getarrayitem((PyObject *)a, i); + return getarrayitem(op, i); } static PyObject * @@ -930,8 +936,9 @@ array_array___deepcopy__(arrayobject *self, PyObject *unused) } static PyObject * -array_concat(arrayobject *a, PyObject *bb) +array_concat(PyObject *op, PyObject *bb) { + arrayobject *a = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(a)); Py_ssize_t size; arrayobject *np; @@ -966,8 +973,9 @@ array_concat(arrayobject *a, PyObject *bb) } static PyObject * -array_repeat(arrayobject *a, Py_ssize_t n) +array_repeat(PyObject *op, Py_ssize_t n) { + arrayobject *a = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(a)); if (n < 0) @@ -1026,8 +1034,9 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) } static int -array_ass_item(arrayobject *a, Py_ssize_t i, PyObject *v) +array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) { + arrayobject *a = arrayobject_CAST(op); if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, "array assignment index out of range"); @@ -1045,7 +1054,7 @@ setarrayitem(PyObject *a, Py_ssize_t i, PyObject *v) array_state *state = find_array_state_by_type(Py_TYPE(a)); assert(array_Check(a, state)); #endif - return array_ass_item((arrayobject *)a, i, v); + return array_ass_item(a, i, v); } static int @@ -1105,8 +1114,9 @@ array_do_extend(array_state *state, arrayobject *self, PyObject *bb) } static PyObject * -array_inplace_concat(arrayobject *self, PyObject *bb) +array_inplace_concat(PyObject *op, PyObject *bb) { + arrayobject *self = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(self)); if (!array_Check(bb, state)) { @@ -1121,8 +1131,9 @@ array_inplace_concat(arrayobject *self, PyObject *bb) } static PyObject * -array_inplace_repeat(arrayobject *self, Py_ssize_t n) +array_inplace_repeat(PyObject *op, Py_ssize_t n) { + arrayobject *self = arrayobject_CAST(op); const Py_ssize_t array_size = Py_SIZE(self); if (array_size > 0 && n != 1 ) { @@ -1236,13 +1247,13 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, } static int -array_contains(arrayobject *self, PyObject *v) +array_contains(PyObject *self, PyObject *v) { Py_ssize_t i; int cmp; for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(self); i++) { - PyObject *selfi = getarrayitem((PyObject *)self, i); + PyObject *selfi = getarrayitem(self, i); if (selfi == NULL) return -1; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -2349,22 +2360,24 @@ array_array___reduce_ex___impl(arrayobject *self, PyTypeObject *cls, } static PyObject * -array_get_typecode(arrayobject *a, void *closure) +array_get_typecode(PyObject *op, void *Py_UNUSED(closure)) { + arrayobject *a = arrayobject_CAST(op); char typecode = a->ob_descr->typecode; return PyUnicode_FromOrdinal(typecode); } static PyObject * -array_get_itemsize(arrayobject *a, void *closure) +array_get_itemsize(PyObject *op, void *Py_UNUSED(closure)) { + arrayobject *a = arrayobject_CAST(op); return PyLong_FromLong((long)a->ob_descr->itemsize); } static PyGetSetDef array_getsets [] = { - {"typecode", (getter) array_get_typecode, NULL, + {"typecode", array_get_typecode, NULL, "the typecode character used to create the array"}, - {"itemsize", (getter) array_get_itemsize, NULL, + {"itemsize", array_get_itemsize, NULL, "the size, in bytes, of one array item"}, {NULL} }; @@ -2398,11 +2411,12 @@ static PyMethodDef array_methods[] = { }; static PyObject * -array_repr(arrayobject *a) +array_repr(PyObject *op) { char typecode; PyObject *s, *v = NULL; Py_ssize_t len; + arrayobject *a = arrayobject_CAST(op); len = Py_SIZE(a); typecode = a->ob_descr->typecode; @@ -2425,8 +2439,9 @@ array_repr(arrayobject *a) } static PyObject* -array_subscr(arrayobject* self, PyObject* item) +array_subscr(PyObject *op, PyObject *item) { + arrayobject *self = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(self)); if (PyIndex_Check(item)) { @@ -2436,7 +2451,7 @@ array_subscr(arrayobject* self, PyObject* item) } if (i < 0) i += Py_SIZE(self); - return array_item(self, i); + return array_item(op, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, i; @@ -2488,9 +2503,10 @@ array_subscr(arrayobject* self, PyObject* item) } static int -array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) +array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) { Py_ssize_t start, stop, step, slicelength, needed; + arrayobject *self = arrayobject_CAST(op); array_state* state = find_array_state_by_type(Py_TYPE(self)); arrayobject* other; int itemsize; @@ -2542,7 +2558,7 @@ array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) value = array_slice(other, 0, needed); if (value == NULL) return -1; - ret = array_ass_subscr(self, item, value); + ret = array_ass_subscr(op, item, value); Py_DECREF(value); return ret; } @@ -2649,7 +2665,7 @@ static const void *emptybuf = ""; static int -array_buffer_getbuf(arrayobject *self, Py_buffer *view, int flags) +array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) { if (view == NULL) { PyErr_SetString(PyExc_BufferError, @@ -2657,6 +2673,7 @@ array_buffer_getbuf(arrayobject *self, Py_buffer *view, int flags) return -1; } + arrayobject *self = arrayobject_CAST(op); view->buf = (void *)self->ob_item; view->obj = Py_NewRef(self); if (view->buf == NULL) @@ -2689,8 +2706,9 @@ array_buffer_getbuf(arrayobject *self, Py_buffer *view, int flags) } static void -array_buffer_relbuf(arrayobject *self, Py_buffer *view) +array_buffer_relbuf(PyObject *op, Py_buffer *Py_UNUSED(view)) { + arrayobject *self = arrayobject_CAST(op); self->ob_exports--; } @@ -2925,7 +2943,7 @@ typecode -- the typecode character used to create the array\n\ itemsize -- the length in bytes of one array item\n\ "); -static PyObject *array_iter(arrayobject *ao); +static PyObject *array_iter(PyObject *op); static struct PyMemberDef array_members[] = { {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(arrayobject, weakreflist), Py_READONLY}, @@ -2985,8 +3003,9 @@ class array.arrayiterator "arrayiterobject *" "find_array_state_by_type(type)->A /*[clinic end generated code: output=da39a3ee5e6b4b0d input=fb46d5ef98dd95ff]*/ static PyObject * -array_iter(arrayobject *ao) +array_iter(PyObject *op) { + arrayobject *ao = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(ao)); arrayiterobject *it; @@ -3007,16 +3026,15 @@ array_iter(arrayobject *ao) } static PyObject * -arrayiter_next(arrayiterobject *it) +arrayiter_next(PyObject *op) { - arrayobject *ao; - + arrayiterobject *it = arrayiterobject_CAST(op); assert(it != NULL); #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(it)); assert(PyObject_TypeCheck(it, state->ArrayIterType)); #endif - ao = it->ao; + arrayobject *ao = it->ao; if (ao == NULL) { return NULL; } @@ -3032,10 +3050,10 @@ arrayiter_next(arrayiterobject *it) } static void -arrayiter_dealloc(arrayiterobject *it) +arrayiter_dealloc(PyObject *op) { + arrayiterobject *it = arrayiterobject_CAST(op); PyTypeObject *tp = Py_TYPE(it); - PyObject_GC_UnTrack(it); Py_XDECREF(it->ao); PyObject_GC_Del(it); @@ -3043,8 +3061,9 @@ arrayiter_dealloc(arrayiterobject *it) } static int -arrayiter_traverse(arrayiterobject *it, visitproc visit, void *arg) +arrayiter_traverse(PyObject *op, visitproc visit, void *arg) { + arrayiterobject *it = arrayiterobject_CAST(op); Py_VISIT(Py_TYPE(it)); Py_VISIT(it->ao); return 0; @@ -3156,7 +3175,7 @@ array_clear(PyObject *module) static void array_free(void *module) { - array_clear((PyObject *)module); + (void)array_clear((PyObject *)module); } /* No functions in array module. */ From 12e1d3011b0ff427b7be36d5f5d2d63014c6a54d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 8 Feb 2025 14:47:19 +0100 Subject: [PATCH 005/305] gh-111178: fix UBSan failures in `Objects/floatobject.c` (GH-129776) fix UBSan failures for `PyFloatObject` --- Objects/floatobject.c | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 7ca43033d722ab..3b72a1e7c37b7f 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -369,8 +369,9 @@ _Py_convert_int_to_double(PyObject **v, double *dbl) } static PyObject * -float_repr(PyFloatObject *v) +float_repr(PyObject *op) { + PyFloatObject *v = _PyFloat_CAST(op); PyObject *result; char *buf; @@ -579,9 +580,10 @@ float_richcompare(PyObject *v, PyObject *w, int op) } static Py_hash_t -float_hash(PyFloatObject *v) +float_hash(PyObject *op) { - return _Py_HashDouble((PyObject *)v, v->ob_fval); + PyFloatObject *v = _PyFloat_CAST(op); + return _Py_HashDouble(op, v->ob_fval); } static PyObject * @@ -851,20 +853,23 @@ float_pow(PyObject *v, PyObject *w, PyObject *z) #undef DOUBLE_IS_ODD_INTEGER static PyObject * -float_neg(PyFloatObject *v) +float_neg(PyObject *op) { + PyFloatObject *v = _PyFloat_CAST(op); return PyFloat_FromDouble(-v->ob_fval); } static PyObject * -float_abs(PyFloatObject *v) +float_abs(PyObject *op) { + PyFloatObject *v = _PyFloat_CAST(op); return PyFloat_FromDouble(fabs(v->ob_fval)); } static int -float_bool(PyFloatObject *v) +float_bool(PyObject *op) { + PyFloatObject *v = _PyFloat_CAST(op); return v->ob_fval != 0.0; } @@ -1206,7 +1211,7 @@ float_hex_impl(PyObject *self) CONVERT_TO_DOUBLE(self, x); if (isnan(x) || isinf(x)) - return float_repr((PyFloatObject *)self); + return float_repr(self); if (x == 0.0) { if (copysign(1.0, x) == -1.0) @@ -1651,7 +1656,7 @@ float_subtype_new(PyTypeObject *type, PyObject *x) } static PyObject * -float_vectorcall(PyObject *type, PyObject * const*args, +float_vectorcall(PyObject *type, PyObject *const *args, size_t nargsf, PyObject *kwnames) { if (!_PyArg_NoKwnames("float", kwnames)) { @@ -1771,13 +1776,13 @@ float___getformat___impl(PyTypeObject *type, const char *typestr) static PyObject * -float_getreal(PyObject *v, void *closure) +float_getreal(PyObject *v, void *Py_UNUSED(closure)) { return float_float(v); } static PyObject * -float_getimag(PyObject *v, void *closure) +float_getimag(PyObject *Py_UNUSED(v), void *Py_UNUSED(closure)) { return PyFloat_FromDouble(0.0); } @@ -1829,11 +1834,11 @@ static PyMethodDef float_methods[] = { static PyGetSetDef float_getset[] = { {"real", - float_getreal, (setter)NULL, + float_getreal, NULL, "the real part of a complex number", NULL}, {"imag", - float_getimag, (setter)NULL, + float_getimag, NULL, "the imaginary part of a complex number", NULL}, {NULL} /* Sentinel */ @@ -1847,10 +1852,10 @@ static PyNumberMethods float_as_number = { float_rem, /* nb_remainder */ float_divmod, /* nb_divmod */ float_pow, /* nb_power */ - (unaryfunc)float_neg, /* nb_negative */ + float_neg, /* nb_negative */ float_float, /* nb_positive */ - (unaryfunc)float_abs, /* nb_absolute */ - (inquiry)float_bool, /* nb_bool */ + float_abs, /* nb_absolute */ + float_bool, /* nb_bool */ 0, /* nb_invert */ 0, /* nb_lshift */ 0, /* nb_rshift */ @@ -1881,16 +1886,16 @@ PyTypeObject PyFloat_Type = { "float", sizeof(PyFloatObject), 0, - (destructor)float_dealloc, /* tp_dealloc */ + float_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)float_repr, /* tp_repr */ + float_repr, /* tp_repr */ &float_as_number, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ - (hashfunc)float_hash, /* tp_hash */ + float_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -1916,7 +1921,7 @@ PyTypeObject PyFloat_Type = { 0, /* tp_init */ 0, /* tp_alloc */ float_new, /* tp_new */ - .tp_vectorcall = (vectorcallfunc)float_vectorcall, + .tp_vectorcall = float_vectorcall, .tp_version_tag = _Py_TYPE_VERSION_FLOAT, }; From d046421f4edacbf64f9436750a727cb0a4bfbd04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 8 Feb 2025 14:54:34 +0100 Subject: [PATCH 006/305] gh-111178: fix UBSan failures in `Objects/frameobject.c` (GH-129777) fix UBSan failures for `PyFrameObject`, `PyFrameLocalsProxyObject` --- Objects/frameobject.c | 214 +++++++++++++++++++++++------------------- 1 file changed, 118 insertions(+), 96 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 15ec4b78235992..8ebcc1a4b5e892 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -16,6 +16,14 @@ #include "pycore_frame.h" #include "opcode.h" // EXTENDED_ARG +#define PyFrameObject_CAST(op) \ + (assert(PyObject_TypeCheck((op), &PyFrame_Type)), (PyFrameObject *)(op)) + +#define PyFrameLocalsProxyObject_CAST(op) \ + ( \ + assert(PyObject_TypeCheck((op), &PyFrameLocalsProxy_Type)), \ + (PyFrameLocalsProxyObject *)(op) \ + ) #define OFF(x) offsetof(PyFrameObject, x) @@ -126,8 +134,8 @@ framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) static PyObject * framelocalsproxy_getitem(PyObject *self, PyObject *key) { - PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; - PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); + PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); int i = framelocalsproxy_getkeyindex(frame, key, true); if (i == -2) { @@ -157,7 +165,7 @@ static int framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) { /* Merge locals into fast locals */ - PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; _PyStackRef *fast = _PyFrame_GetLocalsArray(frame->f_frame); PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); @@ -272,9 +280,9 @@ framelocalsproxy_merge(PyObject* self, PyObject* other) } static PyObject * -framelocalsproxy_keys(PyObject *self, void *Py_UNUSED(ignored)) +framelocalsproxy_keys(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); PyObject *names = PyList_New(0); if (names == NULL) { @@ -314,8 +322,9 @@ framelocalsproxy_keys(PyObject *self, void *Py_UNUSED(ignored)) static void framelocalsproxy_dealloc(PyObject *self) { + PyFrameLocalsProxyObject *proxy = PyFrameLocalsProxyObject_CAST(self); PyObject_GC_UnTrack(self); - Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + Py_CLEAR(proxy->frame); Py_TYPE(self)->tp_free(self); } @@ -355,14 +364,16 @@ framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static int framelocalsproxy_tp_clear(PyObject *self) { - Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + PyFrameLocalsProxyObject *proxy = PyFrameLocalsProxyObject_CAST(self); + Py_CLEAR(proxy->frame); return 0; } static int framelocalsproxy_visit(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(((PyFrameLocalsProxyObject*)self)->frame); + PyFrameLocalsProxyObject *proxy = PyFrameLocalsProxyObject_CAST(self); + Py_VISIT(proxy->frame); return 0; } @@ -381,27 +392,29 @@ framelocalsproxy_iter(PyObject *self) } static PyObject * -framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) +framelocalsproxy_richcompare(PyObject *lhs, PyObject *rhs, int op) { - if (PyFrameLocalsProxy_Check(other)) { - bool result = ((PyFrameLocalsProxyObject*)self)->frame == ((PyFrameLocalsProxyObject*)other)->frame; + PyFrameLocalsProxyObject *self = PyFrameLocalsProxyObject_CAST(lhs); + if (PyFrameLocalsProxy_Check(rhs)) { + PyFrameLocalsProxyObject *other = (PyFrameLocalsProxyObject *)rhs; + bool result = self->frame == other->frame; if (op == Py_EQ) { return PyBool_FromLong(result); } else if (op == Py_NE) { return PyBool_FromLong(!result); } - } else if (PyDict_Check(other)) { + } else if (PyDict_Check(rhs)) { PyObject *dct = PyDict_New(); if (dct == NULL) { return NULL; } - if (PyDict_Update(dct, self) < 0) { + if (PyDict_Update(dct, lhs) < 0) { Py_DECREF(dct); return NULL; } - PyObject *result = PyObject_RichCompare(dct, other, op); + PyObject *result = PyObject_RichCompare(dct, rhs, op); Py_DECREF(dct); return result; } @@ -476,10 +489,10 @@ framelocalsproxy_inplace_or(PyObject *self, PyObject *other) return Py_NewRef(self); } -static PyObject* -framelocalsproxy_values(PyObject *self, void *Py_UNUSED(ignored)) +static PyObject * +framelocalsproxy_values(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); PyObject *values = PyList_New(0); if (values == NULL) { @@ -513,9 +526,9 @@ framelocalsproxy_values(PyObject *self, void *Py_UNUSED(ignored)) } static PyObject * -framelocalsproxy_items(PyObject *self, void *Py_UNUSED(ignored)) +framelocalsproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); PyObject *items = PyList_New(0); if (items == NULL) { @@ -571,7 +584,7 @@ framelocalsproxy_items(PyObject *self, void *Py_UNUSED(ignored)) static Py_ssize_t framelocalsproxy_length(PyObject *self) { - PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); Py_ssize_t size = 0; @@ -591,7 +604,7 @@ framelocalsproxy_length(PyObject *self) static int framelocalsproxy_contains(PyObject *self, PyObject *key) { - PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; int i = framelocalsproxy_getkeyindex(frame, key, true); if (i == -2) { @@ -601,7 +614,7 @@ framelocalsproxy_contains(PyObject *self, PyObject *key) return 1; } - PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; + PyObject *extra = frame->f_extra_locals; if (extra != NULL) { return PyDict_Contains(extra, key); } @@ -702,7 +715,7 @@ framelocalsproxy_pop(PyObject* self, PyObject *const *args, Py_ssize_t nargs) default_value = args[1]; } - PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; int i = framelocalsproxy_getkeyindex(frame, key, false); if (i == -2) { @@ -759,7 +772,7 @@ framelocalsproxy_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) } static PyObject* -framelocalsproxy_reversed(PyObject *self, void *Py_UNUSED(ignored)) +framelocalsproxy_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *result = framelocalsproxy_keys(self, NULL); @@ -784,42 +797,36 @@ static PySequenceMethods framelocalsproxy_as_sequence = { }; static PyMappingMethods framelocalsproxy_as_mapping = { - framelocalsproxy_length, // mp_length - framelocalsproxy_getitem, // mp_subscript - framelocalsproxy_setitem, // mp_ass_subscript + .mp_length = framelocalsproxy_length, + .mp_subscript = framelocalsproxy_getitem, + .mp_ass_subscript = framelocalsproxy_setitem, }; static PyMethodDef framelocalsproxy_methods[] = { - {"__contains__", framelocalsproxy___contains__, METH_O | METH_COEXIST, - NULL}, - {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, - NULL}, - {"update", framelocalsproxy_update, METH_O, - NULL}, - {"__reversed__", _PyCFunction_CAST(framelocalsproxy_reversed), METH_NOARGS, - NULL}, - {"copy", _PyCFunction_CAST(framelocalsproxy_copy), METH_NOARGS, - NULL}, - {"keys", _PyCFunction_CAST(framelocalsproxy_keys), METH_NOARGS, - NULL}, - {"values", _PyCFunction_CAST(framelocalsproxy_values), METH_NOARGS, - NULL}, - {"items", _PyCFunction_CAST(framelocalsproxy_items), METH_NOARGS, - NULL}, - {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, - NULL}, - {"pop", _PyCFunction_CAST(framelocalsproxy_pop), METH_FASTCALL, - NULL}, - {"setdefault", _PyCFunction_CAST(framelocalsproxy_setdefault), METH_FASTCALL, - NULL}, - {NULL, NULL} /* sentinel */ + {"__contains__", framelocalsproxy___contains__, METH_O | METH_COEXIST, NULL}, + {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, NULL}, + {"update", framelocalsproxy_update, METH_O, NULL}, + {"__reversed__", framelocalsproxy_reversed, METH_NOARGS, NULL}, + {"copy", framelocalsproxy_copy, METH_NOARGS, NULL}, + {"keys", framelocalsproxy_keys, METH_NOARGS, NULL}, + {"values", framelocalsproxy_values, METH_NOARGS, NULL}, + {"items", _PyCFunction_CAST(framelocalsproxy_items), METH_NOARGS, NULL}, + {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, NULL}, + {"pop", _PyCFunction_CAST(framelocalsproxy_pop), METH_FASTCALL, NULL}, + { + "setdefault", + _PyCFunction_CAST(framelocalsproxy_setdefault), + METH_FASTCALL, + NULL + }, + {NULL, NULL} /* sentinel */ }; PyTypeObject PyFrameLocalsProxy_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "FrameLocalsProxy", .tp_basicsize = sizeof(PyFrameLocalsProxyObject), - .tp_dealloc = (destructor)framelocalsproxy_dealloc, + .tp_dealloc = framelocalsproxy_dealloc, .tp_repr = &framelocalsproxy_repr, .tp_as_number = &framelocalsproxy_as_number, .tp_as_sequence = &framelocalsproxy_as_sequence, @@ -845,7 +852,7 @@ _PyFrameLocalsProxy_New(PyFrameObject *frame) return NULL; } - PyObject* proxy = (PyObject*)framelocalsproxy_new(&PyFrameLocalsProxy_Type, args, NULL); + PyObject* proxy = framelocalsproxy_new(&PyFrameLocalsProxy_Type, args, NULL); Py_DECREF(args); return proxy; } @@ -856,8 +863,9 @@ static PyMemberDef frame_memberlist[] = { }; static PyObject * -frame_getlocals(PyFrameObject *f, void *closure) +frame_getlocals(PyObject *op, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); if (f == NULL) { PyErr_BadInternalCall(); return NULL; @@ -903,8 +911,9 @@ PyFrame_GetLineNumber(PyFrameObject *f) } static PyObject * -frame_getlineno(PyFrameObject *f, void *closure) +frame_getlineno(PyObject *op, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); int lineno = PyFrame_GetLineNumber(f); if (lineno < 0) { Py_RETURN_NONE; @@ -915,8 +924,9 @@ frame_getlineno(PyFrameObject *f, void *closure) } static PyObject * -frame_getlasti(PyFrameObject *f, void *closure) +frame_getlasti(PyObject *op, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); int lasti = _PyInterpreterFrame_LASTI(f->f_frame); if (lasti < 0) { return PyLong_FromLong(-1); @@ -925,8 +935,9 @@ frame_getlasti(PyFrameObject *f, void *closure) } static PyObject * -frame_getglobals(PyFrameObject *f, void *closure) +frame_getglobals(PyObject *op, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); PyObject *globals = f->f_frame->f_globals; if (globals == NULL) { globals = Py_None; @@ -935,8 +946,9 @@ frame_getglobals(PyFrameObject *f, void *closure) } static PyObject * -frame_getbuiltins(PyFrameObject *f, void *closure) +frame_getbuiltins(PyObject *op, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); PyObject *builtins = f->f_frame->f_builtins; if (builtins == NULL) { builtins = Py_None; @@ -945,8 +957,9 @@ frame_getbuiltins(PyFrameObject *f, void *closure) } static PyObject * -frame_getcode(PyFrameObject *f, void *closure) +frame_getcode(PyObject *op, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); if (PySys_Audit("object.__getattr__", "Os", f, "f_code") < 0) { return NULL; } @@ -954,8 +967,9 @@ frame_getcode(PyFrameObject *f, void *closure) } static PyObject * -frame_getback(PyFrameObject *f, void *closure) +frame_getback(PyObject *op, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); PyObject *res = (PyObject *)PyFrame_GetBack(f); if (res == NULL) { Py_RETURN_NONE; @@ -964,15 +978,17 @@ frame_getback(PyFrameObject *f, void *closure) } static PyObject * -frame_gettrace_opcodes(PyFrameObject *f, void *closure) +frame_gettrace_opcodes(PyObject *op, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); PyObject *result = f->f_trace_opcodes ? Py_True : Py_False; return Py_NewRef(result); } static int -frame_settrace_opcodes(PyFrameObject *f, PyObject* value, void *Py_UNUSED(ignored)) +frame_settrace_opcodes(PyObject *op, PyObject* value, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); if (!PyBool_Check(value)) { PyErr_SetString(PyExc_TypeError, "attribute value type must be bool"); @@ -1464,8 +1480,9 @@ static bool frame_is_suspended(PyFrameObject *frame) * that time. */ static int -frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignored)) +frame_setlineno(PyObject *op, PyObject* p_new_lineno, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); PyCodeObject *code = _PyFrame_GetCode(f->f_frame); if (p_new_lineno == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); @@ -1658,8 +1675,9 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore } static PyObject * -frame_gettrace(PyFrameObject *f, void *closure) +frame_gettrace(PyObject *op, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); PyObject* trace = f->f_trace; if (trace == NULL) trace = Py_None; @@ -1667,8 +1685,9 @@ frame_gettrace(PyFrameObject *f, void *closure) } static int -frame_settrace(PyFrameObject *f, PyObject* v, void *closure) +frame_settrace(PyObject *op, PyObject* v, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); if (v == Py_None) { v = NULL; } @@ -1682,7 +1701,8 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) } static PyObject * -frame_getgenerator(PyFrameObject *f, void *arg) { +frame_getgenerator(PyObject *op, void *Py_UNUSED(closure)) { + PyFrameObject *f = PyFrameObject_CAST(op); if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { PyObject *gen = (PyObject *)_PyGen_GetGeneratorFromFrame(f->f_frame); return Py_NewRef(gen); @@ -1692,26 +1712,25 @@ frame_getgenerator(PyFrameObject *f, void *arg) { static PyGetSetDef frame_getsetlist[] = { - {"f_back", (getter)frame_getback, NULL, NULL}, - {"f_locals", (getter)frame_getlocals, NULL, NULL}, - {"f_lineno", (getter)frame_getlineno, - (setter)frame_setlineno, NULL}, - {"f_trace", (getter)frame_gettrace, (setter)frame_settrace, NULL}, - {"f_lasti", (getter)frame_getlasti, NULL, NULL}, - {"f_globals", (getter)frame_getglobals, NULL, NULL}, - {"f_builtins", (getter)frame_getbuiltins, NULL, NULL}, - {"f_code", (getter)frame_getcode, NULL, NULL}, - {"f_trace_opcodes", (getter)frame_gettrace_opcodes, (setter)frame_settrace_opcodes, NULL}, - {"f_generator", (getter)frame_getgenerator, NULL, NULL}, + {"f_back", frame_getback, NULL, NULL}, + {"f_locals", frame_getlocals, NULL, NULL}, + {"f_lineno", frame_getlineno, frame_setlineno, NULL}, + {"f_trace", frame_gettrace, frame_settrace, NULL}, + {"f_lasti", frame_getlasti, NULL, NULL}, + {"f_globals", frame_getglobals, NULL, NULL}, + {"f_builtins", frame_getbuiltins, NULL, NULL}, + {"f_code", frame_getcode, NULL, NULL}, + {"f_trace_opcodes", frame_gettrace_opcodes, frame_settrace_opcodes, NULL}, + {"f_generator", frame_getgenerator, NULL, NULL}, {0} }; static void -frame_dealloc(PyFrameObject *f) +frame_dealloc(PyObject *op) { /* It is the responsibility of the owning generator/coroutine * to have cleared the generator pointer */ - + PyFrameObject *f = PyFrameObject_CAST(op); if (_PyObject_GC_IS_TRACKED(f)) { _PyObject_GC_UNTRACK(f); } @@ -1744,8 +1763,9 @@ frame_dealloc(PyFrameObject *f) } static int -frame_traverse(PyFrameObject *f, visitproc visit, void *arg) +frame_traverse(PyObject *op, visitproc visit, void *arg) { + PyFrameObject *f = PyFrameObject_CAST(op); Py_VISIT(f->f_back); Py_VISIT(f->f_trace); Py_VISIT(f->f_extra_locals); @@ -1758,8 +1778,9 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) } static int -frame_tp_clear(PyFrameObject *f) +frame_tp_clear(PyObject *op) { + PyFrameObject *f = PyFrameObject_CAST(op); Py_CLEAR(f->f_trace); Py_CLEAR(f->f_extra_locals); Py_CLEAR(f->f_locals_cache); @@ -1778,8 +1799,9 @@ frame_tp_clear(PyFrameObject *f) } static PyObject * -frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) +frame_clear(PyObject *op, PyObject *Py_UNUSED(ignored)) { + PyFrameObject *f = PyFrameObject_CAST(op); if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { PyGenObject *gen = _PyGen_GetGeneratorFromFrame(f->f_frame); if (gen->gi_frame_state == FRAME_EXECUTING) { @@ -1795,7 +1817,7 @@ frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) } else { assert(f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); - (void)frame_tp_clear(f); + (void)frame_tp_clear(op); } Py_RETURN_NONE; running: @@ -1812,8 +1834,9 @@ PyDoc_STRVAR(clear__doc__, "F.clear(): clear all references held by the frame"); static PyObject * -frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) +frame_sizeof(PyObject *op, PyObject *Py_UNUSED(ignored)) { + PyFrameObject *f = PyFrameObject_CAST(op); Py_ssize_t res; res = offsetof(PyFrameObject, _f_frame_data) + offsetof(_PyInterpreterFrame, localsplus); PyCodeObject *code = _PyFrame_GetCode(f->f_frame); @@ -1825,8 +1848,9 @@ PyDoc_STRVAR(sizeof__doc__, "F.__sizeof__() -> size of F in memory, in bytes"); static PyObject * -frame_repr(PyFrameObject *f) +frame_repr(PyObject *op) { + PyFrameObject *f = PyFrameObject_CAST(op); int lineno = PyFrame_GetLineNumber(f); PyCodeObject *code = _PyFrame_GetCode(f->f_frame); return PyUnicode_FromFormat( @@ -1835,11 +1859,9 @@ frame_repr(PyFrameObject *f) } static PyMethodDef frame_methods[] = { - {"clear", (PyCFunction)frame_clear, METH_NOARGS, - clear__doc__}, - {"__sizeof__", (PyCFunction)frame_sizeof, METH_NOARGS, - sizeof__doc__}, - {NULL, NULL} /* sentinel */ + {"clear", frame_clear, METH_NOARGS, clear__doc__}, + {"__sizeof__", frame_sizeof, METH_NOARGS, sizeof__doc__}, + {NULL, NULL} /* sentinel */ }; PyTypeObject PyFrame_Type = { @@ -1848,12 +1870,12 @@ PyTypeObject PyFrame_Type = { offsetof(PyFrameObject, _f_frame_data) + offsetof(_PyInterpreterFrame, localsplus), sizeof(PyObject *), - (destructor)frame_dealloc, /* tp_dealloc */ + frame_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ - (reprfunc)frame_repr, /* tp_repr */ + frame_repr, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ @@ -1865,8 +1887,8 @@ PyTypeObject PyFrame_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)frame_traverse, /* tp_traverse */ - (inquiry)frame_tp_clear, /* tp_clear */ + frame_traverse, /* tp_traverse */ + frame_tp_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -2186,21 +2208,21 @@ PyObject* PyFrame_GetLocals(PyFrameObject *frame) { assert(!_PyFrame_IsIncomplete(frame->f_frame)); - return frame_getlocals(frame, NULL); + return frame_getlocals((PyObject *)frame, NULL); } PyObject* PyFrame_GetGlobals(PyFrameObject *frame) { assert(!_PyFrame_IsIncomplete(frame->f_frame)); - return frame_getglobals(frame, NULL); + return frame_getglobals((PyObject *)frame, NULL); } PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) { assert(!_PyFrame_IsIncomplete(frame->f_frame)); - return frame_getbuiltins(frame, NULL); + return frame_getbuiltins((PyObject *)frame, NULL); } int From 167cf3ace0db553dce50b2068dfae1f9eae4da6e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 8 Feb 2025 15:38:11 +0100 Subject: [PATCH 007/305] gh-115806: Don't mangle gdbm header dependency detection output (#129390) Replace AC_CACHE_VAL/AC_CHECK_HEADER with a cleaner variant using AC_CACHE_CHECK/AC_PREPROC_IFELSE. The former would produce garbled output when config.cache was reused. It also required directly manipulating GNU Autoconf cache variables. --- configure | 41 ++++++++++++++++++++++------------------- configure.ac | 24 ++++++++---------------- 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/configure b/configure index 65b8e711cdccae..d46bc563a67245 100755 --- a/configure +++ b/configure @@ -17162,26 +17162,28 @@ esac { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $have_ndbm ($dbm_ndbm)" >&5 printf "%s\n" "$have_ndbm ($dbm_ndbm)" >&6; } -{ ac_cv_header_gdbm_ndbm_h=; unset ac_cv_header_gdbm_ndbm_h;} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gdbm/ndbm.h" >&5 +printf %s "checking for gdbm/ndbm.h... " >&6; } if test ${ac_cv_header_gdbm_slash_ndbm_h+y} then : printf %s "(cached) " >&6 else case e in #( - e) - ac_fn_c_check_header_compile "$LINENO" "gdbm/ndbm.h" "ac_cv_header_gdbm_ndbm_h" "$ac_includes_default" -if test "x$ac_cv_header_gdbm_ndbm_h" = xyes + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO" then : ac_cv_header_gdbm_slash_ndbm_h=yes else case e in #( - e) ac_cv_header_gdbm_slash_ndbm_h=no - ;; + e) ac_cv_header_gdbm_slash_ndbm_h=no ;; esac fi - - ;; +rm -f conftest.err conftest.i conftest.$ac_ext ;; esac fi - +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_gdbm_slash_ndbm_h" >&5 +printf "%s\n" "$ac_cv_header_gdbm_slash_ndbm_h" >&6; } if test "x$ac_cv_header_gdbm_slash_ndbm_h" = xyes then : @@ -17191,26 +17193,28 @@ printf "%s\n" "#define HAVE_GDBM_NDBM_H 1" >>confdefs.h fi -{ ac_cv_header_gdbm_ndbm_h=; unset ac_cv_header_gdbm_ndbm_h;} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gdbm-ndbm.h" >&5 +printf %s "checking for gdbm-ndbm.h... " >&6; } if test ${ac_cv_header_gdbm_dash_ndbm_h+y} then : printf %s "(cached) " >&6 else case e in #( - e) - ac_fn_c_check_header_compile "$LINENO" "gdbm-ndbm.h" "ac_cv_header_gdbm_ndbm_h" "$ac_includes_default" -if test "x$ac_cv_header_gdbm_ndbm_h" = xyes + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO" then : ac_cv_header_gdbm_dash_ndbm_h=yes else case e in #( - e) ac_cv_header_gdbm_dash_ndbm_h=no - ;; + e) ac_cv_header_gdbm_dash_ndbm_h=no ;; esac fi - - ;; +rm -f conftest.err conftest.i conftest.$ac_ext ;; esac fi - +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_gdbm_dash_ndbm_h" >&5 +printf "%s\n" "$ac_cv_header_gdbm_dash_ndbm_h" >&6; } if test "x$ac_cv_header_gdbm_dash_ndbm_h" = xyes then : @@ -17219,7 +17223,6 @@ printf "%s\n" "#define HAVE_GDBM_DASH_NDBM_H 1" >>confdefs.h fi -{ ac_cv_header_gdbm_ndbm_h=; unset ac_cv_header_gdbm_ndbm_h;} if test "$ac_cv_header_gdbm_slash_ndbm_h" = yes -o "$ac_cv_header_gdbm_dash_ndbm_h" = yes; then { ac_cv_search_dbm_open=; unset ac_cv_search_dbm_open;} diff --git a/configure.ac b/configure.ac index 0c6063d87d654a..faa8909530389d 100644 --- a/configure.ac +++ b/configure.ac @@ -4430,29 +4430,21 @@ AS_CASE([$ac_cv_search_dbm_open], AC_MSG_RESULT([$have_ndbm ($dbm_ndbm)]) dnl "gdbm-ndbm.h" and "gdbm/ndbm.h" are both normalized to "gdbm_ndbm_h" -dnl unset ac_cv_header_gdbm_ndbm_h to prevent false positive cache hits. -AS_UNSET([ac_cv_header_gdbm_ndbm_h]) -AC_CACHE_VAL([ac_cv_header_gdbm_slash_ndbm_h], [ - AC_CHECK_HEADER( - [gdbm/ndbm.h], - [ac_cv_header_gdbm_slash_ndbm_h=yes], [ac_cv_header_gdbm_slash_ndbm_h=no] - ) -]) +AC_CACHE_CHECK([for gdbm/ndbm.h], [ac_cv_header_gdbm_slash_ndbm_h], + [AC_PREPROC_IFELSE([AC_LANG_SOURCE([@%:@include ])], + [ac_cv_header_gdbm_slash_ndbm_h=yes], + [ac_cv_header_gdbm_slash_ndbm_h=no])]) AS_VAR_IF([ac_cv_header_gdbm_slash_ndbm_h], [yes], [ AC_DEFINE([HAVE_GDBM_NDBM_H], [1], [Define to 1 if you have the header file.]) ]) -AS_UNSET([ac_cv_header_gdbm_ndbm_h]) -AC_CACHE_VAL([ac_cv_header_gdbm_dash_ndbm_h], [ - AC_CHECK_HEADER( - [gdbm-ndbm.h], - [ac_cv_header_gdbm_dash_ndbm_h=yes], [ac_cv_header_gdbm_dash_ndbm_h=no] - ) -]) +AC_CACHE_CHECK([for gdbm-ndbm.h], [ac_cv_header_gdbm_dash_ndbm_h], + [AC_PREPROC_IFELSE([AC_LANG_SOURCE([@%:@include ])], + [ac_cv_header_gdbm_dash_ndbm_h=yes], + [ac_cv_header_gdbm_dash_ndbm_h=no])]) AS_VAR_IF([ac_cv_header_gdbm_dash_ndbm_h], [yes], [ AC_DEFINE([HAVE_GDBM_DASH_NDBM_H], [1], [Define to 1 if you have the header file.]) ]) -AS_UNSET([ac_cv_header_gdbm_ndbm_h]) if test "$ac_cv_header_gdbm_slash_ndbm_h" = yes -o "$ac_cv_header_gdbm_dash_ndbm_h" = yes; then AS_UNSET([ac_cv_search_dbm_open]) From a56ead089caf4e975d7e36036ad98e932822d7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 8 Feb 2025 16:01:57 +0100 Subject: [PATCH 008/305] gh-129173: Use `_PyUnicodeError_GetParams` in `PyCodec_NameReplaceErrors` (GH-129135) --- Python/codecs.c | 234 +++++++++++++++++++++++++----------------------- 1 file changed, 123 insertions(+), 111 deletions(-) diff --git a/Python/codecs.c b/Python/codecs.c index 53680a79082634..6c9f8222079ec8 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -676,6 +676,60 @@ wrong_exception_type(PyObject *exc) PyObject_TypeCheck(EXC, (PyTypeObject *)PyExc_UnicodeTranslateError) +// --- codecs handlers: utilities --------------------------------------------- + +/* + * Return the number of characters (including special prefixes) + * needed to represent 'ch' by codec_handler_write_unicode_hex(). + */ +static inline Py_ssize_t +codec_handler_unicode_hex_width(Py_UCS4 ch) +{ + if (ch >= 0x10000) { + // format: '\\' + 'U' + 8 hex digits + return 1 + 1 + 8; + } + else if (ch >= 0x100) { + // format: '\\' + 'u' + 4 hex digits + return 1 + 1 + 4; + } + else { + // format: '\\' + 'x' + 2 hex digits + return 1 + 1 + 2; + } +} + + +/* + * Write the hexadecimal representation of 'ch' to the buffer pointed by 'p' + * using 2, 4, or 8 characters prefixed by '\x', '\u', or '\U' respectively. + */ +static inline void +codec_handler_write_unicode_hex(Py_UCS1 **p, Py_UCS4 ch) +{ + *(*p)++ = '\\'; + if (ch >= 0x10000) { + *(*p)++ = 'U'; + *(*p)++ = Py_hexdigits[(ch >> 28) & 0xf]; + *(*p)++ = Py_hexdigits[(ch >> 24) & 0xf]; + *(*p)++ = Py_hexdigits[(ch >> 20) & 0xf]; + *(*p)++ = Py_hexdigits[(ch >> 16) & 0xf]; + *(*p)++ = Py_hexdigits[(ch >> 12) & 0xf]; + *(*p)++ = Py_hexdigits[(ch >> 8) & 0xf]; + } + else if (ch >= 0x100) { + *(*p)++ = 'u'; + *(*p)++ = Py_hexdigits[(ch >> 12) & 0xf]; + *(*p)++ = Py_hexdigits[(ch >> 8) & 0xf]; + } + else { + *(*p)++ = 'x'; + } + *(*p)++ = Py_hexdigits[(ch >> 4) & 0xf]; + *(*p)++ = Py_hexdigits[ch & 0xf]; +} + + // --- handler: 'strict' ------------------------------------------------------ PyObject *PyCodec_StrictErrors(PyObject *exc) @@ -942,17 +996,8 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) Py_ssize_t ressize = 0; for (Py_ssize_t i = start; i < end; ++i) { - /* object is guaranteed to be "ready" */ Py_UCS4 c = PyUnicode_READ_CHAR(obj, i); - if (c >= 0x10000) { - ressize += 1 + 1 + 8; - } - else if (c >= 0x100) { - ressize += 1 + 1 + 4; - } - else { - ressize += 1 + 1 + 2; - } + ressize += codec_handler_unicode_hex_width(c); } PyObject *res = PyUnicode_New(ressize, 127); if (res == NULL) { @@ -962,122 +1007,86 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res); for (Py_ssize_t i = start; i < end; ++i) { Py_UCS4 c = PyUnicode_READ_CHAR(obj, i); - *outp++ = '\\'; - if (c >= 0x00010000) { - *outp++ = 'U'; - *outp++ = Py_hexdigits[(c >> 28) & 0xf]; - *outp++ = Py_hexdigits[(c >> 24) & 0xf]; - *outp++ = Py_hexdigits[(c >> 20) & 0xf]; - *outp++ = Py_hexdigits[(c >> 16) & 0xf]; - *outp++ = Py_hexdigits[(c >> 12) & 0xf]; - *outp++ = Py_hexdigits[(c >> 8) & 0xf]; - } - else if (c >= 0x100) { - *outp++ = 'u'; - *outp++ = Py_hexdigits[(c >> 12) & 0xf]; - *outp++ = Py_hexdigits[(c >> 8) & 0xf]; - } - else { - *outp++ = 'x'; - } - *outp++ = Py_hexdigits[(c >> 4) & 0xf]; - *outp++ = Py_hexdigits[c & 0xf]; + codec_handler_write_unicode_hex(&outp, c); } assert(_PyUnicode_CheckConsistency(res, 1)); Py_DECREF(obj); return Py_BuildValue("(Nn)", res, end); } + +// --- handler: 'namereplace' ------------------------------------------------- + PyObject *PyCodec_NameReplaceErrors(PyObject *exc) { - if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) { - PyObject *restuple; - PyObject *object; - Py_ssize_t i; - Py_ssize_t start; - Py_ssize_t end; - PyObject *res; - Py_UCS1 *outp; - Py_ssize_t ressize; - int replsize; - Py_UCS4 c; - char buffer[256]; /* NAME_MAXLEN */ - if (PyUnicodeEncodeError_GetStart(exc, &start)) - return NULL; - if (PyUnicodeEncodeError_GetEnd(exc, &end)) - return NULL; - if (!(object = PyUnicodeEncodeError_GetObject(exc))) - return NULL; - _PyUnicode_Name_CAPI *ucnhash_capi = _PyUnicode_GetNameCAPI(); - if (ucnhash_capi == NULL) { - return NULL; + if (!_PyIsUnicodeEncodeError(exc)) { + wrong_exception_type(exc); + return NULL; + } + + _PyUnicode_Name_CAPI *ucnhash_capi = _PyUnicode_GetNameCAPI(); + if (ucnhash_capi == NULL) { + return NULL; + } + + PyObject *obj; + Py_ssize_t start, end; + if (_PyUnicodeError_GetParams(exc, + &obj, NULL, + &start, &end, NULL, false) < 0) + { + return NULL; + } + + char buffer[256]; /* NAME_MAXLEN in unicodename_db.h */ + Py_ssize_t imax = start, ressize = 0, replsize; + for (; imax < end; ++imax) { + Py_UCS4 c = PyUnicode_READ_CHAR(obj, imax); + if (ucnhash_capi->getname(c, buffer, sizeof(buffer), 1)) { + // If 'c' is recognized by getname(), the corresponding replacement + // is '\\' + 'N' + '{' + NAME + '}', i.e. 1 + 1 + 1 + len(NAME) + 1 + // characters. Failures of getname() are ignored by the handler. + replsize = 1 + 1 + 1 + strlen(buffer) + 1; } - for (i = start, ressize = 0; i < end; ++i) { - /* object is guaranteed to be "ready" */ - c = PyUnicode_READ_CHAR(object, i); - if (ucnhash_capi->getname(c, buffer, sizeof(buffer), 1)) { - replsize = 1+1+1+(int)strlen(buffer)+1; - } - else if (c >= 0x10000) { - replsize = 1+1+8; - } - else if (c >= 0x100) { - replsize = 1+1+4; - } - else - replsize = 1+1+2; - if (ressize > PY_SSIZE_T_MAX - replsize) - break; - ressize += replsize; + else { + replsize = codec_handler_unicode_hex_width(c); } - end = i; - res = PyUnicode_New(ressize, 127); - if (res==NULL) - return NULL; - for (i = start, outp = PyUnicode_1BYTE_DATA(res); - i < end; ++i) { - c = PyUnicode_READ_CHAR(object, i); - *outp++ = '\\'; - if (ucnhash_capi->getname(c, buffer, sizeof(buffer), 1)) { - *outp++ = 'N'; - *outp++ = '{'; - strcpy((char *)outp, buffer); - outp += strlen(buffer); - *outp++ = '}'; - continue; - } - if (c >= 0x00010000) { - *outp++ = 'U'; - *outp++ = Py_hexdigits[(c>>28)&0xf]; - *outp++ = Py_hexdigits[(c>>24)&0xf]; - *outp++ = Py_hexdigits[(c>>20)&0xf]; - *outp++ = Py_hexdigits[(c>>16)&0xf]; - *outp++ = Py_hexdigits[(c>>12)&0xf]; - *outp++ = Py_hexdigits[(c>>8)&0xf]; - } - else if (c >= 0x100) { - *outp++ = 'u'; - *outp++ = Py_hexdigits[(c>>12)&0xf]; - *outp++ = Py_hexdigits[(c>>8)&0xf]; - } - else - *outp++ = 'x'; - *outp++ = Py_hexdigits[(c>>4)&0xf]; - *outp++ = Py_hexdigits[c&0xf]; + if (ressize > PY_SSIZE_T_MAX - replsize) { + break; } - - assert(outp == PyUnicode_1BYTE_DATA(res) + ressize); - assert(_PyUnicode_CheckConsistency(res, 1)); - restuple = Py_BuildValue("(Nn)", res, end); - Py_DECREF(object); - return restuple; + ressize += replsize; } - else { - wrong_exception_type(exc); + + PyObject *res = PyUnicode_New(ressize, 127); + if (res == NULL) { + Py_DECREF(obj); return NULL; } + + Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res); + for (Py_ssize_t i = start; i < imax; ++i) { + Py_UCS4 c = PyUnicode_READ_CHAR(obj, i); + if (ucnhash_capi->getname(c, buffer, sizeof(buffer), 1)) { + *outp++ = '\\'; + *outp++ = 'N'; + *outp++ = '{'; + (void)strcpy((char *)outp, buffer); + outp += strlen(buffer); + *outp++ = '}'; + } + else { + codec_handler_write_unicode_hex(&outp, c); + } + } + + assert(outp == PyUnicode_1BYTE_DATA(res) + ressize); + assert(_PyUnicode_CheckConsistency(res, 1)); + PyObject *restuple = Py_BuildValue("(Nn)", res, imax); + Py_DECREF(obj); + return restuple; } + #define ENC_UNKNOWN -1 #define ENC_UTF8 0 #define ENC_UTF16BE 1 @@ -1421,11 +1430,14 @@ static PyObject *backslashreplace_errors(PyObject *self, PyObject *exc) return PyCodec_BackslashReplaceErrors(exc); } -static PyObject *namereplace_errors(PyObject *self, PyObject *exc) + +static inline PyObject * +namereplace_errors(PyObject *Py_UNUSED(self), PyObject *exc) { return PyCodec_NameReplaceErrors(exc); } + static PyObject *surrogatepass_errors(PyObject *self, PyObject *exc) { return PyCodec_SurrogatePassErrors(exc); From 1bccd6c34f82f8373c320792323bfd7f7a328bc7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 8 Feb 2025 17:56:57 +0200 Subject: [PATCH 009/305] gh-128317: Move CLI calendar highlighting to private class (#129625) Co-authored-by: Petr Viktorin --- Doc/library/calendar.rst | 21 +-- Lib/calendar.py | 155 +++++++++++++----- ...-02-03-22-31-43.gh-issue-128317.n2Swnh.rst | 2 + 3 files changed, 120 insertions(+), 58 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-02-03-22-31-43.gh-issue-128317.n2Swnh.rst diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 1c6b5e03af3560..39090e36ed9c0d 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -166,18 +166,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is the specified width, representing an empty day. The *weekday* parameter is unused. - .. method:: formatweek(theweek, w=0, highlight_day=None) + .. method:: formatweek(theweek, w=0) Return a single week in a string with no newline. If *w* is provided, it specifies the width of the date columns, which are centered. Depends on the first weekday as specified in the constructor or set by the :meth:`setfirstweekday` method. - .. versionchanged:: 3.14 - If *highlight_day* is given, this date is highlighted in color. - This can be :ref:`controlled using environment variables - `. - .. method:: formatweekday(weekday, width) @@ -193,7 +188,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is settings and are padded to the specified width. - .. method:: formatmonth(theyear, themonth, w=0, l=0, highlight_day=None) + .. method:: formatmonth(theyear, themonth, w=0, l=0) Return a month's calendar in a multi-line string. If *w* is provided, it specifies the width of the date columns, which are centered. If *l* is @@ -201,11 +196,6 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is on the first weekday as specified in the constructor or set by the :meth:`setfirstweekday` method. - .. versionchanged:: 3.14 - If *highlight_day* is given, this date is highlighted in color. - This can be :ref:`controlled using environment variables - `. - .. method:: formatmonthname(theyear, themonth, width=0, withyear=True) @@ -220,7 +210,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is Print a month's calendar as returned by :meth:`formatmonth`. - .. method:: formatyear(theyear, w=2, l=1, c=6, m=3, highlight_day=None) + .. method:: formatyear(theyear, w=2, l=1, c=6, m=3) Return a *m*-column calendar for an entire year as a multi-line string. Optional parameters *w*, *l*, and *c* are for date column width, lines per @@ -229,11 +219,6 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :meth:`setfirstweekday` method. The earliest year for which a calendar can be generated is platform-dependent. - .. versionchanged:: 3.14 - If *highlight_day* is given, this date is highlighted in color. - This can be :ref:`controlled using environment variables - `. - .. method:: pryear(theyear, w=2, l=1, c=6, m=3) diff --git a/Lib/calendar.py b/Lib/calendar.py index d2e5e08d02dbb9..1105a705a8036a 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -349,27 +349,11 @@ def formatday(self, day, weekday, width): s = '%2i' % day # right-align single-digit days return s.center(width) - def formatweek(self, theweek, width, *, highlight_day=None): + def formatweek(self, theweek, width): """ Returns a single week in a string (no newline). """ - if highlight_day: - from _colorize import get_colors - - ansi = get_colors() - highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" - reset = ansi.RESET - else: - highlight = reset = "" - - return ' '.join( - ( - f"{highlight}{self.formatday(d, wd, width)}{reset}" - if d == highlight_day - else self.formatday(d, wd, width) - ) - for (d, wd) in theweek - ) + return ' '.join(self.formatday(d, wd, width) for (d, wd) in theweek) def formatweekday(self, day, width): """ @@ -404,11 +388,10 @@ def prmonth(self, theyear, themonth, w=0, l=0): """ print(self.formatmonth(theyear, themonth, w, l), end='') - def formatmonth(self, theyear, themonth, w=0, l=0, *, highlight_day=None): + def formatmonth(self, theyear, themonth, w=0, l=0): """ Return a month's calendar string (multi-line). """ - highlight_day = highlight_day.day if highlight_day else None w = max(2, w) l = max(1, l) s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1) @@ -417,11 +400,11 @@ def formatmonth(self, theyear, themonth, w=0, l=0, *, highlight_day=None): s += self.formatweekheader(w).rstrip() s += '\n' * l for week in self.monthdays2calendar(theyear, themonth): - s += self.formatweek(week, w, highlight_day=highlight_day).rstrip() + s += self.formatweek(week, w).rstrip() s += '\n' * l return s - def formatyear(self, theyear, w=2, l=1, c=6, m=3, *, highlight_day=None): + def formatyear(self, theyear, w=2, l=1, c=6, m=3): """ Returns a year's calendar as a multi-line string. """ @@ -446,23 +429,15 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3, *, highlight_day=None): a(formatstring(headers, colwidth, c).rstrip()) a('\n'*l) - if highlight_day and highlight_day.month in months: - month_pos = months.index(highlight_day.month) - else: - month_pos = None - # max number of weeks for this row height = max(len(cal) for cal in row) for j in range(height): weeks = [] - for k, cal in enumerate(row): + for cal in row: if j >= len(cal): weeks.append('') else: - day = highlight_day.day if k == month_pos else None - weeks.append( - self.formatweek(cal[j], w, highlight_day=day) - ) + weeks.append(self.formatweek(cal[j], w)) a(formatstring(weeks, colwidth, c).rstrip()) a('\n' * l) return ''.join(v) @@ -672,6 +647,111 @@ def formatmonthname(self, theyear, themonth, withyear=True): with different_locale(self.locale): return super().formatmonthname(theyear, themonth, withyear) + +class _CLIDemoCalendar(LocaleTextCalendar): + def __init__(self, highlight_day=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.highlight_day = highlight_day + + def formatweek(self, theweek, width, *, highlight_day=None): + """ + Returns a single week in a string (no newline). + """ + if highlight_day: + from _colorize import get_colors + + ansi = get_colors() + highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" + reset = ansi.RESET + else: + highlight = reset = "" + + return ' '.join( + ( + f"{highlight}{self.formatday(d, wd, width)}{reset}" + if d == highlight_day + else self.formatday(d, wd, width) + ) + for (d, wd) in theweek + ) + + def formatmonth(self, theyear, themonth, w=0, l=0): + """ + Return a month's calendar string (multi-line). + """ + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month == themonth + ): + highlight_day = self.highlight_day.day + else: + highlight_day = None + w = max(2, w) + l = max(1, l) + s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1) + s = s.rstrip() + s += '\n' * l + s += self.formatweekheader(w).rstrip() + s += '\n' * l + for week in self.monthdays2calendar(theyear, themonth): + s += self.formatweek(week, w, highlight_day=highlight_day).rstrip() + s += '\n' * l + return s + + def formatyear(self, theyear, w=2, l=1, c=6, m=3): + """ + Returns a year's calendar as a multi-line string. + """ + w = max(2, w) + l = max(1, l) + c = max(2, c) + colwidth = (w + 1) * 7 - 1 + v = [] + a = v.append + a(repr(theyear).center(colwidth*m+c*(m-1)).rstrip()) + a('\n'*l) + header = self.formatweekheader(w) + for (i, row) in enumerate(self.yeardays2calendar(theyear, m)): + # months in this row + months = range(m*i+1, min(m*(i+1)+1, 13)) + a('\n'*l) + names = (self.formatmonthname(theyear, k, colwidth, False) + for k in months) + a(formatstring(names, colwidth, c).rstrip()) + a('\n'*l) + headers = (header for k in months) + a(formatstring(headers, colwidth, c).rstrip()) + a('\n'*l) + + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month in months + ): + month_pos = months.index(self.highlight_day.month) + else: + month_pos = None + + # max number of weeks for this row + height = max(len(cal) for cal in row) + for j in range(height): + weeks = [] + for k, cal in enumerate(row): + if j >= len(cal): + weeks.append('') + else: + day = ( + self.highlight_day.day if k == month_pos else None + ) + weeks.append( + self.formatweek(cal[j], w, highlight_day=day) + ) + a(formatstring(weeks, colwidth, c).rstrip()) + a('\n' * l) + return ''.join(v) + + # Support for old module level interface c = TextCalendar() @@ -813,26 +893,21 @@ def main(args=None): write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: - cal = LocaleTextCalendar(locale=locale) + cal = _CLIDemoCalendar(highlight_day=today, locale=locale) else: - cal = TextCalendar() + cal = _CLIDemoCalendar(highlight_day=today) cal.setfirstweekday(options.first_weekday) optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months - if options.month is not None: + else: _validate_month(options.month) if options.year is None: - optdict["highlight_day"] = today result = cal.formatyear(today.year, **optdict) elif options.month is None: - if options.year == today.year: - optdict["highlight_day"] = today result = cal.formatyear(options.year, **optdict) else: - if options.year == today.year and options.month == today.month: - optdict["highlight_day"] = today result = cal.formatmonth(options.year, options.month, **optdict) write = sys.stdout.write if options.encoding: diff --git a/Misc/NEWS.d/next/Library/2025-02-03-22-31-43.gh-issue-128317.n2Swnh.rst b/Misc/NEWS.d/next/Library/2025-02-03-22-31-43.gh-issue-128317.n2Swnh.rst new file mode 100644 index 00000000000000..eaff03384a976a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-02-03-22-31-43.gh-issue-128317.n2Swnh.rst @@ -0,0 +1,2 @@ +Put CLI calendar highlighting in private class, removing ``highlight_day`` +from public :class:`calendar.TextCalendar` API. Patch by Hugo van Kemenade. From 33a7094aa680bca66582fec4dcda7d150eb90cd8 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 8 Feb 2025 19:26:07 +0000 Subject: [PATCH 010/305] gh-129699: Add description to IDLE doc title (#129727) Also extend the 'idlelib' section header. These additions affect both the displayed idle.html file and the contents.html file displayed by clicking the Complete table of contents link on the main docs.python.org page. (The module index entries are generated from the module name and synopsis within module files.) --------- Co-authored-by: Terry Jan Reedy --- Doc/library/idle.rst | 8 +- Lib/idlelib/help.html | 120 ++++++++++++++--------------- Lib/idlelib/help.py | 4 +- Lib/idlelib/idle_test/test_help.py | 2 +- 4 files changed, 68 insertions(+), 66 deletions(-) diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 59b181aab3e484..830bc33922d5dd 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -1,7 +1,7 @@ .. _idle: -IDLE -==== +IDLE --- Python editor and shell +================================ .. moduleauthor:: Guido van Rossum @@ -971,8 +971,8 @@ information. The only current default extension is zzdummy, an example also used for testing. -idlelib -------- +idlelib --- implementation of IDLE application +---------------------------------------------- .. module:: idlelib :synopsis: Implementation package for the IDLE shell/editor. diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 2a4adc6a4d395f..2200bf29abea66 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -1,24 +1,24 @@ - + - IDLE — Python 3.14.0a0 documentation + IDLE — Python editor and shell — Python 3.14.0a4 documentation - - - + + + - - - + + + @@ -87,7 +87,7 @@ -