diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 32ba5355a5853f..d218f029a990bf 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -97,6 +97,8 @@ Include/pytime.h @pganssle @abalkin /Tools/peg_generator/ @pablogsal @lysnikolaou /Lib/test/test_peg_generator/ @pablogsal @lysnikolaou /Grammar/python.gram @pablogsal @lysnikolaou +/Lib/tokenize.py @pablogsal @lysnikolaou +/Lib/test/test_tokenize.py @pablogsal @lysnikolaou # AST Python/ast.c @isidentical diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 41abddffa5d648..b9797192dba054 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,7 @@ on: push: branches: - 'main' + - '3.12' - '3.11' - '3.10' - '3.9' @@ -16,6 +17,7 @@ on: pull_request: branches: - 'main' + - '3.12' - '3.11' - '3.10' - '3.9' diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 3f7550cc72943b..ec900ce68a1dde 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -5,6 +5,7 @@ on: #push: # branches: # - 'main' + # - '3.12' # - '3.11' # - '3.10' # - '3.9' @@ -15,6 +16,7 @@ on: pull_request: branches: - 'main' + - '3.12' - '3.11' - '3.10' - '3.9' diff --git a/.gitignore b/.gitignore index d9c4a7972f076d..dddf28da016192 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,10 @@ *.gc?? *.profclang? *.profraw +# Copies of binaries before BOLT optimizations. +*.prebolt +# BOLT profile data. +*.fdata *.dyn .gdb_history .purify @@ -57,7 +61,6 @@ Doc/.venv/ Doc/env/ Doc/.env/ Include/pydtrace_probes.h -Lib/lib2to3/*.pickle Lib/site-packages/* !Lib/site-packages/README.txt Lib/test/data/* @@ -124,6 +127,7 @@ Tools/unicode/data/ /platform /profile-clean-stamp /profile-run-stamp +/profile-bolt-stamp /Python/deepfreeze/*.c /pybuilddir.txt /pyconfig.h diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index ab3a2e274d9395..6771f378bfbc31 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -394,98 +394,149 @@ APIs: arguments, calculate the size of the resulting Python Unicode string and return a string with the values formatted into it. The variable arguments must be C types and must correspond exactly to the format characters in the *format* - ASCII-encoded string. The following format characters are allowed: - - .. % This should be exactly the same as the table in PyErr_Format. - - .. tabularcolumns:: |l|l|L| - - +-------------------+---------------------+----------------------------------+ - | Format Characters | Type | Comment | - +===================+=====================+==================================+ - | :attr:`%%` | *n/a* | The literal % character. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%c` | int | A single character, | - | | | represented as a C int. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%d` | int | Equivalent to | - | | | ``printf("%d")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%u` | unsigned int | Equivalent to | - | | | ``printf("%u")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%ld` | long | Equivalent to | - | | | ``printf("%ld")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%li` | long | Equivalent to | - | | | ``printf("%li")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lu` | unsigned long | Equivalent to | - | | | ``printf("%lu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lld` | long long | Equivalent to | - | | | ``printf("%lld")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lli` | long long | Equivalent to | - | | | ``printf("%lli")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%llu` | unsigned long long | Equivalent to | - | | | ``printf("%llu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zd` | :c:type:`\ | Equivalent to | - | | Py_ssize_t` | ``printf("%zd")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zi` | :c:type:`\ | Equivalent to | - | | Py_ssize_t` | ``printf("%zi")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zu` | size_t | Equivalent to | - | | | ``printf("%zu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%i` | int | Equivalent to | - | | | ``printf("%i")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%x` | int | Equivalent to | - | | | ``printf("%x")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%s` | const char\* | A null-terminated C character | - | | | array. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%p` | const void\* | The hex representation of a C | - | | | pointer. Mostly equivalent to | - | | | ``printf("%p")`` except that | - | | | it is guaranteed to start with | - | | | the literal ``0x`` regardless | - | | | of what the platform's | - | | | ``printf`` yields. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%A` | PyObject\* | The result of calling | - | | | :func:`ascii`. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%U` | PyObject\* | A Unicode object. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%V` | PyObject\*, | A Unicode object (which may be | - | | const char\* | ``NULL``) and a null-terminated | - | | | C character array as a second | - | | | parameter (which will be used, | - | | | if the first parameter is | - | | | ``NULL``). | - +-------------------+---------------------+----------------------------------+ - | :attr:`%S` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Str`. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%R` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Repr`. | - +-------------------+---------------------+----------------------------------+ + ASCII-encoded string. + + A conversion specifier contains two or more characters and has the following + components, which must occur in this order: + + #. The ``'%'`` character, which marks the start of the specifier. + + #. Conversion flags (optional), which affect the result of some conversion + types. + + #. Minimum field width (optional). + If specified as an ``'*'`` (asterisk), the actual width is given in the + next argument, which must be of type :c:expr:`int`, and the object to + convert comes after the minimum field width and optional precision. + + #. Precision (optional), given as a ``'.'`` (dot) followed by the precision. + If specified as ``'*'`` (an asterisk), the actual precision is given in + the next argument, which must be of type :c:expr:`int`, and the value to + convert comes after the precision. + + #. Length modifier (optional). + + #. Conversion type. + + The conversion flag characters are: + + .. tabularcolumns:: |l|L| + + +-------+-------------------------------------------------------------+ + | Flag | Meaning | + +=======+=============================================================+ + | ``0`` | The conversion will be zero padded for numeric values. | + +-------+-------------------------------------------------------------+ + | ``-`` | The converted value is left adjusted (overrides the ``0`` | + | | flag if both are given). | + +-------+-------------------------------------------------------------+ + + The length modifiers for following integer conversions (``d``, ``i``, + ``o``, ``u``, ``x``, or ``X``) specify the type of the argument + (:c:expr:`int` by default): + + .. tabularcolumns:: |l|L| + + +----------+-----------------------------------------------------+ + | Modifier | Types | + +==========+=====================================================+ + | ``l`` | :c:expr:`long` or :c:expr:`unsigned long` | + +----------+-----------------------------------------------------+ + | ``ll`` | :c:expr:`long long` or :c:expr:`unsigned long long` | + +----------+-----------------------------------------------------+ + | ``j`` | :c:expr:`intmax_t` or :c:expr:`uintmax_t` | + +----------+-----------------------------------------------------+ + | ``z`` | :c:expr:`size_t` or :c:expr:`ssize_t` | + +----------+-----------------------------------------------------+ + | ``t`` | :c:expr:`ptrdiff_t` | + +----------+-----------------------------------------------------+ + + The length modifier ``l`` for following conversions ``s`` or ``V`` specify + that the type of the argument is :c:expr:`const wchar_t*`. + + The conversion specifiers are: + + .. list-table:: + :widths: auto + :header-rows: 1 + + * - Conversion Specifier + - Type + - Comment + + * - ``%`` + - *n/a* + - The literal ``%`` character. + + * - ``d``, ``i`` + - Specified by the length modifier + - The decimal representation of a signed C integer. + + * - ``u`` + - Specified by the length modifier + - The decimal representation of an unsigned C integer. + + * - ``o`` + - Specified by the length modifier + - The octal representation of an unsigned C integer. + + * - ``x`` + - Specified by the length modifier + - The hexadecimal representation of an unsigned C integer (lowercase). + + * - ``X`` + - Specified by the length modifier + - The hexadecimal representation of an unsigned C integer (uppercase). + + * - ``c`` + - :c:expr:`int` + - A single character. + + * - ``s`` + - :c:expr:`const char*` or :c:expr:`const wchar_t*` + - A null-terminated C character array. + + * - ``p`` + - :c:expr:`const void*` + - The hex representation of a C pointer. + Mostly equivalent to ``printf("%p")`` except that it is guaranteed to + start with the literal ``0x`` regardless of what the platform's + ``printf`` yields. + + * - ``A`` + - :c:expr:`PyObject*` + - The result of calling :func:`ascii`. + + * - ``U`` + - :c:expr:`PyObject*` + - A Unicode object. + + * - ``V`` + - :c:expr:`PyObject*`, :c:expr:`const char*` or :c:expr:`const wchar_t*` + - A Unicode object (which may be ``NULL``) and a null-terminated + C character array as a second parameter (which will be used, + if the first parameter is ``NULL``). + + * - ``S`` + - :c:expr:`PyObject*` + - The result of calling :c:func:`PyObject_Str`. + + * - ``R`` + - :c:expr:`PyObject*` + - The result of calling :c:func:`PyObject_Repr`. .. note:: The width formatter unit is number of characters rather than bytes. - The precision formatter unit is number of bytes for ``"%s"`` and + The precision formatter unit is number of bytes or :c:expr:`wchar_t` + items (if the length modifier ``l`` is used) for ``"%s"`` and ``"%V"`` (if the ``PyObject*`` argument is ``NULL``), and a number of characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"`` (if the ``PyObject*`` argument is not ``NULL``). - .. [1] For integer specifiers (d, u, ld, li, lu, lld, lli, llu, zd, zi, - zu, i, x): the 0-conversion flag has effect even when a precision is given. + .. note:: + Unlike to C :c:func:`printf` the ``0`` flag has effect even when + a precision is given for integer conversions (``d``, ``i``, ``u``, ``o``, + ``x``, or ``X``). .. versionchanged:: 3.2 Support for ``"%lld"`` and ``"%llu"`` added. @@ -498,6 +549,13 @@ APIs: ``"%V"``, ``"%S"``, ``"%R"`` added. .. versionchanged:: 3.12 + Support for conversion specifiers ``o`` and ``X``. + Support for length modifiers ``j`` and ``t``. + Length modifiers are now applied to all integer conversions. + Length modifier ``l`` is now applied to conversion specifiers ``s`` and ``V``. + Support for variable width and precision ``*``. + Support for flag ``-``. + An unrecognized format character now sets a :exc:`SystemError`. In previous versions it caused all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 513856d8a48d70..000a2d3d8790bb 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -167,6 +167,10 @@ the same library that the Python runtime is using. event loops, as done in the :file:`Modules/_tkinter.c` in the Python source code. + .. versionchanged:: 3.12 + This function is only called from the + :ref:`main interpreter `. + .. c:var:: char* (*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *) @@ -187,6 +191,10 @@ the same library that the Python runtime is using. :c:func:`PyMem_RawRealloc`, instead of being allocated by :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. + .. versionchanged:: 3.12 + This function is only called from the + :ref:`main interpreter `. + .. c:function:: PyObject* PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) This is a simplified interface to :c:func:`PyRun_StringFlags` below, leaving diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 53e8cdcae1cd66..8c493f823a6fae 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -22,15 +22,6 @@ Glossary * The :const:`Ellipsis` built-in constant. - 2to3 - A tool that tries to convert Python 2.x code to Python 3.x code by - handling most of the incompatibilities which can be detected by parsing the - source and traversing the parse tree. - - 2to3 is available in the standard library as :mod:`lib2to3`; a standalone - entry point is provided as :file:`Tools/scripts/2to3`. See - :ref:`2to3-reference`. - abstract base class Abstract base classes complement :term:`duck-typing` by providing a way to define interfaces when other techniques like diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 61ba6bd7224fcc..86137fb38c9b93 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -6,13 +6,6 @@ :Author: `Michael Foord `_ -.. note:: - - There is a French translation of an earlier revision of this - HOWTO, available at `urllib2 - Le Manuel manquant - `_. - - Introduction ============ diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst deleted file mode 100644 index d85ad94e9b7fe4..00000000000000 --- a/Doc/library/2to3.rst +++ /dev/null @@ -1,489 +0,0 @@ -.. _2to3-reference: - -2to3 --- Automated Python 2 to 3 code translation -================================================= - -.. sectionauthor:: Benjamin Peterson - -2to3 is a Python program that reads Python 2.x source code and applies a series -of *fixers* to transform it into valid Python 3.x code. The standard library -contains a rich set of fixers that will handle almost all code. 2to3 supporting -library :mod:`lib2to3` is, however, a flexible and generic library, so it is -possible to write your own fixers for 2to3. - -.. deprecated-removed:: 3.11 3.13 - The ``lib2to3`` module was marked pending for deprecation in Python 3.9 - (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated - in Python 3.11 (raising :exc:`DeprecationWarning`). The ``2to3`` tool is - part of that. It will be removed in Python 3.13. - -.. _2to3-using: - -Using 2to3 ----------- - -2to3 will usually be installed with the Python interpreter as a script. It is -also located in the :file:`Tools/scripts` directory of the Python root. - -2to3's basic arguments are a list of files or directories to transform. The -directories are recursively traversed for Python sources. - -Here is a sample Python 2.x source file, :file:`example.py`:: - - def greet(name): - print "Hello, {0}!".format(name) - print "What's your name?" - name = raw_input() - greet(name) - -It can be converted to Python 3.x code via 2to3 on the command line: - -.. code-block:: shell-session - - $ 2to3 example.py - -A diff against the original source file is printed. 2to3 can also write the -needed modifications right back to the source file. (A backup of the original -file is made unless :option:`!-n` is also given.) Writing the changes back is -enabled with the :option:`!-w` flag: - -.. code-block:: shell-session - - $ 2to3 -w example.py - -After transformation, :file:`example.py` looks like this:: - - def greet(name): - print("Hello, {0}!".format(name)) - print("What's your name?") - name = input() - greet(name) - -Comments and exact indentation are preserved throughout the translation process. - -By default, 2to3 runs a set of :ref:`predefined fixers <2to3-fixers>`. The -:option:`!-l` flag lists all available fixers. An explicit set of fixers to run -can be given with :option:`!-f`. Likewise the :option:`!-x` explicitly disables a -fixer. The following example runs only the ``imports`` and ``has_key`` fixers: - -.. code-block:: shell-session - - $ 2to3 -f imports -f has_key example.py - -This command runs every fixer except the ``apply`` fixer: - -.. code-block:: shell-session - - $ 2to3 -x apply example.py - -Some fixers are *explicit*, meaning they aren't run by default and must be -listed on the command line to be run. Here, in addition to the default fixers, -the ``idioms`` fixer is run: - -.. code-block:: shell-session - - $ 2to3 -f all -f idioms example.py - -Notice how passing ``all`` enables all default fixers. - -Sometimes 2to3 will find a place in your source code that needs to be changed, -but 2to3 cannot fix automatically. In this case, 2to3 will print a warning -beneath the diff for a file. You should address the warning in order to have -compliant 3.x code. - -2to3 can also refactor doctests. To enable this mode, use the :option:`!-d` -flag. Note that *only* doctests will be refactored. This also doesn't require -the module to be valid Python. For example, doctest like examples in a reST -document could also be refactored with this option. - -The :option:`!-v` option enables output of more information on the translation -process. - -Since some print statements can be parsed as function calls or statements, 2to3 -cannot always read files containing the print function. When 2to3 detects the -presence of the ``from __future__ import print_function`` compiler directive, it -modifies its internal grammar to interpret :func:`print` as a function. This -change can also be enabled manually with the :option:`!-p` flag. Use -:option:`!-p` to run fixers on code that already has had its print statements -converted. Also :option:`!-e` can be used to make :func:`exec` a function. - -The :option:`!-o` or :option:`!--output-dir` option allows specification of an -alternate directory for processed output files to be written to. The -:option:`!-n` flag is required when using this as backup files do not make sense -when not overwriting the input files. - -.. versionadded:: 3.2.3 - The :option:`!-o` option was added. - -The :option:`!-W` or :option:`!--write-unchanged-files` flag tells 2to3 to always -write output files even if no changes were required to the file. This is most -useful with :option:`!-o` so that an entire Python source tree is copied with -translation from one directory to another. -This option implies the :option:`!-w` flag as it would not make sense otherwise. - -.. versionadded:: 3.2.3 - The :option:`!-W` flag was added. - -The :option:`!--add-suffix` option specifies a string to append to all output -filenames. The :option:`!-n` flag is required when specifying this as backups -are not necessary when writing to different filenames. Example: - -.. code-block:: shell-session - - $ 2to3 -n -W --add-suffix=3 example.py - -Will cause a converted file named ``example.py3`` to be written. - -.. versionadded:: 3.2.3 - The :option:`!--add-suffix` option was added. - -To translate an entire project from one directory tree to another use: - -.. code-block:: shell-session - - $ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode - - -.. _2to3-fixers: - -Fixers ------- - -Each step of transforming code is encapsulated in a fixer. The command ``2to3 --l`` lists them. As :ref:`documented above <2to3-using>`, each can be turned on -and off individually. They are described here in more detail. - - -.. 2to3fixer:: apply - - Removes usage of :func:`apply`. For example ``apply(function, *args, - **kwargs)`` is converted to ``function(*args, **kwargs)``. - -.. 2to3fixer:: asserts - - Replaces deprecated :mod:`unittest` method names with the correct ones. - - ================================ ========================================== - From To - ================================ ========================================== - ``failUnlessEqual(a, b)`` :meth:`assertEqual(a, b) - ` - ``assertEquals(a, b)`` :meth:`assertEqual(a, b) - ` - ``failIfEqual(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``assertNotEquals(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``failUnless(a)`` :meth:`assertTrue(a) - ` - ``assert_(a)`` :meth:`assertTrue(a) - ` - ``failIf(a)`` :meth:`assertFalse(a) - ` - ``failUnlessRaises(exc, cal)`` :meth:`assertRaises(exc, cal) - ` - ``failUnlessAlmostEqual(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``assertAlmostEquals(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``failIfAlmostEqual(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ``assertNotAlmostEquals(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ================================ ========================================== - -.. 2to3fixer:: basestring - - Converts :class:`basestring` to :class:`str`. - -.. 2to3fixer:: buffer - - Converts :class:`buffer` to :class:`memoryview`. This fixer is optional - because the :class:`memoryview` API is similar but not exactly the same as - that of :class:`buffer`. - -.. 2to3fixer:: dict - - Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to - :meth:`dict.items`, :meth:`dict.iterkeys` to :meth:`dict.keys`, and - :meth:`dict.itervalues` to :meth:`dict.values`. Similarly, - :meth:`dict.viewitems`, :meth:`dict.viewkeys` and :meth:`dict.viewvalues` are - converted respectively to :meth:`dict.items`, :meth:`dict.keys` and - :meth:`dict.values`. It also wraps existing usages of :meth:`dict.items`, - :meth:`dict.keys`, and :meth:`dict.values` in a call to :class:`list`. - -.. 2to3fixer:: except - - Converts ``except X, T`` to ``except X as T``. - -.. 2to3fixer:: exec - - Converts the ``exec`` statement to the :func:`exec` function. - -.. 2to3fixer:: execfile - - Removes usage of :func:`execfile`. The argument to :func:`execfile` is - wrapped in calls to :func:`open`, :func:`compile`, and :func:`exec`. - -.. 2to3fixer:: exitfunc - - Changes assignment of :attr:`sys.exitfunc` to use of the :mod:`atexit` - module. - -.. 2to3fixer:: filter - - Wraps :func:`filter` usage in a :class:`list` call. - -.. 2to3fixer:: funcattrs - - Fixes function attributes that have been renamed. For example, - ``my_function.func_closure`` is converted to ``my_function.__closure__``. - -.. 2to3fixer:: future - - Removes ``from __future__ import new_feature`` statements. - -.. 2to3fixer:: getcwdu - - Renames :func:`os.getcwdu` to :func:`os.getcwd`. - -.. 2to3fixer:: has_key - - Changes ``dict.has_key(key)`` to ``key in dict``. - -.. 2to3fixer:: idioms - - This optional fixer performs several transformations that make Python code - more idiomatic. Type comparisons like ``type(x) is SomeClass`` and - ``type(x) == SomeClass`` are converted to ``isinstance(x, SomeClass)``. - ``while 1`` becomes ``while True``. This fixer also tries to make use of - :func:`sorted` in appropriate places. For example, this block :: - - L = list(some_iterable) - L.sort() - - is changed to :: - - L = sorted(some_iterable) - -.. 2to3fixer:: import - - Detects sibling imports and converts them to relative imports. - -.. 2to3fixer:: imports - - Handles module renames in the standard library. - -.. 2to3fixer:: imports2 - - Handles other modules renames in the standard library. It is separate from - the :2to3fixer:`imports` fixer only because of technical limitations. - -.. 2to3fixer:: input - - Converts ``input(prompt)`` to ``eval(input(prompt))``. - -.. 2to3fixer:: intern - - Converts :func:`intern` to :func:`sys.intern`. - -.. 2to3fixer:: isinstance - - Fixes duplicate types in the second argument of :func:`isinstance`. For - example, ``isinstance(x, (int, int))`` is converted to ``isinstance(x, - int)`` and ``isinstance(x, (int, float, int))`` is converted to - ``isinstance(x, (int, float))``. - -.. 2to3fixer:: itertools_imports - - Removes imports of :func:`itertools.ifilter`, :func:`itertools.izip`, and - :func:`itertools.imap`. Imports of :func:`itertools.ifilterfalse` are also - changed to :func:`itertools.filterfalse`. - -.. 2to3fixer:: itertools - - Changes usage of :func:`itertools.ifilter`, :func:`itertools.izip`, and - :func:`itertools.imap` to their built-in equivalents. - :func:`itertools.ifilterfalse` is changed to :func:`itertools.filterfalse`. - -.. 2to3fixer:: long - - Renames :class:`long` to :class:`int`. - -.. 2to3fixer:: map - - Wraps :func:`map` in a :class:`list` call. It also changes ``map(None, x)`` - to ``list(x)``. Using ``from future_builtins import map`` disables this - fixer. - -.. 2to3fixer:: metaclass - - Converts the old metaclass syntax (``__metaclass__ = Meta`` in the class - body) to the new (``class X(metaclass=Meta)``). - -.. 2to3fixer:: methodattrs - - Fixes old method attribute names. For example, ``meth.im_func`` is converted - to ``meth.__func__``. - -.. 2to3fixer:: ne - - Converts the old not-equal syntax, ``<>``, to ``!=``. - -.. 2to3fixer:: next - - Converts the use of iterator's :meth:`~iterator.next` methods to the - :func:`next` function. It also renames :meth:`next` methods to - :meth:`~iterator.__next__`. - -.. 2to3fixer:: nonzero - - Renames definitions of methods called :meth:`__nonzero__` - to :meth:`~object.__bool__`. - -.. 2to3fixer:: numliterals - - Converts octal literals into the new syntax. - -.. 2to3fixer:: operator - - Converts calls to various functions in the :mod:`operator` module to other, - but equivalent, function calls. When needed, the appropriate ``import`` - statements are added, e.g. ``import collections.abc``. The following mapping - are made: - - ================================== ============================================= - From To - ================================== ============================================= - ``operator.isCallable(obj)`` ``callable(obj)`` - ``operator.sequenceIncludes(obj)`` ``operator.contains(obj)`` - ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.abc.Sequence)`` - ``operator.isMappingType(obj)`` ``isinstance(obj, collections.abc.Mapping)`` - ``operator.isNumberType(obj)`` ``isinstance(obj, numbers.Number)`` - ``operator.repeat(obj, n)`` ``operator.mul(obj, n)`` - ``operator.irepeat(obj, n)`` ``operator.imul(obj, n)`` - ================================== ============================================= - -.. 2to3fixer:: paren - - Add extra parenthesis where they are required in list comprehensions. For - example, ``[x for x in 1, 2]`` becomes ``[x for x in (1, 2)]``. - -.. 2to3fixer:: print - - Converts the ``print`` statement to the :func:`print` function. - -.. 2to3fixer:: raise - - Converts ``raise E, V`` to ``raise E(V)``, and ``raise E, V, T`` to ``raise - E(V).with_traceback(T)``. If ``E`` is a tuple, the translation will be - incorrect because substituting tuples for exceptions has been removed in 3.0. - -.. 2to3fixer:: raw_input - - Converts :func:`raw_input` to :func:`input`. - -.. 2to3fixer:: reduce - - Handles the move of :func:`reduce` to :func:`functools.reduce`. - -.. 2to3fixer:: reload - - Converts :func:`reload` to :func:`importlib.reload`. - -.. 2to3fixer:: renames - - Changes :data:`sys.maxint` to :data:`sys.maxsize`. - -.. 2to3fixer:: repr - - Replaces backtick repr with the :func:`repr` function. - -.. 2to3fixer:: set_literal - - Replaces use of the :class:`set` constructor with set literals. This fixer - is optional. - -.. 2to3fixer:: standarderror - - Renames :exc:`StandardError` to :exc:`Exception`. - -.. 2to3fixer:: sys_exc - - Changes the deprecated :data:`sys.exc_value`, :data:`sys.exc_type`, - :data:`sys.exc_traceback` to use :func:`sys.exc_info`. - -.. 2to3fixer:: throw - - Fixes the API change in generator's :meth:`throw` method. - -.. 2to3fixer:: tuple_params - - Removes implicit tuple parameter unpacking. This fixer inserts temporary - variables. - -.. 2to3fixer:: types - - Fixes code broken from the removal of some members in the :mod:`types` - module. - -.. 2to3fixer:: unicode - - Renames :class:`unicode` to :class:`str`. - -.. 2to3fixer:: urllib - - Handles the rename of :mod:`urllib` and :mod:`urllib2` to the :mod:`urllib` - package. - -.. 2to3fixer:: ws_comma - - Removes excess whitespace from comma separated items. This fixer is - optional. - -.. 2to3fixer:: xrange - - Renames :func:`xrange` to :func:`range` and wraps existing :func:`range` - calls with :class:`list`. - -.. 2to3fixer:: xreadlines - - Changes ``for x in file.xreadlines()`` to ``for x in file``. - -.. 2to3fixer:: zip - - Wraps :func:`zip` usage in a :class:`list` call. This is disabled when - ``from future_builtins import zip`` appears. - - -:mod:`lib2to3` --- 2to3's library ---------------------------------- - -.. module:: lib2to3 - :synopsis: The 2to3 library - -.. moduleauthor:: Guido van Rossum -.. moduleauthor:: Collin Winter -.. moduleauthor:: Benjamin Peterson - -**Source code:** :source:`Lib/lib2to3/` - --------------- - -.. deprecated-removed:: 3.11 3.13 - Python 3.9 switched to a PEG parser (see :pep:`617`) while lib2to3 is - using a less flexible LL(1) parser. Python 3.10 includes new language - syntax that is not parsable by lib2to3's LL(1) parser (see :pep:`634`). - The ``lib2to3`` module was marked pending for deprecation in Python 3.9 - (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated - in Python 3.11 (raising :exc:`DeprecationWarning`). - It will be removed from the standard library in Python 3.13. - Consider third-party alternatives such as `LibCST`_ or `parso`_. - -.. note:: - - The :mod:`lib2to3` API should be considered unstable and may change - drastically in the future. - -.. _LibCST: https://libcst.readthedocs.io/ -.. _parso: https://parso.readthedocs.io/ diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index eb6a973cac62d2..b6b1e076c9f08c 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1724,7 +1724,7 @@ Function and class definitions body=[ FunctionDef( name='f', - typeparams=[], + type_params=[], args=arguments( posonlyargs=[], args=[ @@ -1848,7 +1848,7 @@ Function and class definitions body=[ ClassDef( name='Foo', - typeparams=[], + type_params=[], bases=[ Name(id='base1', ctx=Load()), Name(id='base2', ctx=Load())], @@ -1887,7 +1887,7 @@ Async and await body=[ AsyncFunctionDef( name='f', - typeparams=[], + type_params=[], args=arguments( posonlyargs=[], args=[], diff --git a/Doc/library/development.rst b/Doc/library/development.rst index 9edce758688e2d..b1979b921e7d5f 100644 --- a/Doc/library/development.rst +++ b/Doc/library/development.rst @@ -8,8 +8,7 @@ The modules described in this chapter help you write software. For example, the :mod:`pydoc` module takes a module and generates documentation based on the module's contents. The :mod:`doctest` and :mod:`unittest` modules contains frameworks for writing unit tests that automatically exercise code and verify -that the expected output is produced. :program:`2to3` can translate Python 2.x -source code into valid Python 3.x code. +that the expected output is produced. The list of modules described in this chapter is: @@ -23,5 +22,4 @@ The list of modules described in this chapter is: unittest.rst unittest.mock.rst unittest.mock-examples.rst - 2to3.rst test.rst diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index aaf66ea121d39c..e4a473f712846d 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -27,6 +27,5 @@ backwards compatibility. They have been superseded by other modules. sndhdr.rst spwd.rst sunau.rst - telnetlib.rst uu.rst xdrlib.rst diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 891af1bcf7edff..2f330f018a48be 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -318,7 +318,7 @@ be finalized; only the internally used file object will be closed. See the .. versionadded:: 3.2 Added support for the context management protocol. -.. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=1) +.. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=1, stream=False) All following arguments are optional and can be accessed as instance attributes as well. @@ -369,6 +369,9 @@ be finalized; only the internally used file object will be closed. See the The *pax_headers* argument is an optional dictionary of strings which will be added as a pax global header if *format* is :const:`PAX_FORMAT`. + If *stream* is set to :const:`True` then while reading the archive info about files + in the archive are not cached, saving memory. + .. versionchanged:: 3.2 Use ``'surrogateescape'`` as the default for the *errors* argument. @@ -378,6 +381,8 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.6 The *name* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Add the *stream* parameter. .. classmethod:: TarFile.open(...) diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst deleted file mode 100644 index 5a993dc42a5ab2..00000000000000 --- a/Doc/library/telnetlib.rst +++ /dev/null @@ -1,262 +0,0 @@ -:mod:`telnetlib` --- Telnet client -================================== - -.. module:: telnetlib - :synopsis: Telnet client class. - :deprecated: - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/telnetlib.py` - -.. index:: single: protocol; Telnet - -.. deprecated-removed:: 3.11 3.13 - The :mod:`telnetlib` module is deprecated - (see :pep:`PEP 594 <594#telnetlib>` for details and alternatives). - --------------- - -The :mod:`telnetlib` module provides a :class:`Telnet` class that implements the -Telnet protocol. See :rfc:`854` for details about the protocol. In addition, it -provides symbolic constants for the protocol characters (see below), and for the -telnet options. The symbolic names of the telnet options follow the definitions -in ``arpa/telnet.h``, with the leading ``TELOPT_`` removed. For symbolic names -of options which are traditionally not included in ``arpa/telnet.h``, see the -module source itself. - -The symbolic constants for the telnet commands are: IAC, DONT, DO, WONT, WILL, -SE (Subnegotiation End), NOP (No Operation), DM (Data Mark), BRK (Break), IP -(Interrupt process), AO (Abort output), AYT (Are You There), EC (Erase -Character), EL (Erase Line), GA (Go Ahead), SB (Subnegotiation Begin). - -.. include:: ../includes/wasm-notavail.rst - -.. class:: Telnet(host=None, port=0[, timeout]) - - :class:`Telnet` represents a connection to a Telnet server. The instance is - initially not connected by default; the :meth:`~Telnet.open` method must be used to - establish a connection. Alternatively, the host name and optional port - number can be passed to the constructor too, in which case the connection to - the server will be established before the constructor returns. The optional - *timeout* parameter specifies a timeout in seconds for blocking operations - like the connection attempt (if not specified, the global default timeout - setting will be used). - - Do not reopen an already connected instance. - - This class has many :meth:`read_\*` methods. Note that some of them raise - :exc:`EOFError` when the end of the connection is read, because they can return - an empty string for other reasons. See the individual descriptions below. - - A :class:`Telnet` object is a context manager and can be used in a - :keyword:`with` statement. When the :keyword:`!with` block ends, the - :meth:`close` method is called:: - - >>> from telnetlib import Telnet - >>> with Telnet('localhost', 23) as tn: - ... tn.interact() - ... - - .. versionchanged:: 3.6 Context manager support added - - -.. seealso:: - - :rfc:`854` - Telnet Protocol Specification - Definition of the Telnet protocol. - - -.. _telnet-objects: - -Telnet Objects --------------- - -:class:`Telnet` instances have the following methods: - - -.. method:: Telnet.read_until(expected, timeout=None) - - Read until a given byte string, *expected*, is encountered or until *timeout* - seconds have passed. - - When no match is found, return whatever is available instead, possibly empty - bytes. Raise :exc:`EOFError` if the connection is closed and no cooked data - is available. - - -.. method:: Telnet.read_all() - - Read all data until EOF as bytes; block until connection closed. - - -.. method:: Telnet.read_some() - - Read at least one byte of cooked data unless EOF is hit. Return ``b''`` if - EOF is hit. Block if no data is immediately available. - - -.. method:: Telnet.read_very_eager() - - Read everything that can be without blocking in I/O (eager). - - Raise :exc:`EOFError` if connection closed and no cooked data available. - Return ``b''`` if no cooked data available otherwise. Do not block unless in - the midst of an IAC sequence. - - -.. method:: Telnet.read_eager() - - Read readily available data. - - Raise :exc:`EOFError` if connection closed and no cooked data available. - Return ``b''`` if no cooked data available otherwise. Do not block unless in - the midst of an IAC sequence. - - -.. method:: Telnet.read_lazy() - - Process and return data already in the queues (lazy). - - Raise :exc:`EOFError` if connection closed and no data available. Return - ``b''`` if no cooked data available otherwise. Do not block unless in the - midst of an IAC sequence. - - -.. method:: Telnet.read_very_lazy() - - Return any data available in the cooked queue (very lazy). - - Raise :exc:`EOFError` if connection closed and no data available. Return - ``b''`` if no cooked data available otherwise. This method never blocks. - - -.. method:: Telnet.read_sb_data() - - Return the data collected between a SB/SE pair (suboption begin/end). The - callback should access these data when it was invoked with a ``SE`` command. - This method never blocks. - - -.. method:: Telnet.open(host, port=0[, timeout]) - - Connect to a host. The optional second argument is the port number, which - defaults to the standard Telnet port (23). The optional *timeout* parameter - specifies a timeout in seconds for blocking operations like the connection - attempt (if not specified, the global default timeout setting will be used). - - Do not try to reopen an already connected instance. - - .. audit-event:: telnetlib.Telnet.open self,host,port telnetlib.Telnet.open - - -.. method:: Telnet.msg(msg, *args) - - Print a debug message when the debug level is ``>`` 0. If extra arguments are - present, they are substituted in the message using the standard string - formatting operator. - - -.. method:: Telnet.set_debuglevel(debuglevel) - - Set the debug level. The higher the value of *debuglevel*, the more debug - output you get (on ``sys.stdout``). - - -.. method:: Telnet.close() - - Close the connection. - - -.. method:: Telnet.get_socket() - - Return the socket object used internally. - - -.. method:: Telnet.fileno() - - Return the file descriptor of the socket object used internally. - - -.. method:: Telnet.write(buffer) - - Write a byte string to the socket, doubling any IAC characters. This can - block if the connection is blocked. May raise :exc:`OSError` if the - connection is closed. - - .. audit-event:: telnetlib.Telnet.write self,buffer telnetlib.Telnet.write - - .. versionchanged:: 3.3 - This method used to raise :exc:`socket.error`, which is now an alias - of :exc:`OSError`. - - -.. method:: Telnet.interact() - - Interaction function, emulates a very dumb Telnet client. - - -.. method:: Telnet.mt_interact() - - Multithreaded version of :meth:`interact`. - - -.. method:: Telnet.expect(list, timeout=None) - - Read until one from a list of a regular expressions matches. - - The first argument is a list of regular expressions, either compiled - (:ref:`regex objects `) or uncompiled (byte strings). The - optional second argument is a timeout, in seconds; the default is to block - indefinitely. - - Return a tuple of three items: the index in the list of the first regular - expression that matches; the match object returned; and the bytes read up - till and including the match. - - If end of file is found and no bytes were read, raise :exc:`EOFError`. - Otherwise, when nothing matches, return ``(-1, None, data)`` where *data* is - the bytes received so far (may be empty bytes if a timeout happened). - - If a regular expression ends with a greedy match (such as ``.*``) or if more - than one expression can match the same input, the results are - non-deterministic, and may depend on the I/O timing. - - -.. method:: Telnet.set_option_negotiation_callback(callback) - - Each time a telnet option is read on the input flow, this *callback* (if set) is - called with the following parameters: callback(telnet socket, command - (DO/DONT/WILL/WONT), option). No other action is done afterwards by telnetlib. - - -.. _telnet-example: - -Telnet Example --------------- - -.. sectionauthor:: Peter Funk - - -A simple example illustrating typical use:: - - import getpass - import telnetlib - - HOST = "localhost" - user = input("Enter your remote account: ") - password = getpass.getpass() - - tn = telnetlib.Telnet(HOST) - - tn.read_until(b"login: ") - tn.write(user.encode('ascii') + b"\n") - if password: - tn.read_until(b"Password: ") - tn.write(password.encode('ascii') + b"\n") - - tn.write(b"ls\n") - tn.write(b"exit\n") - - print(tn.read_all().decode('ascii')) - diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c90cb411acde07..92943c46ef5132 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2074,10 +2074,6 @@ Other concrete types represent the types of I/O streams such as returned by :func:`open`. - .. deprecated-removed:: 3.8 3.13 - The ``typing.io`` namespace is deprecated and will be removed. - These types should be directly imported from ``typing`` instead. - .. class:: Pattern Match @@ -2088,10 +2084,6 @@ Other concrete types ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or ``Match[bytes]``. - .. deprecated-removed:: 3.8 3.13 - The ``typing.re`` namespace is deprecated and will be removed. - These types should be directly imported from ``typing`` instead. - .. deprecated:: 3.9 Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. See :pep:`585` and :ref:`types-genericalias`. @@ -2981,9 +2973,6 @@ convenience. This is subject to change, and not all deprecations are listed. +----------------------------------+---------------+-------------------+----------------+ | Feature | Deprecated in | Projected removal | PEP/issue | +==================================+===============+===================+================+ -| ``typing.io`` and ``typing.re`` | 3.8 | 3.13 | :issue:`38291` | -| submodules | | | | -+----------------------------------+---------------+-------------------+----------------+ | ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | | collections | | | | +----------------------------------+---------------+-------------------+----------------+ diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b97a08f25d92a2..0c700f908d6878 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -595,12 +595,19 @@ is already executing raises a :exc:`ValueError` exception. .. method:: generator.close() Raises a :exc:`GeneratorExit` at the point where the generator function was - paused. If the generator function then exits gracefully, is already closed, - or raises :exc:`GeneratorExit` (by not catching the exception), close - returns to its caller. If the generator yields a value, a - :exc:`RuntimeError` is raised. If the generator raises any other exception, - it is propagated to the caller. :meth:`close` does nothing if the generator - has already exited due to an exception or normal exit. + paused. If the generator function catches the exception and returns a + value, this value is returned from :meth:`close`. If the generator function + is already closed, or raises :exc:`GeneratorExit` (by not catching the + exception), :meth:`close` returns :const:`None`. If the generator yields a + value, a :exc:`RuntimeError` is raised. If the generator raises any other + exception, it is propagated to the caller. If the generator has already + exited due to an exception or normal exit, :meth:`close` returns + :const:`None` and has no other effect. + + .. versionchanged:: 3.13 + + If a generator returns a value upon being closed, the value is returned + by :meth:`close`. .. index:: single: yield; examples diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 1d3503bf06f085..7a711031d74e9a 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -72,7 +72,6 @@ Doc/howto/sorting.rst Doc/howto/unicode.rst Doc/howto/urllib2.rst Doc/install/index.rst -Doc/library/2to3.rst Doc/library/__future__.rst Doc/library/_thread.rst Doc/library/abc.rst @@ -215,7 +214,6 @@ Doc/library/sys_path_init.rst Doc/library/sysconfig.rst Doc/library/syslog.rst Doc/library/tarfile.rst -Doc/library/telnetlib.rst Doc/library/tempfile.rst Doc/library/termios.rst Doc/library/test.rst diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index cd8d9febb0d13b..8a2eb07a69a69a 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -41,7 +41,7 @@ ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' GH_ISSUE_URI = 'https://github.com/python/cpython/issues/%s' -SOURCE_URI = 'https://github.com/python/cpython/tree/main/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/3.12/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body @@ -710,7 +710,6 @@ def setup(app): app.add_builder(PydocTopicsBuilder) app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command) - app.add_object_type('2to3fixer', '2to3fixer', '%s (2to3 fixer)') app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction) app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index b71c61089e6dc1..42ebf2b3d294a8 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -10,13 +10,13 @@ Using the Python Interpreter Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.12` +The Python interpreter is usually installed as :file:`/usr/local/bin/python3.13` on those machines where it is available; putting :file:`/usr/local/bin` in your Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.12 + python3.13 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -24,7 +24,7 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) On Windows machines where you have installed Python from the :ref:`Microsoft Store -`, the :file:`python3.12` command will be available. If you have +`, the :file:`python3.13` command will be available. If you have the :ref:`py.exe launcher ` installed, you can use the :file:`py` command. See :ref:`setting-envvars` for other ways to launch Python. @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.12 - Python 3.12 (default, April 4 2022, 09:25:04) + $ python3.13 + Python 3.13 (default, April 4 2023, 09:25:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index ce858ab2c8d79e..fbe280d6413170 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -314,6 +314,13 @@ also be used to improve performance. is dependent on a combination of the build environment + the other optimization configure args + the CPU architecture, and not all combinations are supported. + BOLT versions before LLVM 16 are known to crash BOLT under some scenarios. + Use of LLVM 16 or newer for BOLT optimization is strongly encouraged. + + The :envvar:`!BOLT_INSTRUMENT_FLAGS` and :envvar:`!BOLT_APPLY_FLAGS` + :program:`configure` variables can be defined to override the default set of + arguments for :program:`llvm-bolt` to instrument and apply BOLT data to + binaries, respectively. .. versionadded:: 3.12 diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 84bb651e68eed5..7236e1cad7c8c6 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -2465,7 +2465,7 @@ changes, or look through the Subversion logs for all the details. (All changes contributed by Lars Gustäbel). * An optional ``timeout`` parameter was added to the - :class:`telnetlib.Telnet` class constructor, specifying a timeout + :class:`!telnetlib.Telnet` class constructor, specifying a timeout measured in seconds. (Added by Facundo Batista.) * The :class:`tempfile.NamedTemporaryFile` class usually deletes diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index f9ac13036cbc8d..b8cd7c48b359b2 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -911,7 +911,7 @@ best strategy is the following: tests still pass. 3. Run the ``2to3`` source-to-source translator over your source code - tree. (See :ref:`2to3-reference` for more on this tool.) Run the + tree. Run the result of the translation under Python 3.0. Manually fix up any remaining issues, fixing problems until all tests pass again. diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 7a479c6e56a9f0..8aadc2a0a581f2 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1731,7 +1731,7 @@ Modules slated for removal in Python 3.13: +---------------------+---------------------+---------------------+---------------------+---------------------+ - | :mod:`aifc` | :mod:`chunk` | :mod:`msilib` | :mod:`pipes` | :mod:`telnetlib` | + | :mod:`aifc` | :mod:`chunk` | :mod:`msilib` | :mod:`pipes` | :mod:`!telnetlib` | +---------------------+---------------------+---------------------+---------------------+---------------------+ | :mod:`audioop` | :mod:`crypt` | :mod:`nis` | :mod:`sndhdr` | :mod:`uu` | +---------------------+---------------------+---------------------+---------------------+---------------------+ @@ -1748,7 +1748,7 @@ Modules warnings have now been updated to note they will be removed in Python 3.12. (Contributed by Hugo van Kemenade in :issue:`47022`.) -* The :mod:`lib2to3` package and :ref:`2to3 <2to3-reference>` tool +* The :mod:`!lib2to3` package and ``2to3`` tool are now deprecated and may not be able to parse Python 3.10 or newer. See :pep:`617`, introducing the new PEG parser, for details. (Contributed by Victor Stinner in :issue:`40360`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8a0a59ba7301e1..5e07a4caeb9ebe 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -532,6 +532,17 @@ threading profiling functions in all running threads in addition to the calling one. (Contributed by Pablo Galindo in :gh:`93503`.) +tkinter +------- + +* ``tkinter.Canvas.coords()`` now flattens its arguments. + It now accepts not only coordinates as separate arguments + (``x1, y1, x2, y2, ...``) and a sequence of coordinates + (``[x1, y1, x2, y2, ...]``), but also coordinates grouped in pairs + (``(x1, y1), (x2, y2), ...`` and ``[(x1, y1), (x2, y2), ...]``), + like ``create_*()`` methods. + (Contributed by Serhiy Storchaka in :gh:`94473`.) + types ----- @@ -774,6 +785,16 @@ Deprecated ``int``, convert to int explicitly with ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) +* :class:`datetime.datetime`'s + :meth:`~datetime.datetime.utcnow` and + :meth:`~datetime.datetime.utcfromtimestamp` are deprecated and will be + removed in a future version. Instead, use timezone-aware objects to represent + datetimes in UTC: respectively, call + :meth:`~datetime.datetime.now` and + :meth:`~datetime.datetime.fromtimestamp` with the *tz* parameter set to + :attr:`datetime.UTC`. + (Contributed by Paul Ganssle in :gh:`103857`.) + Pending Removal in Python 3.13 ------------------------------ @@ -798,7 +819,7 @@ Modules (see :pep:`594`): * :mod:`sndhdr` * :mod:`spwd` * :mod:`sunau` -* :mod:`telnetlib` +* :mod:`!telnetlib` * :mod:`uu` * :mod:`xdrlib` @@ -1392,6 +1413,12 @@ Porting to Python 3.12 :py:meth:`~class.__subclasses__` (using :c:func:`PyObject_CallMethod`, for example). +* Add support of more formatting options (left aligning, octals, uppercase + hexadecimals, ``intmax_t``, ``ptrdiff_t``, ``wchar_t`` C + strings, variable width and precision) in :c:func:`PyUnicode_FromFormat` and + :c:func:`PyUnicode_FromFormatV`. + (Contributed by Serhiy Storchaka in :gh:`98836`.) + * An unrecognized format character in :c:func:`PyUnicode_FromFormat` and :c:func:`PyUnicode_FromFormatV` now sets a :exc:`SystemError`. In previous versions it caused all the rest of the format string to be @@ -1449,6 +1476,15 @@ Porting to Python 3.12 Note that :c:func:`PyType_FromMetaclass` (added in Python 3.12) already disallows creating classes whose metaclass overrides ``tp_new``. +* :c:var:`PyOS_InputHook` and :c:var:`PyOS_ReadlineFunctionPointer` are no + longer called in :ref:`subinterpreters `. This is + because clients generally rely on process-wide global state (since these + callbacks have no way of recovering extension module state). + + This also avoids situations where extensions may find themselves running in a + subinterpreter that they don't support (or haven't yet been loaded in). See + :gh:`104668` for more info. + Deprecated ---------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst new file mode 100644 index 00000000000000..309f26ea27dbfd --- /dev/null +++ b/Doc/whatsnew/3.13.rst @@ -0,0 +1,146 @@ + +**************************** + What's New In Python 3.13 +**************************** + +:Release: |release| +:Date: |today| + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.13, compared to 3.12. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.13 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.13. + Brevity is key. + + +.. PEP-sized items next. + + + +New Features +============ + + + +Other Language Changes +====================== + + + +New Modules +=========== + +* None yet. + + +Improved Modules +================ + + +Optimizations +============= + + + + +Deprecated +========== + + + +Removed +======= + +* :pep:`594`: Remove the :mod:`!telnetlib` module, deprecated in Python 3.11: + use the projects `telnetlib3 `_ or + `Exscript `_ instead. + (Contributed by Victor Stinner in :gh:`104773`.) + +* Remove the ``2to3`` program and the :mod:`!lib2to3` module, + deprecated in Python 3.11. + (Contributed by Victor Stinner in :gh:`104780`.) + +* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, + are now removed. The items in those namespaces can be imported directly + from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) + +* Remove support for using :class:`pathlib.Path` objects as context managers. + This functionality was deprecated and made a no-op in Python 3.9. + +Porting to Python 3.13 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Build Changes +============= + + +C API Changes +============= + +New Features +------------ + +Porting to Python 3.13 +---------------------- + +Deprecated +---------- + +Removed +------- + diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3a681754e25dd7..1cd33dfb60421a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -1520,7 +1520,7 @@ rather than the version that is being emulated for the process telnetlib --------- -:class:`~telnetlib.Telnet` is now a context manager (contributed by +:class:`!telnetlib.Telnet` is now a context manager (contributed by Stéphane Wirtel in :issue:`25485`). diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index fd86db96302356..532cabdd7a75be 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -935,10 +935,10 @@ Deprecated * :func:`smtpd.MailmanProxy` is now deprecated as it is unusable without an external module, ``mailman``. (Contributed by Samuel Colvin in :issue:`35800`.) -* The :mod:`lib2to3` module now emits a :exc:`PendingDeprecationWarning`. +* The :mod:`!lib2to3` module now emits a :exc:`PendingDeprecationWarning`. Python 3.9 switched to a PEG parser (see :pep:`617`), and Python 3.10 may include new language syntax that is not parsable by lib2to3's LL(1) parser. - The ``lib2to3`` module may be removed from the standard library in a future + The :mod:`!lib2to3` module may be removed from the standard library in a future Python version. Consider third-party alternatives such as `LibCST`_ or `parso`_. (Contributed by Carl Meyer in :issue:`40360`.) diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index bfee225791eee9..b9c19602653219 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.13.rst 3.12.rst 3.11.rst 3.10.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index c79207b9cb51a4..e6a983429e39d8 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -640,12 +640,12 @@ type_alias[stmt_ty]: # Type parameter declaration # -------------------------- -type_params[asdl_typeparam_seq*]: '[' t=type_param_seq ']' { - CHECK_VERSION(asdl_typeparam_seq *, 12, "Type parameter lists are", t) } +type_params[asdl_type_param_seq*]: '[' t=type_param_seq ']' { + CHECK_VERSION(asdl_type_param_seq *, 12, "Type parameter lists are", t) } -type_param_seq[asdl_typeparam_seq*]: a[asdl_typeparam_seq*]=','.type_param+ [','] { a } +type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ [','] { a } -type_param[typeparam_ty] (memo): +type_param[type_param_ty] (memo): | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) } | '*' a=NAME colon=":" e=expression { RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 0f569935fff14a..692c69ba76db2f 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -104,9 +104,10 @@ _PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits); #define _PyLong_SIGN_MASK 3 #define _PyLong_NON_SIZE_BITS 3 + static inline int _PyLong_IsCompact(const PyLongObject* op) { - assert(PyLong_Check(op)); + assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); return op->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS); } @@ -115,7 +116,7 @@ _PyLong_IsCompact(const PyLongObject* op) { static inline Py_ssize_t _PyLong_CompactValue(const PyLongObject *op) { - assert(PyLong_Check(op)); + assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); Py_ssize_t sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); return sign * (Py_ssize_t)op->long_value.ob_digit[0]; diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 9f1cef0541508c..06a40239a2473a 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -51,7 +51,7 @@ typedef struct _pattern *pattern_ty; typedef struct _type_ignore *type_ignore_ty; -typedef struct _typeparam *typeparam_ty; +typedef struct _type_param *type_param_ty; typedef struct { @@ -151,10 +151,11 @@ asdl_type_ignore_seq *_Py_asdl_type_ignore_seq_new(Py_ssize_t size, PyArena typedef struct { _ASDL_SEQ_HEAD - typeparam_ty typed_elements[1]; -} asdl_typeparam_seq; + type_param_ty typed_elements[1]; +} asdl_type_param_seq; -asdl_typeparam_seq *_Py_asdl_typeparam_seq_new(Py_ssize_t size, PyArena *arena); +asdl_type_param_seq *_Py_asdl_type_param_seq_new(Py_ssize_t size, PyArena + *arena); enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3, @@ -197,7 +198,7 @@ struct _stmt { union { struct { identifier name; - asdl_typeparam_seq *typeparams; + asdl_type_param_seq *type_params; arguments_ty args; asdl_stmt_seq *body; asdl_expr_seq *decorator_list; @@ -207,7 +208,7 @@ struct _stmt { struct { identifier name; - asdl_typeparam_seq *typeparams; + asdl_type_param_seq *type_params; arguments_ty args; asdl_stmt_seq *body; asdl_expr_seq *decorator_list; @@ -217,7 +218,7 @@ struct _stmt { struct { identifier name; - asdl_typeparam_seq *typeparams; + asdl_type_param_seq *type_params; asdl_expr_seq *bases; asdl_keyword_seq *keywords; asdl_stmt_seq *body; @@ -240,7 +241,7 @@ struct _stmt { struct { expr_ty name; - asdl_typeparam_seq *typeparams; + asdl_type_param_seq *type_params; expr_ty value; } TypeAlias; @@ -649,9 +650,9 @@ struct _type_ignore { } v; }; -enum _typeparam_kind {TypeVar_kind=1, ParamSpec_kind=2, TypeVarTuple_kind=3}; -struct _typeparam { - enum _typeparam_kind kind; +enum _type_param_kind {TypeVar_kind=1, ParamSpec_kind=2, TypeVarTuple_kind=3}; +struct _type_param { + enum _type_param_kind kind; union { struct { identifier name; @@ -681,18 +682,18 @@ mod_ty _PyAST_Interactive(asdl_stmt_seq * body, PyArena *arena); mod_ty _PyAST_Expression(expr_ty body, PyArena *arena); mod_ty _PyAST_FunctionType(asdl_expr_seq * argtypes, expr_ty returns, PyArena *arena); -stmt_ty _PyAST_FunctionDef(identifier name, asdl_typeparam_seq * typeparams, +stmt_ty _PyAST_FunctionDef(identifier name, asdl_type_param_seq * type_params, arguments_ty args, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, expr_ty returns, string type_comment, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -stmt_ty _PyAST_AsyncFunctionDef(identifier name, asdl_typeparam_seq * - typeparams, arguments_ty args, asdl_stmt_seq * +stmt_ty _PyAST_AsyncFunctionDef(identifier name, asdl_type_param_seq * + type_params, arguments_ty args, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, expr_ty returns, string type_comment, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -stmt_ty _PyAST_ClassDef(identifier name, asdl_typeparam_seq * typeparams, +stmt_ty _PyAST_ClassDef(identifier name, asdl_type_param_seq * type_params, asdl_expr_seq * bases, asdl_keyword_seq * keywords, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, int lineno, int col_offset, int end_lineno, int @@ -704,9 +705,9 @@ stmt_ty _PyAST_Delete(asdl_expr_seq * targets, int lineno, int col_offset, int stmt_ty _PyAST_Assign(asdl_expr_seq * targets, expr_ty value, string type_comment, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -stmt_ty _PyAST_TypeAlias(expr_ty name, asdl_typeparam_seq * typeparams, expr_ty - value, int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); +stmt_ty _PyAST_TypeAlias(expr_ty name, asdl_type_param_seq * type_params, + expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); @@ -891,14 +892,14 @@ pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena); -typeparam_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int - col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -typeparam_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -typeparam_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena - *arena); +type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +type_param_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +type_param_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena + *arena); PyObject* PyAST_mod2obj(mod_ty t); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index e723ead577b888..0c0d53f3e5d7e9 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -248,8 +248,8 @@ struct ast_state { PyObject *type_comment; PyObject *type_ignore_type; PyObject *type_ignores; - PyObject *typeparam_type; - PyObject *typeparams; + PyObject *type_param_type; + PyObject *type_params; PyObject *unaryop_type; PyObject *upper; PyObject *value; diff --git a/Include/object.h b/Include/object.h index 81aeb2d8bd5a69..c4fe2f83ef62bc 100644 --- a/Include/object.h +++ b/Include/object.h @@ -590,7 +590,7 @@ you can count such references to the type object.) extern Py_ssize_t _Py_RefTotal; # define _Py_INC_REFTOTAL() _Py_RefTotal++ # define _Py_DEC_REFTOTAL() _Py_RefTotal-- -# elif !defined(Py_LIMITED_API) || Py_LIMITED_API+0 > 0x030C0000 +# elif !defined(Py_LIMITED_API) || Py_LIMITED_API+0 > 0x030D0000 PyAPI_FUNC(void) _Py_IncRefTotal_DO_NOT_USE_THIS(void); PyAPI_FUNC(void) _Py_DecRefTotal_DO_NOT_USE_THIS(void); # define _Py_INC_REFTOTAL() _Py_IncRefTotal_DO_NOT_USE_THIS() diff --git a/Include/patchlevel.h b/Include/patchlevel.h index aaedd563a905e9..ae9d36c12d6eaf 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -17,13 +17,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 12 +#define PY_MINOR_VERSION 13 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 7 +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.12.0a7+" +#define PY_VERSION "3.13.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/ast.py b/Lib/ast.py index 08904afb203183..226910ecac0b4a 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1051,7 +1051,7 @@ def visit_ClassDef(self, node): self.fill("@") self.traverse(deco) self.fill("class " + node.name) - self._typeparams_helper(node.typeparams) + self._type_params_helper(node.type_params) with self.delimit_if("(", ")", condition = node.bases or node.keywords): comma = False for e in node.bases: @@ -1083,7 +1083,7 @@ def _function_helper(self, node, fill_suffix): self.traverse(deco) def_str = fill_suffix + " " + node.name self.fill(def_str) - self._typeparams_helper(node.typeparams) + self._type_params_helper(node.type_params) with self.delimit("(", ")"): self.traverse(node.args) if node.returns: @@ -1092,10 +1092,10 @@ def _function_helper(self, node, fill_suffix): with self.block(extra=self.get_type_comment(node)): self._write_docstring_and_traverse_body(node) - def _typeparams_helper(self, typeparams): - if typeparams is not None and len(typeparams) > 0: + def _type_params_helper(self, type_params): + if type_params is not None and len(type_params) > 0: with self.delimit("[", "]"): - self.interleave(lambda: self.write(", "), self.traverse, typeparams) + self.interleave(lambda: self.write(", "), self.traverse, type_params) def visit_TypeVar(self, node): self.write(node.name) @@ -1112,7 +1112,7 @@ def visit_ParamSpec(self, node): def visit_TypeAlias(self, node): self.fill("type ") self.traverse(node.name) - self._typeparams_helper(node.typeparams) + self._type_params_helper(node.type_params) self.write(" = ") self.traverse(node.value) diff --git a/Lib/enum.py b/Lib/enum.py index 6e497f7ef6a7de..b50fe50d8258d3 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -388,16 +388,8 @@ def __setitem__(self, key, value): Single underscore (sunder) names are reserved. """ - if _is_internal_class(self._cls_name, value): - import warnings - warnings.warn( - "In 3.13 classes created inside an enum will not become a member. " - "Use the `member` decorator to keep the current behavior.", - DeprecationWarning, - stacklevel=2, - ) if _is_private(self._cls_name, key): - # also do nothing, name will be a normal attribute + # do nothing, name will be a normal attribute pass elif _is_sunder(key): if key not in ( @@ -440,10 +432,9 @@ def __setitem__(self, key, value): value = value.value elif _is_descriptor(value): pass - # TODO: uncomment next three lines in 3.13 - # elif _is_internal_class(self._cls_name, value): - # # do nothing, name will be a normal attribute - # pass + elif _is_internal_class(self._cls_name, value): + # do nothing, name will be a normal attribute + pass else: if key in self: # enum overwriting a descriptor? @@ -1169,28 +1160,13 @@ def _generate_next_value_(name, start, count, last_values): if not last_values: return start try: - last = last_values[-1] - last_values.sort() - if last == last_values[-1]: - # no difference between old and new methods - return last + 1 - else: - # trigger old method (with warning) - raise TypeError + last_value = sorted(last_values).pop() except TypeError: - import warnings - warnings.warn( - "In 3.13 the default `auto()`/`_generate_next_value_` will require all values to be sortable and support adding +1\n" - "and the value returned will be the largest value in the enum incremented by 1", - DeprecationWarning, - stacklevel=3, - ) - for v in last_values: - try: - return v + 1 - except TypeError: - pass - return start + raise TypeError('unable to sort non-numeric values') from None + try: + return last_value + 1 + except TypeError: + raise TypeError('unable to increment %r' % (last_value, )) from None @classmethod def _missing_(cls, value): diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 21402ad7139173..df36be8766016f 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -1643,19 +1643,13 @@ def tokeneater(self, type, token, start, end, line, self.finished = 1 def run(self): - save_tabsize = tokenize.tabsize - tokenize.tabsize = self.tabwidth try: - try: - tokens = tokenize.generate_tokens(self.readline) - for token in tokens: - self.tokeneater(*token) - except (tokenize.TokenError, SyntaxError): - # since we cut off the tokenizer early, we can trigger - # spurious errors - pass - finally: - tokenize.tabsize = save_tabsize + tokens = tokenize.generate_tokens(self.readline) + for token in tokens: + self.tokeneater(*token) + except (tokenize.TokenError, SyntaxError): + # Stopping the tokenizer early can trigger spurious errors. + pass return self.blkopenline, self.indentedline ### end autoindent code ### diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt deleted file mode 100644 index fa7b15061d941c..00000000000000 --- a/Lib/lib2to3/Grammar.txt +++ /dev/null @@ -1,196 +0,0 @@ -# Grammar for 2to3. This grammar supports Python 2.x and 3.x. - -# NOTE WELL: You should also follow all the steps listed at -# https://devguide.python.org/grammar/ - -# Start symbols for the grammar: -# file_input is a module or sequence of commands read from an input file; -# single_input is a single interactive statement; -# eval_input is the input for the eval() and input() functions. -# NB: compound_stmt in single_input is followed by extra NEWLINE! -file_input: (NEWLINE | stmt)* ENDMARKER -single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE -eval_input: testlist NEWLINE* ENDMARKER - -decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE -decorators: decorator+ -decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: ASYNC funcdef -funcdef: 'def' NAME parameters ['->' test] ':' suite -parameters: '(' [typedargslist] ')' - -# The following definition for typedarglist is equivalent to this set of rules: -# -# arguments = argument (',' argument)* -# argument = tfpdef ['=' test] -# kwargs = '**' tname [','] -# args = '*' [tname] -# kwonly_kwargs = (',' argument)* [',' [kwargs]] -# args_kwonly_kwargs = args kwonly_kwargs | kwargs -# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] -# typedargslist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs -# typedarglist = arguments ',' '/' [',' [typedargslist_no_posonly]])|(typedargslist_no_posonly)" -# -# It needs to be fully expanded to allow our LL(1) parser to work on it. - -typedargslist: tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [ - ',' [((tfpdef ['=' test] ',')* ('*' [tname] (',' tname ['=' test])* - [',' ['**' tname [',']]] | '**' tname [',']) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])] - ] | ((tfpdef ['=' test] ',')* ('*' [tname] (',' tname ['=' test])* - [',' ['**' tname [',']]] | '**' tname [',']) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) - -tname: NAME [':' test] -tfpdef: tname | '(' tfplist ')' -tfplist: tfpdef (',' tfpdef)* [','] - -# The following definition for varargslist is equivalent to this set of rules: -# -# arguments = argument (',' argument )* -# argument = vfpdef ['=' test] -# kwargs = '**' vname [','] -# args = '*' [vname] -# kwonly_kwargs = (',' argument )* [',' [kwargs]] -# args_kwonly_kwargs = args kwonly_kwargs | kwargs -# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] -# vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs -# varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] | (vararglist_no_posonly) -# -# It needs to be fully expanded to allow our LL(1) parser to work on it. - -varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ - ((vfpdef ['=' test] ',')* ('*' [vname] (',' vname ['=' test])* - [',' ['**' vname [',']]] | '**' vname [',']) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) - ]] | ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]]| '**' vname [',']) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) - -vname: NAME -vfpdef: vname | '(' vfplist ')' -vfplist: vfpdef (',' vfpdef)* [','] - -stmt: simple_stmt | compound_stmt -simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE -small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | - import_stmt | global_stmt | exec_stmt | assert_stmt) -expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist_star_expr))*) -annassign: ':' test ['=' test] -testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] -augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | - '<<=' | '>>=' | '**=' | '//=') -# For normal and annotated assignments, additional restrictions enforced by the interpreter -print_stmt: 'print' ( [ test (',' test)* [','] ] | - '>>' test [ (',' test)+ [','] ] ) -del_stmt: 'del' exprlist -pass_stmt: 'pass' -flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt -break_stmt: 'break' -continue_stmt: 'continue' -return_stmt: 'return' [testlist_star_expr] -yield_stmt: yield_expr -raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]] -import_stmt: import_name | import_from -import_name: 'import' dotted_as_names -import_from: ('from' ('.'* dotted_name | '.'+) - 'import' ('*' | '(' import_as_names ')' | import_as_names)) -import_as_name: NAME ['as' NAME] -dotted_as_name: dotted_name ['as' NAME] -import_as_names: import_as_name (',' import_as_name)* [','] -dotted_as_names: dotted_as_name (',' dotted_as_name)* -dotted_name: NAME ('.' NAME)* -global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* -exec_stmt: 'exec' expr ['in' test [',' test]] -assert_stmt: 'assert' test [',' test] - -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: ASYNC (funcdef | with_stmt | for_stmt) -if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] -while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] -for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] -try_stmt: ('try' ':' suite - ((except_clause ':' suite)+ - ['else' ':' suite] - ['finally' ':' suite] | - 'finally' ':' suite)) -with_stmt: 'with' with_item (',' with_item)* ':' suite -with_item: test ['as' expr] -with_var: 'as' expr -# NB compile.c makes sure that the default except clause is last -except_clause: 'except' [test [(',' | 'as') test]] -suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT - -# Backward compatibility cruft to support: -# [ x for x in lambda: True, lambda: False if x() ] -# even while also allowing: -# lambda x: 5 if x else 2 -# (But not a mix of the two) -testlist_safe: old_test [(',' old_test)+ [',']] -old_test: or_test | old_lambdef -old_lambdef: 'lambda' [varargslist] ':' old_test - -namedexpr_test: test [':=' test] -test: or_test ['if' or_test 'else' test] | lambdef -or_test: and_test ('or' and_test)* -and_test: not_test ('and' not_test)* -not_test: 'not' not_test | comparison -comparison: expr (comp_op expr)* -comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' -star_expr: '*' expr -expr: xor_expr ('|' xor_expr)* -xor_expr: and_expr ('^' and_expr)* -and_expr: shift_expr ('&' shift_expr)* -shift_expr: arith_expr (('<<'|'>>') arith_expr)* -arith_expr: term (('+'|'-') term)* -term: factor (('*'|'@'|'/'|'%'|'//') factor)* -factor: ('+'|'-'|'~') factor | power -power: [AWAIT] atom trailer* ['**' factor] -atom: ('(' [yield_expr|testlist_gexp] ')' | - '[' [listmaker] ']' | - '{' [dictsetmaker] '}' | - '`' testlist1 '`' | - NAME | NUMBER | STRING+ | '.' '.' '.') -listmaker: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) -testlist_gexp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) -lambdef: 'lambda' [varargslist] ':' test -trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -subscriptlist: subscript (',' subscript)* [','] -subscript: test | [test] ':' [test] [sliceop] -sliceop: ':' [test] -exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] -testlist: test (',' test)* [','] -dictsetmaker: ( ((test ':' test | '**' expr) - (comp_for | (',' (test ':' test | '**' expr))* [','])) | - ((test | star_expr) - (comp_for | (',' (test | star_expr))* [','])) ) - -classdef: 'class' NAME ['(' [arglist] ')'] ':' suite - -arglist: argument (',' argument)* [','] - -# "test '=' test" is really "keyword '=' test", but we have no such token. -# These need to be in a single rule to avoid grammar that is ambiguous -# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, -# we explicitly match '*' here, too, to give it proper precedence. -# Illegal combinations and orderings are blocked in ast.c: -# multiple (test comp_for) arguments are blocked; keyword unpackings -# that precede iterable unpackings are blocked; etc. -argument: ( test [comp_for] | - test ':=' test | - test '=' test | - '**' test | - '*' test ) - -comp_iter: comp_for | comp_if -comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [comp_iter] -comp_if: 'if' old_test [comp_iter] - -testlist1: test (',' test)* - -# not used in grammar, but may appear in "node" passed from Parser to Compiler -encoding_decl: NAME - -yield_expr: 'yield' [yield_arg] -yield_arg: 'from' test | testlist_star_expr diff --git a/Lib/lib2to3/PatternGrammar.txt b/Lib/lib2to3/PatternGrammar.txt deleted file mode 100644 index 36bf8148273bd7..00000000000000 --- a/Lib/lib2to3/PatternGrammar.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# A grammar to describe tree matching patterns. -# Not shown here: -# - 'TOKEN' stands for any token (leaf node) -# - 'any' stands for any node (leaf or interior) -# With 'any' we can still specify the sub-structure. - -# The start symbol is 'Matcher'. - -Matcher: Alternatives ENDMARKER - -Alternatives: Alternative ('|' Alternative)* - -Alternative: (Unit | NegatedUnit)+ - -Unit: [NAME '='] ( STRING [Repeater] - | NAME [Details] [Repeater] - | '(' Alternatives ')' [Repeater] - | '[' Alternatives ']' - ) - -NegatedUnit: 'not' (STRING | NAME [Details] | '(' Alternatives ')') - -Repeater: '*' | '+' | '{' NUMBER [',' NUMBER] '}' - -Details: '<' Alternatives '>' diff --git a/Lib/lib2to3/__init__.py b/Lib/lib2to3/__init__.py deleted file mode 100644 index 177405c8090d3e..00000000000000 --- a/Lib/lib2to3/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -import warnings - - -warnings.warn( - "lib2to3 package is deprecated and may not be able to parse Python 3.10+", - DeprecationWarning, - stacklevel=2, -) diff --git a/Lib/lib2to3/__main__.py b/Lib/lib2to3/__main__.py deleted file mode 100644 index 80688baf27abfc..00000000000000 --- a/Lib/lib2to3/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys -from .main import main - -sys.exit(main("lib2to3.fixes")) diff --git a/Lib/lib2to3/btm_matcher.py b/Lib/lib2to3/btm_matcher.py deleted file mode 100644 index 3b78868038bda0..00000000000000 --- a/Lib/lib2to3/btm_matcher.py +++ /dev/null @@ -1,163 +0,0 @@ -"""A bottom-up tree matching algorithm implementation meant to speed -up 2to3's matching process. After the tree patterns are reduced to -their rarest linear path, a linear Aho-Corasick automaton is -created. The linear automaton traverses the linear paths from the -leaves to the root of the AST and returns a set of nodes for further -matching. This reduces significantly the number of candidate nodes.""" - -__author__ = "George Boutsioukis " - -import logging -import itertools -from collections import defaultdict - -from . import pytree -from .btm_utils import reduce_tree - -class BMNode(object): - """Class for a node of the Aho-Corasick automaton used in matching""" - count = itertools.count() - def __init__(self): - self.transition_table = {} - self.fixers = [] - self.id = next(BMNode.count) - self.content = '' - -class BottomMatcher(object): - """The main matcher class. After instantiating the patterns should - be added using the add_fixer method""" - - def __init__(self): - self.match = set() - self.root = BMNode() - self.nodes = [self.root] - self.fixers = [] - self.logger = logging.getLogger("RefactoringTool") - - def add_fixer(self, fixer): - """Reduces a fixer's pattern tree to a linear path and adds it - to the matcher(a common Aho-Corasick automaton). The fixer is - appended on the matching states and called when they are - reached""" - self.fixers.append(fixer) - tree = reduce_tree(fixer.pattern_tree) - linear = tree.get_linear_subpattern() - match_nodes = self.add(linear, start=self.root) - for match_node in match_nodes: - match_node.fixers.append(fixer) - - def add(self, pattern, start): - "Recursively adds a linear pattern to the AC automaton" - #print("adding pattern", pattern, "to", start) - if not pattern: - #print("empty pattern") - return [start] - if isinstance(pattern[0], tuple): - #alternatives - #print("alternatives") - match_nodes = [] - for alternative in pattern[0]: - #add all alternatives, and add the rest of the pattern - #to each end node - end_nodes = self.add(alternative, start=start) - for end in end_nodes: - match_nodes.extend(self.add(pattern[1:], end)) - return match_nodes - else: - #single token - #not last - if pattern[0] not in start.transition_table: - #transition did not exist, create new - next_node = BMNode() - start.transition_table[pattern[0]] = next_node - else: - #transition exists already, follow - next_node = start.transition_table[pattern[0]] - - if pattern[1:]: - end_nodes = self.add(pattern[1:], start=next_node) - else: - end_nodes = [next_node] - return end_nodes - - def run(self, leaves): - """The main interface with the bottom matcher. The tree is - traversed from the bottom using the constructed - automaton. Nodes are only checked once as the tree is - retraversed. When the automaton fails, we give it one more - shot(in case the above tree matches as a whole with the - rejected leaf), then we break for the next leaf. There is the - special case of multiple arguments(see code comments) where we - recheck the nodes - - Args: - The leaves of the AST tree to be matched - - Returns: - A dictionary of node matches with fixers as the keys - """ - current_ac_node = self.root - results = defaultdict(list) - for leaf in leaves: - current_ast_node = leaf - while current_ast_node: - current_ast_node.was_checked = True - for child in current_ast_node.children: - # multiple statements, recheck - if isinstance(child, pytree.Leaf) and child.value == ";": - current_ast_node.was_checked = False - break - if current_ast_node.type == 1: - #name - node_token = current_ast_node.value - else: - node_token = current_ast_node.type - - if node_token in current_ac_node.transition_table: - #token matches - current_ac_node = current_ac_node.transition_table[node_token] - for fixer in current_ac_node.fixers: - results[fixer].append(current_ast_node) - else: - #matching failed, reset automaton - current_ac_node = self.root - if (current_ast_node.parent is not None - and current_ast_node.parent.was_checked): - #the rest of the tree upwards has been checked, next leaf - break - - #recheck the rejected node once from the root - if node_token in current_ac_node.transition_table: - #token matches - current_ac_node = current_ac_node.transition_table[node_token] - for fixer in current_ac_node.fixers: - results[fixer].append(current_ast_node) - - current_ast_node = current_ast_node.parent - return results - - def print_ac(self): - "Prints a graphviz diagram of the BM automaton(for debugging)" - print("digraph g{") - def print_node(node): - for subnode_key in node.transition_table.keys(): - subnode = node.transition_table[subnode_key] - print("%d -> %d [label=%s] //%s" % - (node.id, subnode.id, type_repr(subnode_key), str(subnode.fixers))) - if subnode_key == 1: - print(subnode.content) - print_node(subnode) - print_node(self.root) - print("}") - -# taken from pytree.py for debugging; only used by print_ac -_type_reprs = {} -def type_repr(type_num): - global _type_reprs - if not _type_reprs: - from .pygram import python_symbols - # printing tokens is possible but not as useful - # from .pgen2 import token // token.__dict__.items(): - for name, val in python_symbols.__dict__.items(): - if type(val) == int: _type_reprs[val] = name - return _type_reprs.setdefault(type_num, type_num) diff --git a/Lib/lib2to3/btm_utils.py b/Lib/lib2to3/btm_utils.py deleted file mode 100644 index b61afdba693071..00000000000000 --- a/Lib/lib2to3/btm_utils.py +++ /dev/null @@ -1,280 +0,0 @@ -"Utility functions used by the btm_matcher module" - -from . import pytree -from .pgen2 import grammar, token -from .pygram import pattern_symbols, python_symbols - -syms = pattern_symbols -pysyms = python_symbols -tokens = grammar.opmap -token_labels = token - -TYPE_ANY = -1 -TYPE_ALTERNATIVES = -2 -TYPE_GROUP = -3 - -class MinNode(object): - """This class serves as an intermediate representation of the - pattern tree during the conversion to sets of leaf-to-root - subpatterns""" - - def __init__(self, type=None, name=None): - self.type = type - self.name = name - self.children = [] - self.leaf = False - self.parent = None - self.alternatives = [] - self.group = [] - - def __repr__(self): - return str(self.type) + ' ' + str(self.name) - - def leaf_to_root(self): - """Internal method. Returns a characteristic path of the - pattern tree. This method must be run for all leaves until the - linear subpatterns are merged into a single""" - node = self - subp = [] - while node: - if node.type == TYPE_ALTERNATIVES: - node.alternatives.append(subp) - if len(node.alternatives) == len(node.children): - #last alternative - subp = [tuple(node.alternatives)] - node.alternatives = [] - node = node.parent - continue - else: - node = node.parent - subp = None - break - - if node.type == TYPE_GROUP: - node.group.append(subp) - #probably should check the number of leaves - if len(node.group) == len(node.children): - subp = get_characteristic_subpattern(node.group) - node.group = [] - node = node.parent - continue - else: - node = node.parent - subp = None - break - - if node.type == token_labels.NAME and node.name: - #in case of type=name, use the name instead - subp.append(node.name) - else: - subp.append(node.type) - - node = node.parent - return subp - - def get_linear_subpattern(self): - """Drives the leaf_to_root method. The reason that - leaf_to_root must be run multiple times is because we need to - reject 'group' matches; for example the alternative form - (a | b c) creates a group [b c] that needs to be matched. Since - matching multiple linear patterns overcomes the automaton's - capabilities, leaf_to_root merges each group into a single - choice based on 'characteristic'ity, - - i.e. (a|b c) -> (a|b) if b more characteristic than c - - Returns: The most 'characteristic'(as defined by - get_characteristic_subpattern) path for the compiled pattern - tree. - """ - - for l in self.leaves(): - subp = l.leaf_to_root() - if subp: - return subp - - def leaves(self): - "Generator that returns the leaves of the tree" - for child in self.children: - yield from child.leaves() - if not self.children: - yield self - -def reduce_tree(node, parent=None): - """ - Internal function. Reduces a compiled pattern tree to an - intermediate representation suitable for feeding the - automaton. This also trims off any optional pattern elements(like - [a], a*). - """ - - new_node = None - #switch on the node type - if node.type == syms.Matcher: - #skip - node = node.children[0] - - if node.type == syms.Alternatives : - #2 cases - if len(node.children) <= 2: - #just a single 'Alternative', skip this node - new_node = reduce_tree(node.children[0], parent) - else: - #real alternatives - new_node = MinNode(type=TYPE_ALTERNATIVES) - #skip odd children('|' tokens) - for child in node.children: - if node.children.index(child)%2: - continue - reduced = reduce_tree(child, new_node) - if reduced is not None: - new_node.children.append(reduced) - elif node.type == syms.Alternative: - if len(node.children) > 1: - - new_node = MinNode(type=TYPE_GROUP) - for child in node.children: - reduced = reduce_tree(child, new_node) - if reduced: - new_node.children.append(reduced) - if not new_node.children: - # delete the group if all of the children were reduced to None - new_node = None - - else: - new_node = reduce_tree(node.children[0], parent) - - elif node.type == syms.Unit: - if (isinstance(node.children[0], pytree.Leaf) and - node.children[0].value == '('): - #skip parentheses - return reduce_tree(node.children[1], parent) - if ((isinstance(node.children[0], pytree.Leaf) and - node.children[0].value == '[') - or - (len(node.children)>1 and - hasattr(node.children[1], "value") and - node.children[1].value == '[')): - #skip whole unit if its optional - return None - - leaf = True - details_node = None - alternatives_node = None - has_repeater = False - repeater_node = None - has_variable_name = False - - for child in node.children: - if child.type == syms.Details: - leaf = False - details_node = child - elif child.type == syms.Repeater: - has_repeater = True - repeater_node = child - elif child.type == syms.Alternatives: - alternatives_node = child - if hasattr(child, 'value') and child.value == '=': # variable name - has_variable_name = True - - #skip variable name - if has_variable_name: - #skip variable name, '=' - name_leaf = node.children[2] - if hasattr(name_leaf, 'value') and name_leaf.value == '(': - # skip parenthesis - name_leaf = node.children[3] - else: - name_leaf = node.children[0] - - #set node type - if name_leaf.type == token_labels.NAME: - #(python) non-name or wildcard - if name_leaf.value == 'any': - new_node = MinNode(type=TYPE_ANY) - else: - if hasattr(token_labels, name_leaf.value): - new_node = MinNode(type=getattr(token_labels, name_leaf.value)) - else: - new_node = MinNode(type=getattr(pysyms, name_leaf.value)) - - elif name_leaf.type == token_labels.STRING: - #(python) name or character; remove the apostrophes from - #the string value - name = name_leaf.value.strip("'") - if name in tokens: - new_node = MinNode(type=tokens[name]) - else: - new_node = MinNode(type=token_labels.NAME, name=name) - elif name_leaf.type == syms.Alternatives: - new_node = reduce_tree(alternatives_node, parent) - - #handle repeaters - if has_repeater: - if repeater_node.children[0].value == '*': - #reduce to None - new_node = None - elif repeater_node.children[0].value == '+': - #reduce to a single occurrence i.e. do nothing - pass - else: - #TODO: handle {min, max} repeaters - raise NotImplementedError - - #add children - if details_node and new_node is not None: - for child in details_node.children[1:-1]: - #skip '<', '>' markers - reduced = reduce_tree(child, new_node) - if reduced is not None: - new_node.children.append(reduced) - if new_node: - new_node.parent = parent - return new_node - - -def get_characteristic_subpattern(subpatterns): - """Picks the most characteristic from a list of linear patterns - Current order used is: - names > common_names > common_chars - """ - if not isinstance(subpatterns, list): - return subpatterns - if len(subpatterns)==1: - return subpatterns[0] - - # first pick out the ones containing variable names - subpatterns_with_names = [] - subpatterns_with_common_names = [] - common_names = ['in', 'for', 'if' , 'not', 'None'] - subpatterns_with_common_chars = [] - common_chars = "[]().,:" - for subpattern in subpatterns: - if any(rec_test(subpattern, lambda x: type(x) is str)): - if any(rec_test(subpattern, - lambda x: isinstance(x, str) and x in common_chars)): - subpatterns_with_common_chars.append(subpattern) - elif any(rec_test(subpattern, - lambda x: isinstance(x, str) and x in common_names)): - subpatterns_with_common_names.append(subpattern) - - else: - subpatterns_with_names.append(subpattern) - - if subpatterns_with_names: - subpatterns = subpatterns_with_names - elif subpatterns_with_common_names: - subpatterns = subpatterns_with_common_names - elif subpatterns_with_common_chars: - subpatterns = subpatterns_with_common_chars - # of the remaining subpatterns pick out the longest one - return max(subpatterns, key=len) - -def rec_test(sequence, test_func): - """Tests test_func on all items of sequence and items of included - sub-iterables""" - for x in sequence: - if isinstance(x, (list, tuple)): - yield from rec_test(x, test_func) - else: - yield test_func(x) diff --git a/Lib/lib2to3/fixer_base.py b/Lib/lib2to3/fixer_base.py deleted file mode 100644 index df581a4deab9e9..00000000000000 --- a/Lib/lib2to3/fixer_base.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Base class for fixers (optional, but recommended).""" - -# Python imports -import itertools - -# Local imports -from .patcomp import PatternCompiler -from . import pygram -from .fixer_util import does_tree_import - -class BaseFix(object): - - """Optional base class for fixers. - - The subclass name must be FixFooBar where FooBar is the result of - removing underscores and capitalizing the words of the fix name. - For example, the class name for a fixer named 'has_key' should be - FixHasKey. - """ - - PATTERN = None # Most subclasses should override with a string literal - pattern = None # Compiled pattern, set by compile_pattern() - pattern_tree = None # Tree representation of the pattern - options = None # Options object passed to initializer - filename = None # The filename (set by set_filename) - numbers = itertools.count(1) # For new_name() - used_names = set() # A set of all used NAMEs - order = "post" # Does the fixer prefer pre- or post-order traversal - explicit = False # Is this ignored by refactor.py -f all? - run_order = 5 # Fixers will be sorted by run order before execution - # Lower numbers will be run first. - _accept_type = None # [Advanced and not public] This tells RefactoringTool - # which node type to accept when there's not a pattern. - - keep_line_order = False # For the bottom matcher: match with the - # original line order - BM_compatible = False # Compatibility with the bottom matching - # module; every fixer should set this - # manually - - # Shortcut for access to Python grammar symbols - syms = pygram.python_symbols - - def __init__(self, options, log): - """Initializer. Subclass may override. - - Args: - options: a dict containing the options passed to RefactoringTool - that could be used to customize the fixer through the command line. - log: a list to append warnings and other messages to. - """ - self.options = options - self.log = log - self.compile_pattern() - - def compile_pattern(self): - """Compiles self.PATTERN into self.pattern. - - Subclass may override if it doesn't want to use - self.{pattern,PATTERN} in .match(). - """ - if self.PATTERN is not None: - PC = PatternCompiler() - self.pattern, self.pattern_tree = PC.compile_pattern(self.PATTERN, - with_tree=True) - - def set_filename(self, filename): - """Set the filename. - - The main refactoring tool should call this. - """ - self.filename = filename - - def match(self, node): - """Returns match for a given parse tree node. - - Should return a true or false object (not necessarily a bool). - It may return a non-empty dict of matching sub-nodes as - returned by a matching pattern. - - Subclass may override. - """ - results = {"node": node} - return self.pattern.match(node, results) and results - - def transform(self, node, results): - """Returns the transformation for a given parse tree node. - - Args: - node: the root of the parse tree that matched the fixer. - results: a dict mapping symbolic names to part of the match. - - Returns: - None, or a node that is a modified copy of the - argument node. The node argument may also be modified in-place to - effect the same change. - - Subclass *must* override. - """ - raise NotImplementedError() - - def new_name(self, template="xxx_todo_changeme"): - """Return a string suitable for use as an identifier - - The new name is guaranteed not to conflict with other identifiers. - """ - name = template - while name in self.used_names: - name = template + str(next(self.numbers)) - self.used_names.add(name) - return name - - def log_message(self, message): - if self.first_log: - self.first_log = False - self.log.append("### In file %s ###" % self.filename) - self.log.append(message) - - def cannot_convert(self, node, reason=None): - """Warn the user that a given chunk of code is not valid Python 3, - but that it cannot be converted automatically. - - First argument is the top-level node for the code in question. - Optional second argument is why it can't be converted. - """ - lineno = node.get_lineno() - for_output = node.clone() - for_output.prefix = "" - msg = "Line %d: could not convert: %s" - self.log_message(msg % (lineno, for_output)) - if reason: - self.log_message(reason) - - def warning(self, node, reason): - """Used for warning the user about possible uncertainty in the - translation. - - First argument is the top-level node for the code in question. - Optional second argument is why it can't be converted. - """ - lineno = node.get_lineno() - self.log_message("Line %d: %s" % (lineno, reason)) - - def start_tree(self, tree, filename): - """Some fixers need to maintain tree-wide state. - This method is called once, at the start of tree fix-up. - - tree - the root node of the tree to be processed. - filename - the name of the file the tree came from. - """ - self.used_names = tree.used_names - self.set_filename(filename) - self.numbers = itertools.count(1) - self.first_log = True - - def finish_tree(self, tree, filename): - """Some fixers need to maintain tree-wide state. - This method is called once, at the conclusion of tree fix-up. - - tree - the root node of the tree to be processed. - filename - the name of the file the tree came from. - """ - pass - - -class ConditionalFix(BaseFix): - """ Base class for fixers which not execute if an import is found. """ - - # This is the name of the import which, if found, will cause the test to be skipped - skip_on = None - - def start_tree(self, *args): - super(ConditionalFix, self).start_tree(*args) - self._should_skip = None - - def should_skip(self, node): - if self._should_skip is not None: - return self._should_skip - pkg = self.skip_on.split(".") - name = pkg[-1] - pkg = ".".join(pkg[:-1]) - self._should_skip = does_tree_import(pkg, name, node) - return self._should_skip diff --git a/Lib/lib2to3/fixer_util.py b/Lib/lib2to3/fixer_util.py deleted file mode 100644 index c2a3a47f503286..00000000000000 --- a/Lib/lib2to3/fixer_util.py +++ /dev/null @@ -1,453 +0,0 @@ -"""Utility functions, node construction macros, etc.""" -# Author: Collin Winter - -# Local imports -from .pgen2 import token -from .pytree import Leaf, Node -from .pygram import python_symbols as syms -from . import patcomp - - -########################################################### -### Common node-construction "macros" -########################################################### - -def KeywordArg(keyword, value): - return Node(syms.argument, - [keyword, Leaf(token.EQUAL, "="), value]) - -def LParen(): - return Leaf(token.LPAR, "(") - -def RParen(): - return Leaf(token.RPAR, ")") - -def Assign(target, source): - """Build an assignment statement""" - if not isinstance(target, list): - target = [target] - if not isinstance(source, list): - source.prefix = " " - source = [source] - - return Node(syms.atom, - target + [Leaf(token.EQUAL, "=", prefix=" ")] + source) - -def Name(name, prefix=None): - """Return a NAME leaf""" - return Leaf(token.NAME, name, prefix=prefix) - -def Attr(obj, attr): - """A node tuple for obj.attr""" - return [obj, Node(syms.trailer, [Dot(), attr])] - -def Comma(): - """A comma leaf""" - return Leaf(token.COMMA, ",") - -def Dot(): - """A period (.) leaf""" - return Leaf(token.DOT, ".") - -def ArgList(args, lparen=LParen(), rparen=RParen()): - """A parenthesised argument list, used by Call()""" - node = Node(syms.trailer, [lparen.clone(), rparen.clone()]) - if args: - node.insert_child(1, Node(syms.arglist, args)) - return node - -def Call(func_name, args=None, prefix=None): - """A function call""" - node = Node(syms.power, [func_name, ArgList(args)]) - if prefix is not None: - node.prefix = prefix - return node - -def Newline(): - """A newline literal""" - return Leaf(token.NEWLINE, "\n") - -def BlankLine(): - """A blank line""" - return Leaf(token.NEWLINE, "") - -def Number(n, prefix=None): - return Leaf(token.NUMBER, n, prefix=prefix) - -def Subscript(index_node): - """A numeric or string subscript""" - return Node(syms.trailer, [Leaf(token.LBRACE, "["), - index_node, - Leaf(token.RBRACE, "]")]) - -def String(string, prefix=None): - """A string leaf""" - return Leaf(token.STRING, string, prefix=prefix) - -def ListComp(xp, fp, it, test=None): - """A list comprehension of the form [xp for fp in it if test]. - - If test is None, the "if test" part is omitted. - """ - xp.prefix = "" - fp.prefix = " " - it.prefix = " " - for_leaf = Leaf(token.NAME, "for") - for_leaf.prefix = " " - in_leaf = Leaf(token.NAME, "in") - in_leaf.prefix = " " - inner_args = [for_leaf, fp, in_leaf, it] - if test: - test.prefix = " " - if_leaf = Leaf(token.NAME, "if") - if_leaf.prefix = " " - inner_args.append(Node(syms.comp_if, [if_leaf, test])) - inner = Node(syms.listmaker, [xp, Node(syms.comp_for, inner_args)]) - return Node(syms.atom, - [Leaf(token.LBRACE, "["), - inner, - Leaf(token.RBRACE, "]")]) - -def FromImport(package_name, name_leafs): - """ Return an import statement in the form: - from package import name_leafs""" - # XXX: May not handle dotted imports properly (eg, package_name='foo.bar') - #assert package_name == '.' or '.' not in package_name, "FromImport has "\ - # "not been tested with dotted package names -- use at your own "\ - # "peril!" - - for leaf in name_leafs: - # Pull the leaves out of their old tree - leaf.remove() - - children = [Leaf(token.NAME, "from"), - Leaf(token.NAME, package_name, prefix=" "), - Leaf(token.NAME, "import", prefix=" "), - Node(syms.import_as_names, name_leafs)] - imp = Node(syms.import_from, children) - return imp - -def ImportAndCall(node, results, names): - """Returns an import statement and calls a method - of the module: - - import module - module.name()""" - obj = results["obj"].clone() - if obj.type == syms.arglist: - newarglist = obj.clone() - else: - newarglist = Node(syms.arglist, [obj.clone()]) - after = results["after"] - if after: - after = [n.clone() for n in after] - new = Node(syms.power, - Attr(Name(names[0]), Name(names[1])) + - [Node(syms.trailer, - [results["lpar"].clone(), - newarglist, - results["rpar"].clone()])] + after) - new.prefix = node.prefix - return new - - -########################################################### -### Determine whether a node represents a given literal -########################################################### - -def is_tuple(node): - """Does the node represent a tuple literal?""" - if isinstance(node, Node) and node.children == [LParen(), RParen()]: - return True - return (isinstance(node, Node) - and len(node.children) == 3 - and isinstance(node.children[0], Leaf) - and isinstance(node.children[1], Node) - and isinstance(node.children[2], Leaf) - and node.children[0].value == "(" - and node.children[2].value == ")") - -def is_list(node): - """Does the node represent a list literal?""" - return (isinstance(node, Node) - and len(node.children) > 1 - and isinstance(node.children[0], Leaf) - and isinstance(node.children[-1], Leaf) - and node.children[0].value == "[" - and node.children[-1].value == "]") - - -########################################################### -### Misc -########################################################### - -def parenthesize(node): - return Node(syms.atom, [LParen(), node, RParen()]) - - -consuming_calls = {"sorted", "list", "set", "any", "all", "tuple", "sum", - "min", "max", "enumerate"} - -def attr_chain(obj, attr): - """Follow an attribute chain. - - If you have a chain of objects where a.foo -> b, b.foo-> c, etc, - use this to iterate over all objects in the chain. Iteration is - terminated by getattr(x, attr) is None. - - Args: - obj: the starting object - attr: the name of the chaining attribute - - Yields: - Each successive object in the chain. - """ - next = getattr(obj, attr) - while next: - yield next - next = getattr(next, attr) - -p0 = """for_stmt< 'for' any 'in' node=any ':' any* > - | comp_for< 'for' any 'in' node=any any* > - """ -p1 = """ -power< - ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' | - 'any' | 'all' | 'enumerate' | (any* trailer< '.' 'join' >) ) - trailer< '(' node=any ')' > - any* -> -""" -p2 = """ -power< - ( 'sorted' | 'enumerate' ) - trailer< '(' arglist ')' > - any* -> -""" -pats_built = False -def in_special_context(node): - """ Returns true if node is in an environment where all that is required - of it is being iterable (ie, it doesn't matter if it returns a list - or an iterator). - See test_map_nochange in test_fixers.py for some examples and tests. - """ - global p0, p1, p2, pats_built - if not pats_built: - p0 = patcomp.compile_pattern(p0) - p1 = patcomp.compile_pattern(p1) - p2 = patcomp.compile_pattern(p2) - pats_built = True - patterns = [p0, p1, p2] - for pattern, parent in zip(patterns, attr_chain(node, "parent")): - results = {} - if pattern.match(parent, results) and results["node"] is node: - return True - return False - -def is_probably_builtin(node): - """ - Check that something isn't an attribute or function name etc. - """ - prev = node.prev_sibling - if prev is not None and prev.type == token.DOT: - # Attribute lookup. - return False - parent = node.parent - if parent.type in (syms.funcdef, syms.classdef): - return False - if parent.type == syms.expr_stmt and parent.children[0] is node: - # Assignment. - return False - if parent.type == syms.parameters or \ - (parent.type == syms.typedargslist and ( - (prev is not None and prev.type == token.COMMA) or - parent.children[0] is node - )): - # The name of an argument. - return False - return True - -def find_indentation(node): - """Find the indentation of *node*.""" - while node is not None: - if node.type == syms.suite and len(node.children) > 2: - indent = node.children[1] - if indent.type == token.INDENT: - return indent.value - node = node.parent - return "" - -########################################################### -### The following functions are to find bindings in a suite -########################################################### - -def make_suite(node): - if node.type == syms.suite: - return node - node = node.clone() - parent, node.parent = node.parent, None - suite = Node(syms.suite, [node]) - suite.parent = parent - return suite - -def find_root(node): - """Find the top level namespace.""" - # Scamper up to the top level namespace - while node.type != syms.file_input: - node = node.parent - if not node: - raise ValueError("root found before file_input node was found.") - return node - -def does_tree_import(package, name, node): - """ Returns true if name is imported from package at the - top level of the tree which node belongs to. - To cover the case of an import like 'import foo', use - None for the package and 'foo' for the name. """ - binding = find_binding(name, find_root(node), package) - return bool(binding) - -def is_import(node): - """Returns true if the node is an import statement.""" - return node.type in (syms.import_name, syms.import_from) - -def touch_import(package, name, node): - """ Works like `does_tree_import` but adds an import statement - if it was not imported. """ - def is_import_stmt(node): - return (node.type == syms.simple_stmt and node.children and - is_import(node.children[0])) - - root = find_root(node) - - if does_tree_import(package, name, root): - return - - # figure out where to insert the new import. First try to find - # the first import and then skip to the last one. - insert_pos = offset = 0 - for idx, node in enumerate(root.children): - if not is_import_stmt(node): - continue - for offset, node2 in enumerate(root.children[idx:]): - if not is_import_stmt(node2): - break - insert_pos = idx + offset - break - - # if there are no imports where we can insert, find the docstring. - # if that also fails, we stick to the beginning of the file - if insert_pos == 0: - for idx, node in enumerate(root.children): - if (node.type == syms.simple_stmt and node.children and - node.children[0].type == token.STRING): - insert_pos = idx + 1 - break - - if package is None: - import_ = Node(syms.import_name, [ - Leaf(token.NAME, "import"), - Leaf(token.NAME, name, prefix=" ") - ]) - else: - import_ = FromImport(package, [Leaf(token.NAME, name, prefix=" ")]) - - children = [import_, Newline()] - root.insert_child(insert_pos, Node(syms.simple_stmt, children)) - - -_def_syms = {syms.classdef, syms.funcdef} -def find_binding(name, node, package=None): - """ Returns the node which binds variable name, otherwise None. - If optional argument package is supplied, only imports will - be returned. - See test cases for examples.""" - for child in node.children: - ret = None - if child.type == syms.for_stmt: - if _find(name, child.children[1]): - return child - n = find_binding(name, make_suite(child.children[-1]), package) - if n: ret = n - elif child.type in (syms.if_stmt, syms.while_stmt): - n = find_binding(name, make_suite(child.children[-1]), package) - if n: ret = n - elif child.type == syms.try_stmt: - n = find_binding(name, make_suite(child.children[2]), package) - if n: - ret = n - else: - for i, kid in enumerate(child.children[3:]): - if kid.type == token.COLON and kid.value == ":": - # i+3 is the colon, i+4 is the suite - n = find_binding(name, make_suite(child.children[i+4]), package) - if n: ret = n - elif child.type in _def_syms and child.children[1].value == name: - ret = child - elif _is_import_binding(child, name, package): - ret = child - elif child.type == syms.simple_stmt: - ret = find_binding(name, child, package) - elif child.type == syms.expr_stmt: - if _find(name, child.children[0]): - ret = child - - if ret: - if not package: - return ret - if is_import(ret): - return ret - return None - -_block_syms = {syms.funcdef, syms.classdef, syms.trailer} -def _find(name, node): - nodes = [node] - while nodes: - node = nodes.pop() - if node.type > 256 and node.type not in _block_syms: - nodes.extend(node.children) - elif node.type == token.NAME and node.value == name: - return node - return None - -def _is_import_binding(node, name, package=None): - """ Will return node if node will import name, or node - will import * from package. None is returned otherwise. - See test cases for examples. """ - - if node.type == syms.import_name and not package: - imp = node.children[1] - if imp.type == syms.dotted_as_names: - for child in imp.children: - if child.type == syms.dotted_as_name: - if child.children[2].value == name: - return node - elif child.type == token.NAME and child.value == name: - return node - elif imp.type == syms.dotted_as_name: - last = imp.children[-1] - if last.type == token.NAME and last.value == name: - return node - elif imp.type == token.NAME and imp.value == name: - return node - elif node.type == syms.import_from: - # str(...) is used to make life easier here, because - # from a.b import parses to ['import', ['a', '.', 'b'], ...] - if package and str(node.children[1]).strip() != package: - return None - n = node.children[3] - if package and _find("as", n): - # See test_from_import_as for explanation - return None - elif n.type == syms.import_as_names and _find(name, n): - return node - elif n.type == syms.import_as_name: - child = n.children[2] - if child.type == token.NAME and child.value == name: - return node - elif n.type == token.NAME and n.value == name: - return node - elif package and n.type == token.STAR: - return node - return None diff --git a/Lib/lib2to3/fixes/__init__.py b/Lib/lib2to3/fixes/__init__.py deleted file mode 100644 index b93054b3ecf3a5..00000000000000 --- a/Lib/lib2to3/fixes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Dummy file to make this directory a package. diff --git a/Lib/lib2to3/fixes/fix_apply.py b/Lib/lib2to3/fixes/fix_apply.py deleted file mode 100644 index 6408582c426477..00000000000000 --- a/Lib/lib2to3/fixes/fix_apply.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for apply(). - -This converts apply(func, v, k) into (func)(*v, **k).""" - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Call, Comma, parenthesize - -class FixApply(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< 'apply' - trailer< - '(' - arglist< - (not argument - ')' - > - > - """ - - def transform(self, node, results): - syms = self.syms - assert results - func = results["func"] - args = results["args"] - kwds = results.get("kwds") - # I feel like we should be able to express this logic in the - # PATTERN above but I don't know how to do it so... - if args: - if (args.type == self.syms.argument and - args.children[0].value in {'**', '*'}): - return # Make no change. - if kwds and (kwds.type == self.syms.argument and - kwds.children[0].value == '**'): - return # Make no change. - prefix = node.prefix - func = func.clone() - if (func.type not in (token.NAME, syms.atom) and - (func.type != syms.power or - func.children[-2].type == token.DOUBLESTAR)): - # Need to parenthesize - func = parenthesize(func) - func.prefix = "" - args = args.clone() - args.prefix = "" - if kwds is not None: - kwds = kwds.clone() - kwds.prefix = "" - l_newargs = [pytree.Leaf(token.STAR, "*"), args] - if kwds is not None: - l_newargs.extend([Comma(), - pytree.Leaf(token.DOUBLESTAR, "**"), - kwds]) - l_newargs[-2].prefix = " " # that's the ** token - # XXX Sometimes we could be cleverer, e.g. apply(f, (x, y) + t) - # can be translated into f(x, y, *t) instead of f(*(x, y) + t) - #new = pytree.Node(syms.power, (func, ArgList(l_newargs))) - return Call(func, l_newargs, prefix=prefix) diff --git a/Lib/lib2to3/fixes/fix_asserts.py b/Lib/lib2to3/fixes/fix_asserts.py deleted file mode 100644 index 5bcec885f52cbf..00000000000000 --- a/Lib/lib2to3/fixes/fix_asserts.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Fixer that replaces deprecated unittest method names.""" - -# Author: Ezio Melotti - -from ..fixer_base import BaseFix -from ..fixer_util import Name - -NAMES = dict( - assert_="assertTrue", - assertEquals="assertEqual", - assertNotEquals="assertNotEqual", - assertAlmostEquals="assertAlmostEqual", - assertNotAlmostEquals="assertNotAlmostEqual", - assertRegexpMatches="assertRegex", - assertRaisesRegexp="assertRaisesRegex", - failUnlessEqual="assertEqual", - failIfEqual="assertNotEqual", - failUnlessAlmostEqual="assertAlmostEqual", - failIfAlmostEqual="assertNotAlmostEqual", - failUnless="assertTrue", - failUnlessRaises="assertRaises", - failIf="assertFalse", -) - - -class FixAsserts(BaseFix): - - PATTERN = """ - power< any+ trailer< '.' meth=(%s)> any* > - """ % '|'.join(map(repr, NAMES)) - - def transform(self, node, results): - name = results["meth"][0] - name.replace(Name(NAMES[str(name)], prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_basestring.py b/Lib/lib2to3/fixes/fix_basestring.py deleted file mode 100644 index 5fe69a0f03b1b8..00000000000000 --- a/Lib/lib2to3/fixes/fix_basestring.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Fixer for basestring -> str.""" -# Author: Christian Heimes - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixBasestring(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = "'basestring'" - - def transform(self, node, results): - return Name("str", prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_buffer.py b/Lib/lib2to3/fixes/fix_buffer.py deleted file mode 100644 index f9a1958ad3b93e..00000000000000 --- a/Lib/lib2to3/fixes/fix_buffer.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes buffer(...) into memoryview(...).""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixBuffer(fixer_base.BaseFix): - BM_compatible = True - - explicit = True # The user must ask for this fixer - - PATTERN = """ - power< name='buffer' trailer< '(' [any] ')' > any* > - """ - - def transform(self, node, results): - name = results["name"] - name.replace(Name("memoryview", prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_dict.py b/Lib/lib2to3/fixes/fix_dict.py deleted file mode 100644 index d3655c9f1b2d9b..00000000000000 --- a/Lib/lib2to3/fixes/fix_dict.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for dict methods. - -d.keys() -> list(d.keys()) -d.items() -> list(d.items()) -d.values() -> list(d.values()) - -d.iterkeys() -> iter(d.keys()) -d.iteritems() -> iter(d.items()) -d.itervalues() -> iter(d.values()) - -d.viewkeys() -> d.keys() -d.viewitems() -> d.items() -d.viewvalues() -> d.values() - -Except in certain very specific contexts: the iter() can be dropped -when the context is list(), sorted(), iter() or for...in; the list() -can be dropped when the context is list() or sorted() (but not iter() -or for...in!). Special contexts that apply to both: list(), sorted(), tuple() -set(), any(), all(), sum(). - -Note: iter(d.keys()) could be written as iter(d) but since the -original d.iterkeys() was also redundant we don't fix this. And there -are (rare) contexts where it makes a difference (e.g. when passing it -as an argument to a function that introspects the argument). -""" - -# Local imports -from .. import pytree -from .. import patcomp -from .. import fixer_base -from ..fixer_util import Name, Call, Dot -from .. import fixer_util - - -iter_exempt = fixer_util.consuming_calls | {"iter"} - - -class FixDict(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< head=any+ - trailer< '.' method=('keys'|'items'|'values'| - 'iterkeys'|'iteritems'|'itervalues'| - 'viewkeys'|'viewitems'|'viewvalues') > - parens=trailer< '(' ')' > - tail=any* - > - """ - - def transform(self, node, results): - head = results["head"] - method = results["method"][0] # Extract node for method name - tail = results["tail"] - syms = self.syms - method_name = method.value - isiter = method_name.startswith("iter") - isview = method_name.startswith("view") - if isiter or isview: - method_name = method_name[4:] - assert method_name in ("keys", "items", "values"), repr(method) - head = [n.clone() for n in head] - tail = [n.clone() for n in tail] - special = not tail and self.in_special_context(node, isiter) - args = head + [pytree.Node(syms.trailer, - [Dot(), - Name(method_name, - prefix=method.prefix)]), - results["parens"].clone()] - new = pytree.Node(syms.power, args) - if not (special or isview): - new.prefix = "" - new = Call(Name("iter" if isiter else "list"), [new]) - if tail: - new = pytree.Node(syms.power, [new] + tail) - new.prefix = node.prefix - return new - - P1 = "power< func=NAME trailer< '(' node=any ')' > any* >" - p1 = patcomp.compile_pattern(P1) - - P2 = """for_stmt< 'for' any 'in' node=any ':' any* > - | comp_for< 'for' any 'in' node=any any* > - """ - p2 = patcomp.compile_pattern(P2) - - def in_special_context(self, node, isiter): - if node.parent is None: - return False - results = {} - if (node.parent.parent is not None and - self.p1.match(node.parent.parent, results) and - results["node"] is node): - if isiter: - # iter(d.iterkeys()) -> iter(d.keys()), etc. - return results["func"].value in iter_exempt - else: - # list(d.keys()) -> list(d.keys()), etc. - return results["func"].value in fixer_util.consuming_calls - if not isiter: - return False - # for ... in d.iterkeys() -> for ... in d.keys(), etc. - return self.p2.match(node.parent, results) and results["node"] is node diff --git a/Lib/lib2to3/fixes/fix_except.py b/Lib/lib2to3/fixes/fix_except.py deleted file mode 100644 index 49bd3d5ab7d6cc..00000000000000 --- a/Lib/lib2to3/fixes/fix_except.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Fixer for except statements with named exceptions. - -The following cases will be converted: - -- "except E, T:" where T is a name: - - except E as T: - -- "except E, T:" where T is not a name, tuple or list: - - except E as t: - T = t - - This is done because the target of an "except" clause must be a - name. - -- "except E, T:" where T is a tuple or list literal: - - except E as t: - T = t.args -""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, syms - -def find_excepts(nodes): - for i, n in enumerate(nodes): - if n.type == syms.except_clause: - if n.children[0].value == 'except': - yield (n, nodes[i+2]) - -class FixExcept(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - try_stmt< 'try' ':' (simple_stmt | suite) - cleanup=(except_clause ':' (simple_stmt | suite))+ - tail=(['except' ':' (simple_stmt | suite)] - ['else' ':' (simple_stmt | suite)] - ['finally' ':' (simple_stmt | suite)]) > - """ - - def transform(self, node, results): - syms = self.syms - - tail = [n.clone() for n in results["tail"]] - - try_cleanup = [ch.clone() for ch in results["cleanup"]] - for except_clause, e_suite in find_excepts(try_cleanup): - if len(except_clause.children) == 4: - (E, comma, N) = except_clause.children[1:4] - comma.replace(Name("as", prefix=" ")) - - if N.type != token.NAME: - # Generate a new N for the except clause - new_N = Name(self.new_name(), prefix=" ") - target = N.clone() - target.prefix = "" - N.replace(new_N) - new_N = new_N.clone() - - # Insert "old_N = new_N" as the first statement in - # the except body. This loop skips leading whitespace - # and indents - #TODO(cwinter) suite-cleanup - suite_stmts = e_suite.children - for i, stmt in enumerate(suite_stmts): - if isinstance(stmt, pytree.Node): - break - - # The assignment is different if old_N is a tuple or list - # In that case, the assignment is old_N = new_N.args - if is_tuple(N) or is_list(N): - assign = Assign(target, Attr(new_N, Name('args'))) - else: - assign = Assign(target, new_N) - - #TODO(cwinter) stopgap until children becomes a smart list - for child in reversed(suite_stmts[:i]): - e_suite.insert_child(0, child) - e_suite.insert_child(i, assign) - elif N.prefix == "": - # No space after a comma is legal; no space after "as", - # not so much. - N.prefix = " " - - #TODO(cwinter) fix this when children becomes a smart list - children = [c.clone() for c in node.children[:3]] + try_cleanup + tail - return pytree.Node(node.type, children) diff --git a/Lib/lib2to3/fixes/fix_exec.py b/Lib/lib2to3/fixes/fix_exec.py deleted file mode 100644 index ab921ee80cdf36..00000000000000 --- a/Lib/lib2to3/fixes/fix_exec.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for exec. - -This converts usages of the exec statement into calls to a built-in -exec() function. - -exec code in ns1, ns2 -> exec(code, ns1, ns2) -""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Comma, Name, Call - - -class FixExec(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - exec_stmt< 'exec' a=any 'in' b=any [',' c=any] > - | - exec_stmt< 'exec' (not atom<'(' [any] ')'>) a=any > - """ - - def transform(self, node, results): - assert results - syms = self.syms - a = results["a"] - b = results.get("b") - c = results.get("c") - args = [a.clone()] - args[0].prefix = "" - if b is not None: - args.extend([Comma(), b.clone()]) - if c is not None: - args.extend([Comma(), c.clone()]) - - return Call(Name("exec"), args, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_execfile.py b/Lib/lib2to3/fixes/fix_execfile.py deleted file mode 100644 index b6c786fd4e8b6a..00000000000000 --- a/Lib/lib2to3/fixes/fix_execfile.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for execfile. - -This converts usages of the execfile function into calls to the built-in -exec() function. -""" - -from .. import fixer_base -from ..fixer_util import (Comma, Name, Call, LParen, RParen, Dot, Node, - ArgList, String, syms) - - -class FixExecfile(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< 'execfile' trailer< '(' arglist< filename=any [',' globals=any [',' locals=any ] ] > ')' > > - | - power< 'execfile' trailer< '(' filename=any ')' > > - """ - - def transform(self, node, results): - assert results - filename = results["filename"] - globals = results.get("globals") - locals = results.get("locals") - - # Copy over the prefix from the right parentheses end of the execfile - # call. - execfile_paren = node.children[-1].children[-1].clone() - # Construct open().read(). - open_args = ArgList([filename.clone(), Comma(), String('"rb"', ' ')], - rparen=execfile_paren) - open_call = Node(syms.power, [Name("open"), open_args]) - read = [Node(syms.trailer, [Dot(), Name('read')]), - Node(syms.trailer, [LParen(), RParen()])] - open_expr = [open_call] + read - # Wrap the open call in a compile call. This is so the filename will be - # preserved in the execed code. - filename_arg = filename.clone() - filename_arg.prefix = " " - exec_str = String("'exec'", " ") - compile_args = open_expr + [Comma(), filename_arg, Comma(), exec_str] - compile_call = Call(Name("compile"), compile_args, "") - # Finally, replace the execfile call with an exec call. - args = [compile_call] - if globals is not None: - args.extend([Comma(), globals.clone()]) - if locals is not None: - args.extend([Comma(), locals.clone()]) - return Call(Name("exec"), args, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_exitfunc.py b/Lib/lib2to3/fixes/fix_exitfunc.py deleted file mode 100644 index 2e47887afead36..00000000000000 --- a/Lib/lib2to3/fixes/fix_exitfunc.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Convert use of sys.exitfunc to use the atexit module. -""" - -# Author: Benjamin Peterson - -from lib2to3 import pytree, fixer_base -from lib2to3.fixer_util import Name, Attr, Call, Comma, Newline, syms - - -class FixExitfunc(fixer_base.BaseFix): - keep_line_order = True - BM_compatible = True - - PATTERN = """ - ( - sys_import=import_name<'import' - ('sys' - | - dotted_as_names< (any ',')* 'sys' (',' any)* > - ) - > - | - expr_stmt< - power< 'sys' trailer< '.' 'exitfunc' > > - '=' func=any > - ) - """ - - def __init__(self, *args): - super(FixExitfunc, self).__init__(*args) - - def start_tree(self, tree, filename): - super(FixExitfunc, self).start_tree(tree, filename) - self.sys_import = None - - def transform(self, node, results): - # First, find the sys import. We'll just hope it's global scope. - if "sys_import" in results: - if self.sys_import is None: - self.sys_import = results["sys_import"] - return - - func = results["func"].clone() - func.prefix = "" - register = pytree.Node(syms.power, - Attr(Name("atexit"), Name("register")) - ) - call = Call(register, [func], node.prefix) - node.replace(call) - - if self.sys_import is None: - # That's interesting. - self.warning(node, "Can't find sys import; Please add an atexit " - "import at the top of your file.") - return - - # Now add an atexit import after the sys import. - names = self.sys_import.children[1] - if names.type == syms.dotted_as_names: - names.append_child(Comma()) - names.append_child(Name("atexit", " ")) - else: - containing_stmt = self.sys_import.parent - position = containing_stmt.children.index(self.sys_import) - stmt_container = containing_stmt.parent - new_import = pytree.Node(syms.import_name, - [Name("import"), Name("atexit", " ")] - ) - new = pytree.Node(syms.simple_stmt, [new_import]) - containing_stmt.insert_child(position + 1, Newline()) - containing_stmt.insert_child(position + 2, new) diff --git a/Lib/lib2to3/fixes/fix_filter.py b/Lib/lib2to3/fixes/fix_filter.py deleted file mode 100644 index 38e9078f11ac88..00000000000000 --- a/Lib/lib2to3/fixes/fix_filter.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes filter(F, X) into list(filter(F, X)). - -We avoid the transformation if the filter() call is directly contained -in iter(<>), list(<>), tuple(<>), sorted(<>), ...join(<>), or -for V in <>:. - -NOTE: This is still not correct if the original code was depending on -filter(F, X) to return a string if X is a string and a tuple if X is a -tuple. That would require type inference, which we don't do. Let -Python 2.6 figure it out. -""" - -# Local imports -from .. import fixer_base -from ..pytree import Node -from ..pygram import python_symbols as syms -from ..fixer_util import Name, ArgList, ListComp, in_special_context, parenthesize - - -class FixFilter(fixer_base.ConditionalFix): - BM_compatible = True - - PATTERN = """ - filter_lambda=power< - 'filter' - trailer< - '(' - arglist< - lambdef< 'lambda' - (fp=NAME | vfpdef< '(' fp=NAME ')'> ) ':' xp=any - > - ',' - it=any - > - ')' - > - [extra_trailers=trailer*] - > - | - power< - 'filter' - trailer< '(' arglist< none='None' ',' seq=any > ')' > - [extra_trailers=trailer*] - > - | - power< - 'filter' - args=trailer< '(' [any] ')' > - [extra_trailers=trailer*] - > - """ - - skip_on = "future_builtins.filter" - - def transform(self, node, results): - if self.should_skip(node): - return - - trailers = [] - if 'extra_trailers' in results: - for t in results['extra_trailers']: - trailers.append(t.clone()) - - if "filter_lambda" in results: - xp = results.get("xp").clone() - if xp.type == syms.test: - xp.prefix = "" - xp = parenthesize(xp) - - new = ListComp(results.get("fp").clone(), - results.get("fp").clone(), - results.get("it").clone(), xp) - new = Node(syms.power, [new] + trailers, prefix="") - - elif "none" in results: - new = ListComp(Name("_f"), - Name("_f"), - results["seq"].clone(), - Name("_f")) - new = Node(syms.power, [new] + trailers, prefix="") - - else: - if in_special_context(node): - return None - - args = results['args'].clone() - new = Node(syms.power, [Name("filter"), args], prefix="") - new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) - new.prefix = "" - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/fixes/fix_funcattrs.py b/Lib/lib2to3/fixes/fix_funcattrs.py deleted file mode 100644 index 67f3e18e061bdb..00000000000000 --- a/Lib/lib2to3/fixes/fix_funcattrs.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Fix function attribute names (f.func_x -> f.__x__).""" -# Author: Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixFuncattrs(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< any+ trailer< '.' attr=('func_closure' | 'func_doc' | 'func_globals' - | 'func_name' | 'func_defaults' | 'func_code' - | 'func_dict') > any* > - """ - - def transform(self, node, results): - attr = results["attr"][0] - attr.replace(Name(("__%s__" % attr.value[5:]), - prefix=attr.prefix)) diff --git a/Lib/lib2to3/fixes/fix_future.py b/Lib/lib2to3/fixes/fix_future.py deleted file mode 100644 index fbcb86af079133..00000000000000 --- a/Lib/lib2to3/fixes/fix_future.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Remove __future__ imports - -from __future__ import foo is replaced with an empty line. -""" -# Author: Christian Heimes - -# Local imports -from .. import fixer_base -from ..fixer_util import BlankLine - -class FixFuture(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """import_from< 'from' module_name="__future__" 'import' any >""" - - # This should be run last -- some things check for the import - run_order = 10 - - def transform(self, node, results): - new = BlankLine() - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/fixes/fix_getcwdu.py b/Lib/lib2to3/fixes/fix_getcwdu.py deleted file mode 100644 index 087eaedcb26f9c..00000000000000 --- a/Lib/lib2to3/fixes/fix_getcwdu.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Fixer that changes os.getcwdu() to os.getcwd(). -""" -# Author: Victor Stinner - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixGetcwdu(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - power< 'os' trailer< dot='.' name='getcwdu' > any* > - """ - - def transform(self, node, results): - name = results["name"] - name.replace(Name("getcwd", prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_has_key.py b/Lib/lib2to3/fixes/fix_has_key.py deleted file mode 100644 index 439708c9923404..00000000000000 --- a/Lib/lib2to3/fixes/fix_has_key.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for has_key(). - -Calls to .has_key() methods are expressed in terms of the 'in' -operator: - - d.has_key(k) -> k in d - -CAVEATS: -1) While the primary target of this fixer is dict.has_key(), the - fixer will change any has_key() method call, regardless of its - class. - -2) Cases like this will not be converted: - - m = d.has_key - if m(k): - ... - - Only *calls* to has_key() are converted. While it is possible to - convert the above to something like - - m = d.__contains__ - if m(k): - ... - - this is currently not done. -""" - -# Local imports -from .. import pytree -from .. import fixer_base -from ..fixer_util import Name, parenthesize - - -class FixHasKey(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - anchor=power< - before=any+ - trailer< '.' 'has_key' > - trailer< - '(' - ( not(arglist | argument) arg=any ','> - ) - ')' - > - after=any* - > - | - negation=not_test< - 'not' - anchor=power< - before=any+ - trailer< '.' 'has_key' > - trailer< - '(' - ( not(arglist | argument) arg=any ','> - ) - ')' - > - > - > - """ - - def transform(self, node, results): - assert results - syms = self.syms - if (node.parent.type == syms.not_test and - self.pattern.match(node.parent)): - # Don't transform a node matching the first alternative of the - # pattern when its parent matches the second alternative - return None - negation = results.get("negation") - anchor = results["anchor"] - prefix = node.prefix - before = [n.clone() for n in results["before"]] - arg = results["arg"].clone() - after = results.get("after") - if after: - after = [n.clone() for n in after] - if arg.type in (syms.comparison, syms.not_test, syms.and_test, - syms.or_test, syms.test, syms.lambdef, syms.argument): - arg = parenthesize(arg) - if len(before) == 1: - before = before[0] - else: - before = pytree.Node(syms.power, before) - before.prefix = " " - n_op = Name("in", prefix=" ") - if negation: - n_not = Name("not", prefix=" ") - n_op = pytree.Node(syms.comp_op, (n_not, n_op)) - new = pytree.Node(syms.comparison, (arg, n_op, before)) - if after: - new = parenthesize(new) - new = pytree.Node(syms.power, (new,) + tuple(after)) - if node.parent.type in (syms.comparison, syms.expr, syms.xor_expr, - syms.and_expr, syms.shift_expr, - syms.arith_expr, syms.term, - syms.factor, syms.power): - new = parenthesize(new) - new.prefix = prefix - return new diff --git a/Lib/lib2to3/fixes/fix_idioms.py b/Lib/lib2to3/fixes/fix_idioms.py deleted file mode 100644 index 6905913d7cb79d..00000000000000 --- a/Lib/lib2to3/fixes/fix_idioms.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Adjust some old Python 2 idioms to their modern counterparts. - -* Change some type comparisons to isinstance() calls: - type(x) == T -> isinstance(x, T) - type(x) is T -> isinstance(x, T) - type(x) != T -> not isinstance(x, T) - type(x) is not T -> not isinstance(x, T) - -* Change "while 1:" into "while True:". - -* Change both - - v = list(EXPR) - v.sort() - foo(v) - -and the more general - - v = EXPR - v.sort() - foo(v) - -into - - v = sorted(EXPR) - foo(v) -""" -# Author: Jacques Frechet, Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Call, Comma, Name, Node, BlankLine, syms - -CMP = "(n='!=' | '==' | 'is' | n=comp_op< 'is' 'not' >)" -TYPE = "power< 'type' trailer< '(' x=any ')' > >" - -class FixIdioms(fixer_base.BaseFix): - explicit = True # The user must ask for this fixer - - PATTERN = r""" - isinstance=comparison< %s %s T=any > - | - isinstance=comparison< T=any %s %s > - | - while_stmt< 'while' while='1' ':' any+ > - | - sorted=any< - any* - simple_stmt< - expr_stmt< id1=any '=' - power< list='list' trailer< '(' (not arglist) any ')' > > - > - '\n' - > - sort= - simple_stmt< - power< id2=any - trailer< '.' 'sort' > trailer< '(' ')' > - > - '\n' - > - next=any* - > - | - sorted=any< - any* - simple_stmt< expr_stmt< id1=any '=' expr=any > '\n' > - sort= - simple_stmt< - power< id2=any - trailer< '.' 'sort' > trailer< '(' ')' > - > - '\n' - > - next=any* - > - """ % (TYPE, CMP, CMP, TYPE) - - def match(self, node): - r = super(FixIdioms, self).match(node) - # If we've matched one of the sort/sorted subpatterns above, we - # want to reject matches where the initial assignment and the - # subsequent .sort() call involve different identifiers. - if r and "sorted" in r: - if r["id1"] == r["id2"]: - return r - return None - return r - - def transform(self, node, results): - if "isinstance" in results: - return self.transform_isinstance(node, results) - elif "while" in results: - return self.transform_while(node, results) - elif "sorted" in results: - return self.transform_sort(node, results) - else: - raise RuntimeError("Invalid match") - - def transform_isinstance(self, node, results): - x = results["x"].clone() # The thing inside of type() - T = results["T"].clone() # The type being compared against - x.prefix = "" - T.prefix = " " - test = Call(Name("isinstance"), [x, Comma(), T]) - if "n" in results: - test.prefix = " " - test = Node(syms.not_test, [Name("not"), test]) - test.prefix = node.prefix - return test - - def transform_while(self, node, results): - one = results["while"] - one.replace(Name("True", prefix=one.prefix)) - - def transform_sort(self, node, results): - sort_stmt = results["sort"] - next_stmt = results["next"] - list_call = results.get("list") - simple_expr = results.get("expr") - - if list_call: - list_call.replace(Name("sorted", prefix=list_call.prefix)) - elif simple_expr: - new = simple_expr.clone() - new.prefix = "" - simple_expr.replace(Call(Name("sorted"), [new], - prefix=simple_expr.prefix)) - else: - raise RuntimeError("should not have reached here") - sort_stmt.remove() - - btwn = sort_stmt.prefix - # Keep any prefix lines between the sort_stmt and the list_call and - # shove them right after the sorted() call. - if "\n" in btwn: - if next_stmt: - # The new prefix should be everything from the sort_stmt's - # prefix up to the last newline, then the old prefix after a new - # line. - prefix_lines = (btwn.rpartition("\n")[0], next_stmt[0].prefix) - next_stmt[0].prefix = "\n".join(prefix_lines) - else: - assert list_call.parent - assert list_call.next_sibling is None - # Put a blank line after list_call and set its prefix. - end_line = BlankLine() - list_call.parent.append_child(end_line) - assert list_call.next_sibling is end_line - # The new prefix should be everything up to the first new line - # of sort_stmt's prefix. - end_line.prefix = btwn.rpartition("\n")[0] diff --git a/Lib/lib2to3/fixes/fix_import.py b/Lib/lib2to3/fixes/fix_import.py deleted file mode 100644 index 734ca294699c36..00000000000000 --- a/Lib/lib2to3/fixes/fix_import.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Fixer for import statements. -If spam is being imported from the local directory, this import: - from spam import eggs -Becomes: - from .spam import eggs - -And this import: - import spam -Becomes: - from . import spam -""" - -# Local imports -from .. import fixer_base -from os.path import dirname, join, exists, sep -from ..fixer_util import FromImport, syms, token - - -def traverse_imports(names): - """ - Walks over all the names imported in a dotted_as_names node. - """ - pending = [names] - while pending: - node = pending.pop() - if node.type == token.NAME: - yield node.value - elif node.type == syms.dotted_name: - yield "".join([ch.value for ch in node.children]) - elif node.type == syms.dotted_as_name: - pending.append(node.children[0]) - elif node.type == syms.dotted_as_names: - pending.extend(node.children[::-2]) - else: - raise AssertionError("unknown node type") - - -class FixImport(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - import_from< 'from' imp=any 'import' ['('] any [')'] > - | - import_name< 'import' imp=any > - """ - - def start_tree(self, tree, name): - super(FixImport, self).start_tree(tree, name) - self.skip = "absolute_import" in tree.future_features - - def transform(self, node, results): - if self.skip: - return - imp = results['imp'] - - if node.type == syms.import_from: - # Some imps are top-level (eg: 'import ham') - # some are first level (eg: 'import ham.eggs') - # some are third level (eg: 'import ham.eggs as spam') - # Hence, the loop - while not hasattr(imp, 'value'): - imp = imp.children[0] - if self.probably_a_local_import(imp.value): - imp.value = "." + imp.value - imp.changed() - else: - have_local = False - have_absolute = False - for mod_name in traverse_imports(imp): - if self.probably_a_local_import(mod_name): - have_local = True - else: - have_absolute = True - if have_absolute: - if have_local: - # We won't handle both sibling and absolute imports in the - # same statement at the moment. - self.warning(node, "absolute and local imports together") - return - - new = FromImport(".", [imp]) - new.prefix = node.prefix - return new - - def probably_a_local_import(self, imp_name): - if imp_name.startswith("."): - # Relative imports are certainly not local imports. - return False - imp_name = imp_name.split(".", 1)[0] - base_path = dirname(self.filename) - base_path = join(base_path, imp_name) - # If there is no __init__.py next to the file its not in a package - # so can't be a relative import. - if not exists(join(dirname(base_path), "__init__.py")): - return False - for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd"]: - if exists(base_path + ext): - return True - return False diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py deleted file mode 100644 index aaf4f2f642efb5..00000000000000 --- a/Lib/lib2to3/fixes/fix_imports.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Fix incompatible imports and module references.""" -# Authors: Collin Winter, Nick Edds - -# Local imports -from .. import fixer_base -from ..fixer_util import Name, attr_chain - -MAPPING = {'StringIO': 'io', - 'cStringIO': 'io', - 'cPickle': 'pickle', - '__builtin__' : 'builtins', - 'copy_reg': 'copyreg', - 'Queue': 'queue', - 'SocketServer': 'socketserver', - 'ConfigParser': 'configparser', - 'repr': 'reprlib', - 'FileDialog': 'tkinter.filedialog', - 'tkFileDialog': 'tkinter.filedialog', - 'SimpleDialog': 'tkinter.simpledialog', - 'tkSimpleDialog': 'tkinter.simpledialog', - 'tkColorChooser': 'tkinter.colorchooser', - 'tkCommonDialog': 'tkinter.commondialog', - 'Dialog': 'tkinter.dialog', - 'Tkdnd': 'tkinter.dnd', - 'tkFont': 'tkinter.font', - 'tkMessageBox': 'tkinter.messagebox', - 'ScrolledText': 'tkinter.scrolledtext', - 'Tkconstants': 'tkinter.constants', - 'Tix': 'tkinter.tix', - 'ttk': 'tkinter.ttk', - 'Tkinter': 'tkinter', - 'markupbase': '_markupbase', - '_winreg': 'winreg', - 'thread': '_thread', - 'dummy_thread': '_dummy_thread', - # anydbm and whichdb are handled by fix_imports2 - 'dbhash': 'dbm.bsd', - 'dumbdbm': 'dbm.dumb', - 'dbm': 'dbm.ndbm', - 'gdbm': 'dbm.gnu', - 'xmlrpclib': 'xmlrpc.client', - 'DocXMLRPCServer': 'xmlrpc.server', - 'SimpleXMLRPCServer': 'xmlrpc.server', - 'httplib': 'http.client', - 'htmlentitydefs' : 'html.entities', - 'HTMLParser' : 'html.parser', - 'Cookie': 'http.cookies', - 'cookielib': 'http.cookiejar', - 'BaseHTTPServer': 'http.server', - 'SimpleHTTPServer': 'http.server', - 'CGIHTTPServer': 'http.server', - #'test.test_support': 'test.support', - 'commands': 'subprocess', - 'UserString' : 'collections', - 'UserList' : 'collections', - 'urlparse' : 'urllib.parse', - 'robotparser' : 'urllib.robotparser', -} - - -def alternates(members): - return "(" + "|".join(map(repr, members)) + ")" - - -def build_pattern(mapping=MAPPING): - mod_list = ' | '.join(["module_name='%s'" % key for key in mapping]) - bare_names = alternates(mapping.keys()) - - yield """name_import=import_name< 'import' ((%s) | - multiple_imports=dotted_as_names< any* (%s) any* >) > - """ % (mod_list, mod_list) - yield """import_from< 'from' (%s) 'import' ['('] - ( any | import_as_name< any 'as' any > | - import_as_names< any* >) [')'] > - """ % mod_list - yield """import_name< 'import' (dotted_as_name< (%s) 'as' any > | - multiple_imports=dotted_as_names< - any* dotted_as_name< (%s) 'as' any > any* >) > - """ % (mod_list, mod_list) - - # Find usages of module members in code e.g. thread.foo(bar) - yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names - - -class FixImports(fixer_base.BaseFix): - - BM_compatible = True - keep_line_order = True - # This is overridden in fix_imports2. - mapping = MAPPING - - # We want to run this fixer late, so fix_import doesn't try to make stdlib - # renames into relative imports. - run_order = 6 - - def build_pattern(self): - return "|".join(build_pattern(self.mapping)) - - def compile_pattern(self): - # We override this, so MAPPING can be pragmatically altered and the - # changes will be reflected in PATTERN. - self.PATTERN = self.build_pattern() - super(FixImports, self).compile_pattern() - - # Don't match the node if it's within another match. - def match(self, node): - match = super(FixImports, self).match - results = match(node) - if results: - # Module usage could be in the trailer of an attribute lookup, so we - # might have nested matches when "bare_with_attr" is present. - if "bare_with_attr" not in results and \ - any(match(obj) for obj in attr_chain(node, "parent")): - return False - return results - return False - - def start_tree(self, tree, filename): - super(FixImports, self).start_tree(tree, filename) - self.replace = {} - - def transform(self, node, results): - import_mod = results.get("module_name") - if import_mod: - mod_name = import_mod.value - new_name = self.mapping[mod_name] - import_mod.replace(Name(new_name, prefix=import_mod.prefix)) - if "name_import" in results: - # If it's not a "from x import x, y" or "import x as y" import, - # marked its usage to be replaced. - self.replace[mod_name] = new_name - if "multiple_imports" in results: - # This is a nasty hack to fix multiple imports on a line (e.g., - # "import StringIO, urlparse"). The problem is that I can't - # figure out an easy way to make a pattern recognize the keys of - # MAPPING randomly sprinkled in an import statement. - results = self.match(node) - if results: - self.transform(node, results) - else: - # Replace usage of the module. - bare_name = results["bare_with_attr"][0] - new_name = self.replace.get(bare_name.value) - if new_name: - bare_name.replace(Name(new_name, prefix=bare_name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_imports2.py b/Lib/lib2to3/fixes/fix_imports2.py deleted file mode 100644 index 9a33c67b1dc194..00000000000000 --- a/Lib/lib2to3/fixes/fix_imports2.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Fix incompatible imports and module references that must be fixed after -fix_imports.""" -from . import fix_imports - - -MAPPING = { - 'whichdb': 'dbm', - 'anydbm': 'dbm', - } - - -class FixImports2(fix_imports.FixImports): - - run_order = 7 - - mapping = MAPPING diff --git a/Lib/lib2to3/fixes/fix_input.py b/Lib/lib2to3/fixes/fix_input.py deleted file mode 100644 index 9cf9a48c471f35..00000000000000 --- a/Lib/lib2to3/fixes/fix_input.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Fixer that changes input(...) into eval(input(...)).""" -# Author: Andre Roberge - -# Local imports -from .. import fixer_base -from ..fixer_util import Call, Name -from .. import patcomp - - -context = patcomp.compile_pattern("power< 'eval' trailer< '(' any ')' > >") - - -class FixInput(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< 'input' args=trailer< '(' [any] ')' > > - """ - - def transform(self, node, results): - # If we're already wrapped in an eval() call, we're done. - if context.match(node.parent.parent): - return - - new = node.clone() - new.prefix = "" - return Call(Name("eval"), [new], prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_intern.py b/Lib/lib2to3/fixes/fix_intern.py deleted file mode 100644 index d752843092aacd..00000000000000 --- a/Lib/lib2to3/fixes/fix_intern.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2006 Georg Brandl. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for intern(). - -intern(s) -> sys.intern(s)""" - -# Local imports -from .. import fixer_base -from ..fixer_util import ImportAndCall, touch_import - - -class FixIntern(fixer_base.BaseFix): - BM_compatible = True - order = "pre" - - PATTERN = """ - power< 'intern' - trailer< lpar='(' - ( not(arglist | argument) any ','> ) - rpar=')' > - after=any* - > - """ - - def transform(self, node, results): - if results: - # I feel like we should be able to express this logic in the - # PATTERN above but I don't know how to do it so... - obj = results['obj'] - if obj: - if (obj.type == self.syms.argument and - obj.children[0].value in {'**', '*'}): - return # Make no change. - names = ('sys', 'intern') - new = ImportAndCall(node, results, names) - touch_import(None, 'sys', node) - return new diff --git a/Lib/lib2to3/fixes/fix_isinstance.py b/Lib/lib2to3/fixes/fix_isinstance.py deleted file mode 100644 index bebb1de120424b..00000000000000 --- a/Lib/lib2to3/fixes/fix_isinstance.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2008 Armin Ronacher. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that cleans up a tuple argument to isinstance after the tokens -in it were fixed. This is mainly used to remove double occurrences of -tokens as a leftover of the long -> int / unicode -> str conversion. - -eg. isinstance(x, (int, long)) -> isinstance(x, (int, int)) - -> isinstance(x, int) -""" - -from .. import fixer_base -from ..fixer_util import token - - -class FixIsinstance(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< - 'isinstance' - trailer< '(' arglist< any ',' atom< '(' - args=testlist_gexp< any+ > - ')' > > ')' > - > - """ - - run_order = 6 - - def transform(self, node, results): - names_inserted = set() - testlist = results["args"] - args = testlist.children - new_args = [] - iterator = enumerate(args) - for idx, arg in iterator: - if arg.type == token.NAME and arg.value in names_inserted: - if idx < len(args) - 1 and args[idx + 1].type == token.COMMA: - next(iterator) - continue - else: - new_args.append(arg) - if arg.type == token.NAME: - names_inserted.add(arg.value) - if new_args and new_args[-1].type == token.COMMA: - del new_args[-1] - if len(new_args) == 1: - atom = testlist.parent - new_args[0].prefix = atom.prefix - atom.replace(new_args[0]) - else: - args[:] = new_args - node.changed() diff --git a/Lib/lib2to3/fixes/fix_itertools.py b/Lib/lib2to3/fixes/fix_itertools.py deleted file mode 100644 index 8e78d6c689f439..00000000000000 --- a/Lib/lib2to3/fixes/fix_itertools.py +++ /dev/null @@ -1,43 +0,0 @@ -""" Fixer for itertools.(imap|ifilter|izip) --> (map|filter|zip) and - itertools.ifilterfalse --> itertools.filterfalse (bugs 2360-2363) - - imports from itertools are fixed in fix_itertools_import.py - - If itertools is imported as something else (ie: import itertools as it; - it.izip(spam, eggs)) method calls will not get fixed. - """ - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixItertools(fixer_base.BaseFix): - BM_compatible = True - it_funcs = "('imap'|'ifilter'|'izip'|'izip_longest'|'ifilterfalse')" - PATTERN = """ - power< it='itertools' - trailer< - dot='.' func=%(it_funcs)s > trailer< '(' [any] ')' > > - | - power< func=%(it_funcs)s trailer< '(' [any] ')' > > - """ %(locals()) - - # Needs to be run after fix_(map|zip|filter) - run_order = 6 - - def transform(self, node, results): - prefix = None - func = results['func'][0] - if ('it' in results and - func.value not in ('ifilterfalse', 'izip_longest')): - dot, it = (results['dot'], results['it']) - # Remove the 'itertools' - prefix = it.prefix - it.remove() - # Replace the node which contains ('.', 'function') with the - # function (to be consistent with the second part of the pattern) - dot.remove() - func.parent.replace(func) - - prefix = prefix or func.prefix - func.replace(Name(func.value[1:], prefix=prefix)) diff --git a/Lib/lib2to3/fixes/fix_itertools_imports.py b/Lib/lib2to3/fixes/fix_itertools_imports.py deleted file mode 100644 index 0ddbc7b8422991..00000000000000 --- a/Lib/lib2to3/fixes/fix_itertools_imports.py +++ /dev/null @@ -1,57 +0,0 @@ -""" Fixer for imports of itertools.(imap|ifilter|izip|ifilterfalse) """ - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import BlankLine, syms, token - - -class FixItertoolsImports(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - import_from< 'from' 'itertools' 'import' imports=any > - """ %(locals()) - - def transform(self, node, results): - imports = results['imports'] - if imports.type == syms.import_as_name or not imports.children: - children = [imports] - else: - children = imports.children - for child in children[::2]: - if child.type == token.NAME: - member = child.value - name_node = child - elif child.type == token.STAR: - # Just leave the import as is. - return - else: - assert child.type == syms.import_as_name - name_node = child.children[0] - member_name = name_node.value - if member_name in ('imap', 'izip', 'ifilter'): - child.value = None - child.remove() - elif member_name in ('ifilterfalse', 'izip_longest'): - node.changed() - name_node.value = ('filterfalse' if member_name[1] == 'f' - else 'zip_longest') - - # Make sure the import statement is still sane - children = imports.children[:] or [imports] - remove_comma = True - for child in children: - if remove_comma and child.type == token.COMMA: - child.remove() - else: - remove_comma ^= True - - while children and children[-1].type == token.COMMA: - children.pop().remove() - - # If there are no imports left, just get rid of the entire statement - if (not (imports.children or getattr(imports, 'value', None)) or - imports.parent is None): - p = node.prefix - node = BlankLine() - node.prefix = p - return node diff --git a/Lib/lib2to3/fixes/fix_long.py b/Lib/lib2to3/fixes/fix_long.py deleted file mode 100644 index f227c9f4981538..00000000000000 --- a/Lib/lib2to3/fixes/fix_long.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that turns 'long' into 'int' everywhere. -""" - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import is_probably_builtin - - -class FixLong(fixer_base.BaseFix): - BM_compatible = True - PATTERN = "'long'" - - def transform(self, node, results): - if is_probably_builtin(node): - node.value = "int" - node.changed() diff --git a/Lib/lib2to3/fixes/fix_map.py b/Lib/lib2to3/fixes/fix_map.py deleted file mode 100644 index 78cf81c6f94098..00000000000000 --- a/Lib/lib2to3/fixes/fix_map.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes map(F, ...) into list(map(F, ...)) unless there -exists a 'from future_builtins import map' statement in the top-level -namespace. - -As a special case, map(None, X) is changed into list(X). (This is -necessary because the semantics are changed in this case -- the new -map(None, X) is equivalent to [(x,) for x in X].) - -We avoid the transformation (except for the special case mentioned -above) if the map() call is directly contained in iter(<>), list(<>), -tuple(<>), sorted(<>), ...join(<>), or for V in <>:. - -NOTE: This is still not correct if the original code was depending on -map(F, X, Y, ...) to go on until the longest argument is exhausted, -substituting None for missing values -- like zip(), it now stops as -soon as the shortest argument is exhausted. -""" - -# Local imports -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, ArgList, Call, ListComp, in_special_context -from ..pygram import python_symbols as syms -from ..pytree import Node - - -class FixMap(fixer_base.ConditionalFix): - BM_compatible = True - - PATTERN = """ - map_none=power< - 'map' - trailer< '(' arglist< 'None' ',' arg=any [','] > ')' > - [extra_trailers=trailer*] - > - | - map_lambda=power< - 'map' - trailer< - '(' - arglist< - lambdef< 'lambda' - (fp=NAME | vfpdef< '(' fp=NAME ')'> ) ':' xp=any - > - ',' - it=any - > - ')' - > - [extra_trailers=trailer*] - > - | - power< - 'map' args=trailer< '(' [any] ')' > - [extra_trailers=trailer*] - > - """ - - skip_on = 'future_builtins.map' - - def transform(self, node, results): - if self.should_skip(node): - return - - trailers = [] - if 'extra_trailers' in results: - for t in results['extra_trailers']: - trailers.append(t.clone()) - - if node.parent.type == syms.simple_stmt: - self.warning(node, "You should use a for loop here") - new = node.clone() - new.prefix = "" - new = Call(Name("list"), [new]) - elif "map_lambda" in results: - new = ListComp(results["xp"].clone(), - results["fp"].clone(), - results["it"].clone()) - new = Node(syms.power, [new] + trailers, prefix="") - - else: - if "map_none" in results: - new = results["arg"].clone() - new.prefix = "" - else: - if "args" in results: - args = results["args"] - if args.type == syms.trailer and \ - args.children[1].type == syms.arglist and \ - args.children[1].children[0].type == token.NAME and \ - args.children[1].children[0].value == "None": - self.warning(node, "cannot convert map(None, ...) " - "with multiple arguments because map() " - "now truncates to the shortest sequence") - return - - new = Node(syms.power, [Name("map"), args.clone()]) - new.prefix = "" - - if in_special_context(node): - return None - - new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) - new.prefix = "" - - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/fixes/fix_metaclass.py b/Lib/lib2to3/fixes/fix_metaclass.py deleted file mode 100644 index fe547b2228072a..00000000000000 --- a/Lib/lib2to3/fixes/fix_metaclass.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Fixer for __metaclass__ = X -> (metaclass=X) methods. - - The various forms of classef (inherits nothing, inherits once, inherits - many) don't parse the same in the CST so we look at ALL classes for - a __metaclass__ and if we find one normalize the inherits to all be - an arglist. - - For one-liner classes ('class X: pass') there is no indent/dedent so - we normalize those into having a suite. - - Moving the __metaclass__ into the classdef can also cause the class - body to be empty so there is some special casing for that as well. - - This fixer also tries very hard to keep original indenting and spacing - in all those corner cases. - -""" -# Author: Jack Diederich - -# Local imports -from .. import fixer_base -from ..pygram import token -from ..fixer_util import syms, Node, Leaf - - -def has_metaclass(parent): - """ we have to check the cls_node without changing it. - There are two possibilities: - 1) clsdef => suite => simple_stmt => expr_stmt => Leaf('__meta') - 2) clsdef => simple_stmt => expr_stmt => Leaf('__meta') - """ - for node in parent.children: - if node.type == syms.suite: - return has_metaclass(node) - elif node.type == syms.simple_stmt and node.children: - expr_node = node.children[0] - if expr_node.type == syms.expr_stmt and expr_node.children: - left_side = expr_node.children[0] - if isinstance(left_side, Leaf) and \ - left_side.value == '__metaclass__': - return True - return False - - -def fixup_parse_tree(cls_node): - """ one-line classes don't get a suite in the parse tree so we add - one to normalize the tree - """ - for node in cls_node.children: - if node.type == syms.suite: - # already in the preferred format, do nothing - return - - # !%@#! one-liners have no suite node, we have to fake one up - for i, node in enumerate(cls_node.children): - if node.type == token.COLON: - break - else: - raise ValueError("No class suite and no ':'!") - - # move everything into a suite node - suite = Node(syms.suite, []) - while cls_node.children[i+1:]: - move_node = cls_node.children[i+1] - suite.append_child(move_node.clone()) - move_node.remove() - cls_node.append_child(suite) - node = suite - - -def fixup_simple_stmt(parent, i, stmt_node): - """ if there is a semi-colon all the parts count as part of the same - simple_stmt. We just want the __metaclass__ part so we move - everything after the semi-colon into its own simple_stmt node - """ - for semi_ind, node in enumerate(stmt_node.children): - if node.type == token.SEMI: # *sigh* - break - else: - return - - node.remove() # kill the semicolon - new_expr = Node(syms.expr_stmt, []) - new_stmt = Node(syms.simple_stmt, [new_expr]) - while stmt_node.children[semi_ind:]: - move_node = stmt_node.children[semi_ind] - new_expr.append_child(move_node.clone()) - move_node.remove() - parent.insert_child(i, new_stmt) - new_leaf1 = new_stmt.children[0].children[0] - old_leaf1 = stmt_node.children[0].children[0] - new_leaf1.prefix = old_leaf1.prefix - - -def remove_trailing_newline(node): - if node.children and node.children[-1].type == token.NEWLINE: - node.children[-1].remove() - - -def find_metas(cls_node): - # find the suite node (Mmm, sweet nodes) - for node in cls_node.children: - if node.type == syms.suite: - break - else: - raise ValueError("No class suite!") - - # look for simple_stmt[ expr_stmt[ Leaf('__metaclass__') ] ] - for i, simple_node in list(enumerate(node.children)): - if simple_node.type == syms.simple_stmt and simple_node.children: - expr_node = simple_node.children[0] - if expr_node.type == syms.expr_stmt and expr_node.children: - # Check if the expr_node is a simple assignment. - left_node = expr_node.children[0] - if isinstance(left_node, Leaf) and \ - left_node.value == '__metaclass__': - # We found an assignment to __metaclass__. - fixup_simple_stmt(node, i, simple_node) - remove_trailing_newline(simple_node) - yield (node, i, simple_node) - - -def fixup_indent(suite): - """ If an INDENT is followed by a thing with a prefix then nuke the prefix - Otherwise we get in trouble when removing __metaclass__ at suite start - """ - kids = suite.children[::-1] - # find the first indent - while kids: - node = kids.pop() - if node.type == token.INDENT: - break - - # find the first Leaf - while kids: - node = kids.pop() - if isinstance(node, Leaf) and node.type != token.DEDENT: - if node.prefix: - node.prefix = '' - return - else: - kids.extend(node.children[::-1]) - - -class FixMetaclass(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - classdef - """ - - def transform(self, node, results): - if not has_metaclass(node): - return - - fixup_parse_tree(node) - - # find metaclasses, keep the last one - last_metaclass = None - for suite, i, stmt in find_metas(node): - last_metaclass = stmt - stmt.remove() - - text_type = node.children[0].type # always Leaf(nnn, 'class') - - # figure out what kind of classdef we have - if len(node.children) == 7: - # Node(classdef, ['class', 'name', '(', arglist, ')', ':', suite]) - # 0 1 2 3 4 5 6 - if node.children[3].type == syms.arglist: - arglist = node.children[3] - # Node(classdef, ['class', 'name', '(', 'Parent', ')', ':', suite]) - else: - parent = node.children[3].clone() - arglist = Node(syms.arglist, [parent]) - node.set_child(3, arglist) - elif len(node.children) == 6: - # Node(classdef, ['class', 'name', '(', ')', ':', suite]) - # 0 1 2 3 4 5 - arglist = Node(syms.arglist, []) - node.insert_child(3, arglist) - elif len(node.children) == 4: - # Node(classdef, ['class', 'name', ':', suite]) - # 0 1 2 3 - arglist = Node(syms.arglist, []) - node.insert_child(2, Leaf(token.RPAR, ')')) - node.insert_child(2, arglist) - node.insert_child(2, Leaf(token.LPAR, '(')) - else: - raise ValueError("Unexpected class definition") - - # now stick the metaclass in the arglist - meta_txt = last_metaclass.children[0].children[0] - meta_txt.value = 'metaclass' - orig_meta_prefix = meta_txt.prefix - - if arglist.children: - arglist.append_child(Leaf(token.COMMA, ',')) - meta_txt.prefix = ' ' - else: - meta_txt.prefix = '' - - # compact the expression "metaclass = Meta" -> "metaclass=Meta" - expr_stmt = last_metaclass.children[0] - assert expr_stmt.type == syms.expr_stmt - expr_stmt.children[1].prefix = '' - expr_stmt.children[2].prefix = '' - - arglist.append_child(last_metaclass) - - fixup_indent(suite) - - # check for empty suite - if not suite.children: - # one-liner that was just __metaclass_ - suite.remove() - pass_leaf = Leaf(text_type, 'pass') - pass_leaf.prefix = orig_meta_prefix - node.append_child(pass_leaf) - node.append_child(Leaf(token.NEWLINE, '\n')) - - elif len(suite.children) > 1 and \ - (suite.children[-2].type == token.INDENT and - suite.children[-1].type == token.DEDENT): - # there was only one line in the class body and it was __metaclass__ - pass_leaf = Leaf(text_type, 'pass') - suite.insert_child(-1, pass_leaf) - suite.insert_child(-1, Leaf(token.NEWLINE, '\n')) diff --git a/Lib/lib2to3/fixes/fix_methodattrs.py b/Lib/lib2to3/fixes/fix_methodattrs.py deleted file mode 100644 index 7f9004f00e6e8f..00000000000000 --- a/Lib/lib2to3/fixes/fix_methodattrs.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Fix bound method attributes (method.im_? -> method.__?__). -""" -# Author: Christian Heimes - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -MAP = { - "im_func" : "__func__", - "im_self" : "__self__", - "im_class" : "__self__.__class__" - } - -class FixMethodattrs(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< any+ trailer< '.' attr=('im_func' | 'im_self' | 'im_class') > any* > - """ - - def transform(self, node, results): - attr = results["attr"][0] - new = MAP[attr.value] - attr.replace(Name(new, prefix=attr.prefix)) diff --git a/Lib/lib2to3/fixes/fix_ne.py b/Lib/lib2to3/fixes/fix_ne.py deleted file mode 100644 index e3ee10f4a63e0c..00000000000000 --- a/Lib/lib2to3/fixes/fix_ne.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that turns <> into !=.""" - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base - - -class FixNe(fixer_base.BaseFix): - # This is so simple that we don't need the pattern compiler. - - _accept_type = token.NOTEQUAL - - def match(self, node): - # Override - return node.value == "<>" - - def transform(self, node, results): - new = pytree.Leaf(token.NOTEQUAL, "!=", prefix=node.prefix) - return new diff --git a/Lib/lib2to3/fixes/fix_next.py b/Lib/lib2to3/fixes/fix_next.py deleted file mode 100644 index 9f6305e1d49dc5..00000000000000 --- a/Lib/lib2to3/fixes/fix_next.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Fixer for it.next() -> next(it), per PEP 3114.""" -# Author: Collin Winter - -# Things that currently aren't covered: -# - listcomp "next" names aren't warned -# - "with" statement targets aren't checked - -# Local imports -from ..pgen2 import token -from ..pygram import python_symbols as syms -from .. import fixer_base -from ..fixer_util import Name, Call, find_binding - -bind_warning = "Calls to builtin next() possibly shadowed by global binding" - - -class FixNext(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< base=any+ trailer< '.' attr='next' > trailer< '(' ')' > > - | - power< head=any+ trailer< '.' attr='next' > not trailer< '(' ')' > > - | - classdef< 'class' any+ ':' - suite< any* - funcdef< 'def' - name='next' - parameters< '(' NAME ')' > any+ > - any* > > - | - global=global_stmt< 'global' any* 'next' any* > - """ - - order = "pre" # Pre-order tree traversal - - def start_tree(self, tree, filename): - super(FixNext, self).start_tree(tree, filename) - - n = find_binding('next', tree) - if n: - self.warning(n, bind_warning) - self.shadowed_next = True - else: - self.shadowed_next = False - - def transform(self, node, results): - assert results - - base = results.get("base") - attr = results.get("attr") - name = results.get("name") - - if base: - if self.shadowed_next: - attr.replace(Name("__next__", prefix=attr.prefix)) - else: - base = [n.clone() for n in base] - base[0].prefix = "" - node.replace(Call(Name("next", prefix=node.prefix), base)) - elif name: - n = Name("__next__", prefix=name.prefix) - name.replace(n) - elif attr: - # We don't do this transformation if we're assigning to "x.next". - # Unfortunately, it doesn't seem possible to do this in PATTERN, - # so it's being done here. - if is_assign_target(node): - head = results["head"] - if "".join([str(n) for n in head]).strip() == '__builtin__': - self.warning(node, bind_warning) - return - attr.replace(Name("__next__")) - elif "global" in results: - self.warning(node, bind_warning) - self.shadowed_next = True - - -### The following functions help test if node is part of an assignment -### target. - -def is_assign_target(node): - assign = find_assign(node) - if assign is None: - return False - - for child in assign.children: - if child.type == token.EQUAL: - return False - elif is_subtree(child, node): - return True - return False - -def find_assign(node): - if node.type == syms.expr_stmt: - return node - if node.type == syms.simple_stmt or node.parent is None: - return None - return find_assign(node.parent) - -def is_subtree(root, node): - if root == node: - return True - return any(is_subtree(c, node) for c in root.children) diff --git a/Lib/lib2to3/fixes/fix_nonzero.py b/Lib/lib2to3/fixes/fix_nonzero.py deleted file mode 100644 index c2295969a7728f..00000000000000 --- a/Lib/lib2to3/fixes/fix_nonzero.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Fixer for __nonzero__ -> __bool__ methods.""" -# Author: Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixNonzero(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - classdef< 'class' any+ ':' - suite< any* - funcdef< 'def' name='__nonzero__' - parameters< '(' NAME ')' > any+ > - any* > > - """ - - def transform(self, node, results): - name = results["name"] - new = Name("__bool__", prefix=name.prefix) - name.replace(new) diff --git a/Lib/lib2to3/fixes/fix_numliterals.py b/Lib/lib2to3/fixes/fix_numliterals.py deleted file mode 100644 index 79207d4aa368ae..00000000000000 --- a/Lib/lib2to3/fixes/fix_numliterals.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Fixer that turns 1L into 1, 0755 into 0o755. -""" -# Copyright 2007 Georg Brandl. -# Licensed to PSF under a Contributor Agreement. - -# Local imports -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Number - - -class FixNumliterals(fixer_base.BaseFix): - # This is so simple that we don't need the pattern compiler. - - _accept_type = token.NUMBER - - def match(self, node): - # Override - return (node.value.startswith("0") or node.value[-1] in "Ll") - - def transform(self, node, results): - val = node.value - if val[-1] in 'Ll': - val = val[:-1] - elif val.startswith('0') and val.isdigit() and len(set(val)) > 1: - val = "0o" + val[1:] - - return Number(val, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_operator.py b/Lib/lib2to3/fixes/fix_operator.py deleted file mode 100644 index d303cd2018befb..00000000000000 --- a/Lib/lib2to3/fixes/fix_operator.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Fixer for operator functions. - -operator.isCallable(obj) -> callable(obj) -operator.sequenceIncludes(obj) -> operator.contains(obj) -operator.isSequenceType(obj) -> isinstance(obj, collections.abc.Sequence) -operator.isMappingType(obj) -> isinstance(obj, collections.abc.Mapping) -operator.isNumberType(obj) -> isinstance(obj, numbers.Number) -operator.repeat(obj, n) -> operator.mul(obj, n) -operator.irepeat(obj, n) -> operator.imul(obj, n) -""" - -import collections.abc - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import Call, Name, String, touch_import - - -def invocation(s): - def dec(f): - f.invocation = s - return f - return dec - - -class FixOperator(fixer_base.BaseFix): - BM_compatible = True - order = "pre" - - methods = """ - method=('isCallable'|'sequenceIncludes' - |'isSequenceType'|'isMappingType'|'isNumberType' - |'repeat'|'irepeat') - """ - obj = "'(' obj=any ')'" - PATTERN = """ - power< module='operator' - trailer< '.' %(methods)s > trailer< %(obj)s > > - | - power< %(methods)s trailer< %(obj)s > > - """ % dict(methods=methods, obj=obj) - - def transform(self, node, results): - method = self._check_method(node, results) - if method is not None: - return method(node, results) - - @invocation("operator.contains(%s)") - def _sequenceIncludes(self, node, results): - return self._handle_rename(node, results, "contains") - - @invocation("callable(%s)") - def _isCallable(self, node, results): - obj = results["obj"] - return Call(Name("callable"), [obj.clone()], prefix=node.prefix) - - @invocation("operator.mul(%s)") - def _repeat(self, node, results): - return self._handle_rename(node, results, "mul") - - @invocation("operator.imul(%s)") - def _irepeat(self, node, results): - return self._handle_rename(node, results, "imul") - - @invocation("isinstance(%s, collections.abc.Sequence)") - def _isSequenceType(self, node, results): - return self._handle_type2abc(node, results, "collections.abc", "Sequence") - - @invocation("isinstance(%s, collections.abc.Mapping)") - def _isMappingType(self, node, results): - return self._handle_type2abc(node, results, "collections.abc", "Mapping") - - @invocation("isinstance(%s, numbers.Number)") - def _isNumberType(self, node, results): - return self._handle_type2abc(node, results, "numbers", "Number") - - def _handle_rename(self, node, results, name): - method = results["method"][0] - method.value = name - method.changed() - - def _handle_type2abc(self, node, results, module, abc): - touch_import(None, module, node) - obj = results["obj"] - args = [obj.clone(), String(", " + ".".join([module, abc]))] - return Call(Name("isinstance"), args, prefix=node.prefix) - - def _check_method(self, node, results): - method = getattr(self, "_" + results["method"][0].value) - if isinstance(method, collections.abc.Callable): - if "module" in results: - return method - else: - sub = (str(results["obj"]),) - invocation_str = method.invocation % sub - self.warning(node, "You should use '%s' here." % invocation_str) - return None diff --git a/Lib/lib2to3/fixes/fix_paren.py b/Lib/lib2to3/fixes/fix_paren.py deleted file mode 100644 index df3da5f5232c9c..00000000000000 --- a/Lib/lib2to3/fixes/fix_paren.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Fixer that adds parentheses where they are required - -This converts ``[x for x in 1, 2]`` to ``[x for x in (1, 2)]``.""" - -# By Taek Joo Kim and Benjamin Peterson - -# Local imports -from .. import fixer_base -from ..fixer_util import LParen, RParen - -# XXX This doesn't support nested for loops like [x for x in 1, 2 for x in 1, 2] -class FixParen(fixer_base.BaseFix): - BM_compatible = True - - PATTERN = """ - atom< ('[' | '(') - (listmaker< any - comp_for< - 'for' NAME 'in' - target=testlist_safe< any (',' any)+ [','] - > - [any] - > - > - | - testlist_gexp< any - comp_for< - 'for' NAME 'in' - target=testlist_safe< any (',' any)+ [','] - > - [any] - > - >) - (']' | ')') > - """ - - def transform(self, node, results): - target = results["target"] - - lparen = LParen() - lparen.prefix = target.prefix - target.prefix = "" # Make it hug the parentheses - target.insert_child(0, lparen) - target.append_child(RParen()) diff --git a/Lib/lib2to3/fixes/fix_print.py b/Lib/lib2to3/fixes/fix_print.py deleted file mode 100644 index 8780322265f6fe..00000000000000 --- a/Lib/lib2to3/fixes/fix_print.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for print. - -Change: - 'print' into 'print()' - 'print ...' into 'print(...)' - 'print ... ,' into 'print(..., end=" ")' - 'print >>x, ...' into 'print(..., file=x)' - -No changes are applied if print_function is imported from __future__ - -""" - -# Local imports -from .. import patcomp -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, Call, Comma, String - - -parend_expr = patcomp.compile_pattern( - """atom< '(' [atom|STRING|NAME] ')' >""" - ) - - -class FixPrint(fixer_base.BaseFix): - - BM_compatible = True - - PATTERN = """ - simple_stmt< any* bare='print' any* > | print_stmt - """ - - def transform(self, node, results): - assert results - - bare_print = results.get("bare") - - if bare_print: - # Special-case print all by itself - bare_print.replace(Call(Name("print"), [], - prefix=bare_print.prefix)) - return - assert node.children[0] == Name("print") - args = node.children[1:] - if len(args) == 1 and parend_expr.match(args[0]): - # We don't want to keep sticking parens around an - # already-parenthesised expression. - return - - sep = end = file = None - if args and args[-1] == Comma(): - args = args[:-1] - end = " " - if args and args[0] == pytree.Leaf(token.RIGHTSHIFT, ">>"): - assert len(args) >= 2 - file = args[1].clone() - args = args[3:] # Strip a possible comma after the file expression - # Now synthesize a print(args, sep=..., end=..., file=...) node. - l_args = [arg.clone() for arg in args] - if l_args: - l_args[0].prefix = "" - if sep is not None or end is not None or file is not None: - if sep is not None: - self.add_kwarg(l_args, "sep", String(repr(sep))) - if end is not None: - self.add_kwarg(l_args, "end", String(repr(end))) - if file is not None: - self.add_kwarg(l_args, "file", file) - n_stmt = Call(Name("print"), l_args) - n_stmt.prefix = node.prefix - return n_stmt - - def add_kwarg(self, l_nodes, s_kwd, n_expr): - # XXX All this prefix-setting may lose comments (though rarely) - n_expr.prefix = "" - n_argument = pytree.Node(self.syms.argument, - (Name(s_kwd), - pytree.Leaf(token.EQUAL, "="), - n_expr)) - if l_nodes: - l_nodes.append(Comma()) - n_argument.prefix = " " - l_nodes.append(n_argument) diff --git a/Lib/lib2to3/fixes/fix_raise.py b/Lib/lib2to3/fixes/fix_raise.py deleted file mode 100644 index 05aa21e74a30ff..00000000000000 --- a/Lib/lib2to3/fixes/fix_raise.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Fixer for 'raise E, V, T' - -raise -> raise -raise E -> raise E -raise E, V -> raise E(V) -raise E, V, T -> raise E(V).with_traceback(T) -raise E, None, T -> raise E.with_traceback(T) - -raise (((E, E'), E''), E'''), V -> raise E(V) -raise "foo", V, T -> warns about string exceptions - - -CAVEATS: -1) "raise E, V" will be incorrectly translated if V is an exception - instance. The correct Python 3 idiom is - - raise E from V - - but since we can't detect instance-hood by syntax alone and since - any client code would have to be changed as well, we don't automate - this. -""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, Call, Attr, ArgList, is_tuple - -class FixRaise(fixer_base.BaseFix): - - BM_compatible = True - PATTERN = """ - raise_stmt< 'raise' exc=any [',' val=any [',' tb=any]] > - """ - - def transform(self, node, results): - syms = self.syms - - exc = results["exc"].clone() - if exc.type == token.STRING: - msg = "Python 3 does not support string exceptions" - self.cannot_convert(node, msg) - return - - # Python 2 supports - # raise ((((E1, E2), E3), E4), E5), V - # as a synonym for - # raise E1, V - # Since Python 3 will not support this, we recurse down any tuple - # literals, always taking the first element. - if is_tuple(exc): - while is_tuple(exc): - # exc.children[1:-1] is the unparenthesized tuple - # exc.children[1].children[0] is the first element of the tuple - exc = exc.children[1].children[0].clone() - exc.prefix = " " - - if "val" not in results: - # One-argument raise - new = pytree.Node(syms.raise_stmt, [Name("raise"), exc]) - new.prefix = node.prefix - return new - - val = results["val"].clone() - if is_tuple(val): - args = [c.clone() for c in val.children[1:-1]] - else: - val.prefix = "" - args = [val] - - if "tb" in results: - tb = results["tb"].clone() - tb.prefix = "" - - e = exc - # If there's a traceback and None is passed as the value, then don't - # add a call, since the user probably just wants to add a - # traceback. See issue #9661. - if val.type != token.NAME or val.value != "None": - e = Call(exc, args) - with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])] - new = pytree.Node(syms.simple_stmt, [Name("raise")] + with_tb) - new.prefix = node.prefix - return new - else: - return pytree.Node(syms.raise_stmt, - [Name("raise"), Call(exc, args)], - prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_raw_input.py b/Lib/lib2to3/fixes/fix_raw_input.py deleted file mode 100644 index a51bb694b9e01e..00000000000000 --- a/Lib/lib2to3/fixes/fix_raw_input.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Fixer that changes raw_input(...) into input(...).""" -# Author: Andre Roberge - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -class FixRawInput(fixer_base.BaseFix): - - BM_compatible = True - PATTERN = """ - power< name='raw_input' trailer< '(' [any] ')' > any* > - """ - - def transform(self, node, results): - name = results["name"] - name.replace(Name("input", prefix=name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_reduce.py b/Lib/lib2to3/fixes/fix_reduce.py deleted file mode 100644 index 00e5aa1c33d482..00000000000000 --- a/Lib/lib2to3/fixes/fix_reduce.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2008 Armin Ronacher. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for reduce(). - -Makes sure reduce() is imported from the functools module if reduce is -used in that module. -""" - -from lib2to3 import fixer_base -from lib2to3.fixer_util import touch_import - - - -class FixReduce(fixer_base.BaseFix): - - BM_compatible = True - order = "pre" - - PATTERN = """ - power< 'reduce' - trailer< '(' - arglist< ( - (not(argument) any ',' - not(argument - > - """ - - def transform(self, node, results): - touch_import('functools', 'reduce', node) diff --git a/Lib/lib2to3/fixes/fix_reload.py b/Lib/lib2to3/fixes/fix_reload.py deleted file mode 100644 index b30841131c51f9..00000000000000 --- a/Lib/lib2to3/fixes/fix_reload.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Fixer for reload(). - -reload(s) -> importlib.reload(s)""" - -# Local imports -from .. import fixer_base -from ..fixer_util import ImportAndCall, touch_import - - -class FixReload(fixer_base.BaseFix): - BM_compatible = True - order = "pre" - - PATTERN = """ - power< 'reload' - trailer< lpar='(' - ( not(arglist | argument) any ','> ) - rpar=')' > - after=any* - > - """ - - def transform(self, node, results): - if results: - # I feel like we should be able to express this logic in the - # PATTERN above but I don't know how to do it so... - obj = results['obj'] - if obj: - if (obj.type == self.syms.argument and - obj.children[0].value in {'**', '*'}): - return # Make no change. - names = ('importlib', 'reload') - new = ImportAndCall(node, results, names) - touch_import(None, 'importlib', node) - return new diff --git a/Lib/lib2to3/fixes/fix_renames.py b/Lib/lib2to3/fixes/fix_renames.py deleted file mode 100644 index c0e3705ab7be19..00000000000000 --- a/Lib/lib2to3/fixes/fix_renames.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Fix incompatible renames - -Fixes: - * sys.maxint -> sys.maxsize -""" -# Author: Christian Heimes -# based on Collin Winter's fix_import - -# Local imports -from .. import fixer_base -from ..fixer_util import Name, attr_chain - -MAPPING = {"sys": {"maxint" : "maxsize"}, - } -LOOKUP = {} - -def alternates(members): - return "(" + "|".join(map(repr, members)) + ")" - - -def build_pattern(): - #bare = set() - for module, replace in list(MAPPING.items()): - for old_attr, new_attr in list(replace.items()): - LOOKUP[(module, old_attr)] = new_attr - #bare.add(module) - #bare.add(old_attr) - #yield """ - # import_name< 'import' (module=%r - # | dotted_as_names< any* module=%r any* >) > - # """ % (module, module) - yield """ - import_from< 'from' module_name=%r 'import' - ( attr_name=%r | import_as_name< attr_name=%r 'as' any >) > - """ % (module, old_attr, old_attr) - yield """ - power< module_name=%r trailer< '.' attr_name=%r > any* > - """ % (module, old_attr) - #yield """bare_name=%s""" % alternates(bare) - - -class FixRenames(fixer_base.BaseFix): - BM_compatible = True - PATTERN = "|".join(build_pattern()) - - order = "pre" # Pre-order tree traversal - - # Don't match the node if it's within another match - def match(self, node): - match = super(FixRenames, self).match - results = match(node) - if results: - if any(match(obj) for obj in attr_chain(node, "parent")): - return False - return results - return False - - #def start_tree(self, tree, filename): - # super(FixRenames, self).start_tree(tree, filename) - # self.replace = {} - - def transform(self, node, results): - mod_name = results.get("module_name") - attr_name = results.get("attr_name") - #bare_name = results.get("bare_name") - #import_mod = results.get("module") - - if mod_name and attr_name: - new_attr = LOOKUP[(mod_name.value, attr_name.value)] - attr_name.replace(Name(new_attr, prefix=attr_name.prefix)) diff --git a/Lib/lib2to3/fixes/fix_repr.py b/Lib/lib2to3/fixes/fix_repr.py deleted file mode 100644 index 1150bb8b9db2af..00000000000000 --- a/Lib/lib2to3/fixes/fix_repr.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that transforms `xyzzy` into repr(xyzzy).""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Call, Name, parenthesize - - -class FixRepr(fixer_base.BaseFix): - - BM_compatible = True - PATTERN = """ - atom < '`' expr=any '`' > - """ - - def transform(self, node, results): - expr = results["expr"].clone() - - if expr.type == self.syms.testlist1: - expr = parenthesize(expr) - return Call(Name("repr"), [expr], prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_set_literal.py b/Lib/lib2to3/fixes/fix_set_literal.py deleted file mode 100644 index 762550cf73dc0b..00000000000000 --- a/Lib/lib2to3/fixes/fix_set_literal.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Optional fixer to transform set() calls to set literals. -""" - -# Author: Benjamin Peterson - -from lib2to3 import fixer_base, pytree -from lib2to3.fixer_util import token, syms - - - -class FixSetLiteral(fixer_base.BaseFix): - - BM_compatible = True - explicit = True - - PATTERN = """power< 'set' trailer< '(' - (atom=atom< '[' (items=listmaker< any ((',' any)* [',']) > - | - single=any) ']' > - | - atom< '(' items=testlist_gexp< any ((',' any)* [',']) > ')' > - ) - ')' > > - """ - - def transform(self, node, results): - single = results.get("single") - if single: - # Make a fake listmaker - fake = pytree.Node(syms.listmaker, [single.clone()]) - single.replace(fake) - items = fake - else: - items = results["items"] - - # Build the contents of the literal - literal = [pytree.Leaf(token.LBRACE, "{")] - literal.extend(n.clone() for n in items.children) - literal.append(pytree.Leaf(token.RBRACE, "}")) - # Set the prefix of the right brace to that of the ')' or ']' - literal[-1].prefix = items.next_sibling.prefix - maker = pytree.Node(syms.dictsetmaker, literal) - maker.prefix = node.prefix - - # If the original was a one tuple, we need to remove the extra comma. - if len(maker.children) == 4: - n = maker.children[2] - n.remove() - maker.children[-1].prefix = n.prefix - - # Finally, replace the set call with our shiny new literal. - return maker diff --git a/Lib/lib2to3/fixes/fix_standarderror.py b/Lib/lib2to3/fixes/fix_standarderror.py deleted file mode 100644 index dc742167e6e9d4..00000000000000 --- a/Lib/lib2to3/fixes/fix_standarderror.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for StandardError -> Exception.""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixStandarderror(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - 'StandardError' - """ - - def transform(self, node, results): - return Name("Exception", prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_sys_exc.py b/Lib/lib2to3/fixes/fix_sys_exc.py deleted file mode 100644 index f6039690374ab2..00000000000000 --- a/Lib/lib2to3/fixes/fix_sys_exc.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Fixer for sys.exc_{type, value, traceback} - -sys.exc_type -> sys.exc_info()[0] -sys.exc_value -> sys.exc_info()[1] -sys.exc_traceback -> sys.exc_info()[2] -""" - -# By Jeff Balogh and Benjamin Peterson - -# Local imports -from .. import fixer_base -from ..fixer_util import Attr, Call, Name, Number, Subscript, Node, syms - -class FixSysExc(fixer_base.BaseFix): - # This order matches the ordering of sys.exc_info(). - exc_info = ["exc_type", "exc_value", "exc_traceback"] - BM_compatible = True - PATTERN = """ - power< 'sys' trailer< dot='.' attribute=(%s) > > - """ % '|'.join("'%s'" % e for e in exc_info) - - def transform(self, node, results): - sys_attr = results["attribute"][0] - index = Number(self.exc_info.index(sys_attr.value)) - - call = Call(Name("exc_info"), prefix=sys_attr.prefix) - attr = Attr(Name("sys"), call) - attr[1].children[0].prefix = results["dot"].prefix - attr.append(Subscript(index)) - return Node(syms.power, attr, prefix=node.prefix) diff --git a/Lib/lib2to3/fixes/fix_throw.py b/Lib/lib2to3/fixes/fix_throw.py deleted file mode 100644 index aac29169b4e98e..00000000000000 --- a/Lib/lib2to3/fixes/fix_throw.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Fixer for generator.throw(E, V, T). - -g.throw(E) -> g.throw(E) -g.throw(E, V) -> g.throw(E(V)) -g.throw(E, V, T) -> g.throw(E(V).with_traceback(T)) - -g.throw("foo"[, V[, T]]) will warn about string exceptions.""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Name, Call, ArgList, Attr, is_tuple - -class FixThrow(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< any trailer< '.' 'throw' > - trailer< '(' args=arglist< exc=any ',' val=any [',' tb=any] > ')' > - > - | - power< any trailer< '.' 'throw' > trailer< '(' exc=any ')' > > - """ - - def transform(self, node, results): - syms = self.syms - - exc = results["exc"].clone() - if exc.type is token.STRING: - self.cannot_convert(node, "Python 3 does not support string exceptions") - return - - # Leave "g.throw(E)" alone - val = results.get("val") - if val is None: - return - - val = val.clone() - if is_tuple(val): - args = [c.clone() for c in val.children[1:-1]] - else: - val.prefix = "" - args = [val] - - throw_args = results["args"] - - if "tb" in results: - tb = results["tb"].clone() - tb.prefix = "" - - e = Call(exc, args) - with_tb = Attr(e, Name('with_traceback')) + [ArgList([tb])] - throw_args.replace(pytree.Node(syms.power, with_tb)) - else: - throw_args.replace(Call(exc, args)) diff --git a/Lib/lib2to3/fixes/fix_tuple_params.py b/Lib/lib2to3/fixes/fix_tuple_params.py deleted file mode 100644 index cad755ffdbefb3..00000000000000 --- a/Lib/lib2to3/fixes/fix_tuple_params.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Fixer for function definitions with tuple parameters. - -def func(((a, b), c), d): - ... - - -> - -def func(x, d): - ((a, b), c) = x - ... - -It will also support lambdas: - - lambda (x, y): x + y -> lambda t: t[0] + t[1] - - # The parens are a syntax error in Python 3 - lambda (x): x + y -> lambda x: x + y -""" -# Author: Collin Winter - -# Local imports -from .. import pytree -from ..pgen2 import token -from .. import fixer_base -from ..fixer_util import Assign, Name, Newline, Number, Subscript, syms - -def is_docstring(stmt): - return isinstance(stmt, pytree.Node) and \ - stmt.children[0].type == token.STRING - -class FixTupleParams(fixer_base.BaseFix): - run_order = 4 #use a lower order since lambda is part of other - #patterns - BM_compatible = True - - PATTERN = """ - funcdef< 'def' any parameters< '(' args=any ')' > - ['->' any] ':' suite=any+ > - | - lambda= - lambdef< 'lambda' args=vfpdef< '(' inner=any ')' > - ':' body=any - > - """ - - def transform(self, node, results): - if "lambda" in results: - return self.transform_lambda(node, results) - - new_lines = [] - suite = results["suite"] - args = results["args"] - # This crap is so "def foo(...): x = 5; y = 7" is handled correctly. - # TODO(cwinter): suite-cleanup - if suite[0].children[1].type == token.INDENT: - start = 2 - indent = suite[0].children[1].value - end = Newline() - else: - start = 0 - indent = "; " - end = pytree.Leaf(token.INDENT, "") - - # We need access to self for new_name(), and making this a method - # doesn't feel right. Closing over self and new_lines makes the - # code below cleaner. - def handle_tuple(tuple_arg, add_prefix=False): - n = Name(self.new_name()) - arg = tuple_arg.clone() - arg.prefix = "" - stmt = Assign(arg, n.clone()) - if add_prefix: - n.prefix = " " - tuple_arg.replace(n) - new_lines.append(pytree.Node(syms.simple_stmt, - [stmt, end.clone()])) - - if args.type == syms.tfpdef: - handle_tuple(args) - elif args.type == syms.typedargslist: - for i, arg in enumerate(args.children): - if arg.type == syms.tfpdef: - # Without add_prefix, the emitted code is correct, - # just ugly. - handle_tuple(arg, add_prefix=(i > 0)) - - if not new_lines: - return - - # This isn't strictly necessary, but it plays nicely with other fixers. - # TODO(cwinter) get rid of this when children becomes a smart list - for line in new_lines: - line.parent = suite[0] - - # TODO(cwinter) suite-cleanup - after = start - if start == 0: - new_lines[0].prefix = " " - elif is_docstring(suite[0].children[start]): - new_lines[0].prefix = indent - after = start + 1 - - for line in new_lines: - line.parent = suite[0] - suite[0].children[after:after] = new_lines - for i in range(after+1, after+len(new_lines)+1): - suite[0].children[i].prefix = indent - suite[0].changed() - - def transform_lambda(self, node, results): - args = results["args"] - body = results["body"] - inner = simplify_args(results["inner"]) - - # Replace lambda ((((x)))): x with lambda x: x - if inner.type == token.NAME: - inner = inner.clone() - inner.prefix = " " - args.replace(inner) - return - - params = find_params(args) - to_index = map_to_index(params) - tup_name = self.new_name(tuple_name(params)) - - new_param = Name(tup_name, prefix=" ") - args.replace(new_param.clone()) - for n in body.post_order(): - if n.type == token.NAME and n.value in to_index: - subscripts = [c.clone() for c in to_index[n.value]] - new = pytree.Node(syms.power, - [new_param.clone()] + subscripts) - new.prefix = n.prefix - n.replace(new) - - -### Helper functions for transform_lambda() - -def simplify_args(node): - if node.type in (syms.vfplist, token.NAME): - return node - elif node.type == syms.vfpdef: - # These look like vfpdef< '(' x ')' > where x is NAME - # or another vfpdef instance (leading to recursion). - while node.type == syms.vfpdef: - node = node.children[1] - return node - raise RuntimeError("Received unexpected node %s" % node) - -def find_params(node): - if node.type == syms.vfpdef: - return find_params(node.children[1]) - elif node.type == token.NAME: - return node.value - return [find_params(c) for c in node.children if c.type != token.COMMA] - -def map_to_index(param_list, prefix=[], d=None): - if d is None: - d = {} - for i, obj in enumerate(param_list): - trailer = [Subscript(Number(str(i)))] - if isinstance(obj, list): - map_to_index(obj, trailer, d=d) - else: - d[obj] = prefix + trailer - return d - -def tuple_name(param_list): - l = [] - for obj in param_list: - if isinstance(obj, list): - l.append(tuple_name(obj)) - else: - l.append(obj) - return "_".join(l) diff --git a/Lib/lib2to3/fixes/fix_types.py b/Lib/lib2to3/fixes/fix_types.py deleted file mode 100644 index 67bf51f2f5b85a..00000000000000 --- a/Lib/lib2to3/fixes/fix_types.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer for removing uses of the types module. - -These work for only the known names in the types module. The forms above -can include types. or not. ie, It is assumed the module is imported either as: - - import types - from types import ... # either * or specific types - -The import statements are not modified. - -There should be another fixer that handles at least the following constants: - - type([]) -> list - type(()) -> tuple - type('') -> str - -""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - -_TYPE_MAPPING = { - 'BooleanType' : 'bool', - 'BufferType' : 'memoryview', - 'ClassType' : 'type', - 'ComplexType' : 'complex', - 'DictType': 'dict', - 'DictionaryType' : 'dict', - 'EllipsisType' : 'type(Ellipsis)', - #'FileType' : 'io.IOBase', - 'FloatType': 'float', - 'IntType': 'int', - 'ListType': 'list', - 'LongType': 'int', - 'ObjectType' : 'object', - 'NoneType': 'type(None)', - 'NotImplementedType' : 'type(NotImplemented)', - 'SliceType' : 'slice', - 'StringType': 'bytes', # XXX ? - 'StringTypes' : '(str,)', # XXX ? - 'TupleType': 'tuple', - 'TypeType' : 'type', - 'UnicodeType': 'str', - 'XRangeType' : 'range', - } - -_pats = ["power< 'types' trailer< '.' name='%s' > >" % t for t in _TYPE_MAPPING] - -class FixTypes(fixer_base.BaseFix): - BM_compatible = True - PATTERN = '|'.join(_pats) - - def transform(self, node, results): - new_value = _TYPE_MAPPING.get(results["name"].value) - if new_value: - return Name(new_value, prefix=node.prefix) - return None diff --git a/Lib/lib2to3/fixes/fix_unicode.py b/Lib/lib2to3/fixes/fix_unicode.py deleted file mode 100644 index c7982c2b97c3e1..00000000000000 --- a/Lib/lib2to3/fixes/fix_unicode.py +++ /dev/null @@ -1,42 +0,0 @@ -r"""Fixer for unicode. - -* Changes unicode to str and unichr to chr. - -* If "...\u..." is not unicode literal change it into "...\\u...". - -* Change u"..." into "...". - -""" - -from ..pgen2 import token -from .. import fixer_base - -_mapping = {"unichr" : "chr", "unicode" : "str"} - -class FixUnicode(fixer_base.BaseFix): - BM_compatible = True - PATTERN = "STRING | 'unicode' | 'unichr'" - - def start_tree(self, tree, filename): - super(FixUnicode, self).start_tree(tree, filename) - self.unicode_literals = 'unicode_literals' in tree.future_features - - def transform(self, node, results): - if node.type == token.NAME: - new = node.clone() - new.value = _mapping[node.value] - return new - elif node.type == token.STRING: - val = node.value - if not self.unicode_literals and val[0] in '\'"' and '\\' in val: - val = r'\\'.join([ - v.replace('\\u', r'\\u').replace('\\U', r'\\U') - for v in val.split(r'\\') - ]) - if val[0] in 'uU': - val = val[1:] - if val == node.value: - return node - new = node.clone() - new.value = val - return new diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py deleted file mode 100644 index ab892bc52494c2..00000000000000 --- a/Lib/lib2to3/fixes/fix_urllib.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Fix changes imports of urllib which are now incompatible. - This is rather similar to fix_imports, but because of the more - complex nature of the fixing for urllib, it has its own fixer. -""" -# Author: Nick Edds - -# Local imports -from lib2to3.fixes.fix_imports import alternates, FixImports -from lib2to3.fixer_util import (Name, Comma, FromImport, Newline, - find_indentation, Node, syms) - -MAPPING = {"urllib": [ - ("urllib.request", - ["URLopener", "FancyURLopener", "urlretrieve", - "_urlopener", "urlopen", "urlcleanup", - "pathname2url", "url2pathname", "getproxies"]), - ("urllib.parse", - ["quote", "quote_plus", "unquote", "unquote_plus", - "urlencode", "splitattr", "splithost", "splitnport", - "splitpasswd", "splitport", "splitquery", "splittag", - "splittype", "splituser", "splitvalue", ]), - ("urllib.error", - ["ContentTooShortError"])], - "urllib2" : [ - ("urllib.request", - ["urlopen", "install_opener", "build_opener", - "Request", "OpenerDirector", "BaseHandler", - "HTTPDefaultErrorHandler", "HTTPRedirectHandler", - "HTTPCookieProcessor", "ProxyHandler", - "HTTPPasswordMgr", - "HTTPPasswordMgrWithDefaultRealm", - "AbstractBasicAuthHandler", - "HTTPBasicAuthHandler", "ProxyBasicAuthHandler", - "AbstractDigestAuthHandler", - "HTTPDigestAuthHandler", "ProxyDigestAuthHandler", - "HTTPHandler", "HTTPSHandler", "FileHandler", - "FTPHandler", "CacheFTPHandler", - "UnknownHandler"]), - ("urllib.error", - ["URLError", "HTTPError"]), - ] -} - -# Duplicate the url parsing functions for urllib2. -MAPPING["urllib2"].append(MAPPING["urllib"][1]) - - -def build_pattern(): - bare = set() - for old_module, changes in MAPPING.items(): - for change in changes: - new_module, members = change - members = alternates(members) - yield """import_name< 'import' (module=%r - | dotted_as_names< any* module=%r any* >) > - """ % (old_module, old_module) - yield """import_from< 'from' mod_member=%r 'import' - ( member=%s | import_as_name< member=%s 'as' any > | - import_as_names< members=any* >) > - """ % (old_module, members, members) - yield """import_from< 'from' module_star=%r 'import' star='*' > - """ % old_module - yield """import_name< 'import' - dotted_as_name< module_as=%r 'as' any > > - """ % old_module - # bare_with_attr has a special significance for FixImports.match(). - yield """power< bare_with_attr=%r trailer< '.' member=%s > any* > - """ % (old_module, members) - - -class FixUrllib(FixImports): - - def build_pattern(self): - return "|".join(build_pattern()) - - def transform_import(self, node, results): - """Transform for the basic import case. Replaces the old - import name with a comma separated list of its - replacements. - """ - import_mod = results.get("module") - pref = import_mod.prefix - - names = [] - - # create a Node list of the replacement modules - for name in MAPPING[import_mod.value][:-1]: - names.extend([Name(name[0], prefix=pref), Comma()]) - names.append(Name(MAPPING[import_mod.value][-1][0], prefix=pref)) - import_mod.replace(names) - - def transform_member(self, node, results): - """Transform for imports of specific module elements. Replaces - the module to be imported from with the appropriate new - module. - """ - mod_member = results.get("mod_member") - pref = mod_member.prefix - member = results.get("member") - - # Simple case with only a single member being imported - if member: - # this may be a list of length one, or just a node - if isinstance(member, list): - member = member[0] - new_name = None - for change in MAPPING[mod_member.value]: - if member.value in change[1]: - new_name = change[0] - break - if new_name: - mod_member.replace(Name(new_name, prefix=pref)) - else: - self.cannot_convert(node, "This is an invalid module element") - - # Multiple members being imported - else: - # a dictionary for replacements, order matters - modules = [] - mod_dict = {} - members = results["members"] - for member in members: - # we only care about the actual members - if member.type == syms.import_as_name: - as_name = member.children[2].value - member_name = member.children[0].value - else: - member_name = member.value - as_name = None - if member_name != ",": - for change in MAPPING[mod_member.value]: - if member_name in change[1]: - if change[0] not in mod_dict: - modules.append(change[0]) - mod_dict.setdefault(change[0], []).append(member) - - new_nodes = [] - indentation = find_indentation(node) - first = True - def handle_name(name, prefix): - if name.type == syms.import_as_name: - kids = [Name(name.children[0].value, prefix=prefix), - name.children[1].clone(), - name.children[2].clone()] - return [Node(syms.import_as_name, kids)] - return [Name(name.value, prefix=prefix)] - for module in modules: - elts = mod_dict[module] - names = [] - for elt in elts[:-1]: - names.extend(handle_name(elt, pref)) - names.append(Comma()) - names.extend(handle_name(elts[-1], pref)) - new = FromImport(module, names) - if not first or node.parent.prefix.endswith(indentation): - new.prefix = indentation - new_nodes.append(new) - first = False - if new_nodes: - nodes = [] - for new_node in new_nodes[:-1]: - nodes.extend([new_node, Newline()]) - nodes.append(new_nodes[-1]) - node.replace(nodes) - else: - self.cannot_convert(node, "All module elements are invalid") - - def transform_dot(self, node, results): - """Transform for calls to module members in code.""" - module_dot = results.get("bare_with_attr") - member = results.get("member") - new_name = None - if isinstance(member, list): - member = member[0] - for change in MAPPING[module_dot.value]: - if member.value in change[1]: - new_name = change[0] - break - if new_name: - module_dot.replace(Name(new_name, - prefix=module_dot.prefix)) - else: - self.cannot_convert(node, "This is an invalid module element") - - def transform(self, node, results): - if results.get("module"): - self.transform_import(node, results) - elif results.get("mod_member"): - self.transform_member(node, results) - elif results.get("bare_with_attr"): - self.transform_dot(node, results) - # Renaming and star imports are not supported for these modules. - elif results.get("module_star"): - self.cannot_convert(node, "Cannot handle star imports.") - elif results.get("module_as"): - self.cannot_convert(node, "This module is now multiple modules") diff --git a/Lib/lib2to3/fixes/fix_ws_comma.py b/Lib/lib2to3/fixes/fix_ws_comma.py deleted file mode 100644 index a54a376c472afb..00000000000000 --- a/Lib/lib2to3/fixes/fix_ws_comma.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Fixer that changes 'a ,b' into 'a, b'. - -This also changes '{a :b}' into '{a: b}', but does not touch other -uses of colons. It does not touch other uses of whitespace. - -""" - -from .. import pytree -from ..pgen2 import token -from .. import fixer_base - -class FixWsComma(fixer_base.BaseFix): - - explicit = True # The user must ask for this fixers - - PATTERN = """ - any<(not(',') any)+ ',' ((not(',') any)+ ',')* [not(',') any]> - """ - - COMMA = pytree.Leaf(token.COMMA, ",") - COLON = pytree.Leaf(token.COLON, ":") - SEPS = (COMMA, COLON) - - def transform(self, node, results): - new = node.clone() - comma = False - for child in new.children: - if child in self.SEPS: - prefix = child.prefix - if prefix.isspace() and "\n" not in prefix: - child.prefix = "" - comma = True - else: - if comma: - prefix = child.prefix - if not prefix: - child.prefix = " " - comma = False - return new diff --git a/Lib/lib2to3/fixes/fix_xrange.py b/Lib/lib2to3/fixes/fix_xrange.py deleted file mode 100644 index 1e491e166a3f1c..00000000000000 --- a/Lib/lib2to3/fixes/fix_xrange.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Fixer that changes xrange(...) into range(...).""" - -# Local imports -from .. import fixer_base -from ..fixer_util import Name, Call, consuming_calls -from .. import patcomp - - -class FixXrange(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< - (name='range'|name='xrange') trailer< '(' args=any ')' > - rest=any* > - """ - - def start_tree(self, tree, filename): - super(FixXrange, self).start_tree(tree, filename) - self.transformed_xranges = set() - - def finish_tree(self, tree, filename): - self.transformed_xranges = None - - def transform(self, node, results): - name = results["name"] - if name.value == "xrange": - return self.transform_xrange(node, results) - elif name.value == "range": - return self.transform_range(node, results) - else: - raise ValueError(repr(name)) - - def transform_xrange(self, node, results): - name = results["name"] - name.replace(Name("range", prefix=name.prefix)) - # This prevents the new range call from being wrapped in a list later. - self.transformed_xranges.add(id(node)) - - def transform_range(self, node, results): - if (id(node) not in self.transformed_xranges and - not self.in_special_context(node)): - range_call = Call(Name("range"), [results["args"].clone()]) - # Encase the range call in list(). - list_call = Call(Name("list"), [range_call], - prefix=node.prefix) - # Put things that were after the range() call after the list call. - for n in results["rest"]: - list_call.append_child(n) - return list_call - - P1 = "power< func=NAME trailer< '(' node=any ')' > any* >" - p1 = patcomp.compile_pattern(P1) - - P2 = """for_stmt< 'for' any 'in' node=any ':' any* > - | comp_for< 'for' any 'in' node=any any* > - | comparison< any 'in' node=any any*> - """ - p2 = patcomp.compile_pattern(P2) - - def in_special_context(self, node): - if node.parent is None: - return False - results = {} - if (node.parent.parent is not None and - self.p1.match(node.parent.parent, results) and - results["node"] is node): - # list(d.keys()) -> list(d.keys()), etc. - return results["func"].value in consuming_calls - # for ... in d.iterkeys() -> for ... in d.keys(), etc. - return self.p2.match(node.parent, results) and results["node"] is node diff --git a/Lib/lib2to3/fixes/fix_xreadlines.py b/Lib/lib2to3/fixes/fix_xreadlines.py deleted file mode 100644 index 3e3f71ab045573..00000000000000 --- a/Lib/lib2to3/fixes/fix_xreadlines.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Fix "for x in f.xreadlines()" -> "for x in f". - -This fixer will also convert g(f.xreadlines) into g(f.__iter__).""" -# Author: Collin Winter - -# Local imports -from .. import fixer_base -from ..fixer_util import Name - - -class FixXreadlines(fixer_base.BaseFix): - BM_compatible = True - PATTERN = """ - power< call=any+ trailer< '.' 'xreadlines' > trailer< '(' ')' > > - | - power< any+ trailer< '.' no_call='xreadlines' > > - """ - - def transform(self, node, results): - no_call = results.get("no_call") - - if no_call: - no_call.replace(Name("__iter__", prefix=no_call.prefix)) - else: - node.replace([x.clone() for x in results["call"]]) diff --git a/Lib/lib2to3/fixes/fix_zip.py b/Lib/lib2to3/fixes/fix_zip.py deleted file mode 100644 index 52c28df6aab411..00000000000000 --- a/Lib/lib2to3/fixes/fix_zip.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Fixer that changes zip(seq0, seq1, ...) into list(zip(seq0, seq1, ...) -unless there exists a 'from future_builtins import zip' statement in the -top-level namespace. - -We avoid the transformation if the zip() call is directly contained in -iter(<>), list(<>), tuple(<>), sorted(<>), ...join(<>), or for V in <>:. -""" - -# Local imports -from .. import fixer_base -from ..pytree import Node -from ..pygram import python_symbols as syms -from ..fixer_util import Name, ArgList, in_special_context - - -class FixZip(fixer_base.ConditionalFix): - - BM_compatible = True - PATTERN = """ - power< 'zip' args=trailer< '(' [any] ')' > [trailers=trailer*] - > - """ - - skip_on = "future_builtins.zip" - - def transform(self, node, results): - if self.should_skip(node): - return - - if in_special_context(node): - return None - - args = results['args'].clone() - args.prefix = "" - - trailers = [] - if 'trailers' in results: - trailers = [n.clone() for n in results['trailers']] - for n in trailers: - n.prefix = "" - - new = Node(syms.power, [Name("zip"), args], prefix="") - new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) - new.prefix = node.prefix - return new diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py deleted file mode 100644 index f2849fd6be3fd9..00000000000000 --- a/Lib/lib2to3/main.py +++ /dev/null @@ -1,273 +0,0 @@ -""" -Main program for 2to3. -""" - -from __future__ import with_statement, print_function - -import sys -import os -import difflib -import logging -import shutil -import optparse - -from . import refactor - - -def diff_texts(a, b, filename): - """Return a unified diff of two strings.""" - a = a.splitlines() - b = b.splitlines() - return difflib.unified_diff(a, b, filename, filename, - "(original)", "(refactored)", - lineterm="") - - -class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool): - """ - A refactoring tool that can avoid overwriting its input files. - Prints output to stdout. - - Output files can optionally be written to a different directory and or - have an extra file suffix appended to their name for use in situations - where you do not want to replace the input files. - """ - - def __init__(self, fixers, options, explicit, nobackups, show_diffs, - input_base_dir='', output_dir='', append_suffix=''): - """ - Args: - fixers: A list of fixers to import. - options: A dict with RefactoringTool configuration. - explicit: A list of fixers to run even if they are explicit. - nobackups: If true no backup '.bak' files will be created for those - files that are being refactored. - show_diffs: Should diffs of the refactoring be printed to stdout? - input_base_dir: The base directory for all input files. This class - will strip this path prefix off of filenames before substituting - it with output_dir. Only meaningful if output_dir is supplied. - All files processed by refactor() must start with this path. - output_dir: If supplied, all converted files will be written into - this directory tree instead of input_base_dir. - append_suffix: If supplied, all files output by this tool will have - this appended to their filename. Useful for changing .py to - .py3 for example by passing append_suffix='3'. - """ - self.nobackups = nobackups - self.show_diffs = show_diffs - if input_base_dir and not input_base_dir.endswith(os.sep): - input_base_dir += os.sep - self._input_base_dir = input_base_dir - self._output_dir = output_dir - self._append_suffix = append_suffix - super(StdoutRefactoringTool, self).__init__(fixers, options, explicit) - - def log_error(self, msg, *args, **kwargs): - self.errors.append((msg, args, kwargs)) - self.logger.error(msg, *args, **kwargs) - - def write_file(self, new_text, filename, old_text, encoding): - orig_filename = filename - if self._output_dir: - if filename.startswith(self._input_base_dir): - filename = os.path.join(self._output_dir, - filename[len(self._input_base_dir):]) - else: - raise ValueError('filename %s does not start with the ' - 'input_base_dir %s' % ( - filename, self._input_base_dir)) - if self._append_suffix: - filename += self._append_suffix - if orig_filename != filename: - output_dir = os.path.dirname(filename) - if not os.path.isdir(output_dir) and output_dir: - os.makedirs(output_dir) - self.log_message('Writing converted %s to %s.', orig_filename, - filename) - if not self.nobackups: - # Make backup - backup = filename + ".bak" - if os.path.lexists(backup): - try: - os.remove(backup) - except OSError: - self.log_message("Can't remove backup %s", backup) - try: - os.rename(filename, backup) - except OSError: - self.log_message("Can't rename %s to %s", filename, backup) - # Actually write the new file - write = super(StdoutRefactoringTool, self).write_file - write(new_text, filename, old_text, encoding) - if not self.nobackups: - shutil.copymode(backup, filename) - if orig_filename != filename: - # Preserve the file mode in the new output directory. - shutil.copymode(orig_filename, filename) - - def print_output(self, old, new, filename, equal): - if equal: - self.log_message("No changes to %s", filename) - else: - self.log_message("Refactored %s", filename) - if self.show_diffs: - diff_lines = diff_texts(old, new, filename) - try: - if self.output_lock is not None: - with self.output_lock: - for line in diff_lines: - print(line) - sys.stdout.flush() - else: - for line in diff_lines: - print(line) - except UnicodeEncodeError: - warn("couldn't encode %s's diff for your terminal" % - (filename,)) - return - -def warn(msg): - print("WARNING: %s" % (msg,), file=sys.stderr) - - -def main(fixer_pkg, args=None): - """Main program. - - Args: - fixer_pkg: the name of a package where the fixers are located. - args: optional; a list of command line arguments. If omitted, - sys.argv[1:] is used. - - Returns a suggested exit status (0, 1, 2). - """ - # Set up option parser - parser = optparse.OptionParser(usage="2to3 [options] file|dir ...") - parser.add_option("-d", "--doctests_only", action="store_true", - help="Fix up doctests only") - parser.add_option("-f", "--fix", action="append", default=[], - help="Each FIX specifies a transformation; default: all") - parser.add_option("-j", "--processes", action="store", default=1, - type="int", help="Run 2to3 concurrently") - parser.add_option("-x", "--nofix", action="append", default=[], - help="Prevent a transformation from being run") - parser.add_option("-l", "--list-fixes", action="store_true", - help="List available transformations") - parser.add_option("-p", "--print-function", action="store_true", - help="Modify the grammar so that print() is a function") - parser.add_option("-e", "--exec-function", action="store_true", - help="Modify the grammar so that exec() is a function") - parser.add_option("-v", "--verbose", action="store_true", - help="More verbose logging") - parser.add_option("--no-diffs", action="store_true", - help="Don't show diffs of the refactoring") - parser.add_option("-w", "--write", action="store_true", - help="Write back modified files") - parser.add_option("-n", "--nobackups", action="store_true", default=False, - help="Don't write backups for modified files") - parser.add_option("-o", "--output-dir", action="store", type="str", - default="", help="Put output files in this directory " - "instead of overwriting the input files. Requires -n.") - parser.add_option("-W", "--write-unchanged-files", action="store_true", - help="Also write files even if no changes were required" - " (useful with --output-dir); implies -w.") - parser.add_option("--add-suffix", action="store", type="str", default="", - help="Append this string to all output filenames." - " Requires -n if non-empty. " - "ex: --add-suffix='3' will generate .py3 files.") - - # Parse command line arguments - refactor_stdin = False - flags = {} - options, args = parser.parse_args(args) - if options.write_unchanged_files: - flags["write_unchanged_files"] = True - if not options.write: - warn("--write-unchanged-files/-W implies -w.") - options.write = True - # If we allowed these, the original files would be renamed to backup names - # but not replaced. - if options.output_dir and not options.nobackups: - parser.error("Can't use --output-dir/-o without -n.") - if options.add_suffix and not options.nobackups: - parser.error("Can't use --add-suffix without -n.") - - if not options.write and options.no_diffs: - warn("not writing files and not printing diffs; that's not very useful") - if not options.write and options.nobackups: - parser.error("Can't use -n without -w") - if options.list_fixes: - print("Available transformations for the -f/--fix option:") - for fixname in refactor.get_all_fix_names(fixer_pkg): - print(fixname) - if not args: - return 0 - if not args: - print("At least one file or directory argument required.", file=sys.stderr) - print("Use --help to show usage.", file=sys.stderr) - return 2 - if "-" in args: - refactor_stdin = True - if options.write: - print("Can't write to stdin.", file=sys.stderr) - return 2 - if options.print_function: - flags["print_function"] = True - - if options.exec_function: - flags["exec_function"] = True - - # Set up logging handler - level = logging.DEBUG if options.verbose else logging.INFO - logging.basicConfig(format='%(name)s: %(message)s', level=level) - logger = logging.getLogger('lib2to3.main') - - # Initialize the refactoring tool - avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg)) - unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix) - explicit = set() - if options.fix: - all_present = False - for fix in options.fix: - if fix == "all": - all_present = True - else: - explicit.add(fixer_pkg + ".fix_" + fix) - requested = avail_fixes.union(explicit) if all_present else explicit - else: - requested = avail_fixes.union(explicit) - fixer_names = requested.difference(unwanted_fixes) - input_base_dir = os.path.commonprefix(args) - if (input_base_dir and not input_base_dir.endswith(os.sep) - and not os.path.isdir(input_base_dir)): - # One or more similar names were passed, their directory is the base. - # os.path.commonprefix() is ignorant of path elements, this corrects - # for that weird API. - input_base_dir = os.path.dirname(input_base_dir) - if options.output_dir: - input_base_dir = input_base_dir.rstrip(os.sep) - logger.info('Output in %r will mirror the input directory %r layout.', - options.output_dir, input_base_dir) - rt = StdoutRefactoringTool( - sorted(fixer_names), flags, sorted(explicit), - options.nobackups, not options.no_diffs, - input_base_dir=input_base_dir, - output_dir=options.output_dir, - append_suffix=options.add_suffix) - - # Refactor all files and directories passed as arguments - if not rt.errors: - if refactor_stdin: - rt.refactor_stdin() - else: - try: - rt.refactor(args, options.write, options.doctests_only, - options.processes) - except refactor.MultiprocessingUnsupported: - assert options.processes > 1 - print("Sorry, -j isn't supported on this platform.", - file=sys.stderr) - return 1 - rt.summarize() - - # Return error status (0 if rt.errors is zero) - return int(bool(rt.errors)) diff --git a/Lib/lib2to3/patcomp.py b/Lib/lib2to3/patcomp.py deleted file mode 100644 index f57f4954b26ce7..00000000000000 --- a/Lib/lib2to3/patcomp.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Pattern compiler. - -The grammar is taken from PatternGrammar.txt. - -The compiler compiles a pattern to a pytree.*Pattern instance. -""" - -__author__ = "Guido van Rossum " - -# Python imports -import io - -# Fairly local imports -from .pgen2 import driver, literals, token, tokenize, parse, grammar - -# Really local imports -from . import pytree -from . import pygram - - -class PatternSyntaxError(Exception): - pass - - -def tokenize_wrapper(input): - """Tokenizes a string suppressing significant whitespace.""" - skip = {token.NEWLINE, token.INDENT, token.DEDENT} - tokens = tokenize.generate_tokens(io.StringIO(input).readline) - for quintuple in tokens: - type, value, start, end, line_text = quintuple - if type not in skip: - yield quintuple - - -class PatternCompiler(object): - - def __init__(self, grammar_file=None): - """Initializer. - - Takes an optional alternative filename for the pattern grammar. - """ - if grammar_file is None: - self.grammar = pygram.pattern_grammar - self.syms = pygram.pattern_symbols - else: - self.grammar = driver.load_grammar(grammar_file) - self.syms = pygram.Symbols(self.grammar) - self.pygrammar = pygram.python_grammar - self.pysyms = pygram.python_symbols - self.driver = driver.Driver(self.grammar, convert=pattern_convert) - - def compile_pattern(self, input, debug=False, with_tree=False): - """Compiles a pattern string to a nested pytree.*Pattern object.""" - tokens = tokenize_wrapper(input) - try: - root = self.driver.parse_tokens(tokens, debug=debug) - except parse.ParseError as e: - raise PatternSyntaxError(str(e)) from None - if with_tree: - return self.compile_node(root), root - else: - return self.compile_node(root) - - def compile_node(self, node): - """Compiles a node, recursively. - - This is one big switch on the node type. - """ - # XXX Optimize certain Wildcard-containing-Wildcard patterns - # that can be merged - if node.type == self.syms.Matcher: - node = node.children[0] # Avoid unneeded recursion - - if node.type == self.syms.Alternatives: - # Skip the odd children since they are just '|' tokens - alts = [self.compile_node(ch) for ch in node.children[::2]] - if len(alts) == 1: - return alts[0] - p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1) - return p.optimize() - - if node.type == self.syms.Alternative: - units = [self.compile_node(ch) for ch in node.children] - if len(units) == 1: - return units[0] - p = pytree.WildcardPattern([units], min=1, max=1) - return p.optimize() - - if node.type == self.syms.NegatedUnit: - pattern = self.compile_basic(node.children[1:]) - p = pytree.NegatedPattern(pattern) - return p.optimize() - - assert node.type == self.syms.Unit - - name = None - nodes = node.children - if len(nodes) >= 3 and nodes[1].type == token.EQUAL: - name = nodes[0].value - nodes = nodes[2:] - repeat = None - if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater: - repeat = nodes[-1] - nodes = nodes[:-1] - - # Now we've reduced it to: STRING | NAME [Details] | (...) | [...] - pattern = self.compile_basic(nodes, repeat) - - if repeat is not None: - assert repeat.type == self.syms.Repeater - children = repeat.children - child = children[0] - if child.type == token.STAR: - min = 0 - max = pytree.HUGE - elif child.type == token.PLUS: - min = 1 - max = pytree.HUGE - elif child.type == token.LBRACE: - assert children[-1].type == token.RBRACE - assert len(children) in (3, 5) - min = max = self.get_int(children[1]) - if len(children) == 5: - max = self.get_int(children[3]) - else: - assert False - if min != 1 or max != 1: - pattern = pattern.optimize() - pattern = pytree.WildcardPattern([[pattern]], min=min, max=max) - - if name is not None: - pattern.name = name - return pattern.optimize() - - def compile_basic(self, nodes, repeat=None): - # Compile STRING | NAME [Details] | (...) | [...] - assert len(nodes) >= 1 - node = nodes[0] - if node.type == token.STRING: - value = str(literals.evalString(node.value)) - return pytree.LeafPattern(_type_of_literal(value), value) - elif node.type == token.NAME: - value = node.value - if value.isupper(): - if value not in TOKEN_MAP: - raise PatternSyntaxError("Invalid token: %r" % value) - if nodes[1:]: - raise PatternSyntaxError("Can't have details for token") - return pytree.LeafPattern(TOKEN_MAP[value]) - else: - if value == "any": - type = None - elif not value.startswith("_"): - type = getattr(self.pysyms, value, None) - if type is None: - raise PatternSyntaxError("Invalid symbol: %r" % value) - if nodes[1:]: # Details present - content = [self.compile_node(nodes[1].children[1])] - else: - content = None - return pytree.NodePattern(type, content) - elif node.value == "(": - return self.compile_node(nodes[1]) - elif node.value == "[": - assert repeat is None - subpattern = self.compile_node(nodes[1]) - return pytree.WildcardPattern([[subpattern]], min=0, max=1) - assert False, node - - def get_int(self, node): - assert node.type == token.NUMBER - return int(node.value) - - -# Map named tokens to the type value for a LeafPattern -TOKEN_MAP = {"NAME": token.NAME, - "STRING": token.STRING, - "NUMBER": token.NUMBER, - "TOKEN": None} - - -def _type_of_literal(value): - if value[0].isalpha(): - return token.NAME - elif value in grammar.opmap: - return grammar.opmap[value] - else: - return None - - -def pattern_convert(grammar, raw_node_info): - """Converts raw node information to a Node or Leaf instance.""" - type, value, context, children = raw_node_info - if children or type in grammar.number2symbol: - return pytree.Node(type, children, context=context) - else: - return pytree.Leaf(type, value, context=context) - - -def compile_pattern(pattern): - return PatternCompiler().compile_pattern(pattern) diff --git a/Lib/lib2to3/pgen2/__init__.py b/Lib/lib2to3/pgen2/__init__.py deleted file mode 100644 index af390484528d87..00000000000000 --- a/Lib/lib2to3/pgen2/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""The pgen2 package.""" diff --git a/Lib/lib2to3/pgen2/conv.py b/Lib/lib2to3/pgen2/conv.py deleted file mode 100644 index ed0cac532e4249..00000000000000 --- a/Lib/lib2to3/pgen2/conv.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Convert graminit.[ch] spit out by pgen to Python code. - -Pgen is the Python parser generator. It is useful to quickly create a -parser from a grammar file in Python's grammar notation. But I don't -want my parsers to be written in C (yet), so I'm translating the -parsing tables to Python data structures and writing a Python parse -engine. - -Note that the token numbers are constants determined by the standard -Python tokenizer. The standard token module defines these numbers and -their names (the names are not used much). The token numbers are -hardcoded into the Python tokenizer and into pgen. A Python -implementation of the Python tokenizer is also available, in the -standard tokenize module. - -On the other hand, symbol numbers (representing the grammar's -non-terminals) are assigned by pgen based on the actual grammar -input. - -Note: this module is pretty much obsolete; the pgen module generates -equivalent grammar tables directly from the Grammar.txt input file -without having to invoke the Python pgen C program. - -""" - -# Python imports -import re - -# Local imports -from pgen2 import grammar, token - - -class Converter(grammar.Grammar): - """Grammar subclass that reads classic pgen output files. - - The run() method reads the tables as produced by the pgen parser - generator, typically contained in two C files, graminit.h and - graminit.c. The other methods are for internal use only. - - See the base class for more documentation. - - """ - - def run(self, graminit_h, graminit_c): - """Load the grammar tables from the text files written by pgen.""" - self.parse_graminit_h(graminit_h) - self.parse_graminit_c(graminit_c) - self.finish_off() - - def parse_graminit_h(self, filename): - """Parse the .h file written by pgen. (Internal) - - This file is a sequence of #define statements defining the - nonterminals of the grammar as numbers. We build two tables - mapping the numbers to names and back. - - """ - try: - f = open(filename) - except OSError as err: - print("Can't open %s: %s" % (filename, err)) - return False - self.symbol2number = {} - self.number2symbol = {} - lineno = 0 - for line in f: - lineno += 1 - mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line) - if not mo and line.strip(): - print("%s(%s): can't parse %s" % (filename, lineno, - line.strip())) - else: - symbol, number = mo.groups() - number = int(number) - assert symbol not in self.symbol2number - assert number not in self.number2symbol - self.symbol2number[symbol] = number - self.number2symbol[number] = symbol - return True - - def parse_graminit_c(self, filename): - """Parse the .c file written by pgen. (Internal) - - The file looks as follows. The first two lines are always this: - - #include "pgenheaders.h" - #include "grammar.h" - - After that come four blocks: - - 1) one or more state definitions - 2) a table defining dfas - 3) a table defining labels - 4) a struct defining the grammar - - A state definition has the following form: - - one or more arc arrays, each of the form: - static arc arcs__[] = { - {, }, - ... - }; - - followed by a state array, of the form: - static state states_[] = { - {, arcs__}, - ... - }; - - """ - try: - f = open(filename) - except OSError as err: - print("Can't open %s: %s" % (filename, err)) - return False - # The code below essentially uses f's iterator-ness! - lineno = 0 - - # Expect the two #include lines - lineno, line = lineno+1, next(f) - assert line == '#include "pgenheaders.h"\n', (lineno, line) - lineno, line = lineno+1, next(f) - assert line == '#include "grammar.h"\n', (lineno, line) - - # Parse the state definitions - lineno, line = lineno+1, next(f) - allarcs = {} - states = [] - while line.startswith("static arc "): - while line.startswith("static arc "): - mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$", - line) - assert mo, (lineno, line) - n, m, k = list(map(int, mo.groups())) - arcs = [] - for _ in range(k): - lineno, line = lineno+1, next(f) - mo = re.match(r"\s+{(\d+), (\d+)},$", line) - assert mo, (lineno, line) - i, j = list(map(int, mo.groups())) - arcs.append((i, j)) - lineno, line = lineno+1, next(f) - assert line == "};\n", (lineno, line) - allarcs[(n, m)] = arcs - lineno, line = lineno+1, next(f) - mo = re.match(r"static state states_(\d+)\[(\d+)\] = {$", line) - assert mo, (lineno, line) - s, t = list(map(int, mo.groups())) - assert s == len(states), (lineno, line) - state = [] - for _ in range(t): - lineno, line = lineno+1, next(f) - mo = re.match(r"\s+{(\d+), arcs_(\d+)_(\d+)},$", line) - assert mo, (lineno, line) - k, n, m = list(map(int, mo.groups())) - arcs = allarcs[n, m] - assert k == len(arcs), (lineno, line) - state.append(arcs) - states.append(state) - lineno, line = lineno+1, next(f) - assert line == "};\n", (lineno, line) - lineno, line = lineno+1, next(f) - self.states = states - - # Parse the dfas - dfas = {} - mo = re.match(r"static dfa dfas\[(\d+)\] = {$", line) - assert mo, (lineno, line) - ndfas = int(mo.group(1)) - for i in range(ndfas): - lineno, line = lineno+1, next(f) - mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$', - line) - assert mo, (lineno, line) - symbol = mo.group(2) - number, x, y, z = list(map(int, mo.group(1, 3, 4, 5))) - assert self.symbol2number[symbol] == number, (lineno, line) - assert self.number2symbol[number] == symbol, (lineno, line) - assert x == 0, (lineno, line) - state = states[z] - assert y == len(state), (lineno, line) - lineno, line = lineno+1, next(f) - mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line) - assert mo, (lineno, line) - first = {} - rawbitset = eval(mo.group(1)) - for i, c in enumerate(rawbitset): - byte = ord(c) - for j in range(8): - if byte & (1<= os.path.getmtime(b) - - -def load_packaged_grammar(package, grammar_source): - """Normally, loads a pickled grammar by doing - pkgutil.get_data(package, pickled_grammar) - where *pickled_grammar* is computed from *grammar_source* by adding the - Python version and using a ``.pickle`` extension. - - However, if *grammar_source* is an extant file, load_grammar(grammar_source) - is called instead. This facilitates using a packaged grammar file when needed - but preserves load_grammar's automatic regeneration behavior when possible. - - """ - if os.path.isfile(grammar_source): - return load_grammar(grammar_source) - pickled_name = _generate_pickle_name(os.path.basename(grammar_source)) - data = pkgutil.get_data(package, pickled_name) - g = grammar.Grammar() - g.loads(data) - return g - - -def main(*args): - """Main program, when run as a script: produce grammar pickle files. - - Calls load_grammar for each argument, a path to a grammar text file. - """ - if not args: - args = sys.argv[1:] - logging.basicConfig(level=logging.INFO, stream=sys.stdout, - format='%(message)s') - for gt in args: - load_grammar(gt, save=True, force=True) - return True - -if __name__ == "__main__": - sys.exit(int(not main())) diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py deleted file mode 100644 index 5d550aeb65e8dd..00000000000000 --- a/Lib/lib2to3/pgen2/grammar.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""This module defines the data structures used to represent a grammar. - -These are a bit arcane because they are derived from the data -structures used by Python's 'pgen' parser generator. - -There's also a table here mapping operators to their names in the -token module; the Python tokenize module reports all operators as the -fallback token code OP, but the parser needs the actual token code. - -""" - -# Python imports -import pickle - -# Local imports -from . import token - - -class Grammar(object): - """Pgen parsing tables conversion class. - - Once initialized, this class supplies the grammar tables for the - parsing engine implemented by parse.py. The parsing engine - accesses the instance variables directly. The class here does not - provide initialization of the tables; several subclasses exist to - do this (see the conv and pgen modules). - - The load() method reads the tables from a pickle file, which is - much faster than the other ways offered by subclasses. The pickle - file is written by calling dump() (after loading the grammar - tables using a subclass). The report() method prints a readable - representation of the tables to stdout, for debugging. - - The instance variables are as follows: - - symbol2number -- a dict mapping symbol names to numbers. Symbol - numbers are always 256 or higher, to distinguish - them from token numbers, which are between 0 and - 255 (inclusive). - - number2symbol -- a dict mapping numbers to symbol names; - these two are each other's inverse. - - states -- a list of DFAs, where each DFA is a list of - states, each state is a list of arcs, and each - arc is a (i, j) pair where i is a label and j is - a state number. The DFA number is the index into - this list. (This name is slightly confusing.) - Final states are represented by a special arc of - the form (0, j) where j is its own state number. - - dfas -- a dict mapping symbol numbers to (DFA, first) - pairs, where DFA is an item from the states list - above, and first is a set of tokens that can - begin this grammar rule (represented by a dict - whose values are always 1). - - labels -- a list of (x, y) pairs where x is either a token - number or a symbol number, and y is either None - or a string; the strings are keywords. The label - number is the index in this list; label numbers - are used to mark state transitions (arcs) in the - DFAs. - - start -- the number of the grammar's start symbol. - - keywords -- a dict mapping keyword strings to arc labels. - - tokens -- a dict mapping token numbers to arc labels. - - """ - - def __init__(self): - self.symbol2number = {} - self.number2symbol = {} - self.states = [] - self.dfas = {} - self.labels = [(0, "EMPTY")] - self.keywords = {} - self.tokens = {} - self.symbol2label = {} - self.start = 256 - - def dump(self, filename): - """Dump the grammar tables to a pickle file.""" - with open(filename, "wb") as f: - pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL) - - def load(self, filename): - """Load the grammar tables from a pickle file.""" - with open(filename, "rb") as f: - d = pickle.load(f) - self.__dict__.update(d) - - def loads(self, pkl): - """Load the grammar tables from a pickle bytes object.""" - self.__dict__.update(pickle.loads(pkl)) - - def copy(self): - """ - Copy the grammar. - """ - new = self.__class__() - for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords", - "tokens", "symbol2label"): - setattr(new, dict_attr, getattr(self, dict_attr).copy()) - new.labels = self.labels[:] - new.states = self.states[:] - new.start = self.start - return new - - def report(self): - """Dump the grammar tables to standard output, for debugging.""" - from pprint import pprint - print("s2n") - pprint(self.symbol2number) - print("n2s") - pprint(self.number2symbol) - print("states") - pprint(self.states) - print("dfas") - pprint(self.dfas) - print("labels") - pprint(self.labels) - print("start", self.start) - - -# Map from operator to number (since tokenize doesn't do this) - -opmap_raw = """ -( LPAR -) RPAR -[ LSQB -] RSQB -: COLON -, COMMA -; SEMI -+ PLUS -- MINUS -* STAR -/ SLASH -| VBAR -& AMPER -< LESS -> GREATER -= EQUAL -. DOT -% PERCENT -` BACKQUOTE -{ LBRACE -} RBRACE -@ AT -@= ATEQUAL -== EQEQUAL -!= NOTEQUAL -<> NOTEQUAL -<= LESSEQUAL ->= GREATEREQUAL -~ TILDE -^ CIRCUMFLEX -<< LEFTSHIFT ->> RIGHTSHIFT -** DOUBLESTAR -+= PLUSEQUAL --= MINEQUAL -*= STAREQUAL -/= SLASHEQUAL -%= PERCENTEQUAL -&= AMPEREQUAL -|= VBAREQUAL -^= CIRCUMFLEXEQUAL -<<= LEFTSHIFTEQUAL ->>= RIGHTSHIFTEQUAL -**= DOUBLESTAREQUAL -// DOUBLESLASH -//= DOUBLESLASHEQUAL --> RARROW -:= COLONEQUAL -""" - -opmap = {} -for line in opmap_raw.splitlines(): - if line: - op, name = line.split() - opmap[op] = getattr(token, name) -del line, op, name diff --git a/Lib/lib2to3/pgen2/literals.py b/Lib/lib2to3/pgen2/literals.py deleted file mode 100644 index b9b63e6e5572c1..00000000000000 --- a/Lib/lib2to3/pgen2/literals.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Safely evaluate Python string literals without using eval().""" - -import re - -simple_escapes = {"a": "\a", - "b": "\b", - "f": "\f", - "n": "\n", - "r": "\r", - "t": "\t", - "v": "\v", - "'": "'", - '"': '"', - "\\": "\\"} - -def escape(m): - all, tail = m.group(0, 1) - assert all.startswith("\\") - esc = simple_escapes.get(tail) - if esc is not None: - return esc - if tail.startswith("x"): - hexes = tail[1:] - if len(hexes) < 2: - raise ValueError("invalid hex string escape ('\\%s')" % tail) - try: - i = int(hexes, 16) - except ValueError: - raise ValueError("invalid hex string escape ('\\%s')" % tail) from None - else: - try: - i = int(tail, 8) - except ValueError: - raise ValueError("invalid octal string escape ('\\%s')" % tail) from None - return chr(i) - -def evalString(s): - assert s.startswith("'") or s.startswith('"'), repr(s[:1]) - q = s[0] - if s[:3] == q*3: - q = q*3 - assert s.endswith(q), repr(s[-len(q):]) - assert len(s) >= 2*len(q) - s = s[len(q):-len(q)] - return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s) - -def test(): - for i in range(256): - c = chr(i) - s = repr(c) - e = evalString(s) - if e != c: - print(i, c, s, e) - - -if __name__ == "__main__": - test() diff --git a/Lib/lib2to3/pgen2/parse.py b/Lib/lib2to3/pgen2/parse.py deleted file mode 100644 index cf3fcf7e99fd11..00000000000000 --- a/Lib/lib2to3/pgen2/parse.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Parser engine for the grammar tables generated by pgen. - -The grammar table must be loaded first. - -See Parser/parser.c in the Python distribution for additional info on -how this parsing engine works. - -""" - -# Local imports -from . import token - -class ParseError(Exception): - """Exception to signal the parser is stuck.""" - - def __init__(self, msg, type, value, context): - Exception.__init__(self, "%s: type=%r, value=%r, context=%r" % - (msg, type, value, context)) - self.msg = msg - self.type = type - self.value = value - self.context = context - - def __reduce__(self): - return type(self), (self.msg, self.type, self.value, self.context) - -class Parser(object): - """Parser engine. - - The proper usage sequence is: - - p = Parser(grammar, [converter]) # create instance - p.setup([start]) # prepare for parsing - : - if p.addtoken(...): # parse a token; may raise ParseError - break - root = p.rootnode # root of abstract syntax tree - - A Parser instance may be reused by calling setup() repeatedly. - - A Parser instance contains state pertaining to the current token - sequence, and should not be used concurrently by different threads - to parse separate token sequences. - - See driver.py for how to get input tokens by tokenizing a file or - string. - - Parsing is complete when addtoken() returns True; the root of the - abstract syntax tree can then be retrieved from the rootnode - instance variable. When a syntax error occurs, addtoken() raises - the ParseError exception. There is no error recovery; the parser - cannot be used after a syntax error was reported (but it can be - reinitialized by calling setup()). - - """ - - def __init__(self, grammar, convert=None): - """Constructor. - - The grammar argument is a grammar.Grammar instance; see the - grammar module for more information. - - The parser is not ready yet for parsing; you must call the - setup() method to get it started. - - The optional convert argument is a function mapping concrete - syntax tree nodes to abstract syntax tree nodes. If not - given, no conversion is done and the syntax tree produced is - the concrete syntax tree. If given, it must be a function of - two arguments, the first being the grammar (a grammar.Grammar - instance), and the second being the concrete syntax tree node - to be converted. The syntax tree is converted from the bottom - up. - - A concrete syntax tree node is a (type, value, context, nodes) - tuple, where type is the node type (a token or symbol number), - value is None for symbols and a string for tokens, context is - None or an opaque value used for error reporting (typically a - (lineno, offset) pair), and nodes is a list of children for - symbols, and None for tokens. - - An abstract syntax tree node may be anything; this is entirely - up to the converter function. - - """ - self.grammar = grammar - self.convert = convert or (lambda grammar, node: node) - - def setup(self, start=None): - """Prepare for parsing. - - This *must* be called before starting to parse. - - The optional argument is an alternative start symbol; it - defaults to the grammar's start symbol. - - You can use a Parser instance to parse any number of programs; - each time you call setup() the parser is reset to an initial - state determined by the (implicit or explicit) start symbol. - - """ - if start is None: - start = self.grammar.start - # Each stack entry is a tuple: (dfa, state, node). - # A node is a tuple: (type, value, context, children), - # where children is a list of nodes or None, and context may be None. - newnode = (start, None, None, []) - stackentry = (self.grammar.dfas[start], 0, newnode) - self.stack = [stackentry] - self.rootnode = None - self.used_names = set() # Aliased to self.rootnode.used_names in pop() - - def addtoken(self, type, value, context): - """Add a token; return True iff this is the end of the program.""" - # Map from token to label - ilabel = self.classify(type, value, context) - # Loop until the token is shifted; may raise exceptions - while True: - dfa, state, node = self.stack[-1] - states, first = dfa - arcs = states[state] - # Look for a state with this label - for i, newstate in arcs: - t, v = self.grammar.labels[i] - if ilabel == i: - # Look it up in the list of labels - assert t < 256 - # Shift a token; we're done with it - self.shift(type, value, newstate, context) - # Pop while we are in an accept-only state - state = newstate - while states[state] == [(0, state)]: - self.pop() - if not self.stack: - # Done parsing! - return True - dfa, state, node = self.stack[-1] - states, first = dfa - # Done with this token - return False - elif t >= 256: - # See if it's a symbol and if we're in its first set - itsdfa = self.grammar.dfas[t] - itsstates, itsfirst = itsdfa - if ilabel in itsfirst: - # Push a symbol - self.push(t, self.grammar.dfas[t], newstate, context) - break # To continue the outer while loop - else: - if (0, state) in arcs: - # An accepting state, pop it and try something else - self.pop() - if not self.stack: - # Done parsing, but another token is input - raise ParseError("too much input", - type, value, context) - else: - # No success finding a transition - raise ParseError("bad input", type, value, context) - - def classify(self, type, value, context): - """Turn a token into a label. (Internal)""" - if type == token.NAME: - # Keep a listing of all used names - self.used_names.add(value) - # Check for reserved words - ilabel = self.grammar.keywords.get(value) - if ilabel is not None: - return ilabel - ilabel = self.grammar.tokens.get(type) - if ilabel is None: - raise ParseError("bad token", type, value, context) - return ilabel - - def shift(self, type, value, newstate, context): - """Shift a token. (Internal)""" - dfa, state, node = self.stack[-1] - newnode = (type, value, context, None) - newnode = self.convert(self.grammar, newnode) - if newnode is not None: - node[-1].append(newnode) - self.stack[-1] = (dfa, newstate, node) - - def push(self, type, newdfa, newstate, context): - """Push a nonterminal. (Internal)""" - dfa, state, node = self.stack[-1] - newnode = (type, None, context, []) - self.stack[-1] = (dfa, newstate, node) - self.stack.append((newdfa, 0, newnode)) - - def pop(self): - """Pop a nonterminal. (Internal)""" - popdfa, popstate, popnode = self.stack.pop() - newnode = self.convert(self.grammar, popnode) - if newnode is not None: - if self.stack: - dfa, state, node = self.stack[-1] - node[-1].append(newnode) - else: - self.rootnode = newnode - self.rootnode.used_names = self.used_names diff --git a/Lib/lib2to3/pgen2/pgen.py b/Lib/lib2to3/pgen2/pgen.py deleted file mode 100644 index 7abd5cef1c36bb..00000000000000 --- a/Lib/lib2to3/pgen2/pgen.py +++ /dev/null @@ -1,386 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# Pgen imports -from . import grammar, token, tokenize - -class PgenGrammar(grammar.Grammar): - pass - -class ParserGenerator(object): - - def __init__(self, filename, stream=None): - close_stream = None - if stream is None: - stream = open(filename, encoding="utf-8") - close_stream = stream.close - self.filename = filename - self.stream = stream - self.generator = tokenize.generate_tokens(stream.readline) - self.gettoken() # Initialize lookahead - self.dfas, self.startsymbol = self.parse() - if close_stream is not None: - close_stream() - self.first = {} # map from symbol name to set of tokens - self.addfirstsets() - - def make_grammar(self): - c = PgenGrammar() - names = list(self.dfas.keys()) - names.sort() - names.remove(self.startsymbol) - names.insert(0, self.startsymbol) - for name in names: - i = 256 + len(c.symbol2number) - c.symbol2number[name] = i - c.number2symbol[i] = name - for name in names: - dfa = self.dfas[name] - states = [] - for state in dfa: - arcs = [] - for label, next in sorted(state.arcs.items()): - arcs.append((self.make_label(c, label), dfa.index(next))) - if state.isfinal: - arcs.append((0, dfa.index(state))) - states.append(arcs) - c.states.append(states) - c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name)) - c.start = c.symbol2number[self.startsymbol] - return c - - def make_first(self, c, name): - rawfirst = self.first[name] - first = {} - for label in sorted(rawfirst): - ilabel = self.make_label(c, label) - ##assert ilabel not in first # XXX failed on <> ... != - first[ilabel] = 1 - return first - - def make_label(self, c, label): - # XXX Maybe this should be a method on a subclass of converter? - ilabel = len(c.labels) - if label[0].isalpha(): - # Either a symbol name or a named token - if label in c.symbol2number: - # A symbol name (a non-terminal) - if label in c.symbol2label: - return c.symbol2label[label] - else: - c.labels.append((c.symbol2number[label], None)) - c.symbol2label[label] = ilabel - return ilabel - else: - # A named token (NAME, NUMBER, STRING) - itoken = getattr(token, label, None) - assert isinstance(itoken, int), label - assert itoken in token.tok_name, label - if itoken in c.tokens: - return c.tokens[itoken] - else: - c.labels.append((itoken, None)) - c.tokens[itoken] = ilabel - return ilabel - else: - # Either a keyword or an operator - assert label[0] in ('"', "'"), label - value = eval(label) - if value[0].isalpha(): - # A keyword - if value in c.keywords: - return c.keywords[value] - else: - c.labels.append((token.NAME, value)) - c.keywords[value] = ilabel - return ilabel - else: - # An operator (any non-numeric token) - itoken = grammar.opmap[value] # Fails if unknown token - if itoken in c.tokens: - return c.tokens[itoken] - else: - c.labels.append((itoken, None)) - c.tokens[itoken] = ilabel - return ilabel - - def addfirstsets(self): - names = list(self.dfas.keys()) - names.sort() - for name in names: - if name not in self.first: - self.calcfirst(name) - #print name, self.first[name].keys() - - def calcfirst(self, name): - dfa = self.dfas[name] - self.first[name] = None # dummy to detect left recursion - state = dfa[0] - totalset = {} - overlapcheck = {} - for label, next in state.arcs.items(): - if label in self.dfas: - if label in self.first: - fset = self.first[label] - if fset is None: - raise ValueError("recursion for rule %r" % name) - else: - self.calcfirst(label) - fset = self.first[label] - totalset.update(fset) - overlapcheck[label] = fset - else: - totalset[label] = 1 - overlapcheck[label] = {label: 1} - inverse = {} - for label, itsfirst in overlapcheck.items(): - for symbol in itsfirst: - if symbol in inverse: - raise ValueError("rule %s is ambiguous; %s is in the" - " first sets of %s as well as %s" % - (name, symbol, label, inverse[symbol])) - inverse[symbol] = label - self.first[name] = totalset - - def parse(self): - dfas = {} - startsymbol = None - # MSTART: (NEWLINE | RULE)* ENDMARKER - while self.type != token.ENDMARKER: - while self.type == token.NEWLINE: - self.gettoken() - # RULE: NAME ':' RHS NEWLINE - name = self.expect(token.NAME) - self.expect(token.OP, ":") - a, z = self.parse_rhs() - self.expect(token.NEWLINE) - #self.dump_nfa(name, a, z) - dfa = self.make_dfa(a, z) - #self.dump_dfa(name, dfa) - oldlen = len(dfa) - self.simplify_dfa(dfa) - newlen = len(dfa) - dfas[name] = dfa - #print name, oldlen, newlen - if startsymbol is None: - startsymbol = name - return dfas, startsymbol - - def make_dfa(self, start, finish): - # To turn an NFA into a DFA, we define the states of the DFA - # to correspond to *sets* of states of the NFA. Then do some - # state reduction. Let's represent sets as dicts with 1 for - # values. - assert isinstance(start, NFAState) - assert isinstance(finish, NFAState) - def closure(state): - base = {} - addclosure(state, base) - return base - def addclosure(state, base): - assert isinstance(state, NFAState) - if state in base: - return - base[state] = 1 - for label, next in state.arcs: - if label is None: - addclosure(next, base) - states = [DFAState(closure(start), finish)] - for state in states: # NB states grows while we're iterating - arcs = {} - for nfastate in state.nfaset: - for label, next in nfastate.arcs: - if label is not None: - addclosure(next, arcs.setdefault(label, {})) - for label, nfaset in sorted(arcs.items()): - for st in states: - if st.nfaset == nfaset: - break - else: - st = DFAState(nfaset, finish) - states.append(st) - state.addarc(st, label) - return states # List of DFAState instances; first one is start - - def dump_nfa(self, name, start, finish): - print("Dump of NFA for", name) - todo = [start] - for i, state in enumerate(todo): - print(" State", i, state is finish and "(final)" or "") - for label, next in state.arcs: - if next in todo: - j = todo.index(next) - else: - j = len(todo) - todo.append(next) - if label is None: - print(" -> %d" % j) - else: - print(" %s -> %d" % (label, j)) - - def dump_dfa(self, name, dfa): - print("Dump of DFA for", name) - for i, state in enumerate(dfa): - print(" State", i, state.isfinal and "(final)" or "") - for label, next in sorted(state.arcs.items()): - print(" %s -> %d" % (label, dfa.index(next))) - - def simplify_dfa(self, dfa): - # This is not theoretically optimal, but works well enough. - # Algorithm: repeatedly look for two states that have the same - # set of arcs (same labels pointing to the same nodes) and - # unify them, until things stop changing. - - # dfa is a list of DFAState instances - changes = True - while changes: - changes = False - for i, state_i in enumerate(dfa): - for j in range(i+1, len(dfa)): - state_j = dfa[j] - if state_i == state_j: - #print " unify", i, j - del dfa[j] - for state in dfa: - state.unifystate(state_j, state_i) - changes = True - break - - def parse_rhs(self): - # RHS: ALT ('|' ALT)* - a, z = self.parse_alt() - if self.value != "|": - return a, z - else: - aa = NFAState() - zz = NFAState() - aa.addarc(a) - z.addarc(zz) - while self.value == "|": - self.gettoken() - a, z = self.parse_alt() - aa.addarc(a) - z.addarc(zz) - return aa, zz - - def parse_alt(self): - # ALT: ITEM+ - a, b = self.parse_item() - while (self.value in ("(", "[") or - self.type in (token.NAME, token.STRING)): - c, d = self.parse_item() - b.addarc(c) - b = d - return a, b - - def parse_item(self): - # ITEM: '[' RHS ']' | ATOM ['+' | '*'] - if self.value == "[": - self.gettoken() - a, z = self.parse_rhs() - self.expect(token.OP, "]") - a.addarc(z) - return a, z - else: - a, z = self.parse_atom() - value = self.value - if value not in ("+", "*"): - return a, z - self.gettoken() - z.addarc(a) - if value == "+": - return a, z - else: - return a, a - - def parse_atom(self): - # ATOM: '(' RHS ')' | NAME | STRING - if self.value == "(": - self.gettoken() - a, z = self.parse_rhs() - self.expect(token.OP, ")") - return a, z - elif self.type in (token.NAME, token.STRING): - a = NFAState() - z = NFAState() - a.addarc(z, self.value) - self.gettoken() - return a, z - else: - self.raise_error("expected (...) or NAME or STRING, got %s/%s", - self.type, self.value) - - def expect(self, type, value=None): - if self.type != type or (value is not None and self.value != value): - self.raise_error("expected %s/%s, got %s/%s", - type, value, self.type, self.value) - value = self.value - self.gettoken() - return value - - def gettoken(self): - tup = next(self.generator) - while tup[0] in (tokenize.COMMENT, tokenize.NL): - tup = next(self.generator) - self.type, self.value, self.begin, self.end, self.line = tup - #print token.tok_name[self.type], repr(self.value) - - def raise_error(self, msg, *args): - if args: - try: - msg = msg % args - except: - msg = " ".join([msg] + list(map(str, args))) - raise SyntaxError(msg, (self.filename, self.end[0], - self.end[1], self.line)) - -class NFAState(object): - - def __init__(self): - self.arcs = [] # list of (label, NFAState) pairs - - def addarc(self, next, label=None): - assert label is None or isinstance(label, str) - assert isinstance(next, NFAState) - self.arcs.append((label, next)) - -class DFAState(object): - - def __init__(self, nfaset, final): - assert isinstance(nfaset, dict) - assert isinstance(next(iter(nfaset)), NFAState) - assert isinstance(final, NFAState) - self.nfaset = nfaset - self.isfinal = final in nfaset - self.arcs = {} # map from label to DFAState - - def addarc(self, next, label): - assert isinstance(label, str) - assert label not in self.arcs - assert isinstance(next, DFAState) - self.arcs[label] = next - - def unifystate(self, old, new): - for label, next in self.arcs.items(): - if next is old: - self.arcs[label] = new - - def __eq__(self, other): - # Equality test -- ignore the nfaset instance variable - assert isinstance(other, DFAState) - if self.isfinal != other.isfinal: - return False - # Can't just return self.arcs == other.arcs, because that - # would invoke this method recursively, with cycles... - if len(self.arcs) != len(other.arcs): - return False - for label, next in self.arcs.items(): - if next is not other.arcs.get(label): - return False - return True - - __hash__ = None # For Py3 compatibility. - -def generate_grammar(filename="Grammar.txt"): - p = ParserGenerator(filename) - return p.make_grammar() diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py deleted file mode 100755 index 2a55138e48237b..00000000000000 --- a/Lib/lib2to3/pgen2/token.py +++ /dev/null @@ -1,86 +0,0 @@ -#! /usr/bin/env python3 - -"""Token constants (from "token.h").""" - -# Taken from Python (r53757) and modified to include some tokens -# originally monkeypatched in by pgen2.tokenize - -#--start constants-- -ENDMARKER = 0 -NAME = 1 -NUMBER = 2 -STRING = 3 -NEWLINE = 4 -INDENT = 5 -DEDENT = 6 -LPAR = 7 -RPAR = 8 -LSQB = 9 -RSQB = 10 -COLON = 11 -COMMA = 12 -SEMI = 13 -PLUS = 14 -MINUS = 15 -STAR = 16 -SLASH = 17 -VBAR = 18 -AMPER = 19 -LESS = 20 -GREATER = 21 -EQUAL = 22 -DOT = 23 -PERCENT = 24 -BACKQUOTE = 25 -LBRACE = 26 -RBRACE = 27 -EQEQUAL = 28 -NOTEQUAL = 29 -LESSEQUAL = 30 -GREATEREQUAL = 31 -TILDE = 32 -CIRCUMFLEX = 33 -LEFTSHIFT = 34 -RIGHTSHIFT = 35 -DOUBLESTAR = 36 -PLUSEQUAL = 37 -MINEQUAL = 38 -STAREQUAL = 39 -SLASHEQUAL = 40 -PERCENTEQUAL = 41 -AMPEREQUAL = 42 -VBAREQUAL = 43 -CIRCUMFLEXEQUAL = 44 -LEFTSHIFTEQUAL = 45 -RIGHTSHIFTEQUAL = 46 -DOUBLESTAREQUAL = 47 -DOUBLESLASH = 48 -DOUBLESLASHEQUAL = 49 -AT = 50 -ATEQUAL = 51 -OP = 52 -COMMENT = 53 -NL = 54 -RARROW = 55 -AWAIT = 56 -ASYNC = 57 -ERRORTOKEN = 58 -COLONEQUAL = 59 -N_TOKENS = 60 -NT_OFFSET = 256 -#--end constants-- - -tok_name = {} -for _name, _value in list(globals().items()): - if isinstance(_value, int): - tok_name[_value] = _name - - -def ISTERMINAL(x): - return x < NT_OFFSET - -def ISNONTERMINAL(x): - return x >= NT_OFFSET - -def ISEOF(x): - return x == ENDMARKER diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py deleted file mode 100644 index 099dfa7798afd4..00000000000000 --- a/Lib/lib2to3/pgen2/tokenize.py +++ /dev/null @@ -1,564 +0,0 @@ -# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation. -# All rights reserved. - -"""Tokenization help for Python programs. - -generate_tokens(readline) is a generator that breaks a stream of -text into Python tokens. It accepts a readline-like method which is called -repeatedly to get the next line of input (or "" for EOF). It generates -5-tuples with these members: - - the token type (see token.py) - the token (a string) - the starting (row, column) indices of the token (a 2-tuple of ints) - the ending (row, column) indices of the token (a 2-tuple of ints) - the original line (string) - -It is designed to match the working of the Python tokenizer exactly, except -that it produces COMMENT tokens for comments and gives type OP for all -operators - -Older entry points - tokenize_loop(readline, tokeneater) - tokenize(readline, tokeneater=printtoken) -are the same, except instead of generating tokens, tokeneater is a callback -function to which the 5 fields described above are passed as 5 arguments, -each time a new token is found.""" - -__author__ = 'Ka-Ping Yee ' -__credits__ = \ - 'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro' - -import string, re -from codecs import BOM_UTF8, lookup -from lib2to3.pgen2.token import * - -from . import token -__all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize", - "generate_tokens", "untokenize"] -del token - -try: - bytes -except NameError: - # Support bytes type in Python <= 2.5, so 2to3 turns itself into - # valid Python 3 code. - bytes = str - -def group(*choices): return '(' + '|'.join(choices) + ')' -def any(*choices): return group(*choices) + '*' -def maybe(*choices): return group(*choices) + '?' -def _combinations(*l): - return set( - x + y for x in l for y in l + ("",) if x.casefold() != y.casefold() - ) - -Whitespace = r'[ \f\t]*' -Comment = r'#[^\r\n]*' -Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) -Name = r'\w+' - -Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' -Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' -Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' -Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') -Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+(?:_\d+)*' -Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) -Expfloat = r'\d+(?:_\d+)*' + Exponent -Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') -Number = group(Imagnumber, Floatnumber, Intnumber) - -# Tail end of ' string. -Single = r"[^'\\]*(?:\\.[^'\\]*)*'" -# Tail end of " string. -Double = r'[^"\\]*(?:\\.[^"\\]*)*"' -# Tail end of ''' string. -Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" -# Tail end of """ string. -Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -_litprefix = r"(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?" -Triple = group(_litprefix + "'''", _litprefix + '"""') -# Single-line ' or " string. -String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') - -# Because of leftmost-then-longest match semantics, be sure to put the -# longest operators first (e.g., if = came before ==, == would get -# recognized as two instances of =). -Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", - r"//=?", r"->", - r"[+\-*/%&@|^=<>]=?", - r"~") - -Bracket = '[][(){}]' -Special = group(r'\r?\n', r':=', r'[:;.,`@]') -Funny = group(Operator, Bracket, Special) - -PlainToken = group(Number, Funny, String, Name) -Token = Ignore + PlainToken - -# First (or only) line of ' or " string. -ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + - group("'", r'\\\r?\n'), - _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + - group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) -PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) - -tokenprog, pseudoprog, single3prog, double3prog = map( - re.compile, (Token, PseudoToken, Single3, Double3)) - -_strprefixes = ( - _combinations('r', 'R', 'f', 'F') | - _combinations('r', 'R', 'b', 'B') | - {'u', 'U', 'ur', 'uR', 'Ur', 'UR'} -) - -endprogs = {"'": re.compile(Single), '"': re.compile(Double), - "'''": single3prog, '"""': double3prog, - **{f"{prefix}'''": single3prog for prefix in _strprefixes}, - **{f'{prefix}"""': double3prog for prefix in _strprefixes}, - **{prefix: None for prefix in _strprefixes}} - -triple_quoted = ( - {"'''", '"""'} | - {f"{prefix}'''" for prefix in _strprefixes} | - {f'{prefix}"""' for prefix in _strprefixes} -) -single_quoted = ( - {"'", '"'} | - {f"{prefix}'" for prefix in _strprefixes} | - {f'{prefix}"' for prefix in _strprefixes} -) - -tabsize = 8 - -class TokenError(Exception): pass - -class StopTokenizing(Exception): pass - -def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing - (srow, scol) = xxx_todo_changeme - (erow, ecol) = xxx_todo_changeme1 - print("%d,%d-%d,%d:\t%s\t%s" % \ - (srow, scol, erow, ecol, tok_name[type], repr(token))) - -def tokenize(readline, tokeneater=printtoken): - """ - The tokenize() function accepts two parameters: one representing the - input stream, and one providing an output mechanism for tokenize(). - - The first parameter, readline, must be a callable object which provides - the same interface as the readline() method of built-in file objects. - Each call to the function should return one line of input as a string. - - The second parameter, tokeneater, must also be a callable object. It is - called once for each token, with five arguments, corresponding to the - tuples generated by generate_tokens(). - """ - try: - tokenize_loop(readline, tokeneater) - except StopTokenizing: - pass - -# backwards compatible interface -def tokenize_loop(readline, tokeneater): - for token_info in generate_tokens(readline): - tokeneater(*token_info) - -class Untokenizer: - - def __init__(self): - self.tokens = [] - self.prev_row = 1 - self.prev_col = 0 - - def add_whitespace(self, start): - row, col = start - assert row <= self.prev_row - col_offset = col - self.prev_col - if col_offset: - self.tokens.append(" " * col_offset) - - def untokenize(self, iterable): - for t in iterable: - if len(t) == 2: - self.compat(t, iterable) - break - tok_type, token, start, end, line = t - self.add_whitespace(start) - self.tokens.append(token) - self.prev_row, self.prev_col = end - if tok_type in (NEWLINE, NL): - self.prev_row += 1 - self.prev_col = 0 - return "".join(self.tokens) - - def compat(self, token, iterable): - startline = False - indents = [] - toks_append = self.tokens.append - toknum, tokval = token - if toknum in (NAME, NUMBER): - tokval += ' ' - if toknum in (NEWLINE, NL): - startline = True - for tok in iterable: - toknum, tokval = tok[:2] - - if toknum in (NAME, NUMBER, ASYNC, AWAIT): - tokval += ' ' - - if toknum == INDENT: - indents.append(tokval) - continue - elif toknum == DEDENT: - indents.pop() - continue - elif toknum in (NEWLINE, NL): - startline = True - elif startline and indents: - toks_append(indents[-1]) - startline = False - toks_append(tokval) - -cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) -blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) - -def _get_normal_name(orig_enc): - """Imitates get_normal_name in tokenizer.c.""" - # Only care about the first 12 characters. - enc = orig_enc[:12].lower().replace("_", "-") - if enc == "utf-8" or enc.startswith("utf-8-"): - return "utf-8" - if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \ - enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")): - return "iso-8859-1" - return orig_enc - -def detect_encoding(readline): - """ - The detect_encoding() function is used to detect the encoding that should - be used to decode a Python source file. It requires one argument, readline, - in the same way as the tokenize() generator. - - It will call readline a maximum of twice, and return the encoding used - (as a string) and a list of any lines (left as bytes) it has read - in. - - It detects the encoding from the presence of a utf-8 bom or an encoding - cookie as specified in pep-0263. If both a bom and a cookie are present, but - disagree, a SyntaxError will be raised. If the encoding cookie is an invalid - charset, raise a SyntaxError. Note that if a utf-8 bom is found, - 'utf-8-sig' is returned. - - If no encoding is specified, then the default of 'utf-8' will be returned. - """ - bom_found = False - encoding = None - default = 'utf-8' - def read_or_stop(): - try: - return readline() - except StopIteration: - return bytes() - - def find_cookie(line): - try: - line_string = line.decode('ascii') - except UnicodeDecodeError: - return None - match = cookie_re.match(line_string) - if not match: - return None - encoding = _get_normal_name(match.group(1)) - try: - codec = lookup(encoding) - except LookupError: - # This behaviour mimics the Python interpreter - raise SyntaxError("unknown encoding: " + encoding) - - if bom_found: - if codec.name != 'utf-8': - # This behaviour mimics the Python interpreter - raise SyntaxError('encoding problem: utf-8') - encoding += '-sig' - return encoding - - first = read_or_stop() - if first.startswith(BOM_UTF8): - bom_found = True - first = first[3:] - default = 'utf-8-sig' - if not first: - return default, [] - - encoding = find_cookie(first) - if encoding: - return encoding, [first] - if not blank_re.match(first): - return default, [first] - - second = read_or_stop() - if not second: - return default, [first] - - encoding = find_cookie(second) - if encoding: - return encoding, [first, second] - - return default, [first, second] - -def untokenize(iterable): - """Transform tokens back into Python source code. - - Each element returned by the iterable must be a token sequence - with at least two elements, a token number and token value. If - only two tokens are passed, the resulting output is poor. - - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited input: - # Output text will tokenize the back to the input - t1 = [tok[:2] for tok in generate_tokens(f.readline)] - newcode = untokenize(t1) - readline = iter(newcode.splitlines(1)).next - t2 = [tok[:2] for tokin generate_tokens(readline)] - assert t1 == t2 - """ - ut = Untokenizer() - return ut.untokenize(iterable) - -def generate_tokens(readline): - """ - The generate_tokens() generator requires one argument, readline, which - must be a callable object which provides the same interface as the - readline() method of built-in file objects. Each call to the function - should return one line of input as a string. Alternately, readline - can be a callable function terminating with StopIteration: - readline = open(myfile).next # Example of alternate readline - - The generator produces 5-tuples with these members: the token type; the - token string; a 2-tuple (srow, scol) of ints specifying the row and - column where the token begins in the source; a 2-tuple (erow, ecol) of - ints specifying the row and column where the token ends in the source; - and the line on which the token was found. The line passed is the - physical line. - """ - lnum = parenlev = continued = 0 - contstr, needcont = '', 0 - contline = None - indents = [0] - - # 'stashed' and 'async_*' are used for async/await parsing - stashed = None - async_def = False - async_def_indent = 0 - async_def_nl = False - - while 1: # loop over lines in stream - try: - line = readline() - except StopIteration: - line = '' - lnum = lnum + 1 - pos, max = 0, len(line) - - if contstr: # continued string - if not line: - raise TokenError("EOF in multi-line string", strstart) - endmatch = endprog.match(line) - if endmatch: - pos = end = endmatch.end(0) - yield (STRING, contstr + line[:end], - strstart, (lnum, end), contline + line) - contstr, needcont = '', 0 - contline = None - elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': - yield (ERRORTOKEN, contstr + line, - strstart, (lnum, len(line)), contline) - contstr = '' - contline = None - continue - else: - contstr = contstr + line - contline = contline + line - continue - - elif parenlev == 0 and not continued: # new statement - if not line: break - column = 0 - while pos < max: # measure leading whitespace - if line[pos] == ' ': column = column + 1 - elif line[pos] == '\t': column = (column//tabsize + 1)*tabsize - elif line[pos] == '\f': column = 0 - else: break - pos = pos + 1 - if pos == max: break - - if stashed: - yield stashed - stashed = None - - if line[pos] in '#\r\n': # skip comments or blank lines - if line[pos] == '#': - comment_token = line[pos:].rstrip('\r\n') - nl_pos = pos + len(comment_token) - yield (COMMENT, comment_token, - (lnum, pos), (lnum, pos + len(comment_token)), line) - yield (NL, line[nl_pos:], - (lnum, nl_pos), (lnum, len(line)), line) - else: - yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], - (lnum, pos), (lnum, len(line)), line) - continue - - if column > indents[-1]: # count indents or dedents - indents.append(column) - yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) - while column < indents[-1]: - if column not in indents: - raise IndentationError( - "unindent does not match any outer indentation level", - ("", lnum, pos, line)) - indents = indents[:-1] - - if async_def and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - - yield (DEDENT, '', (lnum, pos), (lnum, pos), line) - - if async_def and async_def_nl and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - - else: # continued statement - if not line: - raise TokenError("EOF in multi-line statement", (lnum, 0)) - continued = 0 - - while pos < max: - pseudomatch = pseudoprog.match(line, pos) - if pseudomatch: # scan for tokens - start, end = pseudomatch.span(1) - spos, epos, pos = (lnum, start), (lnum, end), end - token, initial = line[start:end], line[start] - - if initial in string.digits or \ - (initial == '.' and token != '.'): # ordinary number - yield (NUMBER, token, spos, epos, line) - elif initial in '\r\n': - newline = NEWLINE - if parenlev > 0: - newline = NL - elif async_def: - async_def_nl = True - if stashed: - yield stashed - stashed = None - yield (newline, token, spos, epos, line) - - elif initial == '#': - assert not token.endswith("\n") - if stashed: - yield stashed - stashed = None - yield (COMMENT, token, spos, epos, line) - elif token in triple_quoted: - endprog = endprogs[token] - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - pos = endmatch.end(0) - token = line[start:pos] - if stashed: - yield stashed - stashed = None - yield (STRING, token, spos, (lnum, pos), line) - else: - strstart = (lnum, start) # multiple lines - contstr = line[start:] - contline = line - break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: - if token[-1] == '\n': # continued string - strstart = (lnum, start) - endprog = (endprogs[initial] or endprogs[token[1]] or - endprogs[token[2]]) - contstr, needcont = line[start:], 1 - contline = line - break - else: # ordinary string - if stashed: - yield stashed - stashed = None - yield (STRING, token, spos, epos, line) - elif initial.isidentifier(): # ordinary name - if token in ('async', 'await'): - if async_def: - yield (ASYNC if token == 'async' else AWAIT, - token, spos, epos, line) - continue - - tok = (NAME, token, spos, epos, line) - if token == 'async' and not stashed: - stashed = tok - continue - - if token in ('def', 'for'): - if (stashed - and stashed[0] == NAME - and stashed[1] == 'async'): - - if token == 'def': - async_def = True - async_def_indent = indents[-1] - - yield (ASYNC, stashed[1], - stashed[2], stashed[3], - stashed[4]) - stashed = None - - if stashed: - yield stashed - stashed = None - - yield tok - elif initial == '\\': # continued stmt - # This yield is new; needed for better idempotency: - if stashed: - yield stashed - stashed = None - yield (NL, token, spos, (lnum, pos), line) - continued = 1 - else: - if initial in '([{': parenlev = parenlev + 1 - elif initial in ')]}': parenlev = parenlev - 1 - if stashed: - yield stashed - stashed = None - yield (OP, token, spos, epos, line) - else: - yield (ERRORTOKEN, line[pos], - (lnum, pos), (lnum, pos+1), line) - pos = pos + 1 - - if stashed: - yield stashed - stashed = None - - for indent in indents[1:]: # pop remaining indent levels - yield (DEDENT, '', (lnum, 0), (lnum, 0), '') - yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') - -if __name__ == '__main__': # testing - import sys - if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) - else: tokenize(sys.stdin.readline) diff --git a/Lib/lib2to3/pygram.py b/Lib/lib2to3/pygram.py deleted file mode 100644 index 24d9db9217f131..00000000000000 --- a/Lib/lib2to3/pygram.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Export the Python grammar and symbols.""" - -# Python imports -import os - -# Local imports -from .pgen2 import token -from .pgen2 import driver -from . import pytree - -# The grammar file -_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt") -_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), - "PatternGrammar.txt") - - -class Symbols(object): - - def __init__(self, grammar): - """Initializer. - - Creates an attribute for each grammar symbol (nonterminal), - whose value is the symbol's type (an int >= 256). - """ - for name, symbol in grammar.symbol2number.items(): - setattr(self, name, symbol) - - -python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE) - -python_symbols = Symbols(python_grammar) - -python_grammar_no_print_statement = python_grammar.copy() -del python_grammar_no_print_statement.keywords["print"] - -python_grammar_no_print_and_exec_statement = python_grammar_no_print_statement.copy() -del python_grammar_no_print_and_exec_statement.keywords["exec"] - -pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE) -pattern_symbols = Symbols(pattern_grammar) diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py deleted file mode 100644 index 729023df0284e1..00000000000000 --- a/Lib/lib2to3/pytree.py +++ /dev/null @@ -1,853 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -""" -Python parse tree definitions. - -This is a very concrete parse tree; we need to keep every token and -even the comments and whitespace between tokens. - -There's also a pattern matching implementation here. -""" - -__author__ = "Guido van Rossum " - -import sys -from io import StringIO - -HUGE = 0x7FFFFFFF # maximum repeat count, default max - -_type_reprs = {} -def type_repr(type_num): - global _type_reprs - if not _type_reprs: - from .pygram import python_symbols - # printing tokens is possible but not as useful - # from .pgen2 import token // token.__dict__.items(): - for name, val in python_symbols.__dict__.items(): - if type(val) == int: _type_reprs[val] = name - return _type_reprs.setdefault(type_num, type_num) - -class Base(object): - - """ - Abstract base class for Node and Leaf. - - This provides some default functionality and boilerplate using the - template pattern. - - A node may be a subnode of at most one parent. - """ - - # Default values for instance variables - type = None # int: token number (< 256) or symbol number (>= 256) - parent = None # Parent node pointer, or None - children = () # Tuple of subnodes - was_changed = False - was_checked = False - - def __new__(cls, *args, **kwds): - """Constructor that prevents Base from being instantiated.""" - assert cls is not Base, "Cannot instantiate Base" - return object.__new__(cls) - - def __eq__(self, other): - """ - Compare two nodes for equality. - - This calls the method _eq(). - """ - if self.__class__ is not other.__class__: - return NotImplemented - return self._eq(other) - - __hash__ = None # For Py3 compatibility. - - def _eq(self, other): - """ - Compare two nodes for equality. - - This is called by __eq__ and __ne__. It is only called if the two nodes - have the same type. This must be implemented by the concrete subclass. - Nodes should be considered equal if they have the same structure, - ignoring the prefix string and other context information. - """ - raise NotImplementedError - - def clone(self): - """ - Return a cloned (deep) copy of self. - - This must be implemented by the concrete subclass. - """ - raise NotImplementedError - - def post_order(self): - """ - Return a post-order iterator for the tree. - - This must be implemented by the concrete subclass. - """ - raise NotImplementedError - - def pre_order(self): - """ - Return a pre-order iterator for the tree. - - This must be implemented by the concrete subclass. - """ - raise NotImplementedError - - def replace(self, new): - """Replace this node with a new one in the parent.""" - assert self.parent is not None, str(self) - assert new is not None - if not isinstance(new, list): - new = [new] - l_children = [] - found = False - for ch in self.parent.children: - if ch is self: - assert not found, (self.parent.children, self, new) - if new is not None: - l_children.extend(new) - found = True - else: - l_children.append(ch) - assert found, (self.children, self, new) - self.parent.changed() - self.parent.children = l_children - for x in new: - x.parent = self.parent - self.parent = None - - def get_lineno(self): - """Return the line number which generated the invocant node.""" - node = self - while not isinstance(node, Leaf): - if not node.children: - return - node = node.children[0] - return node.lineno - - def changed(self): - if self.parent: - self.parent.changed() - self.was_changed = True - - def remove(self): - """ - Remove the node from the tree. Returns the position of the node in its - parent's children before it was removed. - """ - if self.parent: - for i, node in enumerate(self.parent.children): - if node is self: - self.parent.changed() - del self.parent.children[i] - self.parent = None - return i - - @property - def next_sibling(self): - """ - The node immediately following the invocant in their parent's children - list. If the invocant does not have a next sibling, it is None - """ - if self.parent is None: - return None - - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - try: - return self.parent.children[i+1] - except IndexError: - return None - - @property - def prev_sibling(self): - """ - The node immediately preceding the invocant in their parent's children - list. If the invocant does not have a previous sibling, it is None. - """ - if self.parent is None: - return None - - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - if i == 0: - return None - return self.parent.children[i-1] - - def leaves(self): - for child in self.children: - yield from child.leaves() - - def depth(self): - if self.parent is None: - return 0 - return 1 + self.parent.depth() - - def get_suffix(self): - """ - Return the string immediately following the invocant node. This is - effectively equivalent to node.next_sibling.prefix - """ - next_sib = self.next_sibling - if next_sib is None: - return "" - return next_sib.prefix - - if sys.version_info < (3, 0): - def __str__(self): - return str(self).encode("ascii") - -class Node(Base): - - """Concrete implementation for interior nodes.""" - - def __init__(self,type, children, - context=None, - prefix=None, - fixers_applied=None): - """ - Initializer. - - Takes a type constant (a symbol number >= 256), a sequence of - child nodes, and an optional context keyword argument. - - As a side effect, the parent pointers of the children are updated. - """ - assert type >= 256, type - self.type = type - self.children = list(children) - for ch in self.children: - assert ch.parent is None, repr(ch) - ch.parent = self - if prefix is not None: - self.prefix = prefix - if fixers_applied: - self.fixers_applied = fixers_applied[:] - else: - self.fixers_applied = None - - def __repr__(self): - """Return a canonical string representation.""" - return "%s(%s, %r)" % (self.__class__.__name__, - type_repr(self.type), - self.children) - - def __unicode__(self): - """ - Return a pretty string representation. - - This reproduces the input source exactly. - """ - return "".join(map(str, self.children)) - - if sys.version_info > (3, 0): - __str__ = __unicode__ - - def _eq(self, other): - """Compare two nodes for equality.""" - return (self.type, self.children) == (other.type, other.children) - - def clone(self): - """Return a cloned (deep) copy of self.""" - return Node(self.type, [ch.clone() for ch in self.children], - fixers_applied=self.fixers_applied) - - def post_order(self): - """Return a post-order iterator for the tree.""" - for child in self.children: - yield from child.post_order() - yield self - - def pre_order(self): - """Return a pre-order iterator for the tree.""" - yield self - for child in self.children: - yield from child.pre_order() - - @property - def prefix(self): - """ - The whitespace and comments preceding this node in the input. - """ - if not self.children: - return "" - return self.children[0].prefix - - @prefix.setter - def prefix(self, prefix): - if self.children: - self.children[0].prefix = prefix - - def set_child(self, i, child): - """ - Equivalent to 'node.children[i] = child'. This method also sets the - child's parent attribute appropriately. - """ - child.parent = self - self.children[i].parent = None - self.children[i] = child - self.changed() - - def insert_child(self, i, child): - """ - Equivalent to 'node.children.insert(i, child)'. This method also sets - the child's parent attribute appropriately. - """ - child.parent = self - self.children.insert(i, child) - self.changed() - - def append_child(self, child): - """ - Equivalent to 'node.children.append(child)'. This method also sets the - child's parent attribute appropriately. - """ - child.parent = self - self.children.append(child) - self.changed() - - -class Leaf(Base): - - """Concrete implementation for leaf nodes.""" - - # Default values for instance variables - _prefix = "" # Whitespace and comments preceding this token in the input - lineno = 0 # Line where this token starts in the input - column = 0 # Column where this token tarts in the input - - def __init__(self, type, value, - context=None, - prefix=None, - fixers_applied=[]): - """ - Initializer. - - Takes a type constant (a token number < 256), a string value, and an - optional context keyword argument. - """ - assert 0 <= type < 256, type - if context is not None: - self._prefix, (self.lineno, self.column) = context - self.type = type - self.value = value - if prefix is not None: - self._prefix = prefix - self.fixers_applied = fixers_applied[:] - - def __repr__(self): - """Return a canonical string representation.""" - return "%s(%r, %r)" % (self.__class__.__name__, - self.type, - self.value) - - def __unicode__(self): - """ - Return a pretty string representation. - - This reproduces the input source exactly. - """ - return self.prefix + str(self.value) - - if sys.version_info > (3, 0): - __str__ = __unicode__ - - def _eq(self, other): - """Compare two nodes for equality.""" - return (self.type, self.value) == (other.type, other.value) - - def clone(self): - """Return a cloned (deep) copy of self.""" - return Leaf(self.type, self.value, - (self.prefix, (self.lineno, self.column)), - fixers_applied=self.fixers_applied) - - def leaves(self): - yield self - - def post_order(self): - """Return a post-order iterator for the tree.""" - yield self - - def pre_order(self): - """Return a pre-order iterator for the tree.""" - yield self - - @property - def prefix(self): - """ - The whitespace and comments preceding this token in the input. - """ - return self._prefix - - @prefix.setter - def prefix(self, prefix): - self.changed() - self._prefix = prefix - -def convert(gr, raw_node): - """ - Convert raw node information to a Node or Leaf instance. - - This is passed to the parser driver which calls it whenever a reduction of a - grammar rule produces a new complete node, so that the tree is build - strictly bottom-up. - """ - type, value, context, children = raw_node - if children or type in gr.number2symbol: - # If there's exactly one child, return that child instead of - # creating a new node. - if len(children) == 1: - return children[0] - return Node(type, children, context=context) - else: - return Leaf(type, value, context=context) - - -class BasePattern(object): - - """ - A pattern is a tree matching pattern. - - It looks for a specific node type (token or symbol), and - optionally for a specific content. - - This is an abstract base class. There are three concrete - subclasses: - - - LeafPattern matches a single leaf node; - - NodePattern matches a single node (usually non-leaf); - - WildcardPattern matches a sequence of nodes of variable length. - """ - - # Defaults for instance variables - type = None # Node type (token if < 256, symbol if >= 256) - content = None # Optional content matching pattern - name = None # Optional name used to store match in results dict - - def __new__(cls, *args, **kwds): - """Constructor that prevents BasePattern from being instantiated.""" - assert cls is not BasePattern, "Cannot instantiate BasePattern" - return object.__new__(cls) - - def __repr__(self): - args = [type_repr(self.type), self.content, self.name] - while args and args[-1] is None: - del args[-1] - return "%s(%s)" % (self.__class__.__name__, ", ".join(map(repr, args))) - - def optimize(self): - """ - A subclass can define this as a hook for optimizations. - - Returns either self or another node with the same effect. - """ - return self - - def match(self, node, results=None): - """ - Does this pattern exactly match a node? - - Returns True if it matches, False if not. - - If results is not None, it must be a dict which will be - updated with the nodes matching named subpatterns. - - Default implementation for non-wildcard patterns. - """ - if self.type is not None and node.type != self.type: - return False - if self.content is not None: - r = None - if results is not None: - r = {} - if not self._submatch(node, r): - return False - if r: - results.update(r) - if results is not None and self.name: - results[self.name] = node - return True - - def match_seq(self, nodes, results=None): - """ - Does this pattern exactly match a sequence of nodes? - - Default implementation for non-wildcard patterns. - """ - if len(nodes) != 1: - return False - return self.match(nodes[0], results) - - def generate_matches(self, nodes): - """ - Generator yielding all matches for this pattern. - - Default implementation for non-wildcard patterns. - """ - r = {} - if nodes and self.match(nodes[0], r): - yield 1, r - - -class LeafPattern(BasePattern): - - def __init__(self, type=None, content=None, name=None): - """ - Initializer. Takes optional type, content, and name. - - The type, if given must be a token type (< 256). If not given, - this matches any *leaf* node; the content may still be required. - - The content, if given, must be a string. - - If a name is given, the matching node is stored in the results - dict under that key. - """ - if type is not None: - assert 0 <= type < 256, type - if content is not None: - assert isinstance(content, str), repr(content) - self.type = type - self.content = content - self.name = name - - def match(self, node, results=None): - """Override match() to insist on a leaf node.""" - if not isinstance(node, Leaf): - return False - return BasePattern.match(self, node, results) - - def _submatch(self, node, results=None): - """ - Match the pattern's content to the node's children. - - This assumes the node type matches and self.content is not None. - - Returns True if it matches, False if not. - - If results is not None, it must be a dict which will be - updated with the nodes matching named subpatterns. - - When returning False, the results dict may still be updated. - """ - return self.content == node.value - - -class NodePattern(BasePattern): - - wildcards = False - - def __init__(self, type=None, content=None, name=None): - """ - Initializer. Takes optional type, content, and name. - - The type, if given, must be a symbol type (>= 256). If the - type is None this matches *any* single node (leaf or not), - except if content is not None, in which it only matches - non-leaf nodes that also match the content pattern. - - The content, if not None, must be a sequence of Patterns that - must match the node's children exactly. If the content is - given, the type must not be None. - - If a name is given, the matching node is stored in the results - dict under that key. - """ - if type is not None: - assert type >= 256, type - if content is not None: - assert not isinstance(content, str), repr(content) - content = list(content) - for i, item in enumerate(content): - assert isinstance(item, BasePattern), (i, item) - if isinstance(item, WildcardPattern): - self.wildcards = True - self.type = type - self.content = content - self.name = name - - def _submatch(self, node, results=None): - """ - Match the pattern's content to the node's children. - - This assumes the node type matches and self.content is not None. - - Returns True if it matches, False if not. - - If results is not None, it must be a dict which will be - updated with the nodes matching named subpatterns. - - When returning False, the results dict may still be updated. - """ - if self.wildcards: - for c, r in generate_matches(self.content, node.children): - if c == len(node.children): - if results is not None: - results.update(r) - return True - return False - if len(self.content) != len(node.children): - return False - for subpattern, child in zip(self.content, node.children): - if not subpattern.match(child, results): - return False - return True - - -class WildcardPattern(BasePattern): - - """ - A wildcard pattern can match zero or more nodes. - - This has all the flexibility needed to implement patterns like: - - .* .+ .? .{m,n} - (a b c | d e | f) - (...)* (...)+ (...)? (...){m,n} - - except it always uses non-greedy matching. - """ - - def __init__(self, content=None, min=0, max=HUGE, name=None): - """ - Initializer. - - Args: - content: optional sequence of subsequences of patterns; - if absent, matches one node; - if present, each subsequence is an alternative [*] - min: optional minimum number of times to match, default 0 - max: optional maximum number of times to match, default HUGE - name: optional name assigned to this match - - [*] Thus, if content is [[a, b, c], [d, e], [f, g, h]] this is - equivalent to (a b c | d e | f g h); if content is None, - this is equivalent to '.' in regular expression terms. - The min and max parameters work as follows: - min=0, max=maxint: .* - min=1, max=maxint: .+ - min=0, max=1: .? - min=1, max=1: . - If content is not None, replace the dot with the parenthesized - list of alternatives, e.g. (a b c | d e | f g h)* - """ - assert 0 <= min <= max <= HUGE, (min, max) - if content is not None: - content = tuple(map(tuple, content)) # Protect against alterations - # Check sanity of alternatives - assert len(content), repr(content) # Can't have zero alternatives - for alt in content: - assert len(alt), repr(alt) # Can have empty alternatives - self.content = content - self.min = min - self.max = max - self.name = name - - def optimize(self): - """Optimize certain stacked wildcard patterns.""" - subpattern = None - if (self.content is not None and - len(self.content) == 1 and len(self.content[0]) == 1): - subpattern = self.content[0][0] - if self.min == 1 and self.max == 1: - if self.content is None: - return NodePattern(name=self.name) - if subpattern is not None and self.name == subpattern.name: - return subpattern.optimize() - if (self.min <= 1 and isinstance(subpattern, WildcardPattern) and - subpattern.min <= 1 and self.name == subpattern.name): - return WildcardPattern(subpattern.content, - self.min*subpattern.min, - self.max*subpattern.max, - subpattern.name) - return self - - def match(self, node, results=None): - """Does this pattern exactly match a node?""" - return self.match_seq([node], results) - - def match_seq(self, nodes, results=None): - """Does this pattern exactly match a sequence of nodes?""" - for c, r in self.generate_matches(nodes): - if c == len(nodes): - if results is not None: - results.update(r) - if self.name: - results[self.name] = list(nodes) - return True - return False - - def generate_matches(self, nodes): - """ - Generator yielding matches for a sequence of nodes. - - Args: - nodes: sequence of nodes - - Yields: - (count, results) tuples where: - count: the match comprises nodes[:count]; - results: dict containing named submatches. - """ - if self.content is None: - # Shortcut for special case (see __init__.__doc__) - for count in range(self.min, 1 + min(len(nodes), self.max)): - r = {} - if self.name: - r[self.name] = nodes[:count] - yield count, r - elif self.name == "bare_name": - yield self._bare_name_matches(nodes) - else: - # The reason for this is that hitting the recursion limit usually - # results in some ugly messages about how RuntimeErrors are being - # ignored. We only have to do this on CPython, though, because other - # implementations don't have this nasty bug in the first place. - if hasattr(sys, "getrefcount"): - save_stderr = sys.stderr - sys.stderr = StringIO() - try: - for count, r in self._recursive_matches(nodes, 0): - if self.name: - r[self.name] = nodes[:count] - yield count, r - except RuntimeError: - # Fall back to the iterative pattern matching scheme if the - # recursive scheme hits the recursion limit (RecursionError). - for count, r in self._iterative_matches(nodes): - if self.name: - r[self.name] = nodes[:count] - yield count, r - finally: - if hasattr(sys, "getrefcount"): - sys.stderr = save_stderr - - def _iterative_matches(self, nodes): - """Helper to iteratively yield the matches.""" - nodelen = len(nodes) - if 0 >= self.min: - yield 0, {} - - results = [] - # generate matches that use just one alt from self.content - for alt in self.content: - for c, r in generate_matches(alt, nodes): - yield c, r - results.append((c, r)) - - # for each match, iterate down the nodes - while results: - new_results = [] - for c0, r0 in results: - # stop if the entire set of nodes has been matched - if c0 < nodelen and c0 <= self.max: - for alt in self.content: - for c1, r1 in generate_matches(alt, nodes[c0:]): - if c1 > 0: - r = {} - r.update(r0) - r.update(r1) - yield c0 + c1, r - new_results.append((c0 + c1, r)) - results = new_results - - def _bare_name_matches(self, nodes): - """Special optimized matcher for bare_name.""" - count = 0 - r = {} - done = False - max = len(nodes) - while not done and count < max: - done = True - for leaf in self.content: - if leaf[0].match(nodes[count], r): - count += 1 - done = False - break - r[self.name] = nodes[:count] - return count, r - - def _recursive_matches(self, nodes, count): - """Helper to recursively yield the matches.""" - assert self.content is not None - if count >= self.min: - yield 0, {} - if count < self.max: - for alt in self.content: - for c0, r0 in generate_matches(alt, nodes): - for c1, r1 in self._recursive_matches(nodes[c0:], count+1): - r = {} - r.update(r0) - r.update(r1) - yield c0 + c1, r - - -class NegatedPattern(BasePattern): - - def __init__(self, content=None): - """ - Initializer. - - The argument is either a pattern or None. If it is None, this - only matches an empty sequence (effectively '$' in regex - lingo). If it is not None, this matches whenever the argument - pattern doesn't have any matches. - """ - if content is not None: - assert isinstance(content, BasePattern), repr(content) - self.content = content - - def match(self, node): - # We never match a node in its entirety - return False - - def match_seq(self, nodes): - # We only match an empty sequence of nodes in its entirety - return len(nodes) == 0 - - def generate_matches(self, nodes): - if self.content is None: - # Return a match if there is an empty sequence - if len(nodes) == 0: - yield 0, {} - else: - # Return a match if the argument pattern has no matches - for c, r in self.content.generate_matches(nodes): - return - yield 0, {} - - -def generate_matches(patterns, nodes): - """ - Generator yielding matches for a sequence of patterns and nodes. - - Args: - patterns: a sequence of patterns - nodes: a sequence of nodes - - Yields: - (count, results) tuples where: - count: the entire sequence of patterns matches nodes[:count]; - results: dict containing named submatches. - """ - if not patterns: - yield 0, {} - else: - p, rest = patterns[0], patterns[1:] - for c0, r0 in p.generate_matches(nodes): - if not rest: - yield c0, r0 - else: - for c1, r1 in generate_matches(rest, nodes[c0:]): - r = {} - r.update(r0) - r.update(r1) - yield c0 + c1, r diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py deleted file mode 100644 index 3a5aafffc6df07..00000000000000 --- a/Lib/lib2to3/refactor.py +++ /dev/null @@ -1,732 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Refactoring framework. - -Used as a main program, this can refactor any number of files and/or -recursively descend down directories. Imported as a module, this -provides infrastructure to write your own refactoring tool. -""" - -__author__ = "Guido van Rossum " - - -# Python imports -import io -import os -import pkgutil -import sys -import logging -import operator -import collections -from itertools import chain - -# Local imports -from .pgen2 import driver, tokenize, token -from .fixer_util import find_root -from . import pytree, pygram -from . import btm_matcher as bm - - -def get_all_fix_names(fixer_pkg, remove_prefix=True): - """Return a sorted list of all available fix names in the given package.""" - pkg = __import__(fixer_pkg, [], [], ["*"]) - fix_names = [] - for finder, name, ispkg in pkgutil.iter_modules(pkg.__path__): - if name.startswith("fix_"): - if remove_prefix: - name = name[4:] - fix_names.append(name) - return fix_names - - -class _EveryNode(Exception): - pass - - -def _get_head_types(pat): - """ Accepts a pytree Pattern Node and returns a set - of the pattern types which will match first. """ - - if isinstance(pat, (pytree.NodePattern, pytree.LeafPattern)): - # NodePatters must either have no type and no content - # or a type and content -- so they don't get any farther - # Always return leafs - if pat.type is None: - raise _EveryNode - return {pat.type} - - if isinstance(pat, pytree.NegatedPattern): - if pat.content: - return _get_head_types(pat.content) - raise _EveryNode # Negated Patterns don't have a type - - if isinstance(pat, pytree.WildcardPattern): - # Recurse on each node in content - r = set() - for p in pat.content: - for x in p: - r.update(_get_head_types(x)) - return r - - raise Exception("Oh no! I don't understand pattern %s" %(pat)) - - -def _get_headnode_dict(fixer_list): - """ Accepts a list of fixers and returns a dictionary - of head node type --> fixer list. """ - head_nodes = collections.defaultdict(list) - every = [] - for fixer in fixer_list: - if fixer.pattern: - try: - heads = _get_head_types(fixer.pattern) - except _EveryNode: - every.append(fixer) - else: - for node_type in heads: - head_nodes[node_type].append(fixer) - else: - if fixer._accept_type is not None: - head_nodes[fixer._accept_type].append(fixer) - else: - every.append(fixer) - for node_type in chain(pygram.python_grammar.symbol2number.values(), - pygram.python_grammar.tokens): - head_nodes[node_type].extend(every) - return dict(head_nodes) - - -def get_fixers_from_package(pkg_name): - """ - Return the fully qualified names for fixers in the package pkg_name. - """ - return [pkg_name + "." + fix_name - for fix_name in get_all_fix_names(pkg_name, False)] - -def _identity(obj): - return obj - - -def _detect_future_features(source): - have_docstring = False - gen = tokenize.generate_tokens(io.StringIO(source).readline) - def advance(): - tok = next(gen) - return tok[0], tok[1] - ignore = frozenset({token.NEWLINE, tokenize.NL, token.COMMENT}) - features = set() - try: - while True: - tp, value = advance() - if tp in ignore: - continue - elif tp == token.STRING: - if have_docstring: - break - have_docstring = True - elif tp == token.NAME and value == "from": - tp, value = advance() - if tp != token.NAME or value != "__future__": - break - tp, value = advance() - if tp != token.NAME or value != "import": - break - tp, value = advance() - if tp == token.OP and value == "(": - tp, value = advance() - while tp == token.NAME: - features.add(value) - tp, value = advance() - if tp != token.OP or value != ",": - break - tp, value = advance() - else: - break - except StopIteration: - pass - return frozenset(features) - - -class FixerError(Exception): - """A fixer could not be loaded.""" - - -class RefactoringTool(object): - - _default_options = {"print_function" : False, - "exec_function": False, - "write_unchanged_files" : False} - - CLASS_PREFIX = "Fix" # The prefix for fixer classes - FILE_PREFIX = "fix_" # The prefix for modules with a fixer within - - def __init__(self, fixer_names, options=None, explicit=None): - """Initializer. - - Args: - fixer_names: a list of fixers to import - options: a dict with configuration. - explicit: a list of fixers to run even if they are explicit. - """ - self.fixers = fixer_names - self.explicit = explicit or [] - self.options = self._default_options.copy() - if options is not None: - self.options.update(options) - self.grammar = pygram.python_grammar.copy() - - if self.options['print_function']: - del self.grammar.keywords["print"] - elif self.options['exec_function']: - del self.grammar.keywords["exec"] - - # When this is True, the refactor*() methods will call write_file() for - # files processed even if they were not changed during refactoring. If - # and only if the refactor method's write parameter was True. - self.write_unchanged_files = self.options.get("write_unchanged_files") - self.errors = [] - self.logger = logging.getLogger("RefactoringTool") - self.fixer_log = [] - self.wrote = False - self.driver = driver.Driver(self.grammar, - convert=pytree.convert, - logger=self.logger) - self.pre_order, self.post_order = self.get_fixers() - - - self.files = [] # List of files that were or should be modified - - self.BM = bm.BottomMatcher() - self.bmi_pre_order = [] # Bottom Matcher incompatible fixers - self.bmi_post_order = [] - - for fixer in chain(self.post_order, self.pre_order): - if fixer.BM_compatible: - self.BM.add_fixer(fixer) - # remove fixers that will be handled by the bottom-up - # matcher - elif fixer in self.pre_order: - self.bmi_pre_order.append(fixer) - elif fixer in self.post_order: - self.bmi_post_order.append(fixer) - - self.bmi_pre_order_heads = _get_headnode_dict(self.bmi_pre_order) - self.bmi_post_order_heads = _get_headnode_dict(self.bmi_post_order) - - - - def get_fixers(self): - """Inspects the options to load the requested patterns and handlers. - - Returns: - (pre_order, post_order), where pre_order is the list of fixers that - want a pre-order AST traversal, and post_order is the list that want - post-order traversal. - """ - pre_order_fixers = [] - post_order_fixers = [] - for fix_mod_path in self.fixers: - mod = __import__(fix_mod_path, {}, {}, ["*"]) - fix_name = fix_mod_path.rsplit(".", 1)[-1] - if fix_name.startswith(self.FILE_PREFIX): - fix_name = fix_name[len(self.FILE_PREFIX):] - parts = fix_name.split("_") - class_name = self.CLASS_PREFIX + "".join([p.title() for p in parts]) - try: - fix_class = getattr(mod, class_name) - except AttributeError: - raise FixerError("Can't find %s.%s" % (fix_name, class_name)) from None - fixer = fix_class(self.options, self.fixer_log) - if fixer.explicit and self.explicit is not True and \ - fix_mod_path not in self.explicit: - self.log_message("Skipping optional fixer: %s", fix_name) - continue - - self.log_debug("Adding transformation: %s", fix_name) - if fixer.order == "pre": - pre_order_fixers.append(fixer) - elif fixer.order == "post": - post_order_fixers.append(fixer) - else: - raise FixerError("Illegal fixer order: %r" % fixer.order) - - key_func = operator.attrgetter("run_order") - pre_order_fixers.sort(key=key_func) - post_order_fixers.sort(key=key_func) - return (pre_order_fixers, post_order_fixers) - - def log_error(self, msg, *args, **kwds): - """Called when an error occurs.""" - raise - - def log_message(self, msg, *args): - """Hook to log a message.""" - if args: - msg = msg % args - self.logger.info(msg) - - def log_debug(self, msg, *args): - if args: - msg = msg % args - self.logger.debug(msg) - - def print_output(self, old_text, new_text, filename, equal): - """Called with the old version, new version, and filename of a - refactored file.""" - pass - - def refactor(self, items, write=False, doctests_only=False): - """Refactor a list of files and directories.""" - - for dir_or_file in items: - if os.path.isdir(dir_or_file): - self.refactor_dir(dir_or_file, write, doctests_only) - else: - self.refactor_file(dir_or_file, write, doctests_only) - - def refactor_dir(self, dir_name, write=False, doctests_only=False): - """Descends down a directory and refactor every Python file found. - - Python files are assumed to have a .py extension. - - Files and subdirectories starting with '.' are skipped. - """ - py_ext = os.extsep + "py" - for dirpath, dirnames, filenames in os.walk(dir_name): - self.log_debug("Descending into %s", dirpath) - dirnames.sort() - filenames.sort() - for name in filenames: - if (not name.startswith(".") and - os.path.splitext(name)[1] == py_ext): - fullname = os.path.join(dirpath, name) - self.refactor_file(fullname, write, doctests_only) - # Modify dirnames in-place to remove subdirs with leading dots - dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")] - - def _read_python_source(self, filename): - """ - Do our best to decode a Python source file correctly. - """ - try: - f = open(filename, "rb") - except OSError as err: - self.log_error("Can't open %s: %s", filename, err) - return None, None - try: - encoding = tokenize.detect_encoding(f.readline)[0] - finally: - f.close() - with io.open(filename, "r", encoding=encoding, newline='') as f: - return f.read(), encoding - - def refactor_file(self, filename, write=False, doctests_only=False): - """Refactors a file.""" - input, encoding = self._read_python_source(filename) - if input is None: - # Reading the file failed. - return - input += "\n" # Silence certain parse errors - if doctests_only: - self.log_debug("Refactoring doctests in %s", filename) - output = self.refactor_docstring(input, filename) - if self.write_unchanged_files or output != input: - self.processed_file(output, filename, input, write, encoding) - else: - self.log_debug("No doctest changes in %s", filename) - else: - tree = self.refactor_string(input, filename) - if self.write_unchanged_files or (tree and tree.was_changed): - # The [:-1] is to take off the \n we added earlier - self.processed_file(str(tree)[:-1], filename, - write=write, encoding=encoding) - else: - self.log_debug("No changes in %s", filename) - - def refactor_string(self, data, name): - """Refactor a given input string. - - Args: - data: a string holding the code to be refactored. - name: a human-readable name for use in error/log messages. - - Returns: - An AST corresponding to the refactored input stream; None if - there were errors during the parse. - """ - features = _detect_future_features(data) - if "print_function" in features: - self.driver.grammar = pygram.python_grammar_no_print_statement - try: - tree = self.driver.parse_string(data) - except Exception as err: - self.log_error("Can't parse %s: %s: %s", - name, err.__class__.__name__, err) - return - finally: - self.driver.grammar = self.grammar - tree.future_features = features - self.log_debug("Refactoring %s", name) - self.refactor_tree(tree, name) - return tree - - def refactor_stdin(self, doctests_only=False): - input = sys.stdin.read() - if doctests_only: - self.log_debug("Refactoring doctests in stdin") - output = self.refactor_docstring(input, "") - if self.write_unchanged_files or output != input: - self.processed_file(output, "", input) - else: - self.log_debug("No doctest changes in stdin") - else: - tree = self.refactor_string(input, "") - if self.write_unchanged_files or (tree and tree.was_changed): - self.processed_file(str(tree), "", input) - else: - self.log_debug("No changes in stdin") - - def refactor_tree(self, tree, name): - """Refactors a parse tree (modifying the tree in place). - - For compatible patterns the bottom matcher module is - used. Otherwise the tree is traversed node-to-node for - matches. - - Args: - tree: a pytree.Node instance representing the root of the tree - to be refactored. - name: a human-readable name for this tree. - - Returns: - True if the tree was modified, False otherwise. - """ - - for fixer in chain(self.pre_order, self.post_order): - fixer.start_tree(tree, name) - - #use traditional matching for the incompatible fixers - self.traverse_by(self.bmi_pre_order_heads, tree.pre_order()) - self.traverse_by(self.bmi_post_order_heads, tree.post_order()) - - # obtain a set of candidate nodes - match_set = self.BM.run(tree.leaves()) - - while any(match_set.values()): - for fixer in self.BM.fixers: - if fixer in match_set and match_set[fixer]: - #sort by depth; apply fixers from bottom(of the AST) to top - match_set[fixer].sort(key=pytree.Base.depth, reverse=True) - - if fixer.keep_line_order: - #some fixers(eg fix_imports) must be applied - #with the original file's line order - match_set[fixer].sort(key=pytree.Base.get_lineno) - - for node in list(match_set[fixer]): - if node in match_set[fixer]: - match_set[fixer].remove(node) - - try: - find_root(node) - except ValueError: - # this node has been cut off from a - # previous transformation ; skip - continue - - if node.fixers_applied and fixer in node.fixers_applied: - # do not apply the same fixer again - continue - - results = fixer.match(node) - - if results: - new = fixer.transform(node, results) - if new is not None: - node.replace(new) - #new.fixers_applied.append(fixer) - for node in new.post_order(): - # do not apply the fixer again to - # this or any subnode - if not node.fixers_applied: - node.fixers_applied = [] - node.fixers_applied.append(fixer) - - # update the original match set for - # the added code - new_matches = self.BM.run(new.leaves()) - for fxr in new_matches: - if not fxr in match_set: - match_set[fxr]=[] - - match_set[fxr].extend(new_matches[fxr]) - - for fixer in chain(self.pre_order, self.post_order): - fixer.finish_tree(tree, name) - return tree.was_changed - - def traverse_by(self, fixers, traversal): - """Traverse an AST, applying a set of fixers to each node. - - This is a helper method for refactor_tree(). - - Args: - fixers: a list of fixer instances. - traversal: a generator that yields AST nodes. - - Returns: - None - """ - if not fixers: - return - for node in traversal: - for fixer in fixers[node.type]: - results = fixer.match(node) - if results: - new = fixer.transform(node, results) - if new is not None: - node.replace(new) - node = new - - def processed_file(self, new_text, filename, old_text=None, write=False, - encoding=None): - """ - Called when a file has been refactored and there may be changes. - """ - self.files.append(filename) - if old_text is None: - old_text = self._read_python_source(filename)[0] - if old_text is None: - return - equal = old_text == new_text - self.print_output(old_text, new_text, filename, equal) - if equal: - self.log_debug("No changes to %s", filename) - if not self.write_unchanged_files: - return - if write: - self.write_file(new_text, filename, old_text, encoding) - else: - self.log_debug("Not writing changes to %s", filename) - - def write_file(self, new_text, filename, old_text, encoding=None): - """Writes a string to a file. - - It first shows a unified diff between the old text and the new text, and - then rewrites the file; the latter is only done if the write option is - set. - """ - try: - fp = io.open(filename, "w", encoding=encoding, newline='') - except OSError as err: - self.log_error("Can't create %s: %s", filename, err) - return - - with fp: - try: - fp.write(new_text) - except OSError as err: - self.log_error("Can't write %s: %s", filename, err) - self.log_debug("Wrote changes to %s", filename) - self.wrote = True - - PS1 = ">>> " - PS2 = "... " - - def refactor_docstring(self, input, filename): - """Refactors a docstring, looking for doctests. - - This returns a modified version of the input string. It looks - for doctests, which start with a ">>>" prompt, and may be - continued with "..." prompts, as long as the "..." is indented - the same as the ">>>". - - (Unfortunately we can't use the doctest module's parser, - since, like most parsers, it is not geared towards preserving - the original source.) - """ - result = [] - block = None - block_lineno = None - indent = None - lineno = 0 - for line in input.splitlines(keepends=True): - lineno += 1 - if line.lstrip().startswith(self.PS1): - if block is not None: - result.extend(self.refactor_doctest(block, block_lineno, - indent, filename)) - block_lineno = lineno - block = [line] - i = line.find(self.PS1) - indent = line[:i] - elif (indent is not None and - (line.startswith(indent + self.PS2) or - line == indent + self.PS2.rstrip() + "\n")): - block.append(line) - else: - if block is not None: - result.extend(self.refactor_doctest(block, block_lineno, - indent, filename)) - block = None - indent = None - result.append(line) - if block is not None: - result.extend(self.refactor_doctest(block, block_lineno, - indent, filename)) - return "".join(result) - - def refactor_doctest(self, block, lineno, indent, filename): - """Refactors one doctest. - - A doctest is given as a block of lines, the first of which starts - with ">>>" (possibly indented), while the remaining lines start - with "..." (identically indented). - - """ - try: - tree = self.parse_block(block, lineno, indent) - except Exception as err: - if self.logger.isEnabledFor(logging.DEBUG): - for line in block: - self.log_debug("Source: %s", line.rstrip("\n")) - self.log_error("Can't parse docstring in %s line %s: %s: %s", - filename, lineno, err.__class__.__name__, err) - return block - if self.refactor_tree(tree, filename): - new = str(tree).splitlines(keepends=True) - # Undo the adjustment of the line numbers in wrap_toks() below. - clipped, new = new[:lineno-1], new[lineno-1:] - assert clipped == ["\n"] * (lineno-1), clipped - if not new[-1].endswith("\n"): - new[-1] += "\n" - block = [indent + self.PS1 + new.pop(0)] - if new: - block += [indent + self.PS2 + line for line in new] - return block - - def summarize(self): - if self.wrote: - were = "were" - else: - were = "need to be" - if not self.files: - self.log_message("No files %s modified.", were) - else: - self.log_message("Files that %s modified:", were) - for file in self.files: - self.log_message(file) - if self.fixer_log: - self.log_message("Warnings/messages while refactoring:") - for message in self.fixer_log: - self.log_message(message) - if self.errors: - if len(self.errors) == 1: - self.log_message("There was 1 error:") - else: - self.log_message("There were %d errors:", len(self.errors)) - for msg, args, kwds in self.errors: - self.log_message(msg, *args, **kwds) - - def parse_block(self, block, lineno, indent): - """Parses a block into a tree. - - This is necessary to get correct line number / offset information - in the parser diagnostics and embedded into the parse tree. - """ - tree = self.driver.parse_tokens(self.wrap_toks(block, lineno, indent)) - tree.future_features = frozenset() - return tree - - def wrap_toks(self, block, lineno, indent): - """Wraps a tokenize stream to systematically modify start/end.""" - tokens = tokenize.generate_tokens(self.gen_lines(block, indent).__next__) - for type, value, (line0, col0), (line1, col1), line_text in tokens: - line0 += lineno - 1 - line1 += lineno - 1 - # Don't bother updating the columns; this is too complicated - # since line_text would also have to be updated and it would - # still break for tokens spanning lines. Let the user guess - # that the column numbers for doctests are relative to the - # end of the prompt string (PS1 or PS2). - yield type, value, (line0, col0), (line1, col1), line_text - - - def gen_lines(self, block, indent): - """Generates lines as expected by tokenize from a list of lines. - - This strips the first len(indent + self.PS1) characters off each line. - """ - prefix1 = indent + self.PS1 - prefix2 = indent + self.PS2 - prefix = prefix1 - for line in block: - if line.startswith(prefix): - yield line[len(prefix):] - elif line == prefix.rstrip() + "\n": - yield "\n" - else: - raise AssertionError("line=%r, prefix=%r" % (line, prefix)) - prefix = prefix2 - while True: - yield "" - - -class MultiprocessingUnsupported(Exception): - pass - - -class MultiprocessRefactoringTool(RefactoringTool): - - def __init__(self, *args, **kwargs): - super(MultiprocessRefactoringTool, self).__init__(*args, **kwargs) - self.queue = None - self.output_lock = None - - def refactor(self, items, write=False, doctests_only=False, - num_processes=1): - if num_processes == 1: - return super(MultiprocessRefactoringTool, self).refactor( - items, write, doctests_only) - try: - import multiprocessing - except ImportError: - raise MultiprocessingUnsupported - if self.queue is not None: - raise RuntimeError("already doing multiple processes") - self.queue = multiprocessing.JoinableQueue() - self.output_lock = multiprocessing.Lock() - processes = [multiprocessing.Process(target=self._child) - for i in range(num_processes)] - try: - for p in processes: - p.start() - super(MultiprocessRefactoringTool, self).refactor(items, write, - doctests_only) - finally: - self.queue.join() - for i in range(num_processes): - self.queue.put(None) - for p in processes: - if p.is_alive(): - p.join() - self.queue = None - - def _child(self): - task = self.queue.get() - while task is not None: - args, kwargs = task - try: - super(MultiprocessRefactoringTool, self).refactor_file( - *args, **kwargs) - finally: - self.queue.task_done() - task = self.queue.get() - - def refactor_file(self, *args, **kwargs): - if self.queue is not None: - self.queue.put((args, kwargs)) - else: - return super(MultiprocessRefactoringTool, self).refactor_file( - *args, **kwargs) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index c03c859baa795b..271ba3fd325138 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -61,7 +61,7 @@ def parent_process(): def _cleanup(): # check for processes which have finished for p in list(_children): - if p._popen.poll() is not None: + if (child_popen := p._popen) and child_popen.poll() is not None: _children.discard(p) # diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 3d68c161603d08..3a7a1241ba77f6 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1080,25 +1080,6 @@ def __new__(cls, *args, **kwargs): cls = WindowsPath if os.name == 'nt' else PosixPath return object.__new__(cls) - def __enter__(self): - # In previous versions of pathlib, __exit__() marked this path as - # closed; subsequent attempts to perform I/O would raise an IOError. - # This functionality was never documented, and had the effect of - # making Path objects mutable, contrary to PEP 428. - # In Python 3.9 __exit__() was made a no-op. - # In Python 3.11 __enter__() began emitting DeprecationWarning. - # In Python 3.13 __enter__() and __exit__() should be removed. - warnings.warn("pathlib.Path.__enter__() is deprecated and scheduled " - "for removal in Python 3.13; Path objects as a context " - "manager is a no-op", - DeprecationWarning, stacklevel=2) - return self - - def __exit__(self, t, v, tb): - pass - - # Public API - @classmethod def cwd(cls): """Return a new path pointing to the current working directory.""" diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 3aaaee67fa35a6..cb742992a48e8f 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Apr 4 17:52:21 2023 +# Autogenerated by Sphinx on Mon May 22 14:02:15 2023 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -2573,9 +2573,12 @@ '\n' 'Any remaining exceptions that were not handled by any "except*" ' 'clause\n' - 'are re-raised at the end, combined into an exception group along ' - 'with\n' - 'all exceptions that were raised from within "except*" clauses.\n' + 'are re-raised at the end, along with all exceptions that were ' + 'raised\n' + 'from within the "except*" clauses. If this list contains more ' + 'than one\n' + 'exception to reraise, they are combined into an exception ' + 'group.\n' '\n' 'If the raised exception is not an exception group and its type ' 'matches\n' @@ -4587,8 +4590,7 @@ 'case\n' ' performance of a dict insertion, O(n^2) complexity. ' 'See\n' - ' http://www.ocert.org/advisories/ocert-2011-003.html ' - 'for\n' + ' http://ocert.org/advisories/ocert-2011-003.html for\n' ' details.Changing hash values affects the iteration ' 'order of sets.\n' ' Python has never made guarantees about this ordering ' @@ -4651,20 +4653,41 @@ 'traces of\n' ' Python programs.\n' '\n' - 'The debugger’s prompt is "(Pdb)". Typical usage to run a program ' - 'under\n' - 'control of the debugger is:\n' + 'The typical usage to break into the debugger is to insert:\n' '\n' - ' >>> import pdb\n' - ' >>> import mymodule\n' - " >>> pdb.run('mymodule.test()')\n" - ' > (0)?()\n' - ' (Pdb) continue\n' - ' > (1)?()\n' + ' import pdb; pdb.set_trace()\n' + '\n' + 'Or:\n' + '\n' + ' breakpoint()\n' + '\n' + 'at the location you want to break into the debugger, and then ' + 'run the\n' + 'program. You can then step through the code following this ' + 'statement,\n' + 'and continue running without the debugger using the "continue"\n' + 'command.\n' + '\n' + 'New in version 3.7: The built-in "breakpoint()", when called ' + 'with\n' + 'defaults, can be used instead of "import pdb; pdb.set_trace()".\n' + '\n' + ' def double(x):\n' + ' breakpoint()\n' + ' return x * 2\n' + ' val = 3\n' + ' print(f"{val} * 2 is {double(val)}")\n' + '\n' + 'The debugger’s prompt is "(Pdb)", which is the indicator that ' + 'you are\n' + 'in debug mode:\n' + '\n' + ' > ...(3)double()\n' + ' -> return x * 2\n' + ' (Pdb) p x\n' + ' 3\n' ' (Pdb) continue\n' - " NameError: 'spam'\n" - ' > (1)?()\n' - ' (Pdb)\n' + ' 3 * 2 is 6\n' '\n' 'Changed in version 3.3: Tab-completion via the "readline" module ' 'is\n' @@ -4672,13 +4695,12 @@ 'global\n' 'and local names are offered as arguments of the "p" command.\n' '\n' - '"pdb.py" can also be invoked as a script to debug other ' - 'scripts. For\n' - 'example:\n' + 'You can also invoke "pdb" from the command line to debug other\n' + 'scripts. For example:\n' '\n' ' python -m pdb myscript.py\n' '\n' - 'When invoked as a script, pdb will automatically enter ' + 'When invoked as a module, pdb will automatically enter ' 'post-mortem\n' 'debugging if the program being debugged exits abnormally. After ' 'post-\n' @@ -4690,47 +4712,43 @@ 'the\n' 'debugger upon program’s exit.\n' '\n' - 'New in version 3.2: "pdb.py" now accepts a "-c" option that ' - 'executes\n' - 'commands as if given in a ".pdbrc" file, see Debugger Commands.\n' + 'New in version 3.2: "-c" option is introduced to execute ' + 'commands as\n' + 'if given in a ".pdbrc" file, see Debugger Commands.\n' '\n' - 'New in version 3.7: "pdb.py" now accepts a "-m" option that ' - 'execute\n' - 'modules similar to the way "python -m" does. As with a script, ' - 'the\n' - 'debugger will pause execution just before the first line of the\n' - 'module.\n' - '\n' - 'The typical usage to break into the debugger is to insert:\n' + 'New in version 3.7: "-m" option is introduced to execute ' + 'modules\n' + 'similar to the way "python -m" does. As with a script, the ' + 'debugger\n' + 'will pause execution just before the first line of the module.\n' '\n' - ' import pdb; pdb.set_trace()\n' + 'Typical usage to execute a statement under control of the ' + 'debugger is:\n' '\n' - 'at the location you want to break into the debugger, and then ' - 'run the\n' - 'program. You can then step through the code following this ' - 'statement,\n' - 'and continue running without the debugger using the "continue"\n' - 'command.\n' - '\n' - 'New in version 3.7: The built-in "breakpoint()", when called ' - 'with\n' - 'defaults, can be used instead of "import pdb; pdb.set_trace()".\n' + ' >>> import pdb\n' + ' >>> def f(x):\n' + ' ... print(1 / x)\n' + ' >>> pdb.run("f(2)")\n' + ' > (1)()\n' + ' (Pdb) continue\n' + ' 0.5\n' + ' >>>\n' '\n' 'The typical usage to inspect a crashed program is:\n' '\n' ' >>> import pdb\n' - ' >>> import mymodule\n' - ' >>> mymodule.test()\n' + ' >>> def f(x):\n' + ' ... print(1 / x)\n' + ' ...\n' + ' >>> f(0)\n' ' Traceback (most recent call last):\n' ' File "", line 1, in \n' - ' File "./mymodule.py", line 4, in test\n' - ' test2()\n' - ' File "./mymodule.py", line 3, in test2\n' - ' print(spam)\n' - ' NameError: spam\n' + ' File "", line 2, in f\n' + ' ZeroDivisionError: division by zero\n' ' >>> pdb.pm()\n' - ' > ./mymodule.py(3)test2()\n' - ' -> print(spam)\n' + ' > (2)f()\n' + ' (Pdb) p x\n' + ' 0\n' ' (Pdb)\n' '\n' 'The module defines the following functions; each enters the ' @@ -4914,6 +4932,29 @@ 'implicit\n' 'string concatenation "\';\'\';\'" or "";"";"".\n' '\n' + 'To set a temporary global variable, use a *convenience ' + 'variable*. A\n' + '*convenience variable* is a variable whose name starts with ' + '"$". For\n' + 'example, "$foo = 1" sets a global variable "$foo" which you can ' + 'use in\n' + 'the debugger session. The *convenience variables* are cleared ' + 'when\n' + 'the program resumes execution so it’s less likely to interfere ' + 'with\n' + 'your program compared to using normal variables like "foo = 1".\n' + '\n' + 'There are three preset *convenience variables*:\n' + '\n' + '* "$_frame": the current frame you are debugging\n' + '\n' + '* "$_retval": the return value if the frame is returning\n' + '\n' + '* "$_exception": the exception if the frame is raising an ' + 'exception\n' + '\n' + 'New in version 3.12.\n' + '\n' 'If a file ".pdbrc" exists in the user’s home directory or in ' 'the\n' 'current directory, it is read with "\'utf-8\'" encoding and ' @@ -4949,9 +4990,9 @@ '\n' ' Print a stack trace, with the most recent frame at the ' 'bottom. An\n' - ' arrow indicates the current frame, which determines the ' - 'context of\n' - ' most commands.\n' + ' arrow (">") indicates the current frame, which determines ' + 'the\n' + ' context of most commands.\n' '\n' 'd(own) [count]\n' '\n' @@ -5007,7 +5048,7 @@ 'first\n' ' ask confirmation).\n' '\n' - 'disable [bpnumber ...]\n' + 'disable bpnumber [bpnumber ...]\n' '\n' ' Disable the breakpoints given as a space separated list of\n' ' breakpoint numbers. Disabling a breakpoint means it cannot ' @@ -5016,7 +5057,7 @@ 'breakpoint, it\n' ' remains in the list of breakpoints and can be (re-)enabled.\n' '\n' - 'enable [bpnumber ...]\n' + 'enable bpnumber [bpnumber ...]\n' '\n' ' Enable the breakpoints specified.\n' '\n' @@ -5179,7 +5220,9 @@ '\n' 'a(rgs)\n' '\n' - ' Print the argument list of the current function.\n' + ' Print the arguments of the current function and their ' + 'current\n' + ' values.\n' '\n' 'p expression\n' '\n' @@ -5217,6 +5260,54 @@ 'current\n' ' frame.\n' '\n' + ' Note:\n' + '\n' + ' Display evaluates *expression* and compares to the result ' + 'of the\n' + ' previous evaluation of *expression*, so when the result is\n' + ' mutable, display may not be able to pick up the changes.\n' + '\n' + ' Example:\n' + '\n' + ' lst = []\n' + ' breakpoint()\n' + ' pass\n' + ' lst.append(1)\n' + ' print(lst)\n' + '\n' + ' Display won’t realize "lst" has been changed because the ' + 'result of\n' + ' evaluation is modified in place by "lst.append(1)" before ' + 'being\n' + ' compared:\n' + '\n' + ' > example.py(3)()\n' + ' -> pass\n' + ' (Pdb) display lst\n' + ' display lst: []\n' + ' (Pdb) n\n' + ' > example.py(4)()\n' + ' -> lst.append(1)\n' + ' (Pdb) n\n' + ' > example.py(5)()\n' + ' -> print(lst)\n' + ' (Pdb)\n' + '\n' + ' You can do some tricks with copy mechanism to make it work:\n' + '\n' + ' > example.py(3)()\n' + ' -> pass\n' + ' (Pdb) display lst[:]\n' + ' display lst[:]: []\n' + ' (Pdb) n\n' + ' > example.py(4)()\n' + ' -> lst.append(1)\n' + ' (Pdb) n\n' + ' > example.py(5)()\n' + ' -> print(lst)\n' + ' display lst[:]: [1] [old: []]\n' + ' (Pdb)\n' + '\n' ' New in version 3.2.\n' '\n' 'undisplay [expression]\n' @@ -5283,14 +5374,14 @@ 'current\n' ' stack frame. The exclamation point can be omitted unless the ' 'first\n' - ' word of the statement resembles a debugger command, e.g.:' + ' word of the statement resembles a debugger command, e.g.:\n' '\n' ' (Pdb) ! n=42\n' ' (Pdb)\n' '\n' - ' To set a global variable, you can prefix the assignment command ' - ' with \n' - ' a "global" statement on the same line, e.g.:\n' + ' To set a global variable, you can prefix the assignment ' + 'command\n' + ' with a "global" statement on the same line, e.g.:\n' '\n' " (Pdb) global list_options; list_options = ['-l']\n" ' (Pdb)\n' @@ -5321,7 +5412,8 @@ '\n' 'retval\n' '\n' - ' Print the return value for the last return of a function.\n' + ' Print the return value for the last return of the current ' + 'function.\n' '\n' '-[ Footnotes ]-\n' '\n' @@ -9509,8 +9601,7 @@ ' by carefully chosen inputs that exploit the worst case\n' ' performance of a dict insertion, O(n^2) complexity. ' 'See\n' - ' http://www.ocert.org/advisories/ocert-2011-003.html ' - 'for\n' + ' http://ocert.org/advisories/ocert-2011-003.html for\n' ' details.Changing hash values affects the iteration ' 'order of sets.\n' ' Python has never made guarantees about this ordering ' @@ -10164,20 +10255,37 @@ 'Resolving MRO entries\n' '---------------------\n' '\n' - 'If a base that appears in class definition is not an ' + 'object.__mro_entries__(self, bases)\n' + '\n' + ' If a base that appears in a class definition is not an ' 'instance of\n' - '"type", then an "__mro_entries__" method is searched on it. ' - 'If found,\n' - 'it is called with the original bases tuple. This method must ' - 'return a\n' - 'tuple of classes that will be used instead of this base. The ' - 'tuple may\n' - 'be empty, in such case the original base is ignored.\n' + ' "type", then an "__mro_entries__()" method is searched on ' + 'the base.\n' + ' If an "__mro_entries__()" method is found, the base is ' + 'substituted\n' + ' with the result of a call to "__mro_entries__()" when ' + 'creating the\n' + ' class. The method is called with the original bases tuple ' + 'passed to\n' + ' the *bases* parameter, and must return a tuple of classes ' + 'that will\n' + ' be used instead of the base. The returned tuple may be ' + 'empty: in\n' + ' these cases, the original base is ignored.\n' '\n' 'See also:\n' '\n' - ' **PEP 560** - Core support for typing module and generic ' - 'types\n' + ' "types.resolve_bases()"\n' + ' Dynamically resolve bases that are not instances of ' + '"type".\n' + '\n' + ' "types.get_original_bases()"\n' + ' Retrieve a class’s “original bases” prior to ' + 'modifications by\n' + ' "__mro_entries__()".\n' + '\n' + ' **PEP 560**\n' + ' Core support for typing module and generic types.\n' '\n' '\n' 'Determining the appropriate metaclass\n' @@ -11153,6 +11261,61 @@ ' The specification for the Python "match" statement.\n' '\n' '\n' + 'Emulating buffer types\n' + '======================\n' + '\n' + 'The buffer protocol provides a way for Python objects to ' + 'expose\n' + 'efficient access to a low-level memory array. This protocol ' + 'is\n' + 'implemented by builtin types such as "bytes" and ' + '"memoryview", and\n' + 'third-party libraries may define additional buffer types.\n' + '\n' + 'While buffer types are usually implemented in C, it is also ' + 'possible\n' + 'to implement the protocol in Python.\n' + '\n' + 'object.__buffer__(self, flags)\n' + '\n' + ' Called when a buffer is requested from *self* (for ' + 'example, by the\n' + ' "memoryview" constructor). The *flags* argument is an ' + 'integer\n' + ' representing the kind of buffer requested, affecting for ' + 'example\n' + ' whether the returned buffer is read-only or writable.\n' + ' "inspect.BufferFlags" provides a convenient way to ' + 'interpret the\n' + ' flags. The method must return a "memoryview" object.\n' + '\n' + 'object.__release_buffer__(self, buffer)\n' + '\n' + ' Called when a buffer is no longer needed. The *buffer* ' + 'argument is\n' + ' a "memoryview" object that was previously returned by\n' + ' "__buffer__()". The method must release any resources ' + 'associated\n' + ' with the buffer. This method should return "None". Buffer ' + 'objects\n' + ' that do not need to perform any cleanup are not required ' + 'to\n' + ' implement this method.\n' + '\n' + 'New in version 3.12.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 688** - Making the buffer protocol accessible in ' + 'Python\n' + ' Introduces the Python "__buffer__" and ' + '"__release_buffer__"\n' + ' methods.\n' + '\n' + ' "collections.abc.Buffer"\n' + ' ABC for buffer types.\n' + '\n' + '\n' 'Special method lookup\n' '=====================\n' '\n' @@ -11300,8 +11463,8 @@ ' "casefold()" converts it to ""ss"".\n' '\n' ' The casefolding algorithm is described in section 3.13 ' - 'of the\n' - ' Unicode Standard.\n' + '‘Default\n' + ' Case Folding’ of the Unicode Standard.\n' '\n' ' New in version 3.3.\n' '\n' @@ -11519,8 +11682,9 @@ ' being one of “Lm”, “Lt”, “Lu”, “Ll”, or “Lo”. Note ' 'that this is\n' ' different from the Alphabetic property defined in the ' - 'Unicode\n' - ' Standard.\n' + 'section 4.10\n' + ' ‘Letters, Alphabetic, and Ideographic’ of the Unicode ' + 'Standard.\n' '\n' 'str.isascii()\n' '\n' @@ -11692,8 +11856,8 @@ ' converted to lowercase.\n' '\n' ' The lowercasing algorithm used is described in section ' - '3.13 of the\n' - ' Unicode Standard.\n' + '3.13\n' + ' ‘Default Case Folding’ of the Unicode Standard.\n' '\n' 'str.lstrip([chars])\n' '\n' @@ -12159,8 +12323,8 @@ ' uppercase), but e.g. “Lt” (Letter, titlecase).\n' '\n' ' The uppercasing algorithm used is described in section ' - '3.13 of the\n' - ' Unicode Standard.\n' + '3.13\n' + ' ‘Default Case Folding’ of the Unicode Standard.\n' '\n' 'str.zfill(width)\n' '\n' @@ -12704,9 +12868,10 @@ '\n' 'Any remaining exceptions that were not handled by any "except*" ' 'clause\n' - 'are re-raised at the end, combined into an exception group along ' - 'with\n' - 'all exceptions that were raised from within "except*" clauses.\n' + 'are re-raised at the end, along with all exceptions that were raised\n' + 'from within the "except*" clauses. If this list contains more than ' + 'one\n' + 'exception to reraise, they are combined into an exception group.\n' '\n' 'If the raised exception is not an exception group and its type ' 'matches\n' diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 7781a430839ea5..df4e41f7a0d23a 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1633,7 +1633,7 @@ class TarFile(object): def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, - errorlevel=None, copybufsize=None): + errorlevel=None, copybufsize=None, stream=False): """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to read from an existing archive, 'a' to append data to an existing file or 'w' to create a new file overwriting an existing one. `mode' @@ -1665,6 +1665,8 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, self.name = os.path.abspath(name) if name else None self.fileobj = fileobj + self.stream = stream + # Init attributes. if format is not None: self.format = format @@ -2631,7 +2633,9 @@ def next(self): break if tarinfo is not None: - self.members.append(tarinfo) + # if streaming the file we do not want to cache the tarinfo + if not self.stream: + self.members.append(tarinfo) else: self._loaded = True @@ -2682,11 +2686,12 @@ def _getmember(self, name, tarinfo=None, normalize=False): def _load(self): """Read through the entire archive file and look for readable - members. + members. This should not run if the file is set to stream. """ - while self.next() is not None: - pass - self._loaded = True + if not self.stream: + while self.next() is not None: + pass + self._loaded = True def _check(self, mode=None): """Check if TarFile is still open, and if the operation's mode diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py deleted file mode 100644 index 62d636129853ad..00000000000000 --- a/Lib/telnetlib.py +++ /dev/null @@ -1,679 +0,0 @@ -r"""TELNET client class. - -Based on RFC 854: TELNET Protocol Specification, by J. Postel and -J. Reynolds - -Example: - ->>> from telnetlib import Telnet ->>> tn = Telnet('www.python.org', 79) # connect to finger port ->>> tn.write(b'guido\r\n') ->>> print(tn.read_all()) -Login Name TTY Idle When Where -guido Guido van Rossum pts/2 snag.cnri.reston.. - ->>> - -Note that read_all() won't read until eof -- it just reads some data --- but it guarantees to read at least one byte unless EOF is hit. - -It is possible to pass a Telnet object to a selector in order to wait until -more data is available. Note that in this case, read_eager() may return b'' -even if there was data on the socket, because the protocol negotiation may have -eaten the data. This is why EOFError is needed in some cases to distinguish -between "no data" and "connection closed" (since the socket also appears ready -for reading when it is closed). - -To do: -- option negotiation -- timeout should be intrinsic to the connection object instead of an - option on one of the read calls only - -""" - - -# Imported modules -import sys -import socket -import selectors -from time import monotonic as _time -import warnings - -warnings._deprecated(__name__, remove=(3, 13)) - -__all__ = ["Telnet"] - -# Tunable parameters -DEBUGLEVEL = 0 - -# Telnet protocol defaults -TELNET_PORT = 23 - -# Telnet protocol characters (don't change) -IAC = bytes([255]) # "Interpret As Command" -DONT = bytes([254]) -DO = bytes([253]) -WONT = bytes([252]) -WILL = bytes([251]) -theNULL = bytes([0]) - -SE = bytes([240]) # Subnegotiation End -NOP = bytes([241]) # No Operation -DM = bytes([242]) # Data Mark -BRK = bytes([243]) # Break -IP = bytes([244]) # Interrupt process -AO = bytes([245]) # Abort output -AYT = bytes([246]) # Are You There -EC = bytes([247]) # Erase Character -EL = bytes([248]) # Erase Line -GA = bytes([249]) # Go Ahead -SB = bytes([250]) # Subnegotiation Begin - - -# Telnet protocol options code (don't change) -# These ones all come from arpa/telnet.h -BINARY = bytes([0]) # 8-bit data path -ECHO = bytes([1]) # echo -RCP = bytes([2]) # prepare to reconnect -SGA = bytes([3]) # suppress go ahead -NAMS = bytes([4]) # approximate message size -STATUS = bytes([5]) # give status -TM = bytes([6]) # timing mark -RCTE = bytes([7]) # remote controlled transmission and echo -NAOL = bytes([8]) # negotiate about output line width -NAOP = bytes([9]) # negotiate about output page size -NAOCRD = bytes([10]) # negotiate about CR disposition -NAOHTS = bytes([11]) # negotiate about horizontal tabstops -NAOHTD = bytes([12]) # negotiate about horizontal tab disposition -NAOFFD = bytes([13]) # negotiate about formfeed disposition -NAOVTS = bytes([14]) # negotiate about vertical tab stops -NAOVTD = bytes([15]) # negotiate about vertical tab disposition -NAOLFD = bytes([16]) # negotiate about output LF disposition -XASCII = bytes([17]) # extended ascii character set -LOGOUT = bytes([18]) # force logout -BM = bytes([19]) # byte macro -DET = bytes([20]) # data entry terminal -SUPDUP = bytes([21]) # supdup protocol -SUPDUPOUTPUT = bytes([22]) # supdup output -SNDLOC = bytes([23]) # send location -TTYPE = bytes([24]) # terminal type -EOR = bytes([25]) # end or record -TUID = bytes([26]) # TACACS user identification -OUTMRK = bytes([27]) # output marking -TTYLOC = bytes([28]) # terminal location number -VT3270REGIME = bytes([29]) # 3270 regime -X3PAD = bytes([30]) # X.3 PAD -NAWS = bytes([31]) # window size -TSPEED = bytes([32]) # terminal speed -LFLOW = bytes([33]) # remote flow control -LINEMODE = bytes([34]) # Linemode option -XDISPLOC = bytes([35]) # X Display Location -OLD_ENVIRON = bytes([36]) # Old - Environment variables -AUTHENTICATION = bytes([37]) # Authenticate -ENCRYPT = bytes([38]) # Encryption option -NEW_ENVIRON = bytes([39]) # New - Environment variables -# the following ones come from -# http://www.iana.org/assignments/telnet-options -# Unfortunately, that document does not assign identifiers -# to all of them, so we are making them up -TN3270E = bytes([40]) # TN3270E -XAUTH = bytes([41]) # XAUTH -CHARSET = bytes([42]) # CHARSET -RSP = bytes([43]) # Telnet Remote Serial Port -COM_PORT_OPTION = bytes([44]) # Com Port Control Option -SUPPRESS_LOCAL_ECHO = bytes([45]) # Telnet Suppress Local Echo -TLS = bytes([46]) # Telnet Start TLS -KERMIT = bytes([47]) # KERMIT -SEND_URL = bytes([48]) # SEND-URL -FORWARD_X = bytes([49]) # FORWARD_X -PRAGMA_LOGON = bytes([138]) # TELOPT PRAGMA LOGON -SSPI_LOGON = bytes([139]) # TELOPT SSPI LOGON -PRAGMA_HEARTBEAT = bytes([140]) # TELOPT PRAGMA HEARTBEAT -EXOPL = bytes([255]) # Extended-Options-List -NOOPT = bytes([0]) - - -# poll/select have the advantage of not requiring any extra file descriptor, -# contrarily to epoll/kqueue (also, they require a single syscall). -if hasattr(selectors, 'PollSelector'): - _TelnetSelector = selectors.PollSelector -else: - _TelnetSelector = selectors.SelectSelector - - -class Telnet: - - """Telnet interface class. - - An instance of this class represents a connection to a telnet - server. The instance is initially not connected; the open() - method must be used to establish a connection. Alternatively, the - host name and optional port number can be passed to the - constructor, too. - - Don't try to reopen an already connected instance. - - This class has many read_*() methods. Note that some of them - raise EOFError when the end of the connection is read, because - they can return an empty string for other reasons. See the - individual doc strings. - - read_until(expected, [timeout]) - Read until the expected string has been seen, or a timeout is - hit (default is no timeout); may block. - - read_all() - Read all data until EOF; may block. - - read_some() - Read at least one byte or EOF; may block. - - read_very_eager() - Read all data available already queued or on the socket, - without blocking. - - read_eager() - Read either data already queued or some data available on the - socket, without blocking. - - read_lazy() - Read all data in the raw queue (processing it first), without - doing any socket I/O. - - read_very_lazy() - Reads all data in the cooked queue, without doing any socket - I/O. - - read_sb_data() - Reads available data between SB ... SE sequence. Don't block. - - set_option_negotiation_callback(callback) - Each time a telnet option is read on the input flow, this callback - (if set) is called with the following parameters : - callback(telnet socket, command, option) - option will be chr(0) when there is no option. - No other action is done afterwards by telnetlib. - - """ - - def __init__(self, host=None, port=0, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - """Constructor. - - When called without arguments, create an unconnected instance. - With a hostname argument, it connects the instance; port number - and timeout are optional. - """ - self.debuglevel = DEBUGLEVEL - self.host = host - self.port = port - self.timeout = timeout - self.sock = None - self.rawq = b'' - self.irawq = 0 - self.cookedq = b'' - self.eof = 0 - self.iacseq = b'' # Buffer for IAC sequence. - self.sb = 0 # flag for SB and SE sequence. - self.sbdataq = b'' - self.option_callback = None - if host is not None: - self.open(host, port, timeout) - - def open(self, host, port=0, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - """Connect to a host. - - The optional second argument is the port number, which - defaults to the standard telnet port (23). - - Don't try to reopen an already connected instance. - """ - self.eof = 0 - if not port: - port = TELNET_PORT - self.host = host - self.port = port - self.timeout = timeout - sys.audit("telnetlib.Telnet.open", self, host, port) - self.sock = socket.create_connection((host, port), timeout) - - def __del__(self): - """Destructor -- close the connection.""" - self.close() - - def msg(self, msg, *args): - """Print a debug message, when the debug level is > 0. - - If extra arguments are present, they are substituted in the - message using the standard string formatting operator. - - """ - if self.debuglevel > 0: - print('Telnet(%s,%s):' % (self.host, self.port), end=' ') - if args: - print(msg % args) - else: - print(msg) - - def set_debuglevel(self, debuglevel): - """Set the debug level. - - The higher it is, the more debug output you get (on sys.stdout). - - """ - self.debuglevel = debuglevel - - def close(self): - """Close the connection.""" - sock = self.sock - self.sock = None - self.eof = True - self.iacseq = b'' - self.sb = 0 - if sock: - sock.close() - - def get_socket(self): - """Return the socket object used internally.""" - return self.sock - - def fileno(self): - """Return the fileno() of the socket object used internally.""" - return self.sock.fileno() - - def write(self, buffer): - """Write a string to the socket, doubling any IAC characters. - - Can block if the connection is blocked. May raise - OSError if the connection is closed. - - """ - if IAC in buffer: - buffer = buffer.replace(IAC, IAC+IAC) - sys.audit("telnetlib.Telnet.write", self, buffer) - self.msg("send %r", buffer) - self.sock.sendall(buffer) - - def read_until(self, match, timeout=None): - """Read until a given string is encountered or until timeout. - - When no match is found, return whatever is available instead, - possibly the empty string. Raise EOFError if the connection - is closed and no cooked data is available. - - """ - n = len(match) - self.process_rawq() - i = self.cookedq.find(match) - if i >= 0: - i = i+n - buf = self.cookedq[:i] - self.cookedq = self.cookedq[i:] - return buf - if timeout is not None: - deadline = _time() + timeout - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - while not self.eof: - if selector.select(timeout): - i = max(0, len(self.cookedq)-n) - self.fill_rawq() - self.process_rawq() - i = self.cookedq.find(match, i) - if i >= 0: - i = i+n - buf = self.cookedq[:i] - self.cookedq = self.cookedq[i:] - return buf - if timeout is not None: - timeout = deadline - _time() - if timeout < 0: - break - return self.read_very_lazy() - - def read_all(self): - """Read all data until EOF; block until connection closed.""" - self.process_rawq() - while not self.eof: - self.fill_rawq() - self.process_rawq() - buf = self.cookedq - self.cookedq = b'' - return buf - - def read_some(self): - """Read at least one byte of cooked data unless EOF is hit. - - Return b'' if EOF is hit. Block if no data is immediately - available. - - """ - self.process_rawq() - while not self.cookedq and not self.eof: - self.fill_rawq() - self.process_rawq() - buf = self.cookedq - self.cookedq = b'' - return buf - - def read_very_eager(self): - """Read everything that's possible without blocking in I/O (eager). - - Raise EOFError if connection closed and no cooked data - available. Return b'' if no cooked data available otherwise. - Don't block unless in the midst of an IAC sequence. - - """ - self.process_rawq() - while not self.eof and self.sock_avail(): - self.fill_rawq() - self.process_rawq() - return self.read_very_lazy() - - def read_eager(self): - """Read readily available data. - - Raise EOFError if connection closed and no cooked data - available. Return b'' if no cooked data available otherwise. - Don't block unless in the midst of an IAC sequence. - - """ - self.process_rawq() - while not self.cookedq and not self.eof and self.sock_avail(): - self.fill_rawq() - self.process_rawq() - return self.read_very_lazy() - - def read_lazy(self): - """Process and return data that's already in the queues (lazy). - - Raise EOFError if connection closed and no data available. - Return b'' if no cooked data available otherwise. Don't block - unless in the midst of an IAC sequence. - - """ - self.process_rawq() - return self.read_very_lazy() - - def read_very_lazy(self): - """Return any data available in the cooked queue (very lazy). - - Raise EOFError if connection closed and no data available. - Return b'' if no cooked data available otherwise. Don't block. - - """ - buf = self.cookedq - self.cookedq = b'' - if not buf and self.eof and not self.rawq: - raise EOFError('telnet connection closed') - return buf - - def read_sb_data(self): - """Return any data available in the SB ... SE queue. - - Return b'' if no SB ... SE available. Should only be called - after seeing a SB or SE command. When a new SB command is - found, old unread SB data will be discarded. Don't block. - - """ - buf = self.sbdataq - self.sbdataq = b'' - return buf - - def set_option_negotiation_callback(self, callback): - """Provide a callback function called after each receipt of a telnet option.""" - self.option_callback = callback - - def process_rawq(self): - """Transfer from raw queue to cooked queue. - - Set self.eof when connection is closed. Don't block unless in - the midst of an IAC sequence. - - """ - buf = [b'', b''] - try: - while self.rawq: - c = self.rawq_getchar() - if not self.iacseq: - if c == theNULL: - continue - if c == b"\021": - continue - if c != IAC: - buf[self.sb] = buf[self.sb] + c - continue - else: - self.iacseq += c - elif len(self.iacseq) == 1: - # 'IAC: IAC CMD [OPTION only for WILL/WONT/DO/DONT]' - if c in (DO, DONT, WILL, WONT): - self.iacseq += c - continue - - self.iacseq = b'' - if c == IAC: - buf[self.sb] = buf[self.sb] + c - else: - if c == SB: # SB ... SE start. - self.sb = 1 - self.sbdataq = b'' - elif c == SE: - self.sb = 0 - self.sbdataq = self.sbdataq + buf[1] - buf[1] = b'' - if self.option_callback: - # Callback is supposed to look into - # the sbdataq - self.option_callback(self.sock, c, NOOPT) - else: - # We can't offer automatic processing of - # suboptions. Alas, we should not get any - # unless we did a WILL/DO before. - self.msg('IAC %d not recognized' % ord(c)) - elif len(self.iacseq) == 2: - cmd = self.iacseq[1:2] - self.iacseq = b'' - opt = c - if cmd in (DO, DONT): - self.msg('IAC %s %d', - cmd == DO and 'DO' or 'DONT', ord(opt)) - if self.option_callback: - self.option_callback(self.sock, cmd, opt) - else: - self.sock.sendall(IAC + WONT + opt) - elif cmd in (WILL, WONT): - self.msg('IAC %s %d', - cmd == WILL and 'WILL' or 'WONT', ord(opt)) - if self.option_callback: - self.option_callback(self.sock, cmd, opt) - else: - self.sock.sendall(IAC + DONT + opt) - except EOFError: # raised by self.rawq_getchar() - self.iacseq = b'' # Reset on EOF - self.sb = 0 - self.cookedq = self.cookedq + buf[0] - self.sbdataq = self.sbdataq + buf[1] - - def rawq_getchar(self): - """Get next char from raw queue. - - Block if no data is immediately available. Raise EOFError - when connection is closed. - - """ - if not self.rawq: - self.fill_rawq() - if self.eof: - raise EOFError - c = self.rawq[self.irawq:self.irawq+1] - self.irawq = self.irawq + 1 - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - return c - - def fill_rawq(self): - """Fill raw queue from exactly one recv() system call. - - Block if no data is immediately available. Set self.eof when - connection is closed. - - """ - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - # The buffer size should be fairly small so as to avoid quadratic - # behavior in process_rawq() above - buf = self.sock.recv(50) - self.msg("recv %r", buf) - self.eof = (not buf) - self.rawq = self.rawq + buf - - def sock_avail(self): - """Test whether data is available on the socket.""" - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - return bool(selector.select(0)) - - def interact(self): - """Interaction function, emulates a very dumb telnet client.""" - if sys.platform == "win32": - self.mt_interact() - return - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - selector.register(sys.stdin, selectors.EVENT_READ) - - while True: - for key, events in selector.select(): - if key.fileobj is self: - try: - text = self.read_eager() - except EOFError: - print('*** Connection closed by remote host ***') - return - if text: - sys.stdout.write(text.decode('ascii')) - sys.stdout.flush() - elif key.fileobj is sys.stdin: - line = sys.stdin.readline().encode('ascii') - if not line: - return - self.write(line) - - def mt_interact(self): - """Multithreaded version of interact().""" - import _thread - _thread.start_new_thread(self.listener, ()) - while 1: - line = sys.stdin.readline() - if not line: - break - self.write(line.encode('ascii')) - - def listener(self): - """Helper for mt_interact() -- this executes in the other thread.""" - while 1: - try: - data = self.read_eager() - except EOFError: - print('*** Connection closed by remote host ***') - return - if data: - sys.stdout.write(data.decode('ascii')) - else: - sys.stdout.flush() - - def expect(self, list, timeout=None): - """Read until one from a list of a regular expressions matches. - - The first argument is a list of regular expressions, either - compiled (re.Pattern instances) or uncompiled (strings). - The optional second argument is a timeout, in seconds; default - is no timeout. - - Return a tuple of three items: the index in the list of the - first regular expression that matches; the re.Match object - returned; and the text read up till and including the match. - - If EOF is read and no text was read, raise EOFError. - Otherwise, when nothing matches, return (-1, None, text) where - text is the text received so far (may be the empty string if a - timeout happened). - - If a regular expression ends with a greedy match (e.g. '.*') - or if more than one expression can match the same input, the - results are undeterministic, and may depend on the I/O timing. - - """ - re = None - list = list[:] - indices = range(len(list)) - for i in indices: - if not hasattr(list[i], "search"): - if not re: import re - list[i] = re.compile(list[i]) - if timeout is not None: - deadline = _time() + timeout - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - while not self.eof: - self.process_rawq() - for i in indices: - m = list[i].search(self.cookedq) - if m: - e = m.end() - text = self.cookedq[:e] - self.cookedq = self.cookedq[e:] - return (i, m, text) - if timeout is not None: - ready = selector.select(timeout) - timeout = deadline - _time() - if not ready: - if timeout < 0: - break - else: - continue - self.fill_rawq() - text = self.read_very_lazy() - if not text and self.eof: - raise EOFError - return (-1, None, text) - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - -def test(): - """Test program for telnetlib. - - Usage: python telnetlib.py [-d] ... [host [port]] - - Default host is localhost; default port is 23. - - """ - debuglevel = 0 - while sys.argv[1:] and sys.argv[1] == '-d': - debuglevel = debuglevel+1 - del sys.argv[1] - host = 'localhost' - if sys.argv[1:]: - host = sys.argv[1] - port = 0 - if sys.argv[2:]: - portstr = sys.argv[2] - try: - port = int(portstr) - except ValueError: - port = socket.getservbyname(portstr, 'tcp') - with Telnet() as tn: - tn.set_debuglevel(debuglevel) - tn.open(host, port, timeout=0.5) - tn.interact() - -if __name__ == '__main__': - test() diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 709cac7a27a449..a6ea2f378b37ac 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -8,18 +8,12 @@ from collections import UserList import random + class Sequence: def __init__(self, seq='wxyz'): self.seq = seq def __len__(self): return len(self.seq) def __getitem__(self, i): return self.seq[i] -class BadSeq1(Sequence): - def __init__(self): self.seq = [7, 'hello', 123] - def __str__(self): return '{0} {1} {2}'.format(*self.seq) - -class BadSeq2(Sequence): - def __init__(self): self.seq = ['a', 'b', 'c'] - def __len__(self): return 8 class BaseTest: # These tests are for buffers of values (bytes) and not @@ -27,7 +21,7 @@ class BaseTest: # and various string implementations # The type to be tested - # Change in subclasses to change the behaviour of fixtesttype() + # Change in subclasses to change the behaviour of fixtype() type2test = None # Whether the "contained items" of the container are integers in @@ -36,7 +30,7 @@ class BaseTest: contains_bytes = False # All tests pass their arguments to the testing methods - # as str objects. fixtesttype() can be used to propagate + # as str objects. fixtype() can be used to propagate # these arguments to the appropriate type def fixtype(self, obj): if isinstance(obj, str): @@ -1096,7 +1090,7 @@ def test_splitlines(self): self.checkraises(TypeError, 'abc', 'splitlines', 42, 42) -class CommonTest(BaseTest): +class StringLikeTest(BaseTest): # This testcase contains tests that can be used in all # stringlike classes. Currently this is str and UserString. @@ -1127,11 +1121,6 @@ def test_capitalize_nonascii(self): self.checkequal('\u019b\u1d00\u1d86\u0221\u1fb7', '\u019b\u1d00\u1d86\u0221\u1fb7', 'capitalize') - -class MixinStrUnicodeUserStringTest: - # additional tests that only work for - # stringlike objects, i.e. str, UserString - def test_startswith(self): self.checkequal(True, 'hello', 'startswith', 'he') self.checkequal(True, 'hello', 'startswith', 'hello') @@ -1313,8 +1302,11 @@ def test_join(self): self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join', ('a' * i,) * i) - #self.checkequal(str(BadSeq1()), ' ', 'join', BadSeq1()) - self.checkequal('a b c', ' ', 'join', BadSeq2()) + class LiesAboutLengthSeq(Sequence): + def __init__(self): self.seq = ['a', 'b', 'c'] + def __len__(self): return 8 + + self.checkequal('a b c', ' ', 'join', LiesAboutLengthSeq()) self.checkraises(TypeError, ' ', 'join') self.checkraises(TypeError, ' ', 'join', None) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 1257b529038afb..f5a5c037f1bf1b 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1614,7 +1614,7 @@ def test_setattr(self): msg = r"^attribute name must be string, not 'int'$" self.assertRaisesRegex(TypeError, msg, setattr, sys, 1, 'spam') - # test_str(): see test_unicode.py and test_bytes.py for str() tests. + # test_str(): see test_str.py and test_bytes.py for str() tests. def test_sum(self): self.assertEqual(sum([]), 0) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 00807d968a7c43..9c7662065689ea 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -319,12 +319,17 @@ def test_fromobject(self): def test_from_format(self): """Test PyUnicode_FromFormat()""" + # Length modifiers "j" and "t" are not tested here because ctypes does + # not expose types for intmax_t and ptrdiff_t. + # _testcapi.test_string_from_format() has a wider coverage of all + # formats. import_helper.import_module('ctypes') from ctypes import ( c_char_p, pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, - c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) + c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p, + sizeof, c_wchar, c_wchar_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) _PyUnicode_FromFormat.argtypes = (c_char_p,) @@ -449,37 +454,28 @@ def check_format(expected, format, *args): check_format("repr= 12", b'repr=%5.2V', None, b'123') - # test integer formats (%i, %d, %u) + # test integer formats (%i, %d, %u, %o, %x, %X) check_format('010', b'%03i', c_int(10)) check_format('0010', b'%0.4i', c_int(10)) - check_format('-123', - b'%i', c_int(-123)) - check_format('-123', - b'%li', c_long(-123)) - check_format('-123', - b'%lli', c_longlong(-123)) - check_format('-123', - b'%zi', c_ssize_t(-123)) - - check_format('-123', - b'%d', c_int(-123)) - check_format('-123', - b'%ld', c_long(-123)) - check_format('-123', - b'%lld', c_longlong(-123)) - check_format('-123', - b'%zd', c_ssize_t(-123)) - - check_format('123', - b'%u', c_uint(123)) - check_format('123', - b'%lu', c_ulong(123)) - check_format('123', - b'%llu', c_ulonglong(123)) - check_format('123', - b'%zu', c_size_t(123)) + for conv, signed, value, expected in [ + (b'i', True, -123, '-123'), + (b'd', True, -123, '-123'), + (b'u', False, 123, '123'), + (b'o', False, 0o123, '123'), + (b'x', False, 0xabc, 'abc'), + (b'X', False, 0xabc, 'ABC'), + ]: + for mod, ctype in [ + (b'', c_int if signed else c_uint), + (b'l', c_long if signed else c_ulong), + (b'll', c_longlong if signed else c_ulonglong), + (b'z', c_ssize_t if signed else c_size_t), + ]: + with self.subTest(format=b'%' + mod + conv): + check_format(expected, + b'%' + mod + conv, ctype(value)) # test long output min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) @@ -494,40 +490,144 @@ def check_format(expected, format, *args): PyUnicode_FromFormat(b'%p', c_void_p(-1)) # test padding (width and/or precision) - check_format('123'.rjust(10, '0'), - b'%010i', c_int(123)) - check_format('123'.rjust(100), - b'%100i', c_int(123)) - check_format('123'.rjust(100, '0'), - b'%.100i', c_int(123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80i', c_int(123)) - - check_format('123'.rjust(10, '0'), - b'%010u', c_uint(123)) - check_format('123'.rjust(100), - b'%100u', c_uint(123)) - check_format('123'.rjust(100, '0'), - b'%.100u', c_uint(123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80u', c_uint(123)) - - check_format('123'.rjust(10, '0'), - b'%010x', c_int(0x123)) - check_format('123'.rjust(100), - b'%100x', c_int(0x123)) - check_format('123'.rjust(100, '0'), - b'%.100x', c_int(0x123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80x', c_int(0x123)) + check_format('123', b'%2i', c_int(123)) + check_format(' 123', b'%10i', c_int(123)) + check_format('0000000123', b'%010i', c_int(123)) + check_format('123 ', b'%-10i', c_int(123)) + check_format('123 ', b'%-010i', c_int(123)) + check_format('123', b'%.2i', c_int(123)) + check_format('0000123', b'%.7i', c_int(123)) + check_format(' 123', b'%10.2i', c_int(123)) + check_format(' 0000123', b'%10.7i', c_int(123)) + check_format('0000000123', b'%010.7i', c_int(123)) + check_format('0000123 ', b'%-10.7i', c_int(123)) + check_format('0000123 ', b'%-010.7i', c_int(123)) + + check_format('-123', b'%2i', c_int(-123)) + check_format(' -123', b'%10i', c_int(-123)) + check_format('-000000123', b'%010i', c_int(-123)) + check_format('-123 ', b'%-10i', c_int(-123)) + check_format('-123 ', b'%-010i', c_int(-123)) + check_format('-123', b'%.2i', c_int(-123)) + check_format('-0000123', b'%.7i', c_int(-123)) + check_format(' -123', b'%10.2i', c_int(-123)) + check_format(' -0000123', b'%10.7i', c_int(-123)) + check_format('-000000123', b'%010.7i', c_int(-123)) + check_format('-0000123 ', b'%-10.7i', c_int(-123)) + check_format('-0000123 ', b'%-010.7i', c_int(-123)) + + check_format('123', b'%2u', c_uint(123)) + check_format(' 123', b'%10u', c_uint(123)) + check_format('0000000123', b'%010u', c_uint(123)) + check_format('123 ', b'%-10u', c_uint(123)) + check_format('123 ', b'%-010u', c_uint(123)) + check_format('123', b'%.2u', c_uint(123)) + check_format('0000123', b'%.7u', c_uint(123)) + check_format(' 123', b'%10.2u', c_uint(123)) + check_format(' 0000123', b'%10.7u', c_uint(123)) + check_format('0000000123', b'%010.7u', c_uint(123)) + check_format('0000123 ', b'%-10.7u', c_uint(123)) + check_format('0000123 ', b'%-010.7u', c_uint(123)) + + check_format('123', b'%2o', c_uint(0o123)) + check_format(' 123', b'%10o', c_uint(0o123)) + check_format('0000000123', b'%010o', c_uint(0o123)) + check_format('123 ', b'%-10o', c_uint(0o123)) + check_format('123 ', b'%-010o', c_uint(0o123)) + check_format('123', b'%.2o', c_uint(0o123)) + check_format('0000123', b'%.7o', c_uint(0o123)) + check_format(' 123', b'%10.2o', c_uint(0o123)) + check_format(' 0000123', b'%10.7o', c_uint(0o123)) + check_format('0000000123', b'%010.7o', c_uint(0o123)) + check_format('0000123 ', b'%-10.7o', c_uint(0o123)) + check_format('0000123 ', b'%-010.7o', c_uint(0o123)) + + check_format('abc', b'%2x', c_uint(0xabc)) + check_format(' abc', b'%10x', c_uint(0xabc)) + check_format('0000000abc', b'%010x', c_uint(0xabc)) + check_format('abc ', b'%-10x', c_uint(0xabc)) + check_format('abc ', b'%-010x', c_uint(0xabc)) + check_format('abc', b'%.2x', c_uint(0xabc)) + check_format('0000abc', b'%.7x', c_uint(0xabc)) + check_format(' abc', b'%10.2x', c_uint(0xabc)) + check_format(' 0000abc', b'%10.7x', c_uint(0xabc)) + check_format('0000000abc', b'%010.7x', c_uint(0xabc)) + check_format('0000abc ', b'%-10.7x', c_uint(0xabc)) + check_format('0000abc ', b'%-010.7x', c_uint(0xabc)) + + check_format('ABC', b'%2X', c_uint(0xabc)) + check_format(' ABC', b'%10X', c_uint(0xabc)) + check_format('0000000ABC', b'%010X', c_uint(0xabc)) + check_format('ABC ', b'%-10X', c_uint(0xabc)) + check_format('ABC ', b'%-010X', c_uint(0xabc)) + check_format('ABC', b'%.2X', c_uint(0xabc)) + check_format('0000ABC', b'%.7X', c_uint(0xabc)) + check_format(' ABC', b'%10.2X', c_uint(0xabc)) + check_format(' 0000ABC', b'%10.7X', c_uint(0xabc)) + check_format('0000000ABC', b'%010.7X', c_uint(0xabc)) + check_format('0000ABC ', b'%-10.7X', c_uint(0xabc)) + check_format('0000ABC ', b'%-010.7X', c_uint(0xabc)) # test %A check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') # test %V - check_format('repr=abc', - b'repr=%V', 'abc', b'xyz') + check_format('abc', + b'%V', 'abc', b'xyz') + check_format('xyz', + b'%V', None, b'xyz') + + # test %ls + check_format('abc', b'%ls', c_wchar_p('abc')) + check_format('\u4eba\u6c11', b'%ls', c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb+\U0001f40d', + b'%ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format(' ab', b'%5.2ls', c_wchar_p('abc')) + check_format(' \u4eba\u6c11', b'%5ls', c_wchar_p('\u4eba\u6c11')) + check_format(' \U0001f4bb+\U0001f40d', + b'%5ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\u4eba', b'%.1ls', c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d', + b'%.1ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', + b'%.2ls', c_wchar_p('\U0001f4bb+\U0001f40d')) + + # test %lV + check_format('abc', + b'%lV', 'abc', c_wchar_p('xyz')) + check_format('xyz', + b'%lV', None, c_wchar_p('xyz')) + check_format('\u4eba\u6c11', + b'%lV', None, c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb+\U0001f40d', + b'%lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format(' ab', + b'%5.2lV', None, c_wchar_p('abc')) + check_format(' \u4eba\u6c11', + b'%5lV', None, c_wchar_p('\u4eba\u6c11')) + check_format(' \U0001f4bb+\U0001f40d', + b'%5lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\u4eba', + b'%.1lV', None, c_wchar_p('\u4eba\u6c11')) + check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d', + b'%.1lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', + b'%.2lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) + + # test variable width and precision + check_format(' abc', b'%*s', c_int(5), b'abc') + check_format('ab', b'%.*s', c_int(2), b'abc') + check_format(' ab', b'%*.*s', c_int(5), c_int(2), b'abc') + check_format(' abc', b'%*U', c_int(5), 'abc') + check_format('ab', b'%.*U', c_int(2), 'abc') + check_format(' ab', b'%*.*U', c_int(5), c_int(2), 'abc') + check_format(' ab', b'%*.*V', c_int(5), c_int(2), None, b'abc') + check_format(' ab', b'%*.*lV', c_int(5), c_int(2), + None, c_wchar_p('abc')) + check_format(' 123', b'%*i', c_int(8), c_int(123)) + check_format('00123', b'%.*i', c_int(5), c_int(123)) + check_format(' 00123', b'%*.*i', c_int(8), c_int(5), c_int(123)) # test %p # We cannot test the exact result, @@ -564,10 +664,11 @@ def check_format(expected, format, *args): check_format('', b'%s', b'') - # check for crashes + # test invalid format strings. these tests are just here + # to check for crashes and should not be considered as specifications for fmt in (b'%', b'%0', b'%01', b'%.', b'%.1', b'%0%s', b'%1%s', b'%.%s', b'%.1%s', b'%1abc', - b'%l', b'%ll', b'%z', b'%ls', b'%lls', b'%zs'): + b'%l', b'%ll', b'%z', b'%lls', b'%zs'): with self.subTest(fmt=fmt): self.assertRaisesRegex(SystemError, 'invalid format string', PyUnicode_FromFormat, fmt, b'abc') diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f72cb0442f35ff..bea7303805567f 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -774,6 +774,45 @@ def test_legacy_converters(self): module, function = block.signatures self.assertIsInstance((function.parameters['path']).converter, clinic.str_converter) + def test_legacy_converters_non_string_constant_annotation(self): + expected_failure_message = """\ +Error on line 0: +Annotations must be either a name, a function call, or a string. +""" + + s = self.parse_function_should_fail('module os\nos.access\n path: 42') + self.assertEqual(s, expected_failure_message) + + s = self.parse_function_should_fail('module os\nos.access\n path: 42.42') + self.assertEqual(s, expected_failure_message) + + s = self.parse_function_should_fail('module os\nos.access\n path: 42j') + self.assertEqual(s, expected_failure_message) + + s = self.parse_function_should_fail('module os\nos.access\n path: b"42"') + self.assertEqual(s, expected_failure_message) + + def test_other_bizarre_things_in_annotations_fail(self): + expected_failure_message = """\ +Error on line 0: +Annotations must be either a name, a function call, or a string. +""" + + s = self.parse_function_should_fail( + 'module os\nos.access\n path: {"some": "dictionary"}' + ) + self.assertEqual(s, expected_failure_message) + + s = self.parse_function_should_fail( + 'module os\nos.access\n path: ["list", "of", "strings"]' + ) + self.assertEqual(s, expected_failure_message) + + s = self.parse_function_should_fail( + 'module os\nos.access\n path: (x for x in range(42))' + ) + self.assertEqual(s, expected_failure_message) + def test_unused_param(self): block = self.parse(""" module foo diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index fb7a016c9007f8..fe701025b70f48 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4265,7 +4265,7 @@ class Color(Enum): 'mixed types with auto() will raise in 3.13', ) def test_auto_garbage_fail(self): - with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): + with self.assertRaisesRegex(TypeError, "unable to increment 'red'"): class Color(Enum): red = 'red' blue = auto() @@ -4275,7 +4275,7 @@ class Color(Enum): 'mixed types with auto() will raise in 3.13', ) def test_auto_garbage_corrected_fail(self): - with self.assertRaisesRegex(TypeError, 'will require all values to be sortable'): + with self.assertRaisesRegex(TypeError, 'unable to sort non-numeric values'): class Color(Enum): red = 'red' blue = 2 diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index fcb12d25ff9dd6..3ba2f943e6e968 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -661,15 +661,50 @@ def test_comments(self): self.assertEqual(f'{"#"}', '#') self.assertEqual(f'{d["#"]}', 'hash') - self.assertAllRaise(SyntaxError, "f-string expression part cannot include '#'", - ["f'{1#}'", # error because the expression becomes "(1#)" - "f'{3(#)}'", + self.assertAllRaise(SyntaxError, "'{' was never closed", + ["f'{1#}'", # error because everything after '#' is a comment "f'{#}'", + "f'one: {1#}'", + "f'{1# one} {2 this is a comment still#}'", ]) self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'", ["f'{)#}'", # When wrapped in parens, this becomes # '()#)'. Make sure that doesn't compile. ]) + self.assertEqual(f'''A complex trick: { +2 # two +}''', 'A complex trick: 2') + self.assertEqual(f''' +{ +40 # fourty ++ # plus +2 # two +}''', '\n42') + self.assertEqual(f''' +{ +40 # fourty ++ # plus +2 # two +}''', '\n42') + + self.assertEqual(f''' +# this is not a comment +{ # the following operation it's +3 # this is a number +* 2}''', '\n# this is not a comment\n6') + self.assertEqual(f''' +{# f'a {comment}' +86 # constant +# nothing more +}''', '\n86') + + self.assertAllRaise(SyntaxError, r"f-string: valid expression required before '}'", + ["""f''' +{ +# only a comment +}''' +""", # this is equivalent to f'{}' + ]) def test_many_expressions(self): # Create a string with many expressions in it. Note that diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 31680b5a92e0f3..a8a344ab8de48d 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -451,6 +451,88 @@ def g(): self.assertEqual(cm.exception.value.value, 2) +class GeneratorCloseTest(unittest.TestCase): + + def test_close_no_return_value(self): + def f(): + yield + + gen = f() + gen.send(None) + self.assertIsNone(gen.close()) + + def test_close_return_value(self): + def f(): + try: + yield + # close() raises GeneratorExit here, which is caught + except GeneratorExit: + return 0 + + gen = f() + gen.send(None) + self.assertEqual(gen.close(), 0) + + def test_close_not_catching_exit(self): + def f(): + yield + # close() raises GeneratorExit here, which isn't caught and + # therefore propagates -- no return value + return 0 + + gen = f() + gen.send(None) + self.assertIsNone(gen.close()) + + def test_close_not_started(self): + def f(): + try: + yield + except GeneratorExit: + return 0 + + gen = f() + self.assertIsNone(gen.close()) + + def test_close_exhausted(self): + def f(): + try: + yield + except GeneratorExit: + return 0 + + gen = f() + next(gen) + with self.assertRaises(StopIteration): + next(gen) + self.assertIsNone(gen.close()) + + def test_close_closed(self): + def f(): + try: + yield + except GeneratorExit: + return 0 + + gen = f() + gen.send(None) + self.assertEqual(gen.close(), 0) + self.assertIsNone(gen.close()) + + def test_close_raises(self): + def f(): + try: + yield + except GeneratorExit: + pass + raise RuntimeError + + gen = f() + gen.send(None) + with self.assertRaises(RuntimeError): + gen.close() + + class GeneratorThrowTest(unittest.TestCase): def test_exception_context_with_yield(self): diff --git a/Lib/test/test_lib2to3/__init__.py b/Lib/test/test_lib2to3/__init__.py deleted file mode 100644 index f323c2355b2287..00000000000000 --- a/Lib/test/test_lib2to3/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# Author: Collin Winter - -import os -import warnings - -from test.support import load_package_tests - -def load_tests(*args): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=DeprecationWarning, message='lib2to3') - return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_lib2to3/__main__.py b/Lib/test/test_lib2to3/__main__.py deleted file mode 100644 index 40a23a297ec2b4..00000000000000 --- a/Lib/test/test_lib2to3/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from . import load_tests -import unittest - -unittest.main() diff --git a/Lib/test/test_lib2to3/data/README b/Lib/test/test_lib2to3/data/README deleted file mode 100644 index 7aa47e40aa97f5..00000000000000 --- a/Lib/test/test_lib2to3/data/README +++ /dev/null @@ -1,6 +0,0 @@ -In this directory: -- py2_test_grammar.py -- test file that exercises most/all of Python 2.x's grammar. -- py3_test_grammar.py -- test file that exercises most/all of Python 3.x's grammar. -- infinite_recursion.py -- test file that causes lib2to3's faster recursive pattern matching - scheme to fail, but passes when lib2to3 falls back to iterative pattern matching. -- fixes/ -- for use by test_refactor.py diff --git a/Lib/test/test_lib2to3/data/bom.py b/Lib/test/test_lib2to3/data/bom.py deleted file mode 100644 index 9bc3975a42f601..00000000000000 --- a/Lib/test/test_lib2to3/data/bom.py +++ /dev/null @@ -1,2 +0,0 @@ -# coding: utf-8 -print "BOM BOOM!" diff --git a/Lib/test/test_lib2to3/data/crlf.py b/Lib/test/test_lib2to3/data/crlf.py deleted file mode 100644 index a83ca8f0a2068f..00000000000000 --- a/Lib/test/test_lib2to3/data/crlf.py +++ /dev/null @@ -1,3 +0,0 @@ -print "hi" - -print "Like bad Windows newlines?" diff --git a/Lib/test/test_lib2to3/data/different_encoding.py b/Lib/test/test_lib2to3/data/different_encoding.py deleted file mode 100755 index 9f32bd04dc28db..00000000000000 --- a/Lib/test/test_lib2to3/data/different_encoding.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -print u'ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ' - -def f(x): - print '%s\t-> α(%2i):%s β(%s)' diff --git a/Lib/test/test_lib2to3/data/false_encoding.py b/Lib/test/test_lib2to3/data/false_encoding.py deleted file mode 100755 index f4e59e787da1e5..00000000000000 --- a/Lib/test/test_lib2to3/data/false_encoding.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -print '#coding=0' diff --git a/Lib/test/test_lib2to3/data/fixers/bad_order.py b/Lib/test/test_lib2to3/data/fixers/bad_order.py deleted file mode 100644 index 061bbf209a0dee..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/bad_order.py +++ /dev/null @@ -1,5 +0,0 @@ -from lib2to3.fixer_base import BaseFix - -class FixBadOrder(BaseFix): - - order = "crazy" diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/__init__.py b/Lib/test/test_lib2to3/data/fixers/myfixes/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_explicit.py b/Lib/test/test_lib2to3/data/fixers/myfixes/fix_explicit.py deleted file mode 100644 index cbe16f6f1f0dfc..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_explicit.py +++ /dev/null @@ -1,6 +0,0 @@ -from lib2to3.fixer_base import BaseFix - -class FixExplicit(BaseFix): - explicit = True - - def match(self): return False diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_first.py b/Lib/test/test_lib2to3/data/fixers/myfixes/fix_first.py deleted file mode 100644 index a88821f06b2d14..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_first.py +++ /dev/null @@ -1,6 +0,0 @@ -from lib2to3.fixer_base import BaseFix - -class FixFirst(BaseFix): - run_order = 1 - - def match(self, node): return False diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_last.py b/Lib/test/test_lib2to3/data/fixers/myfixes/fix_last.py deleted file mode 100644 index 9a077d41594da6..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_last.py +++ /dev/null @@ -1,7 +0,0 @@ -from lib2to3.fixer_base import BaseFix - -class FixLast(BaseFix): - - run_order = 10 - - def match(self, node): return False diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_parrot.py b/Lib/test/test_lib2to3/data/fixers/myfixes/fix_parrot.py deleted file mode 100644 index 6db79adf8d80c1..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_parrot.py +++ /dev/null @@ -1,13 +0,0 @@ -from lib2to3.fixer_base import BaseFix -from lib2to3.fixer_util import Name - -class FixParrot(BaseFix): - """ - Change functions named 'parrot' to 'cheese'. - """ - - PATTERN = """funcdef < 'def' name='parrot' any* >""" - - def transform(self, node, results): - name = results["name"] - name.replace(Name("cheese", name.prefix)) diff --git a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_preorder.py b/Lib/test/test_lib2to3/data/fixers/myfixes/fix_preorder.py deleted file mode 100644 index b9bfbba9d5e900..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/myfixes/fix_preorder.py +++ /dev/null @@ -1,6 +0,0 @@ -from lib2to3.fixer_base import BaseFix - -class FixPreorder(BaseFix): - order = "pre" - - def match(self, node): return False diff --git a/Lib/test/test_lib2to3/data/fixers/no_fixer_cls.py b/Lib/test/test_lib2to3/data/fixers/no_fixer_cls.py deleted file mode 100644 index 506f7949517058..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/no_fixer_cls.py +++ /dev/null @@ -1 +0,0 @@ -# This is empty so trying to fetch the fixer class gives an AttributeError diff --git a/Lib/test/test_lib2to3/data/fixers/parrot_example.py b/Lib/test/test_lib2to3/data/fixers/parrot_example.py deleted file mode 100644 index 08529280cdb4a1..00000000000000 --- a/Lib/test/test_lib2to3/data/fixers/parrot_example.py +++ /dev/null @@ -1,2 +0,0 @@ -def parrot(): - pass diff --git a/Lib/test/test_lib2to3/data/infinite_recursion.py b/Lib/test/test_lib2to3/data/infinite_recursion.py deleted file mode 100644 index acc62ed4f1a44c..00000000000000 --- a/Lib/test/test_lib2to3/data/infinite_recursion.py +++ /dev/null @@ -1,2669 +0,0 @@ -# Verify that 2to3 falls back from the recursive pattern matching scheme to a -# slower, iterative scheme in the event of a RecursionError. -from ctypes import * -STRING = c_char_p - - -OSUnknownByteOrder = 0 -UIT_PROMPT = 1 -P_PGID = 2 -P_PID = 1 -UIT_ERROR = 5 -UIT_INFO = 4 -UIT_NONE = 0 -P_ALL = 0 -UIT_VERIFY = 2 -OSBigEndian = 2 -UIT_BOOLEAN = 3 -OSLittleEndian = 1 -__darwin_nl_item = c_int -__darwin_wctrans_t = c_int -__darwin_wctype_t = c_ulong -__int8_t = c_byte -__uint8_t = c_ubyte -__int16_t = c_short -__uint16_t = c_ushort -__int32_t = c_int -__uint32_t = c_uint -__int64_t = c_longlong -__uint64_t = c_ulonglong -__darwin_intptr_t = c_long -__darwin_natural_t = c_uint -__darwin_ct_rune_t = c_int -class __mbstate_t(Union): - pass -__mbstate_t._pack_ = 4 -__mbstate_t._fields_ = [ - ('__mbstate8', c_char * 128), - ('_mbstateL', c_longlong), -] -assert sizeof(__mbstate_t) == 128, sizeof(__mbstate_t) -assert alignment(__mbstate_t) == 4, alignment(__mbstate_t) -__darwin_mbstate_t = __mbstate_t -__darwin_ptrdiff_t = c_int -__darwin_size_t = c_ulong -__darwin_va_list = STRING -__darwin_wchar_t = c_int -__darwin_rune_t = __darwin_wchar_t -__darwin_wint_t = c_int -__darwin_clock_t = c_ulong -__darwin_socklen_t = __uint32_t -__darwin_ssize_t = c_long -__darwin_time_t = c_long -sig_atomic_t = c_int -class sigcontext(Structure): - pass -sigcontext._fields_ = [ - ('sc_onstack', c_int), - ('sc_mask', c_int), - ('sc_eax', c_uint), - ('sc_ebx', c_uint), - ('sc_ecx', c_uint), - ('sc_edx', c_uint), - ('sc_edi', c_uint), - ('sc_esi', c_uint), - ('sc_ebp', c_uint), - ('sc_esp', c_uint), - ('sc_ss', c_uint), - ('sc_eflags', c_uint), - ('sc_eip', c_uint), - ('sc_cs', c_uint), - ('sc_ds', c_uint), - ('sc_es', c_uint), - ('sc_fs', c_uint), - ('sc_gs', c_uint), -] -assert sizeof(sigcontext) == 72, sizeof(sigcontext) -assert alignment(sigcontext) == 4, alignment(sigcontext) -u_int8_t = c_ubyte -u_int16_t = c_ushort -u_int32_t = c_uint -u_int64_t = c_ulonglong -int32_t = c_int -register_t = int32_t -user_addr_t = u_int64_t -user_size_t = u_int64_t -int64_t = c_longlong -user_ssize_t = int64_t -user_long_t = int64_t -user_ulong_t = u_int64_t -user_time_t = int64_t -syscall_arg_t = u_int64_t - -# values for unnamed enumeration -class aes_key_st(Structure): - pass -aes_key_st._fields_ = [ - ('rd_key', c_ulong * 60), - ('rounds', c_int), -] -assert sizeof(aes_key_st) == 244, sizeof(aes_key_st) -assert alignment(aes_key_st) == 4, alignment(aes_key_st) -AES_KEY = aes_key_st -class asn1_ctx_st(Structure): - pass -asn1_ctx_st._fields_ = [ - ('p', POINTER(c_ubyte)), - ('eos', c_int), - ('error', c_int), - ('inf', c_int), - ('tag', c_int), - ('xclass', c_int), - ('slen', c_long), - ('max', POINTER(c_ubyte)), - ('q', POINTER(c_ubyte)), - ('pp', POINTER(POINTER(c_ubyte))), - ('line', c_int), -] -assert sizeof(asn1_ctx_st) == 44, sizeof(asn1_ctx_st) -assert alignment(asn1_ctx_st) == 4, alignment(asn1_ctx_st) -ASN1_CTX = asn1_ctx_st -class asn1_object_st(Structure): - pass -asn1_object_st._fields_ = [ - ('sn', STRING), - ('ln', STRING), - ('nid', c_int), - ('length', c_int), - ('data', POINTER(c_ubyte)), - ('flags', c_int), -] -assert sizeof(asn1_object_st) == 24, sizeof(asn1_object_st) -assert alignment(asn1_object_st) == 4, alignment(asn1_object_st) -ASN1_OBJECT = asn1_object_st -class asn1_string_st(Structure): - pass -asn1_string_st._fields_ = [ - ('length', c_int), - ('type', c_int), - ('data', POINTER(c_ubyte)), - ('flags', c_long), -] -assert sizeof(asn1_string_st) == 16, sizeof(asn1_string_st) -assert alignment(asn1_string_st) == 4, alignment(asn1_string_st) -ASN1_STRING = asn1_string_st -class ASN1_ENCODING_st(Structure): - pass -ASN1_ENCODING_st._fields_ = [ - ('enc', POINTER(c_ubyte)), - ('len', c_long), - ('modified', c_int), -] -assert sizeof(ASN1_ENCODING_st) == 12, sizeof(ASN1_ENCODING_st) -assert alignment(ASN1_ENCODING_st) == 4, alignment(ASN1_ENCODING_st) -ASN1_ENCODING = ASN1_ENCODING_st -class asn1_string_table_st(Structure): - pass -asn1_string_table_st._fields_ = [ - ('nid', c_int), - ('minsize', c_long), - ('maxsize', c_long), - ('mask', c_ulong), - ('flags', c_ulong), -] -assert sizeof(asn1_string_table_st) == 20, sizeof(asn1_string_table_st) -assert alignment(asn1_string_table_st) == 4, alignment(asn1_string_table_st) -ASN1_STRING_TABLE = asn1_string_table_st -class ASN1_TEMPLATE_st(Structure): - pass -ASN1_TEMPLATE_st._fields_ = [ -] -ASN1_TEMPLATE = ASN1_TEMPLATE_st -class ASN1_ITEM_st(Structure): - pass -ASN1_ITEM = ASN1_ITEM_st -ASN1_ITEM_st._fields_ = [ -] -class ASN1_TLC_st(Structure): - pass -ASN1_TLC = ASN1_TLC_st -ASN1_TLC_st._fields_ = [ -] -class ASN1_VALUE_st(Structure): - pass -ASN1_VALUE_st._fields_ = [ -] -ASN1_VALUE = ASN1_VALUE_st -ASN1_ITEM_EXP = ASN1_ITEM -class asn1_type_st(Structure): - pass -class N12asn1_type_st4DOLLAR_11E(Union): - pass -ASN1_BOOLEAN = c_int -ASN1_INTEGER = asn1_string_st -ASN1_ENUMERATED = asn1_string_st -ASN1_BIT_STRING = asn1_string_st -ASN1_OCTET_STRING = asn1_string_st -ASN1_PRINTABLESTRING = asn1_string_st -ASN1_T61STRING = asn1_string_st -ASN1_IA5STRING = asn1_string_st -ASN1_GENERALSTRING = asn1_string_st -ASN1_BMPSTRING = asn1_string_st -ASN1_UNIVERSALSTRING = asn1_string_st -ASN1_UTCTIME = asn1_string_st -ASN1_GENERALIZEDTIME = asn1_string_st -ASN1_VISIBLESTRING = asn1_string_st -ASN1_UTF8STRING = asn1_string_st -N12asn1_type_st4DOLLAR_11E._fields_ = [ - ('ptr', STRING), - ('boolean', ASN1_BOOLEAN), - ('asn1_string', POINTER(ASN1_STRING)), - ('object', POINTER(ASN1_OBJECT)), - ('integer', POINTER(ASN1_INTEGER)), - ('enumerated', POINTER(ASN1_ENUMERATED)), - ('bit_string', POINTER(ASN1_BIT_STRING)), - ('octet_string', POINTER(ASN1_OCTET_STRING)), - ('printablestring', POINTER(ASN1_PRINTABLESTRING)), - ('t61string', POINTER(ASN1_T61STRING)), - ('ia5string', POINTER(ASN1_IA5STRING)), - ('generalstring', POINTER(ASN1_GENERALSTRING)), - ('bmpstring', POINTER(ASN1_BMPSTRING)), - ('universalstring', POINTER(ASN1_UNIVERSALSTRING)), - ('utctime', POINTER(ASN1_UTCTIME)), - ('generalizedtime', POINTER(ASN1_GENERALIZEDTIME)), - ('visiblestring', POINTER(ASN1_VISIBLESTRING)), - ('utf8string', POINTER(ASN1_UTF8STRING)), - ('set', POINTER(ASN1_STRING)), - ('sequence', POINTER(ASN1_STRING)), -] -assert sizeof(N12asn1_type_st4DOLLAR_11E) == 4, sizeof(N12asn1_type_st4DOLLAR_11E) -assert alignment(N12asn1_type_st4DOLLAR_11E) == 4, alignment(N12asn1_type_st4DOLLAR_11E) -asn1_type_st._fields_ = [ - ('type', c_int), - ('value', N12asn1_type_st4DOLLAR_11E), -] -assert sizeof(asn1_type_st) == 8, sizeof(asn1_type_st) -assert alignment(asn1_type_st) == 4, alignment(asn1_type_st) -ASN1_TYPE = asn1_type_st -class asn1_method_st(Structure): - pass -asn1_method_st._fields_ = [ - ('i2d', CFUNCTYPE(c_int)), - ('d2i', CFUNCTYPE(STRING)), - ('create', CFUNCTYPE(STRING)), - ('destroy', CFUNCTYPE(None)), -] -assert sizeof(asn1_method_st) == 16, sizeof(asn1_method_st) -assert alignment(asn1_method_st) == 4, alignment(asn1_method_st) -ASN1_METHOD = asn1_method_st -class asn1_header_st(Structure): - pass -asn1_header_st._fields_ = [ - ('header', POINTER(ASN1_OCTET_STRING)), - ('data', STRING), - ('meth', POINTER(ASN1_METHOD)), -] -assert sizeof(asn1_header_st) == 12, sizeof(asn1_header_st) -assert alignment(asn1_header_st) == 4, alignment(asn1_header_st) -ASN1_HEADER = asn1_header_st -class BIT_STRING_BITNAME_st(Structure): - pass -BIT_STRING_BITNAME_st._fields_ = [ - ('bitnum', c_int), - ('lname', STRING), - ('sname', STRING), -] -assert sizeof(BIT_STRING_BITNAME_st) == 12, sizeof(BIT_STRING_BITNAME_st) -assert alignment(BIT_STRING_BITNAME_st) == 4, alignment(BIT_STRING_BITNAME_st) -BIT_STRING_BITNAME = BIT_STRING_BITNAME_st -class bio_st(Structure): - pass -BIO = bio_st -bio_info_cb = CFUNCTYPE(None, POINTER(bio_st), c_int, STRING, c_int, c_long, c_long) -class bio_method_st(Structure): - pass -bio_method_st._fields_ = [ - ('type', c_int), - ('name', STRING), - ('bwrite', CFUNCTYPE(c_int, POINTER(BIO), STRING, c_int)), - ('bread', CFUNCTYPE(c_int, POINTER(BIO), STRING, c_int)), - ('bputs', CFUNCTYPE(c_int, POINTER(BIO), STRING)), - ('bgets', CFUNCTYPE(c_int, POINTER(BIO), STRING, c_int)), - ('ctrl', CFUNCTYPE(c_long, POINTER(BIO), c_int, c_long, c_void_p)), - ('create', CFUNCTYPE(c_int, POINTER(BIO))), - ('destroy', CFUNCTYPE(c_int, POINTER(BIO))), - ('callback_ctrl', CFUNCTYPE(c_long, POINTER(BIO), c_int, POINTER(bio_info_cb))), -] -assert sizeof(bio_method_st) == 40, sizeof(bio_method_st) -assert alignment(bio_method_st) == 4, alignment(bio_method_st) -BIO_METHOD = bio_method_st -class crypto_ex_data_st(Structure): - pass -class stack_st(Structure): - pass -STACK = stack_st -crypto_ex_data_st._fields_ = [ - ('sk', POINTER(STACK)), - ('dummy', c_int), -] -assert sizeof(crypto_ex_data_st) == 8, sizeof(crypto_ex_data_st) -assert alignment(crypto_ex_data_st) == 4, alignment(crypto_ex_data_st) -CRYPTO_EX_DATA = crypto_ex_data_st -bio_st._fields_ = [ - ('method', POINTER(BIO_METHOD)), - ('callback', CFUNCTYPE(c_long, POINTER(bio_st), c_int, STRING, c_int, c_long, c_long)), - ('cb_arg', STRING), - ('init', c_int), - ('shutdown', c_int), - ('flags', c_int), - ('retry_reason', c_int), - ('num', c_int), - ('ptr', c_void_p), - ('next_bio', POINTER(bio_st)), - ('prev_bio', POINTER(bio_st)), - ('references', c_int), - ('num_read', c_ulong), - ('num_write', c_ulong), - ('ex_data', CRYPTO_EX_DATA), -] -assert sizeof(bio_st) == 64, sizeof(bio_st) -assert alignment(bio_st) == 4, alignment(bio_st) -class bio_f_buffer_ctx_struct(Structure): - pass -bio_f_buffer_ctx_struct._fields_ = [ - ('ibuf_size', c_int), - ('obuf_size', c_int), - ('ibuf', STRING), - ('ibuf_len', c_int), - ('ibuf_off', c_int), - ('obuf', STRING), - ('obuf_len', c_int), - ('obuf_off', c_int), -] -assert sizeof(bio_f_buffer_ctx_struct) == 32, sizeof(bio_f_buffer_ctx_struct) -assert alignment(bio_f_buffer_ctx_struct) == 4, alignment(bio_f_buffer_ctx_struct) -BIO_F_BUFFER_CTX = bio_f_buffer_ctx_struct -class hostent(Structure): - pass -hostent._fields_ = [ -] -class bf_key_st(Structure): - pass -bf_key_st._fields_ = [ - ('P', c_uint * 18), - ('S', c_uint * 1024), -] -assert sizeof(bf_key_st) == 4168, sizeof(bf_key_st) -assert alignment(bf_key_st) == 4, alignment(bf_key_st) -BF_KEY = bf_key_st -class bignum_st(Structure): - pass -bignum_st._fields_ = [ - ('d', POINTER(c_ulong)), - ('top', c_int), - ('dmax', c_int), - ('neg', c_int), - ('flags', c_int), -] -assert sizeof(bignum_st) == 20, sizeof(bignum_st) -assert alignment(bignum_st) == 4, alignment(bignum_st) -BIGNUM = bignum_st -class bignum_ctx(Structure): - pass -bignum_ctx._fields_ = [ -] -BN_CTX = bignum_ctx -class bn_blinding_st(Structure): - pass -bn_blinding_st._fields_ = [ - ('init', c_int), - ('A', POINTER(BIGNUM)), - ('Ai', POINTER(BIGNUM)), - ('mod', POINTER(BIGNUM)), - ('thread_id', c_ulong), -] -assert sizeof(bn_blinding_st) == 20, sizeof(bn_blinding_st) -assert alignment(bn_blinding_st) == 4, alignment(bn_blinding_st) -BN_BLINDING = bn_blinding_st -class bn_mont_ctx_st(Structure): - pass -bn_mont_ctx_st._fields_ = [ - ('ri', c_int), - ('RR', BIGNUM), - ('N', BIGNUM), - ('Ni', BIGNUM), - ('n0', c_ulong), - ('flags', c_int), -] -assert sizeof(bn_mont_ctx_st) == 72, sizeof(bn_mont_ctx_st) -assert alignment(bn_mont_ctx_st) == 4, alignment(bn_mont_ctx_st) -BN_MONT_CTX = bn_mont_ctx_st -class bn_recp_ctx_st(Structure): - pass -bn_recp_ctx_st._fields_ = [ - ('N', BIGNUM), - ('Nr', BIGNUM), - ('num_bits', c_int), - ('shift', c_int), - ('flags', c_int), -] -assert sizeof(bn_recp_ctx_st) == 52, sizeof(bn_recp_ctx_st) -assert alignment(bn_recp_ctx_st) == 4, alignment(bn_recp_ctx_st) -BN_RECP_CTX = bn_recp_ctx_st -class buf_mem_st(Structure): - pass -buf_mem_st._fields_ = [ - ('length', c_int), - ('data', STRING), - ('max', c_int), -] -assert sizeof(buf_mem_st) == 12, sizeof(buf_mem_st) -assert alignment(buf_mem_st) == 4, alignment(buf_mem_st) -BUF_MEM = buf_mem_st -class cast_key_st(Structure): - pass -cast_key_st._fields_ = [ - ('data', c_ulong * 32), - ('short_key', c_int), -] -assert sizeof(cast_key_st) == 132, sizeof(cast_key_st) -assert alignment(cast_key_st) == 4, alignment(cast_key_st) -CAST_KEY = cast_key_st -class comp_method_st(Structure): - pass -comp_method_st._fields_ = [ - ('type', c_int), - ('name', STRING), - ('init', CFUNCTYPE(c_int)), - ('finish', CFUNCTYPE(None)), - ('compress', CFUNCTYPE(c_int)), - ('expand', CFUNCTYPE(c_int)), - ('ctrl', CFUNCTYPE(c_long)), - ('callback_ctrl', CFUNCTYPE(c_long)), -] -assert sizeof(comp_method_st) == 32, sizeof(comp_method_st) -assert alignment(comp_method_st) == 4, alignment(comp_method_st) -COMP_METHOD = comp_method_st -class comp_ctx_st(Structure): - pass -comp_ctx_st._fields_ = [ - ('meth', POINTER(COMP_METHOD)), - ('compress_in', c_ulong), - ('compress_out', c_ulong), - ('expand_in', c_ulong), - ('expand_out', c_ulong), - ('ex_data', CRYPTO_EX_DATA), -] -assert sizeof(comp_ctx_st) == 28, sizeof(comp_ctx_st) -assert alignment(comp_ctx_st) == 4, alignment(comp_ctx_st) -COMP_CTX = comp_ctx_st -class CRYPTO_dynlock_value(Structure): - pass -CRYPTO_dynlock_value._fields_ = [ -] -class CRYPTO_dynlock(Structure): - pass -CRYPTO_dynlock._fields_ = [ - ('references', c_int), - ('data', POINTER(CRYPTO_dynlock_value)), -] -assert sizeof(CRYPTO_dynlock) == 8, sizeof(CRYPTO_dynlock) -assert alignment(CRYPTO_dynlock) == 4, alignment(CRYPTO_dynlock) -BIO_dummy = bio_st -CRYPTO_EX_new = CFUNCTYPE(c_int, c_void_p, c_void_p, POINTER(CRYPTO_EX_DATA), c_int, c_long, c_void_p) -CRYPTO_EX_free = CFUNCTYPE(None, c_void_p, c_void_p, POINTER(CRYPTO_EX_DATA), c_int, c_long, c_void_p) -CRYPTO_EX_dup = CFUNCTYPE(c_int, POINTER(CRYPTO_EX_DATA), POINTER(CRYPTO_EX_DATA), c_void_p, c_int, c_long, c_void_p) -class crypto_ex_data_func_st(Structure): - pass -crypto_ex_data_func_st._fields_ = [ - ('argl', c_long), - ('argp', c_void_p), - ('new_func', POINTER(CRYPTO_EX_new)), - ('free_func', POINTER(CRYPTO_EX_free)), - ('dup_func', POINTER(CRYPTO_EX_dup)), -] -assert sizeof(crypto_ex_data_func_st) == 20, sizeof(crypto_ex_data_func_st) -assert alignment(crypto_ex_data_func_st) == 4, alignment(crypto_ex_data_func_st) -CRYPTO_EX_DATA_FUNCS = crypto_ex_data_func_st -class st_CRYPTO_EX_DATA_IMPL(Structure): - pass -CRYPTO_EX_DATA_IMPL = st_CRYPTO_EX_DATA_IMPL -st_CRYPTO_EX_DATA_IMPL._fields_ = [ -] -CRYPTO_MEM_LEAK_CB = CFUNCTYPE(c_void_p, c_ulong, STRING, c_int, c_int, c_void_p) -DES_cblock = c_ubyte * 8 -const_DES_cblock = c_ubyte * 8 -class DES_ks(Structure): - pass -class N6DES_ks3DOLLAR_9E(Union): - pass -N6DES_ks3DOLLAR_9E._fields_ = [ - ('cblock', DES_cblock), - ('deslong', c_ulong * 2), -] -assert sizeof(N6DES_ks3DOLLAR_9E) == 8, sizeof(N6DES_ks3DOLLAR_9E) -assert alignment(N6DES_ks3DOLLAR_9E) == 4, alignment(N6DES_ks3DOLLAR_9E) -DES_ks._fields_ = [ - ('ks', N6DES_ks3DOLLAR_9E * 16), -] -assert sizeof(DES_ks) == 128, sizeof(DES_ks) -assert alignment(DES_ks) == 4, alignment(DES_ks) -DES_key_schedule = DES_ks -_ossl_old_des_cblock = c_ubyte * 8 -class _ossl_old_des_ks_struct(Structure): - pass -class N23_ossl_old_des_ks_struct4DOLLAR_10E(Union): - pass -N23_ossl_old_des_ks_struct4DOLLAR_10E._fields_ = [ - ('_', _ossl_old_des_cblock), - ('pad', c_ulong * 2), -] -assert sizeof(N23_ossl_old_des_ks_struct4DOLLAR_10E) == 8, sizeof(N23_ossl_old_des_ks_struct4DOLLAR_10E) -assert alignment(N23_ossl_old_des_ks_struct4DOLLAR_10E) == 4, alignment(N23_ossl_old_des_ks_struct4DOLLAR_10E) -_ossl_old_des_ks_struct._fields_ = [ - ('ks', N23_ossl_old_des_ks_struct4DOLLAR_10E), -] -assert sizeof(_ossl_old_des_ks_struct) == 8, sizeof(_ossl_old_des_ks_struct) -assert alignment(_ossl_old_des_ks_struct) == 4, alignment(_ossl_old_des_ks_struct) -_ossl_old_des_key_schedule = _ossl_old_des_ks_struct * 16 -class dh_st(Structure): - pass -DH = dh_st -class dh_method(Structure): - pass -dh_method._fields_ = [ - ('name', STRING), - ('generate_key', CFUNCTYPE(c_int, POINTER(DH))), - ('compute_key', CFUNCTYPE(c_int, POINTER(c_ubyte), POINTER(BIGNUM), POINTER(DH))), - ('bn_mod_exp', CFUNCTYPE(c_int, POINTER(DH), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BN_CTX), POINTER(BN_MONT_CTX))), - ('init', CFUNCTYPE(c_int, POINTER(DH))), - ('finish', CFUNCTYPE(c_int, POINTER(DH))), - ('flags', c_int), - ('app_data', STRING), -] -assert sizeof(dh_method) == 32, sizeof(dh_method) -assert alignment(dh_method) == 4, alignment(dh_method) -DH_METHOD = dh_method -class engine_st(Structure): - pass -ENGINE = engine_st -dh_st._fields_ = [ - ('pad', c_int), - ('version', c_int), - ('p', POINTER(BIGNUM)), - ('g', POINTER(BIGNUM)), - ('length', c_long), - ('pub_key', POINTER(BIGNUM)), - ('priv_key', POINTER(BIGNUM)), - ('flags', c_int), - ('method_mont_p', STRING), - ('q', POINTER(BIGNUM)), - ('j', POINTER(BIGNUM)), - ('seed', POINTER(c_ubyte)), - ('seedlen', c_int), - ('counter', POINTER(BIGNUM)), - ('references', c_int), - ('ex_data', CRYPTO_EX_DATA), - ('meth', POINTER(DH_METHOD)), - ('engine', POINTER(ENGINE)), -] -assert sizeof(dh_st) == 76, sizeof(dh_st) -assert alignment(dh_st) == 4, alignment(dh_st) -class dsa_st(Structure): - pass -DSA = dsa_st -class DSA_SIG_st(Structure): - pass -DSA_SIG_st._fields_ = [ - ('r', POINTER(BIGNUM)), - ('s', POINTER(BIGNUM)), -] -assert sizeof(DSA_SIG_st) == 8, sizeof(DSA_SIG_st) -assert alignment(DSA_SIG_st) == 4, alignment(DSA_SIG_st) -DSA_SIG = DSA_SIG_st -class dsa_method(Structure): - pass -dsa_method._fields_ = [ - ('name', STRING), - ('dsa_do_sign', CFUNCTYPE(POINTER(DSA_SIG), POINTER(c_ubyte), c_int, POINTER(DSA))), - ('dsa_sign_setup', CFUNCTYPE(c_int, POINTER(DSA), POINTER(BN_CTX), POINTER(POINTER(BIGNUM)), POINTER(POINTER(BIGNUM)))), - ('dsa_do_verify', CFUNCTYPE(c_int, POINTER(c_ubyte), c_int, POINTER(DSA_SIG), POINTER(DSA))), - ('dsa_mod_exp', CFUNCTYPE(c_int, POINTER(DSA), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BN_CTX), POINTER(BN_MONT_CTX))), - ('bn_mod_exp', CFUNCTYPE(c_int, POINTER(DSA), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BN_CTX), POINTER(BN_MONT_CTX))), - ('init', CFUNCTYPE(c_int, POINTER(DSA))), - ('finish', CFUNCTYPE(c_int, POINTER(DSA))), - ('flags', c_int), - ('app_data', STRING), -] -assert sizeof(dsa_method) == 40, sizeof(dsa_method) -assert alignment(dsa_method) == 4, alignment(dsa_method) -DSA_METHOD = dsa_method -dsa_st._fields_ = [ - ('pad', c_int), - ('version', c_long), - ('write_params', c_int), - ('p', POINTER(BIGNUM)), - ('q', POINTER(BIGNUM)), - ('g', POINTER(BIGNUM)), - ('pub_key', POINTER(BIGNUM)), - ('priv_key', POINTER(BIGNUM)), - ('kinv', POINTER(BIGNUM)), - ('r', POINTER(BIGNUM)), - ('flags', c_int), - ('method_mont_p', STRING), - ('references', c_int), - ('ex_data', CRYPTO_EX_DATA), - ('meth', POINTER(DSA_METHOD)), - ('engine', POINTER(ENGINE)), -] -assert sizeof(dsa_st) == 68, sizeof(dsa_st) -assert alignment(dsa_st) == 4, alignment(dsa_st) -class evp_pkey_st(Structure): - pass -class N11evp_pkey_st4DOLLAR_12E(Union): - pass -class rsa_st(Structure): - pass -N11evp_pkey_st4DOLLAR_12E._fields_ = [ - ('ptr', STRING), - ('rsa', POINTER(rsa_st)), - ('dsa', POINTER(dsa_st)), - ('dh', POINTER(dh_st)), -] -assert sizeof(N11evp_pkey_st4DOLLAR_12E) == 4, sizeof(N11evp_pkey_st4DOLLAR_12E) -assert alignment(N11evp_pkey_st4DOLLAR_12E) == 4, alignment(N11evp_pkey_st4DOLLAR_12E) -evp_pkey_st._fields_ = [ - ('type', c_int), - ('save_type', c_int), - ('references', c_int), - ('pkey', N11evp_pkey_st4DOLLAR_12E), - ('save_parameters', c_int), - ('attributes', POINTER(STACK)), -] -assert sizeof(evp_pkey_st) == 24, sizeof(evp_pkey_st) -assert alignment(evp_pkey_st) == 4, alignment(evp_pkey_st) -class env_md_st(Structure): - pass -class env_md_ctx_st(Structure): - pass -EVP_MD_CTX = env_md_ctx_st -env_md_st._fields_ = [ - ('type', c_int), - ('pkey_type', c_int), - ('md_size', c_int), - ('flags', c_ulong), - ('init', CFUNCTYPE(c_int, POINTER(EVP_MD_CTX))), - ('update', CFUNCTYPE(c_int, POINTER(EVP_MD_CTX), c_void_p, c_ulong)), - ('final', CFUNCTYPE(c_int, POINTER(EVP_MD_CTX), POINTER(c_ubyte))), - ('copy', CFUNCTYPE(c_int, POINTER(EVP_MD_CTX), POINTER(EVP_MD_CTX))), - ('cleanup', CFUNCTYPE(c_int, POINTER(EVP_MD_CTX))), - ('sign', CFUNCTYPE(c_int)), - ('verify', CFUNCTYPE(c_int)), - ('required_pkey_type', c_int * 5), - ('block_size', c_int), - ('ctx_size', c_int), -] -assert sizeof(env_md_st) == 72, sizeof(env_md_st) -assert alignment(env_md_st) == 4, alignment(env_md_st) -EVP_MD = env_md_st -env_md_ctx_st._fields_ = [ - ('digest', POINTER(EVP_MD)), - ('engine', POINTER(ENGINE)), - ('flags', c_ulong), - ('md_data', c_void_p), -] -assert sizeof(env_md_ctx_st) == 16, sizeof(env_md_ctx_st) -assert alignment(env_md_ctx_st) == 4, alignment(env_md_ctx_st) -class evp_cipher_st(Structure): - pass -class evp_cipher_ctx_st(Structure): - pass -EVP_CIPHER_CTX = evp_cipher_ctx_st -evp_cipher_st._fields_ = [ - ('nid', c_int), - ('block_size', c_int), - ('key_len', c_int), - ('iv_len', c_int), - ('flags', c_ulong), - ('init', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), POINTER(c_ubyte), POINTER(c_ubyte), c_int)), - ('do_cipher', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), POINTER(c_ubyte), POINTER(c_ubyte), c_uint)), - ('cleanup', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX))), - ('ctx_size', c_int), - ('set_asn1_parameters', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), POINTER(ASN1_TYPE))), - ('get_asn1_parameters', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), POINTER(ASN1_TYPE))), - ('ctrl', CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), c_int, c_int, c_void_p)), - ('app_data', c_void_p), -] -assert sizeof(evp_cipher_st) == 52, sizeof(evp_cipher_st) -assert alignment(evp_cipher_st) == 4, alignment(evp_cipher_st) -class evp_cipher_info_st(Structure): - pass -EVP_CIPHER = evp_cipher_st -evp_cipher_info_st._fields_ = [ - ('cipher', POINTER(EVP_CIPHER)), - ('iv', c_ubyte * 16), -] -assert sizeof(evp_cipher_info_st) == 20, sizeof(evp_cipher_info_st) -assert alignment(evp_cipher_info_st) == 4, alignment(evp_cipher_info_st) -EVP_CIPHER_INFO = evp_cipher_info_st -evp_cipher_ctx_st._fields_ = [ - ('cipher', POINTER(EVP_CIPHER)), - ('engine', POINTER(ENGINE)), - ('encrypt', c_int), - ('buf_len', c_int), - ('oiv', c_ubyte * 16), - ('iv', c_ubyte * 16), - ('buf', c_ubyte * 32), - ('num', c_int), - ('app_data', c_void_p), - ('key_len', c_int), - ('flags', c_ulong), - ('cipher_data', c_void_p), - ('final_used', c_int), - ('block_mask', c_int), - ('final', c_ubyte * 32), -] -assert sizeof(evp_cipher_ctx_st) == 140, sizeof(evp_cipher_ctx_st) -assert alignment(evp_cipher_ctx_st) == 4, alignment(evp_cipher_ctx_st) -class evp_Encode_Ctx_st(Structure): - pass -evp_Encode_Ctx_st._fields_ = [ - ('num', c_int), - ('length', c_int), - ('enc_data', c_ubyte * 80), - ('line_num', c_int), - ('expect_nl', c_int), -] -assert sizeof(evp_Encode_Ctx_st) == 96, sizeof(evp_Encode_Ctx_st) -assert alignment(evp_Encode_Ctx_st) == 4, alignment(evp_Encode_Ctx_st) -EVP_ENCODE_CTX = evp_Encode_Ctx_st -EVP_PBE_KEYGEN = CFUNCTYPE(c_int, POINTER(EVP_CIPHER_CTX), STRING, c_int, POINTER(ASN1_TYPE), POINTER(EVP_CIPHER), POINTER(EVP_MD), c_int) -class lhash_node_st(Structure): - pass -lhash_node_st._fields_ = [ - ('data', c_void_p), - ('next', POINTER(lhash_node_st)), - ('hash', c_ulong), -] -assert sizeof(lhash_node_st) == 12, sizeof(lhash_node_st) -assert alignment(lhash_node_st) == 4, alignment(lhash_node_st) -LHASH_NODE = lhash_node_st -LHASH_COMP_FN_TYPE = CFUNCTYPE(c_int, c_void_p, c_void_p) -LHASH_HASH_FN_TYPE = CFUNCTYPE(c_ulong, c_void_p) -LHASH_DOALL_FN_TYPE = CFUNCTYPE(None, c_void_p) -LHASH_DOALL_ARG_FN_TYPE = CFUNCTYPE(None, c_void_p, c_void_p) -class lhash_st(Structure): - pass -lhash_st._fields_ = [ - ('b', POINTER(POINTER(LHASH_NODE))), - ('comp', LHASH_COMP_FN_TYPE), - ('hash', LHASH_HASH_FN_TYPE), - ('num_nodes', c_uint), - ('num_alloc_nodes', c_uint), - ('p', c_uint), - ('pmax', c_uint), - ('up_load', c_ulong), - ('down_load', c_ulong), - ('num_items', c_ulong), - ('num_expands', c_ulong), - ('num_expand_reallocs', c_ulong), - ('num_contracts', c_ulong), - ('num_contract_reallocs', c_ulong), - ('num_hash_calls', c_ulong), - ('num_comp_calls', c_ulong), - ('num_insert', c_ulong), - ('num_replace', c_ulong), - ('num_delete', c_ulong), - ('num_no_delete', c_ulong), - ('num_retrieve', c_ulong), - ('num_retrieve_miss', c_ulong), - ('num_hash_comps', c_ulong), - ('error', c_int), -] -assert sizeof(lhash_st) == 96, sizeof(lhash_st) -assert alignment(lhash_st) == 4, alignment(lhash_st) -LHASH = lhash_st -class MD2state_st(Structure): - pass -MD2state_st._fields_ = [ - ('num', c_int), - ('data', c_ubyte * 16), - ('cksm', c_uint * 16), - ('state', c_uint * 16), -] -assert sizeof(MD2state_st) == 148, sizeof(MD2state_st) -assert alignment(MD2state_st) == 4, alignment(MD2state_st) -MD2_CTX = MD2state_st -class MD4state_st(Structure): - pass -MD4state_st._fields_ = [ - ('A', c_uint), - ('B', c_uint), - ('C', c_uint), - ('D', c_uint), - ('Nl', c_uint), - ('Nh', c_uint), - ('data', c_uint * 16), - ('num', c_int), -] -assert sizeof(MD4state_st) == 92, sizeof(MD4state_st) -assert alignment(MD4state_st) == 4, alignment(MD4state_st) -MD4_CTX = MD4state_st -class MD5state_st(Structure): - pass -MD5state_st._fields_ = [ - ('A', c_uint), - ('B', c_uint), - ('C', c_uint), - ('D', c_uint), - ('Nl', c_uint), - ('Nh', c_uint), - ('data', c_uint * 16), - ('num', c_int), -] -assert sizeof(MD5state_st) == 92, sizeof(MD5state_st) -assert alignment(MD5state_st) == 4, alignment(MD5state_st) -MD5_CTX = MD5state_st -class mdc2_ctx_st(Structure): - pass -mdc2_ctx_st._fields_ = [ - ('num', c_int), - ('data', c_ubyte * 8), - ('h', DES_cblock), - ('hh', DES_cblock), - ('pad_type', c_int), -] -assert sizeof(mdc2_ctx_st) == 32, sizeof(mdc2_ctx_st) -assert alignment(mdc2_ctx_st) == 4, alignment(mdc2_ctx_st) -MDC2_CTX = mdc2_ctx_st -class obj_name_st(Structure): - pass -obj_name_st._fields_ = [ - ('type', c_int), - ('alias', c_int), - ('name', STRING), - ('data', STRING), -] -assert sizeof(obj_name_st) == 16, sizeof(obj_name_st) -assert alignment(obj_name_st) == 4, alignment(obj_name_st) -OBJ_NAME = obj_name_st -ASN1_TIME = asn1_string_st -ASN1_NULL = c_int -EVP_PKEY = evp_pkey_st -class x509_st(Structure): - pass -X509 = x509_st -class X509_algor_st(Structure): - pass -X509_ALGOR = X509_algor_st -class X509_crl_st(Structure): - pass -X509_CRL = X509_crl_st -class X509_name_st(Structure): - pass -X509_NAME = X509_name_st -class x509_store_st(Structure): - pass -X509_STORE = x509_store_st -class x509_store_ctx_st(Structure): - pass -X509_STORE_CTX = x509_store_ctx_st -engine_st._fields_ = [ -] -class PEM_Encode_Seal_st(Structure): - pass -PEM_Encode_Seal_st._fields_ = [ - ('encode', EVP_ENCODE_CTX), - ('md', EVP_MD_CTX), - ('cipher', EVP_CIPHER_CTX), -] -assert sizeof(PEM_Encode_Seal_st) == 252, sizeof(PEM_Encode_Seal_st) -assert alignment(PEM_Encode_Seal_st) == 4, alignment(PEM_Encode_Seal_st) -PEM_ENCODE_SEAL_CTX = PEM_Encode_Seal_st -class pem_recip_st(Structure): - pass -pem_recip_st._fields_ = [ - ('name', STRING), - ('dn', POINTER(X509_NAME)), - ('cipher', c_int), - ('key_enc', c_int), -] -assert sizeof(pem_recip_st) == 16, sizeof(pem_recip_st) -assert alignment(pem_recip_st) == 4, alignment(pem_recip_st) -PEM_USER = pem_recip_st -class pem_ctx_st(Structure): - pass -class N10pem_ctx_st4DOLLAR_16E(Structure): - pass -N10pem_ctx_st4DOLLAR_16E._fields_ = [ - ('version', c_int), - ('mode', c_int), -] -assert sizeof(N10pem_ctx_st4DOLLAR_16E) == 8, sizeof(N10pem_ctx_st4DOLLAR_16E) -assert alignment(N10pem_ctx_st4DOLLAR_16E) == 4, alignment(N10pem_ctx_st4DOLLAR_16E) -class N10pem_ctx_st4DOLLAR_17E(Structure): - pass -N10pem_ctx_st4DOLLAR_17E._fields_ = [ - ('cipher', c_int), -] -assert sizeof(N10pem_ctx_st4DOLLAR_17E) == 4, sizeof(N10pem_ctx_st4DOLLAR_17E) -assert alignment(N10pem_ctx_st4DOLLAR_17E) == 4, alignment(N10pem_ctx_st4DOLLAR_17E) -pem_ctx_st._fields_ = [ - ('type', c_int), - ('proc_type', N10pem_ctx_st4DOLLAR_16E), - ('domain', STRING), - ('DEK_info', N10pem_ctx_st4DOLLAR_17E), - ('originator', POINTER(PEM_USER)), - ('num_recipient', c_int), - ('recipient', POINTER(POINTER(PEM_USER))), - ('x509_chain', POINTER(STACK)), - ('md', POINTER(EVP_MD)), - ('md_enc', c_int), - ('md_len', c_int), - ('md_data', STRING), - ('dec', POINTER(EVP_CIPHER)), - ('key_len', c_int), - ('key', POINTER(c_ubyte)), - ('data_enc', c_int), - ('data_len', c_int), - ('data', POINTER(c_ubyte)), -] -assert sizeof(pem_ctx_st) == 76, sizeof(pem_ctx_st) -assert alignment(pem_ctx_st) == 4, alignment(pem_ctx_st) -PEM_CTX = pem_ctx_st -pem_password_cb = CFUNCTYPE(c_int, STRING, c_int, c_int, c_void_p) -class pkcs7_issuer_and_serial_st(Structure): - pass -pkcs7_issuer_and_serial_st._fields_ = [ - ('issuer', POINTER(X509_NAME)), - ('serial', POINTER(ASN1_INTEGER)), -] -assert sizeof(pkcs7_issuer_and_serial_st) == 8, sizeof(pkcs7_issuer_and_serial_st) -assert alignment(pkcs7_issuer_and_serial_st) == 4, alignment(pkcs7_issuer_and_serial_st) -PKCS7_ISSUER_AND_SERIAL = pkcs7_issuer_and_serial_st -class pkcs7_signer_info_st(Structure): - pass -pkcs7_signer_info_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('issuer_and_serial', POINTER(PKCS7_ISSUER_AND_SERIAL)), - ('digest_alg', POINTER(X509_ALGOR)), - ('auth_attr', POINTER(STACK)), - ('digest_enc_alg', POINTER(X509_ALGOR)), - ('enc_digest', POINTER(ASN1_OCTET_STRING)), - ('unauth_attr', POINTER(STACK)), - ('pkey', POINTER(EVP_PKEY)), -] -assert sizeof(pkcs7_signer_info_st) == 32, sizeof(pkcs7_signer_info_st) -assert alignment(pkcs7_signer_info_st) == 4, alignment(pkcs7_signer_info_st) -PKCS7_SIGNER_INFO = pkcs7_signer_info_st -class pkcs7_recip_info_st(Structure): - pass -pkcs7_recip_info_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('issuer_and_serial', POINTER(PKCS7_ISSUER_AND_SERIAL)), - ('key_enc_algor', POINTER(X509_ALGOR)), - ('enc_key', POINTER(ASN1_OCTET_STRING)), - ('cert', POINTER(X509)), -] -assert sizeof(pkcs7_recip_info_st) == 20, sizeof(pkcs7_recip_info_st) -assert alignment(pkcs7_recip_info_st) == 4, alignment(pkcs7_recip_info_st) -PKCS7_RECIP_INFO = pkcs7_recip_info_st -class pkcs7_signed_st(Structure): - pass -class pkcs7_st(Structure): - pass -pkcs7_signed_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('md_algs', POINTER(STACK)), - ('cert', POINTER(STACK)), - ('crl', POINTER(STACK)), - ('signer_info', POINTER(STACK)), - ('contents', POINTER(pkcs7_st)), -] -assert sizeof(pkcs7_signed_st) == 24, sizeof(pkcs7_signed_st) -assert alignment(pkcs7_signed_st) == 4, alignment(pkcs7_signed_st) -PKCS7_SIGNED = pkcs7_signed_st -class pkcs7_enc_content_st(Structure): - pass -pkcs7_enc_content_st._fields_ = [ - ('content_type', POINTER(ASN1_OBJECT)), - ('algorithm', POINTER(X509_ALGOR)), - ('enc_data', POINTER(ASN1_OCTET_STRING)), - ('cipher', POINTER(EVP_CIPHER)), -] -assert sizeof(pkcs7_enc_content_st) == 16, sizeof(pkcs7_enc_content_st) -assert alignment(pkcs7_enc_content_st) == 4, alignment(pkcs7_enc_content_st) -PKCS7_ENC_CONTENT = pkcs7_enc_content_st -class pkcs7_enveloped_st(Structure): - pass -pkcs7_enveloped_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('recipientinfo', POINTER(STACK)), - ('enc_data', POINTER(PKCS7_ENC_CONTENT)), -] -assert sizeof(pkcs7_enveloped_st) == 12, sizeof(pkcs7_enveloped_st) -assert alignment(pkcs7_enveloped_st) == 4, alignment(pkcs7_enveloped_st) -PKCS7_ENVELOPE = pkcs7_enveloped_st -class pkcs7_signedandenveloped_st(Structure): - pass -pkcs7_signedandenveloped_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('md_algs', POINTER(STACK)), - ('cert', POINTER(STACK)), - ('crl', POINTER(STACK)), - ('signer_info', POINTER(STACK)), - ('enc_data', POINTER(PKCS7_ENC_CONTENT)), - ('recipientinfo', POINTER(STACK)), -] -assert sizeof(pkcs7_signedandenveloped_st) == 28, sizeof(pkcs7_signedandenveloped_st) -assert alignment(pkcs7_signedandenveloped_st) == 4, alignment(pkcs7_signedandenveloped_st) -PKCS7_SIGN_ENVELOPE = pkcs7_signedandenveloped_st -class pkcs7_digest_st(Structure): - pass -pkcs7_digest_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('md', POINTER(X509_ALGOR)), - ('contents', POINTER(pkcs7_st)), - ('digest', POINTER(ASN1_OCTET_STRING)), -] -assert sizeof(pkcs7_digest_st) == 16, sizeof(pkcs7_digest_st) -assert alignment(pkcs7_digest_st) == 4, alignment(pkcs7_digest_st) -PKCS7_DIGEST = pkcs7_digest_st -class pkcs7_encrypted_st(Structure): - pass -pkcs7_encrypted_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('enc_data', POINTER(PKCS7_ENC_CONTENT)), -] -assert sizeof(pkcs7_encrypted_st) == 8, sizeof(pkcs7_encrypted_st) -assert alignment(pkcs7_encrypted_st) == 4, alignment(pkcs7_encrypted_st) -PKCS7_ENCRYPT = pkcs7_encrypted_st -class N8pkcs7_st4DOLLAR_15E(Union): - pass -N8pkcs7_st4DOLLAR_15E._fields_ = [ - ('ptr', STRING), - ('data', POINTER(ASN1_OCTET_STRING)), - ('sign', POINTER(PKCS7_SIGNED)), - ('enveloped', POINTER(PKCS7_ENVELOPE)), - ('signed_and_enveloped', POINTER(PKCS7_SIGN_ENVELOPE)), - ('digest', POINTER(PKCS7_DIGEST)), - ('encrypted', POINTER(PKCS7_ENCRYPT)), - ('other', POINTER(ASN1_TYPE)), -] -assert sizeof(N8pkcs7_st4DOLLAR_15E) == 4, sizeof(N8pkcs7_st4DOLLAR_15E) -assert alignment(N8pkcs7_st4DOLLAR_15E) == 4, alignment(N8pkcs7_st4DOLLAR_15E) -pkcs7_st._fields_ = [ - ('asn1', POINTER(c_ubyte)), - ('length', c_long), - ('state', c_int), - ('detached', c_int), - ('type', POINTER(ASN1_OBJECT)), - ('d', N8pkcs7_st4DOLLAR_15E), -] -assert sizeof(pkcs7_st) == 24, sizeof(pkcs7_st) -assert alignment(pkcs7_st) == 4, alignment(pkcs7_st) -PKCS7 = pkcs7_st -class rc2_key_st(Structure): - pass -rc2_key_st._fields_ = [ - ('data', c_uint * 64), -] -assert sizeof(rc2_key_st) == 256, sizeof(rc2_key_st) -assert alignment(rc2_key_st) == 4, alignment(rc2_key_st) -RC2_KEY = rc2_key_st -class rc4_key_st(Structure): - pass -rc4_key_st._fields_ = [ - ('x', c_ubyte), - ('y', c_ubyte), - ('data', c_ubyte * 256), -] -assert sizeof(rc4_key_st) == 258, sizeof(rc4_key_st) -assert alignment(rc4_key_st) == 1, alignment(rc4_key_st) -RC4_KEY = rc4_key_st -class rc5_key_st(Structure): - pass -rc5_key_st._fields_ = [ - ('rounds', c_int), - ('data', c_ulong * 34), -] -assert sizeof(rc5_key_st) == 140, sizeof(rc5_key_st) -assert alignment(rc5_key_st) == 4, alignment(rc5_key_st) -RC5_32_KEY = rc5_key_st -class RIPEMD160state_st(Structure): - pass -RIPEMD160state_st._fields_ = [ - ('A', c_uint), - ('B', c_uint), - ('C', c_uint), - ('D', c_uint), - ('E', c_uint), - ('Nl', c_uint), - ('Nh', c_uint), - ('data', c_uint * 16), - ('num', c_int), -] -assert sizeof(RIPEMD160state_st) == 96, sizeof(RIPEMD160state_st) -assert alignment(RIPEMD160state_st) == 4, alignment(RIPEMD160state_st) -RIPEMD160_CTX = RIPEMD160state_st -RSA = rsa_st -class rsa_meth_st(Structure): - pass -rsa_meth_st._fields_ = [ - ('name', STRING), - ('rsa_pub_enc', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), POINTER(c_ubyte), POINTER(RSA), c_int)), - ('rsa_pub_dec', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), POINTER(c_ubyte), POINTER(RSA), c_int)), - ('rsa_priv_enc', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), POINTER(c_ubyte), POINTER(RSA), c_int)), - ('rsa_priv_dec', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), POINTER(c_ubyte), POINTER(RSA), c_int)), - ('rsa_mod_exp', CFUNCTYPE(c_int, POINTER(BIGNUM), POINTER(BIGNUM), POINTER(RSA))), - ('bn_mod_exp', CFUNCTYPE(c_int, POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BIGNUM), POINTER(BN_CTX), POINTER(BN_MONT_CTX))), - ('init', CFUNCTYPE(c_int, POINTER(RSA))), - ('finish', CFUNCTYPE(c_int, POINTER(RSA))), - ('flags', c_int), - ('app_data', STRING), - ('rsa_sign', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), c_uint, POINTER(c_ubyte), POINTER(c_uint), POINTER(RSA))), - ('rsa_verify', CFUNCTYPE(c_int, c_int, POINTER(c_ubyte), c_uint, POINTER(c_ubyte), c_uint, POINTER(RSA))), -] -assert sizeof(rsa_meth_st) == 52, sizeof(rsa_meth_st) -assert alignment(rsa_meth_st) == 4, alignment(rsa_meth_st) -RSA_METHOD = rsa_meth_st -rsa_st._fields_ = [ - ('pad', c_int), - ('version', c_long), - ('meth', POINTER(RSA_METHOD)), - ('engine', POINTER(ENGINE)), - ('n', POINTER(BIGNUM)), - ('e', POINTER(BIGNUM)), - ('d', POINTER(BIGNUM)), - ('p', POINTER(BIGNUM)), - ('q', POINTER(BIGNUM)), - ('dmp1', POINTER(BIGNUM)), - ('dmq1', POINTER(BIGNUM)), - ('iqmp', POINTER(BIGNUM)), - ('ex_data', CRYPTO_EX_DATA), - ('references', c_int), - ('flags', c_int), - ('_method_mod_n', POINTER(BN_MONT_CTX)), - ('_method_mod_p', POINTER(BN_MONT_CTX)), - ('_method_mod_q', POINTER(BN_MONT_CTX)), - ('bignum_data', STRING), - ('blinding', POINTER(BN_BLINDING)), -] -assert sizeof(rsa_st) == 84, sizeof(rsa_st) -assert alignment(rsa_st) == 4, alignment(rsa_st) -openssl_fptr = CFUNCTYPE(None) -class SHAstate_st(Structure): - pass -SHAstate_st._fields_ = [ - ('h0', c_uint), - ('h1', c_uint), - ('h2', c_uint), - ('h3', c_uint), - ('h4', c_uint), - ('Nl', c_uint), - ('Nh', c_uint), - ('data', c_uint * 16), - ('num', c_int), -] -assert sizeof(SHAstate_st) == 96, sizeof(SHAstate_st) -assert alignment(SHAstate_st) == 4, alignment(SHAstate_st) -SHA_CTX = SHAstate_st -class ssl_st(Structure): - pass -ssl_crock_st = POINTER(ssl_st) -class ssl_cipher_st(Structure): - pass -ssl_cipher_st._fields_ = [ - ('valid', c_int), - ('name', STRING), - ('id', c_ulong), - ('algorithms', c_ulong), - ('algo_strength', c_ulong), - ('algorithm2', c_ulong), - ('strength_bits', c_int), - ('alg_bits', c_int), - ('mask', c_ulong), - ('mask_strength', c_ulong), -] -assert sizeof(ssl_cipher_st) == 40, sizeof(ssl_cipher_st) -assert alignment(ssl_cipher_st) == 4, alignment(ssl_cipher_st) -SSL_CIPHER = ssl_cipher_st -SSL = ssl_st -class ssl_ctx_st(Structure): - pass -SSL_CTX = ssl_ctx_st -class ssl_method_st(Structure): - pass -class ssl3_enc_method(Structure): - pass -ssl_method_st._fields_ = [ - ('version', c_int), - ('ssl_new', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_clear', CFUNCTYPE(None, POINTER(SSL))), - ('ssl_free', CFUNCTYPE(None, POINTER(SSL))), - ('ssl_accept', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_connect', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_read', CFUNCTYPE(c_int, POINTER(SSL), c_void_p, c_int)), - ('ssl_peek', CFUNCTYPE(c_int, POINTER(SSL), c_void_p, c_int)), - ('ssl_write', CFUNCTYPE(c_int, POINTER(SSL), c_void_p, c_int)), - ('ssl_shutdown', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_renegotiate', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_renegotiate_check', CFUNCTYPE(c_int, POINTER(SSL))), - ('ssl_ctrl', CFUNCTYPE(c_long, POINTER(SSL), c_int, c_long, c_void_p)), - ('ssl_ctx_ctrl', CFUNCTYPE(c_long, POINTER(SSL_CTX), c_int, c_long, c_void_p)), - ('get_cipher_by_char', CFUNCTYPE(POINTER(SSL_CIPHER), POINTER(c_ubyte))), - ('put_cipher_by_char', CFUNCTYPE(c_int, POINTER(SSL_CIPHER), POINTER(c_ubyte))), - ('ssl_pending', CFUNCTYPE(c_int, POINTER(SSL))), - ('num_ciphers', CFUNCTYPE(c_int)), - ('get_cipher', CFUNCTYPE(POINTER(SSL_CIPHER), c_uint)), - ('get_ssl_method', CFUNCTYPE(POINTER(ssl_method_st), c_int)), - ('get_timeout', CFUNCTYPE(c_long)), - ('ssl3_enc', POINTER(ssl3_enc_method)), - ('ssl_version', CFUNCTYPE(c_int)), - ('ssl_callback_ctrl', CFUNCTYPE(c_long, POINTER(SSL), c_int, CFUNCTYPE(None))), - ('ssl_ctx_callback_ctrl', CFUNCTYPE(c_long, POINTER(SSL_CTX), c_int, CFUNCTYPE(None))), -] -assert sizeof(ssl_method_st) == 100, sizeof(ssl_method_st) -assert alignment(ssl_method_st) == 4, alignment(ssl_method_st) -ssl3_enc_method._fields_ = [ -] -SSL_METHOD = ssl_method_st -class ssl_session_st(Structure): - pass -class sess_cert_st(Structure): - pass -ssl_session_st._fields_ = [ - ('ssl_version', c_int), - ('key_arg_length', c_uint), - ('key_arg', c_ubyte * 8), - ('master_key_length', c_int), - ('master_key', c_ubyte * 48), - ('session_id_length', c_uint), - ('session_id', c_ubyte * 32), - ('sid_ctx_length', c_uint), - ('sid_ctx', c_ubyte * 32), - ('not_resumable', c_int), - ('sess_cert', POINTER(sess_cert_st)), - ('peer', POINTER(X509)), - ('verify_result', c_long), - ('references', c_int), - ('timeout', c_long), - ('time', c_long), - ('compress_meth', c_int), - ('cipher', POINTER(SSL_CIPHER)), - ('cipher_id', c_ulong), - ('ciphers', POINTER(STACK)), - ('ex_data', CRYPTO_EX_DATA), - ('prev', POINTER(ssl_session_st)), - ('next', POINTER(ssl_session_st)), -] -assert sizeof(ssl_session_st) == 200, sizeof(ssl_session_st) -assert alignment(ssl_session_st) == 4, alignment(ssl_session_st) -sess_cert_st._fields_ = [ -] -SSL_SESSION = ssl_session_st -GEN_SESSION_CB = CFUNCTYPE(c_int, POINTER(SSL), POINTER(c_ubyte), POINTER(c_uint)) -class ssl_comp_st(Structure): - pass -ssl_comp_st._fields_ = [ - ('id', c_int), - ('name', STRING), - ('method', POINTER(COMP_METHOD)), -] -assert sizeof(ssl_comp_st) == 12, sizeof(ssl_comp_st) -assert alignment(ssl_comp_st) == 4, alignment(ssl_comp_st) -SSL_COMP = ssl_comp_st -class N10ssl_ctx_st4DOLLAR_18E(Structure): - pass -N10ssl_ctx_st4DOLLAR_18E._fields_ = [ - ('sess_connect', c_int), - ('sess_connect_renegotiate', c_int), - ('sess_connect_good', c_int), - ('sess_accept', c_int), - ('sess_accept_renegotiate', c_int), - ('sess_accept_good', c_int), - ('sess_miss', c_int), - ('sess_timeout', c_int), - ('sess_cache_full', c_int), - ('sess_hit', c_int), - ('sess_cb_hit', c_int), -] -assert sizeof(N10ssl_ctx_st4DOLLAR_18E) == 44, sizeof(N10ssl_ctx_st4DOLLAR_18E) -assert alignment(N10ssl_ctx_st4DOLLAR_18E) == 4, alignment(N10ssl_ctx_st4DOLLAR_18E) -class cert_st(Structure): - pass -ssl_ctx_st._fields_ = [ - ('method', POINTER(SSL_METHOD)), - ('cipher_list', POINTER(STACK)), - ('cipher_list_by_id', POINTER(STACK)), - ('cert_store', POINTER(x509_store_st)), - ('sessions', POINTER(lhash_st)), - ('session_cache_size', c_ulong), - ('session_cache_head', POINTER(ssl_session_st)), - ('session_cache_tail', POINTER(ssl_session_st)), - ('session_cache_mode', c_int), - ('session_timeout', c_long), - ('new_session_cb', CFUNCTYPE(c_int, POINTER(ssl_st), POINTER(SSL_SESSION))), - ('remove_session_cb', CFUNCTYPE(None, POINTER(ssl_ctx_st), POINTER(SSL_SESSION))), - ('get_session_cb', CFUNCTYPE(POINTER(SSL_SESSION), POINTER(ssl_st), POINTER(c_ubyte), c_int, POINTER(c_int))), - ('stats', N10ssl_ctx_st4DOLLAR_18E), - ('references', c_int), - ('app_verify_callback', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), c_void_p)), - ('app_verify_arg', c_void_p), - ('default_passwd_callback', POINTER(pem_password_cb)), - ('default_passwd_callback_userdata', c_void_p), - ('client_cert_cb', CFUNCTYPE(c_int, POINTER(SSL), POINTER(POINTER(X509)), POINTER(POINTER(EVP_PKEY)))), - ('ex_data', CRYPTO_EX_DATA), - ('rsa_md5', POINTER(EVP_MD)), - ('md5', POINTER(EVP_MD)), - ('sha1', POINTER(EVP_MD)), - ('extra_certs', POINTER(STACK)), - ('comp_methods', POINTER(STACK)), - ('info_callback', CFUNCTYPE(None, POINTER(SSL), c_int, c_int)), - ('client_CA', POINTER(STACK)), - ('options', c_ulong), - ('mode', c_ulong), - ('max_cert_list', c_long), - ('cert', POINTER(cert_st)), - ('read_ahead', c_int), - ('msg_callback', CFUNCTYPE(None, c_int, c_int, c_int, c_void_p, c_ulong, POINTER(SSL), c_void_p)), - ('msg_callback_arg', c_void_p), - ('verify_mode', c_int), - ('verify_depth', c_int), - ('sid_ctx_length', c_uint), - ('sid_ctx', c_ubyte * 32), - ('default_verify_callback', CFUNCTYPE(c_int, c_int, POINTER(X509_STORE_CTX))), - ('generate_session_id', GEN_SESSION_CB), - ('purpose', c_int), - ('trust', c_int), - ('quiet_shutdown', c_int), -] -assert sizeof(ssl_ctx_st) == 248, sizeof(ssl_ctx_st) -assert alignment(ssl_ctx_st) == 4, alignment(ssl_ctx_st) -cert_st._fields_ = [ -] -class ssl2_state_st(Structure): - pass -class ssl3_state_st(Structure): - pass -ssl_st._fields_ = [ - ('version', c_int), - ('type', c_int), - ('method', POINTER(SSL_METHOD)), - ('rbio', POINTER(BIO)), - ('wbio', POINTER(BIO)), - ('bbio', POINTER(BIO)), - ('rwstate', c_int), - ('in_handshake', c_int), - ('handshake_func', CFUNCTYPE(c_int)), - ('server', c_int), - ('new_session', c_int), - ('quiet_shutdown', c_int), - ('shutdown', c_int), - ('state', c_int), - ('rstate', c_int), - ('init_buf', POINTER(BUF_MEM)), - ('init_msg', c_void_p), - ('init_num', c_int), - ('init_off', c_int), - ('packet', POINTER(c_ubyte)), - ('packet_length', c_uint), - ('s2', POINTER(ssl2_state_st)), - ('s3', POINTER(ssl3_state_st)), - ('read_ahead', c_int), - ('msg_callback', CFUNCTYPE(None, c_int, c_int, c_int, c_void_p, c_ulong, POINTER(SSL), c_void_p)), - ('msg_callback_arg', c_void_p), - ('hit', c_int), - ('purpose', c_int), - ('trust', c_int), - ('cipher_list', POINTER(STACK)), - ('cipher_list_by_id', POINTER(STACK)), - ('enc_read_ctx', POINTER(EVP_CIPHER_CTX)), - ('read_hash', POINTER(EVP_MD)), - ('expand', POINTER(COMP_CTX)), - ('enc_write_ctx', POINTER(EVP_CIPHER_CTX)), - ('write_hash', POINTER(EVP_MD)), - ('compress', POINTER(COMP_CTX)), - ('cert', POINTER(cert_st)), - ('sid_ctx_length', c_uint), - ('sid_ctx', c_ubyte * 32), - ('session', POINTER(SSL_SESSION)), - ('generate_session_id', GEN_SESSION_CB), - ('verify_mode', c_int), - ('verify_depth', c_int), - ('verify_callback', CFUNCTYPE(c_int, c_int, POINTER(X509_STORE_CTX))), - ('info_callback', CFUNCTYPE(None, POINTER(SSL), c_int, c_int)), - ('error', c_int), - ('error_code', c_int), - ('ctx', POINTER(SSL_CTX)), - ('debug', c_int), - ('verify_result', c_long), - ('ex_data', CRYPTO_EX_DATA), - ('client_CA', POINTER(STACK)), - ('references', c_int), - ('options', c_ulong), - ('mode', c_ulong), - ('max_cert_list', c_long), - ('first_packet', c_int), - ('client_version', c_int), -] -assert sizeof(ssl_st) == 268, sizeof(ssl_st) -assert alignment(ssl_st) == 4, alignment(ssl_st) -class N13ssl2_state_st4DOLLAR_19E(Structure): - pass -N13ssl2_state_st4DOLLAR_19E._fields_ = [ - ('conn_id_length', c_uint), - ('cert_type', c_uint), - ('cert_length', c_uint), - ('csl', c_uint), - ('clear', c_uint), - ('enc', c_uint), - ('ccl', c_ubyte * 32), - ('cipher_spec_length', c_uint), - ('session_id_length', c_uint), - ('clen', c_uint), - ('rlen', c_uint), -] -assert sizeof(N13ssl2_state_st4DOLLAR_19E) == 72, sizeof(N13ssl2_state_st4DOLLAR_19E) -assert alignment(N13ssl2_state_st4DOLLAR_19E) == 4, alignment(N13ssl2_state_st4DOLLAR_19E) -ssl2_state_st._fields_ = [ - ('three_byte_header', c_int), - ('clear_text', c_int), - ('escape', c_int), - ('ssl2_rollback', c_int), - ('wnum', c_uint), - ('wpend_tot', c_int), - ('wpend_buf', POINTER(c_ubyte)), - ('wpend_off', c_int), - ('wpend_len', c_int), - ('wpend_ret', c_int), - ('rbuf_left', c_int), - ('rbuf_offs', c_int), - ('rbuf', POINTER(c_ubyte)), - ('wbuf', POINTER(c_ubyte)), - ('write_ptr', POINTER(c_ubyte)), - ('padding', c_uint), - ('rlength', c_uint), - ('ract_data_length', c_int), - ('wlength', c_uint), - ('wact_data_length', c_int), - ('ract_data', POINTER(c_ubyte)), - ('wact_data', POINTER(c_ubyte)), - ('mac_data', POINTER(c_ubyte)), - ('read_key', POINTER(c_ubyte)), - ('write_key', POINTER(c_ubyte)), - ('challenge_length', c_uint), - ('challenge', c_ubyte * 32), - ('conn_id_length', c_uint), - ('conn_id', c_ubyte * 16), - ('key_material_length', c_uint), - ('key_material', c_ubyte * 48), - ('read_sequence', c_ulong), - ('write_sequence', c_ulong), - ('tmp', N13ssl2_state_st4DOLLAR_19E), -] -assert sizeof(ssl2_state_st) == 288, sizeof(ssl2_state_st) -assert alignment(ssl2_state_st) == 4, alignment(ssl2_state_st) -SSL2_STATE = ssl2_state_st -class ssl3_record_st(Structure): - pass -ssl3_record_st._fields_ = [ - ('type', c_int), - ('length', c_uint), - ('off', c_uint), - ('data', POINTER(c_ubyte)), - ('input', POINTER(c_ubyte)), - ('comp', POINTER(c_ubyte)), -] -assert sizeof(ssl3_record_st) == 24, sizeof(ssl3_record_st) -assert alignment(ssl3_record_st) == 4, alignment(ssl3_record_st) -SSL3_RECORD = ssl3_record_st -class ssl3_buffer_st(Structure): - pass -size_t = __darwin_size_t -ssl3_buffer_st._fields_ = [ - ('buf', POINTER(c_ubyte)), - ('len', size_t), - ('offset', c_int), - ('left', c_int), -] -assert sizeof(ssl3_buffer_st) == 16, sizeof(ssl3_buffer_st) -assert alignment(ssl3_buffer_st) == 4, alignment(ssl3_buffer_st) -SSL3_BUFFER = ssl3_buffer_st -class N13ssl3_state_st4DOLLAR_20E(Structure): - pass -N13ssl3_state_st4DOLLAR_20E._fields_ = [ - ('cert_verify_md', c_ubyte * 72), - ('finish_md', c_ubyte * 72), - ('finish_md_len', c_int), - ('peer_finish_md', c_ubyte * 72), - ('peer_finish_md_len', c_int), - ('message_size', c_ulong), - ('message_type', c_int), - ('new_cipher', POINTER(SSL_CIPHER)), - ('dh', POINTER(DH)), - ('next_state', c_int), - ('reuse_message', c_int), - ('cert_req', c_int), - ('ctype_num', c_int), - ('ctype', c_char * 7), - ('ca_names', POINTER(STACK)), - ('use_rsa_tmp', c_int), - ('key_block_length', c_int), - ('key_block', POINTER(c_ubyte)), - ('new_sym_enc', POINTER(EVP_CIPHER)), - ('new_hash', POINTER(EVP_MD)), - ('new_compression', POINTER(SSL_COMP)), - ('cert_request', c_int), -] -assert sizeof(N13ssl3_state_st4DOLLAR_20E) == 296, sizeof(N13ssl3_state_st4DOLLAR_20E) -assert alignment(N13ssl3_state_st4DOLLAR_20E) == 4, alignment(N13ssl3_state_st4DOLLAR_20E) -ssl3_state_st._fields_ = [ - ('flags', c_long), - ('delay_buf_pop_ret', c_int), - ('read_sequence', c_ubyte * 8), - ('read_mac_secret', c_ubyte * 36), - ('write_sequence', c_ubyte * 8), - ('write_mac_secret', c_ubyte * 36), - ('server_random', c_ubyte * 32), - ('client_random', c_ubyte * 32), - ('need_empty_fragments', c_int), - ('empty_fragment_done', c_int), - ('rbuf', SSL3_BUFFER), - ('wbuf', SSL3_BUFFER), - ('rrec', SSL3_RECORD), - ('wrec', SSL3_RECORD), - ('alert_fragment', c_ubyte * 2), - ('alert_fragment_len', c_uint), - ('handshake_fragment', c_ubyte * 4), - ('handshake_fragment_len', c_uint), - ('wnum', c_uint), - ('wpend_tot', c_int), - ('wpend_type', c_int), - ('wpend_ret', c_int), - ('wpend_buf', POINTER(c_ubyte)), - ('finish_dgst1', EVP_MD_CTX), - ('finish_dgst2', EVP_MD_CTX), - ('change_cipher_spec', c_int), - ('warn_alert', c_int), - ('fatal_alert', c_int), - ('alert_dispatch', c_int), - ('send_alert', c_ubyte * 2), - ('renegotiate', c_int), - ('total_renegotiations', c_int), - ('num_renegotiations', c_int), - ('in_read_app_data', c_int), - ('tmp', N13ssl3_state_st4DOLLAR_20E), -] -assert sizeof(ssl3_state_st) == 648, sizeof(ssl3_state_st) -assert alignment(ssl3_state_st) == 4, alignment(ssl3_state_st) -SSL3_STATE = ssl3_state_st -stack_st._fields_ = [ - ('num', c_int), - ('data', POINTER(STRING)), - ('sorted', c_int), - ('num_alloc', c_int), - ('comp', CFUNCTYPE(c_int, POINTER(STRING), POINTER(STRING))), -] -assert sizeof(stack_st) == 20, sizeof(stack_st) -assert alignment(stack_st) == 4, alignment(stack_st) -class ui_st(Structure): - pass -ui_st._fields_ = [ -] -UI = ui_st -class ui_method_st(Structure): - pass -ui_method_st._fields_ = [ -] -UI_METHOD = ui_method_st -class ui_string_st(Structure): - pass -ui_string_st._fields_ = [ -] -UI_STRING = ui_string_st - -# values for enumeration 'UI_string_types' -UI_string_types = c_int # enum -class X509_objects_st(Structure): - pass -X509_objects_st._fields_ = [ - ('nid', c_int), - ('a2i', CFUNCTYPE(c_int)), - ('i2a', CFUNCTYPE(c_int)), -] -assert sizeof(X509_objects_st) == 12, sizeof(X509_objects_st) -assert alignment(X509_objects_st) == 4, alignment(X509_objects_st) -X509_OBJECTS = X509_objects_st -X509_algor_st._fields_ = [ - ('algorithm', POINTER(ASN1_OBJECT)), - ('parameter', POINTER(ASN1_TYPE)), -] -assert sizeof(X509_algor_st) == 8, sizeof(X509_algor_st) -assert alignment(X509_algor_st) == 4, alignment(X509_algor_st) -class X509_val_st(Structure): - pass -X509_val_st._fields_ = [ - ('notBefore', POINTER(ASN1_TIME)), - ('notAfter', POINTER(ASN1_TIME)), -] -assert sizeof(X509_val_st) == 8, sizeof(X509_val_st) -assert alignment(X509_val_st) == 4, alignment(X509_val_st) -X509_VAL = X509_val_st -class X509_pubkey_st(Structure): - pass -X509_pubkey_st._fields_ = [ - ('algor', POINTER(X509_ALGOR)), - ('public_key', POINTER(ASN1_BIT_STRING)), - ('pkey', POINTER(EVP_PKEY)), -] -assert sizeof(X509_pubkey_st) == 12, sizeof(X509_pubkey_st) -assert alignment(X509_pubkey_st) == 4, alignment(X509_pubkey_st) -X509_PUBKEY = X509_pubkey_st -class X509_sig_st(Structure): - pass -X509_sig_st._fields_ = [ - ('algor', POINTER(X509_ALGOR)), - ('digest', POINTER(ASN1_OCTET_STRING)), -] -assert sizeof(X509_sig_st) == 8, sizeof(X509_sig_st) -assert alignment(X509_sig_st) == 4, alignment(X509_sig_st) -X509_SIG = X509_sig_st -class X509_name_entry_st(Structure): - pass -X509_name_entry_st._fields_ = [ - ('object', POINTER(ASN1_OBJECT)), - ('value', POINTER(ASN1_STRING)), - ('set', c_int), - ('size', c_int), -] -assert sizeof(X509_name_entry_st) == 16, sizeof(X509_name_entry_st) -assert alignment(X509_name_entry_st) == 4, alignment(X509_name_entry_st) -X509_NAME_ENTRY = X509_name_entry_st -X509_name_st._fields_ = [ - ('entries', POINTER(STACK)), - ('modified', c_int), - ('bytes', POINTER(BUF_MEM)), - ('hash', c_ulong), -] -assert sizeof(X509_name_st) == 16, sizeof(X509_name_st) -assert alignment(X509_name_st) == 4, alignment(X509_name_st) -class X509_extension_st(Structure): - pass -X509_extension_st._fields_ = [ - ('object', POINTER(ASN1_OBJECT)), - ('critical', ASN1_BOOLEAN), - ('value', POINTER(ASN1_OCTET_STRING)), -] -assert sizeof(X509_extension_st) == 12, sizeof(X509_extension_st) -assert alignment(X509_extension_st) == 4, alignment(X509_extension_st) -X509_EXTENSION = X509_extension_st -class x509_attributes_st(Structure): - pass -class N18x509_attributes_st4DOLLAR_13E(Union): - pass -N18x509_attributes_st4DOLLAR_13E._fields_ = [ - ('ptr', STRING), - ('set', POINTER(STACK)), - ('single', POINTER(ASN1_TYPE)), -] -assert sizeof(N18x509_attributes_st4DOLLAR_13E) == 4, sizeof(N18x509_attributes_st4DOLLAR_13E) -assert alignment(N18x509_attributes_st4DOLLAR_13E) == 4, alignment(N18x509_attributes_st4DOLLAR_13E) -x509_attributes_st._fields_ = [ - ('object', POINTER(ASN1_OBJECT)), - ('single', c_int), - ('value', N18x509_attributes_st4DOLLAR_13E), -] -assert sizeof(x509_attributes_st) == 12, sizeof(x509_attributes_st) -assert alignment(x509_attributes_st) == 4, alignment(x509_attributes_st) -X509_ATTRIBUTE = x509_attributes_st -class X509_req_info_st(Structure): - pass -X509_req_info_st._fields_ = [ - ('enc', ASN1_ENCODING), - ('version', POINTER(ASN1_INTEGER)), - ('subject', POINTER(X509_NAME)), - ('pubkey', POINTER(X509_PUBKEY)), - ('attributes', POINTER(STACK)), -] -assert sizeof(X509_req_info_st) == 28, sizeof(X509_req_info_st) -assert alignment(X509_req_info_st) == 4, alignment(X509_req_info_st) -X509_REQ_INFO = X509_req_info_st -class X509_req_st(Structure): - pass -X509_req_st._fields_ = [ - ('req_info', POINTER(X509_REQ_INFO)), - ('sig_alg', POINTER(X509_ALGOR)), - ('signature', POINTER(ASN1_BIT_STRING)), - ('references', c_int), -] -assert sizeof(X509_req_st) == 16, sizeof(X509_req_st) -assert alignment(X509_req_st) == 4, alignment(X509_req_st) -X509_REQ = X509_req_st -class x509_cinf_st(Structure): - pass -x509_cinf_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('serialNumber', POINTER(ASN1_INTEGER)), - ('signature', POINTER(X509_ALGOR)), - ('issuer', POINTER(X509_NAME)), - ('validity', POINTER(X509_VAL)), - ('subject', POINTER(X509_NAME)), - ('key', POINTER(X509_PUBKEY)), - ('issuerUID', POINTER(ASN1_BIT_STRING)), - ('subjectUID', POINTER(ASN1_BIT_STRING)), - ('extensions', POINTER(STACK)), -] -assert sizeof(x509_cinf_st) == 40, sizeof(x509_cinf_st) -assert alignment(x509_cinf_st) == 4, alignment(x509_cinf_st) -X509_CINF = x509_cinf_st -class x509_cert_aux_st(Structure): - pass -x509_cert_aux_st._fields_ = [ - ('trust', POINTER(STACK)), - ('reject', POINTER(STACK)), - ('alias', POINTER(ASN1_UTF8STRING)), - ('keyid', POINTER(ASN1_OCTET_STRING)), - ('other', POINTER(STACK)), -] -assert sizeof(x509_cert_aux_st) == 20, sizeof(x509_cert_aux_st) -assert alignment(x509_cert_aux_st) == 4, alignment(x509_cert_aux_st) -X509_CERT_AUX = x509_cert_aux_st -class AUTHORITY_KEYID_st(Structure): - pass -x509_st._fields_ = [ - ('cert_info', POINTER(X509_CINF)), - ('sig_alg', POINTER(X509_ALGOR)), - ('signature', POINTER(ASN1_BIT_STRING)), - ('valid', c_int), - ('references', c_int), - ('name', STRING), - ('ex_data', CRYPTO_EX_DATA), - ('ex_pathlen', c_long), - ('ex_flags', c_ulong), - ('ex_kusage', c_ulong), - ('ex_xkusage', c_ulong), - ('ex_nscert', c_ulong), - ('skid', POINTER(ASN1_OCTET_STRING)), - ('akid', POINTER(AUTHORITY_KEYID_st)), - ('sha1_hash', c_ubyte * 20), - ('aux', POINTER(X509_CERT_AUX)), -] -assert sizeof(x509_st) == 84, sizeof(x509_st) -assert alignment(x509_st) == 4, alignment(x509_st) -AUTHORITY_KEYID_st._fields_ = [ -] -class x509_trust_st(Structure): - pass -x509_trust_st._fields_ = [ - ('trust', c_int), - ('flags', c_int), - ('check_trust', CFUNCTYPE(c_int, POINTER(x509_trust_st), POINTER(X509), c_int)), - ('name', STRING), - ('arg1', c_int), - ('arg2', c_void_p), -] -assert sizeof(x509_trust_st) == 24, sizeof(x509_trust_st) -assert alignment(x509_trust_st) == 4, alignment(x509_trust_st) -X509_TRUST = x509_trust_st -class X509_revoked_st(Structure): - pass -X509_revoked_st._fields_ = [ - ('serialNumber', POINTER(ASN1_INTEGER)), - ('revocationDate', POINTER(ASN1_TIME)), - ('extensions', POINTER(STACK)), - ('sequence', c_int), -] -assert sizeof(X509_revoked_st) == 16, sizeof(X509_revoked_st) -assert alignment(X509_revoked_st) == 4, alignment(X509_revoked_st) -X509_REVOKED = X509_revoked_st -class X509_crl_info_st(Structure): - pass -X509_crl_info_st._fields_ = [ - ('version', POINTER(ASN1_INTEGER)), - ('sig_alg', POINTER(X509_ALGOR)), - ('issuer', POINTER(X509_NAME)), - ('lastUpdate', POINTER(ASN1_TIME)), - ('nextUpdate', POINTER(ASN1_TIME)), - ('revoked', POINTER(STACK)), - ('extensions', POINTER(STACK)), - ('enc', ASN1_ENCODING), -] -assert sizeof(X509_crl_info_st) == 40, sizeof(X509_crl_info_st) -assert alignment(X509_crl_info_st) == 4, alignment(X509_crl_info_st) -X509_CRL_INFO = X509_crl_info_st -X509_crl_st._fields_ = [ - ('crl', POINTER(X509_CRL_INFO)), - ('sig_alg', POINTER(X509_ALGOR)), - ('signature', POINTER(ASN1_BIT_STRING)), - ('references', c_int), -] -assert sizeof(X509_crl_st) == 16, sizeof(X509_crl_st) -assert alignment(X509_crl_st) == 4, alignment(X509_crl_st) -class private_key_st(Structure): - pass -private_key_st._fields_ = [ - ('version', c_int), - ('enc_algor', POINTER(X509_ALGOR)), - ('enc_pkey', POINTER(ASN1_OCTET_STRING)), - ('dec_pkey', POINTER(EVP_PKEY)), - ('key_length', c_int), - ('key_data', STRING), - ('key_free', c_int), - ('cipher', EVP_CIPHER_INFO), - ('references', c_int), -] -assert sizeof(private_key_st) == 52, sizeof(private_key_st) -assert alignment(private_key_st) == 4, alignment(private_key_st) -X509_PKEY = private_key_st -class X509_info_st(Structure): - pass -X509_info_st._fields_ = [ - ('x509', POINTER(X509)), - ('crl', POINTER(X509_CRL)), - ('x_pkey', POINTER(X509_PKEY)), - ('enc_cipher', EVP_CIPHER_INFO), - ('enc_len', c_int), - ('enc_data', STRING), - ('references', c_int), -] -assert sizeof(X509_info_st) == 44, sizeof(X509_info_st) -assert alignment(X509_info_st) == 4, alignment(X509_info_st) -X509_INFO = X509_info_st -class Netscape_spkac_st(Structure): - pass -Netscape_spkac_st._fields_ = [ - ('pubkey', POINTER(X509_PUBKEY)), - ('challenge', POINTER(ASN1_IA5STRING)), -] -assert sizeof(Netscape_spkac_st) == 8, sizeof(Netscape_spkac_st) -assert alignment(Netscape_spkac_st) == 4, alignment(Netscape_spkac_st) -NETSCAPE_SPKAC = Netscape_spkac_st -class Netscape_spki_st(Structure): - pass -Netscape_spki_st._fields_ = [ - ('spkac', POINTER(NETSCAPE_SPKAC)), - ('sig_algor', POINTER(X509_ALGOR)), - ('signature', POINTER(ASN1_BIT_STRING)), -] -assert sizeof(Netscape_spki_st) == 12, sizeof(Netscape_spki_st) -assert alignment(Netscape_spki_st) == 4, alignment(Netscape_spki_st) -NETSCAPE_SPKI = Netscape_spki_st -class Netscape_certificate_sequence(Structure): - pass -Netscape_certificate_sequence._fields_ = [ - ('type', POINTER(ASN1_OBJECT)), - ('certs', POINTER(STACK)), -] -assert sizeof(Netscape_certificate_sequence) == 8, sizeof(Netscape_certificate_sequence) -assert alignment(Netscape_certificate_sequence) == 4, alignment(Netscape_certificate_sequence) -NETSCAPE_CERT_SEQUENCE = Netscape_certificate_sequence -class PBEPARAM_st(Structure): - pass -PBEPARAM_st._fields_ = [ - ('salt', POINTER(ASN1_OCTET_STRING)), - ('iter', POINTER(ASN1_INTEGER)), -] -assert sizeof(PBEPARAM_st) == 8, sizeof(PBEPARAM_st) -assert alignment(PBEPARAM_st) == 4, alignment(PBEPARAM_st) -PBEPARAM = PBEPARAM_st -class PBE2PARAM_st(Structure): - pass -PBE2PARAM_st._fields_ = [ - ('keyfunc', POINTER(X509_ALGOR)), - ('encryption', POINTER(X509_ALGOR)), -] -assert sizeof(PBE2PARAM_st) == 8, sizeof(PBE2PARAM_st) -assert alignment(PBE2PARAM_st) == 4, alignment(PBE2PARAM_st) -PBE2PARAM = PBE2PARAM_st -class PBKDF2PARAM_st(Structure): - pass -PBKDF2PARAM_st._fields_ = [ - ('salt', POINTER(ASN1_TYPE)), - ('iter', POINTER(ASN1_INTEGER)), - ('keylength', POINTER(ASN1_INTEGER)), - ('prf', POINTER(X509_ALGOR)), -] -assert sizeof(PBKDF2PARAM_st) == 16, sizeof(PBKDF2PARAM_st) -assert alignment(PBKDF2PARAM_st) == 4, alignment(PBKDF2PARAM_st) -PBKDF2PARAM = PBKDF2PARAM_st -class pkcs8_priv_key_info_st(Structure): - pass -pkcs8_priv_key_info_st._fields_ = [ - ('broken', c_int), - ('version', POINTER(ASN1_INTEGER)), - ('pkeyalg', POINTER(X509_ALGOR)), - ('pkey', POINTER(ASN1_TYPE)), - ('attributes', POINTER(STACK)), -] -assert sizeof(pkcs8_priv_key_info_st) == 20, sizeof(pkcs8_priv_key_info_st) -assert alignment(pkcs8_priv_key_info_st) == 4, alignment(pkcs8_priv_key_info_st) -PKCS8_PRIV_KEY_INFO = pkcs8_priv_key_info_st -class x509_hash_dir_st(Structure): - pass -x509_hash_dir_st._fields_ = [ - ('num_dirs', c_int), - ('dirs', POINTER(STRING)), - ('dirs_type', POINTER(c_int)), - ('num_dirs_alloced', c_int), -] -assert sizeof(x509_hash_dir_st) == 16, sizeof(x509_hash_dir_st) -assert alignment(x509_hash_dir_st) == 4, alignment(x509_hash_dir_st) -X509_HASH_DIR_CTX = x509_hash_dir_st -class x509_file_st(Structure): - pass -x509_file_st._fields_ = [ - ('num_paths', c_int), - ('num_alloced', c_int), - ('paths', POINTER(STRING)), - ('path_type', POINTER(c_int)), -] -assert sizeof(x509_file_st) == 16, sizeof(x509_file_st) -assert alignment(x509_file_st) == 4, alignment(x509_file_st) -X509_CERT_FILE_CTX = x509_file_st -class x509_object_st(Structure): - pass -class N14x509_object_st4DOLLAR_14E(Union): - pass -N14x509_object_st4DOLLAR_14E._fields_ = [ - ('ptr', STRING), - ('x509', POINTER(X509)), - ('crl', POINTER(X509_CRL)), - ('pkey', POINTER(EVP_PKEY)), -] -assert sizeof(N14x509_object_st4DOLLAR_14E) == 4, sizeof(N14x509_object_st4DOLLAR_14E) -assert alignment(N14x509_object_st4DOLLAR_14E) == 4, alignment(N14x509_object_st4DOLLAR_14E) -x509_object_st._fields_ = [ - ('type', c_int), - ('data', N14x509_object_st4DOLLAR_14E), -] -assert sizeof(x509_object_st) == 8, sizeof(x509_object_st) -assert alignment(x509_object_st) == 4, alignment(x509_object_st) -X509_OBJECT = x509_object_st -class x509_lookup_st(Structure): - pass -X509_LOOKUP = x509_lookup_st -class x509_lookup_method_st(Structure): - pass -x509_lookup_method_st._fields_ = [ - ('name', STRING), - ('new_item', CFUNCTYPE(c_int, POINTER(X509_LOOKUP))), - ('free', CFUNCTYPE(None, POINTER(X509_LOOKUP))), - ('init', CFUNCTYPE(c_int, POINTER(X509_LOOKUP))), - ('shutdown', CFUNCTYPE(c_int, POINTER(X509_LOOKUP))), - ('ctrl', CFUNCTYPE(c_int, POINTER(X509_LOOKUP), c_int, STRING, c_long, POINTER(STRING))), - ('get_by_subject', CFUNCTYPE(c_int, POINTER(X509_LOOKUP), c_int, POINTER(X509_NAME), POINTER(X509_OBJECT))), - ('get_by_issuer_serial', CFUNCTYPE(c_int, POINTER(X509_LOOKUP), c_int, POINTER(X509_NAME), POINTER(ASN1_INTEGER), POINTER(X509_OBJECT))), - ('get_by_fingerprint', CFUNCTYPE(c_int, POINTER(X509_LOOKUP), c_int, POINTER(c_ubyte), c_int, POINTER(X509_OBJECT))), - ('get_by_alias', CFUNCTYPE(c_int, POINTER(X509_LOOKUP), c_int, STRING, c_int, POINTER(X509_OBJECT))), -] -assert sizeof(x509_lookup_method_st) == 40, sizeof(x509_lookup_method_st) -assert alignment(x509_lookup_method_st) == 4, alignment(x509_lookup_method_st) -X509_LOOKUP_METHOD = x509_lookup_method_st -x509_store_st._fields_ = [ - ('cache', c_int), - ('objs', POINTER(STACK)), - ('get_cert_methods', POINTER(STACK)), - ('flags', c_ulong), - ('purpose', c_int), - ('trust', c_int), - ('verify', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('verify_cb', CFUNCTYPE(c_int, c_int, POINTER(X509_STORE_CTX))), - ('get_issuer', CFUNCTYPE(c_int, POINTER(POINTER(X509)), POINTER(X509_STORE_CTX), POINTER(X509))), - ('check_issued', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509), POINTER(X509))), - ('check_revocation', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('get_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(POINTER(X509_CRL)), POINTER(X509))), - ('check_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509_CRL))), - ('cert_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509_CRL), POINTER(X509))), - ('cleanup', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('ex_data', CRYPTO_EX_DATA), - ('references', c_int), - ('depth', c_int), -] -assert sizeof(x509_store_st) == 76, sizeof(x509_store_st) -assert alignment(x509_store_st) == 4, alignment(x509_store_st) -x509_lookup_st._fields_ = [ - ('init', c_int), - ('skip', c_int), - ('method', POINTER(X509_LOOKUP_METHOD)), - ('method_data', STRING), - ('store_ctx', POINTER(X509_STORE)), -] -assert sizeof(x509_lookup_st) == 20, sizeof(x509_lookup_st) -assert alignment(x509_lookup_st) == 4, alignment(x509_lookup_st) -time_t = __darwin_time_t -x509_store_ctx_st._fields_ = [ - ('ctx', POINTER(X509_STORE)), - ('current_method', c_int), - ('cert', POINTER(X509)), - ('untrusted', POINTER(STACK)), - ('purpose', c_int), - ('trust', c_int), - ('check_time', time_t), - ('flags', c_ulong), - ('other_ctx', c_void_p), - ('verify', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('verify_cb', CFUNCTYPE(c_int, c_int, POINTER(X509_STORE_CTX))), - ('get_issuer', CFUNCTYPE(c_int, POINTER(POINTER(X509)), POINTER(X509_STORE_CTX), POINTER(X509))), - ('check_issued', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509), POINTER(X509))), - ('check_revocation', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('get_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(POINTER(X509_CRL)), POINTER(X509))), - ('check_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509_CRL))), - ('cert_crl', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX), POINTER(X509_CRL), POINTER(X509))), - ('cleanup', CFUNCTYPE(c_int, POINTER(X509_STORE_CTX))), - ('depth', c_int), - ('valid', c_int), - ('last_untrusted', c_int), - ('chain', POINTER(STACK)), - ('error_depth', c_int), - ('error', c_int), - ('current_cert', POINTER(X509)), - ('current_issuer', POINTER(X509)), - ('current_crl', POINTER(X509_CRL)), - ('ex_data', CRYPTO_EX_DATA), -] -assert sizeof(x509_store_ctx_st) == 116, sizeof(x509_store_ctx_st) -assert alignment(x509_store_ctx_st) == 4, alignment(x509_store_ctx_st) -va_list = __darwin_va_list -__darwin_off_t = __int64_t -fpos_t = __darwin_off_t -class __sbuf(Structure): - pass -__sbuf._fields_ = [ - ('_base', POINTER(c_ubyte)), - ('_size', c_int), -] -assert sizeof(__sbuf) == 8, sizeof(__sbuf) -assert alignment(__sbuf) == 4, alignment(__sbuf) -class __sFILEX(Structure): - pass -__sFILEX._fields_ = [ -] -class __sFILE(Structure): - pass -__sFILE._pack_ = 4 -__sFILE._fields_ = [ - ('_p', POINTER(c_ubyte)), - ('_r', c_int), - ('_w', c_int), - ('_flags', c_short), - ('_file', c_short), - ('_bf', __sbuf), - ('_lbfsize', c_int), - ('_cookie', c_void_p), - ('_close', CFUNCTYPE(c_int, c_void_p)), - ('_read', CFUNCTYPE(c_int, c_void_p, STRING, c_int)), - ('_seek', CFUNCTYPE(fpos_t, c_void_p, c_longlong, c_int)), - ('_write', CFUNCTYPE(c_int, c_void_p, STRING, c_int)), - ('_ub', __sbuf), - ('_extra', POINTER(__sFILEX)), - ('_ur', c_int), - ('_ubuf', c_ubyte * 3), - ('_nbuf', c_ubyte * 1), - ('_lb', __sbuf), - ('_blksize', c_int), - ('_offset', fpos_t), -] -assert sizeof(__sFILE) == 88, sizeof(__sFILE) -assert alignment(__sFILE) == 4, alignment(__sFILE) -FILE = __sFILE -ct_rune_t = __darwin_ct_rune_t -rune_t = __darwin_rune_t -class div_t(Structure): - pass -div_t._fields_ = [ - ('quot', c_int), - ('rem', c_int), -] -assert sizeof(div_t) == 8, sizeof(div_t) -assert alignment(div_t) == 4, alignment(div_t) -class ldiv_t(Structure): - pass -ldiv_t._fields_ = [ - ('quot', c_long), - ('rem', c_long), -] -assert sizeof(ldiv_t) == 8, sizeof(ldiv_t) -assert alignment(ldiv_t) == 4, alignment(ldiv_t) -class lldiv_t(Structure): - pass -lldiv_t._pack_ = 4 -lldiv_t._fields_ = [ - ('quot', c_longlong), - ('rem', c_longlong), -] -assert sizeof(lldiv_t) == 16, sizeof(lldiv_t) -assert alignment(lldiv_t) == 4, alignment(lldiv_t) -__darwin_dev_t = __int32_t -dev_t = __darwin_dev_t -__darwin_mode_t = __uint16_t -mode_t = __darwin_mode_t -class mcontext(Structure): - pass -mcontext._fields_ = [ -] -class mcontext64(Structure): - pass -mcontext64._fields_ = [ -] -class __darwin_pthread_handler_rec(Structure): - pass -__darwin_pthread_handler_rec._fields_ = [ - ('__routine', CFUNCTYPE(None, c_void_p)), - ('__arg', c_void_p), - ('__next', POINTER(__darwin_pthread_handler_rec)), -] -assert sizeof(__darwin_pthread_handler_rec) == 12, sizeof(__darwin_pthread_handler_rec) -assert alignment(__darwin_pthread_handler_rec) == 4, alignment(__darwin_pthread_handler_rec) -class _opaque_pthread_attr_t(Structure): - pass -_opaque_pthread_attr_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 36), -] -assert sizeof(_opaque_pthread_attr_t) == 40, sizeof(_opaque_pthread_attr_t) -assert alignment(_opaque_pthread_attr_t) == 4, alignment(_opaque_pthread_attr_t) -class _opaque_pthread_cond_t(Structure): - pass -_opaque_pthread_cond_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 24), -] -assert sizeof(_opaque_pthread_cond_t) == 28, sizeof(_opaque_pthread_cond_t) -assert alignment(_opaque_pthread_cond_t) == 4, alignment(_opaque_pthread_cond_t) -class _opaque_pthread_condattr_t(Structure): - pass -_opaque_pthread_condattr_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 4), -] -assert sizeof(_opaque_pthread_condattr_t) == 8, sizeof(_opaque_pthread_condattr_t) -assert alignment(_opaque_pthread_condattr_t) == 4, alignment(_opaque_pthread_condattr_t) -class _opaque_pthread_mutex_t(Structure): - pass -_opaque_pthread_mutex_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 40), -] -assert sizeof(_opaque_pthread_mutex_t) == 44, sizeof(_opaque_pthread_mutex_t) -assert alignment(_opaque_pthread_mutex_t) == 4, alignment(_opaque_pthread_mutex_t) -class _opaque_pthread_mutexattr_t(Structure): - pass -_opaque_pthread_mutexattr_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 8), -] -assert sizeof(_opaque_pthread_mutexattr_t) == 12, sizeof(_opaque_pthread_mutexattr_t) -assert alignment(_opaque_pthread_mutexattr_t) == 4, alignment(_opaque_pthread_mutexattr_t) -class _opaque_pthread_once_t(Structure): - pass -_opaque_pthread_once_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 4), -] -assert sizeof(_opaque_pthread_once_t) == 8, sizeof(_opaque_pthread_once_t) -assert alignment(_opaque_pthread_once_t) == 4, alignment(_opaque_pthread_once_t) -class _opaque_pthread_rwlock_t(Structure): - pass -_opaque_pthread_rwlock_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 124), -] -assert sizeof(_opaque_pthread_rwlock_t) == 128, sizeof(_opaque_pthread_rwlock_t) -assert alignment(_opaque_pthread_rwlock_t) == 4, alignment(_opaque_pthread_rwlock_t) -class _opaque_pthread_rwlockattr_t(Structure): - pass -_opaque_pthread_rwlockattr_t._fields_ = [ - ('__sig', c_long), - ('__opaque', c_char * 12), -] -assert sizeof(_opaque_pthread_rwlockattr_t) == 16, sizeof(_opaque_pthread_rwlockattr_t) -assert alignment(_opaque_pthread_rwlockattr_t) == 4, alignment(_opaque_pthread_rwlockattr_t) -class _opaque_pthread_t(Structure): - pass -_opaque_pthread_t._fields_ = [ - ('__sig', c_long), - ('__cleanup_stack', POINTER(__darwin_pthread_handler_rec)), - ('__opaque', c_char * 596), -] -assert sizeof(_opaque_pthread_t) == 604, sizeof(_opaque_pthread_t) -assert alignment(_opaque_pthread_t) == 4, alignment(_opaque_pthread_t) -__darwin_blkcnt_t = __int64_t -__darwin_blksize_t = __int32_t -__darwin_fsblkcnt_t = c_uint -__darwin_fsfilcnt_t = c_uint -__darwin_gid_t = __uint32_t -__darwin_id_t = __uint32_t -__darwin_ino_t = __uint32_t -__darwin_mach_port_name_t = __darwin_natural_t -__darwin_mach_port_t = __darwin_mach_port_name_t -__darwin_mcontext_t = POINTER(mcontext) -__darwin_mcontext64_t = POINTER(mcontext64) -__darwin_pid_t = __int32_t -__darwin_pthread_attr_t = _opaque_pthread_attr_t -__darwin_pthread_cond_t = _opaque_pthread_cond_t -__darwin_pthread_condattr_t = _opaque_pthread_condattr_t -__darwin_pthread_key_t = c_ulong -__darwin_pthread_mutex_t = _opaque_pthread_mutex_t -__darwin_pthread_mutexattr_t = _opaque_pthread_mutexattr_t -__darwin_pthread_once_t = _opaque_pthread_once_t -__darwin_pthread_rwlock_t = _opaque_pthread_rwlock_t -__darwin_pthread_rwlockattr_t = _opaque_pthread_rwlockattr_t -__darwin_pthread_t = POINTER(_opaque_pthread_t) -__darwin_sigset_t = __uint32_t -__darwin_suseconds_t = __int32_t -__darwin_uid_t = __uint32_t -__darwin_useconds_t = __uint32_t -__darwin_uuid_t = c_ubyte * 16 -class sigaltstack(Structure): - pass -sigaltstack._fields_ = [ - ('ss_sp', c_void_p), - ('ss_size', __darwin_size_t), - ('ss_flags', c_int), -] -assert sizeof(sigaltstack) == 12, sizeof(sigaltstack) -assert alignment(sigaltstack) == 4, alignment(sigaltstack) -__darwin_stack_t = sigaltstack -class ucontext(Structure): - pass -ucontext._fields_ = [ - ('uc_onstack', c_int), - ('uc_sigmask', __darwin_sigset_t), - ('uc_stack', __darwin_stack_t), - ('uc_link', POINTER(ucontext)), - ('uc_mcsize', __darwin_size_t), - ('uc_mcontext', __darwin_mcontext_t), -] -assert sizeof(ucontext) == 32, sizeof(ucontext) -assert alignment(ucontext) == 4, alignment(ucontext) -__darwin_ucontext_t = ucontext -class ucontext64(Structure): - pass -ucontext64._fields_ = [ - ('uc_onstack', c_int), - ('uc_sigmask', __darwin_sigset_t), - ('uc_stack', __darwin_stack_t), - ('uc_link', POINTER(ucontext64)), - ('uc_mcsize', __darwin_size_t), - ('uc_mcontext64', __darwin_mcontext64_t), -] -assert sizeof(ucontext64) == 32, sizeof(ucontext64) -assert alignment(ucontext64) == 4, alignment(ucontext64) -__darwin_ucontext64_t = ucontext64 -class timeval(Structure): - pass -timeval._fields_ = [ - ('tv_sec', __darwin_time_t), - ('tv_usec', __darwin_suseconds_t), -] -assert sizeof(timeval) == 8, sizeof(timeval) -assert alignment(timeval) == 4, alignment(timeval) -rlim_t = __int64_t -class rusage(Structure): - pass -rusage._fields_ = [ - ('ru_utime', timeval), - ('ru_stime', timeval), - ('ru_maxrss', c_long), - ('ru_ixrss', c_long), - ('ru_idrss', c_long), - ('ru_isrss', c_long), - ('ru_minflt', c_long), - ('ru_majflt', c_long), - ('ru_nswap', c_long), - ('ru_inblock', c_long), - ('ru_oublock', c_long), - ('ru_msgsnd', c_long), - ('ru_msgrcv', c_long), - ('ru_nsignals', c_long), - ('ru_nvcsw', c_long), - ('ru_nivcsw', c_long), -] -assert sizeof(rusage) == 72, sizeof(rusage) -assert alignment(rusage) == 4, alignment(rusage) -class rlimit(Structure): - pass -rlimit._pack_ = 4 -rlimit._fields_ = [ - ('rlim_cur', rlim_t), - ('rlim_max', rlim_t), -] -assert sizeof(rlimit) == 16, sizeof(rlimit) -assert alignment(rlimit) == 4, alignment(rlimit) -mcontext_t = __darwin_mcontext_t -mcontext64_t = __darwin_mcontext64_t -pthread_attr_t = __darwin_pthread_attr_t -sigset_t = __darwin_sigset_t -ucontext_t = __darwin_ucontext_t -ucontext64_t = __darwin_ucontext64_t -uid_t = __darwin_uid_t -class sigval(Union): - pass -sigval._fields_ = [ - ('sival_int', c_int), - ('sival_ptr', c_void_p), -] -assert sizeof(sigval) == 4, sizeof(sigval) -assert alignment(sigval) == 4, alignment(sigval) -class sigevent(Structure): - pass -sigevent._fields_ = [ - ('sigev_notify', c_int), - ('sigev_signo', c_int), - ('sigev_value', sigval), - ('sigev_notify_function', CFUNCTYPE(None, sigval)), - ('sigev_notify_attributes', POINTER(pthread_attr_t)), -] -assert sizeof(sigevent) == 20, sizeof(sigevent) -assert alignment(sigevent) == 4, alignment(sigevent) -class __siginfo(Structure): - pass -pid_t = __darwin_pid_t -__siginfo._fields_ = [ - ('si_signo', c_int), - ('si_errno', c_int), - ('si_code', c_int), - ('si_pid', pid_t), - ('si_uid', uid_t), - ('si_status', c_int), - ('si_addr', c_void_p), - ('si_value', sigval), - ('si_band', c_long), - ('pad', c_ulong * 7), -] -assert sizeof(__siginfo) == 64, sizeof(__siginfo) -assert alignment(__siginfo) == 4, alignment(__siginfo) -siginfo_t = __siginfo -class __sigaction_u(Union): - pass -__sigaction_u._fields_ = [ - ('__sa_handler', CFUNCTYPE(None, c_int)), - ('__sa_sigaction', CFUNCTYPE(None, c_int, POINTER(__siginfo), c_void_p)), -] -assert sizeof(__sigaction_u) == 4, sizeof(__sigaction_u) -assert alignment(__sigaction_u) == 4, alignment(__sigaction_u) -class __sigaction(Structure): - pass -__sigaction._fields_ = [ - ('__sigaction_u', __sigaction_u), - ('sa_tramp', CFUNCTYPE(None, c_void_p, c_int, c_int, POINTER(siginfo_t), c_void_p)), - ('sa_mask', sigset_t), - ('sa_flags', c_int), -] -assert sizeof(__sigaction) == 16, sizeof(__sigaction) -assert alignment(__sigaction) == 4, alignment(__sigaction) -class sigaction(Structure): - pass -sigaction._fields_ = [ - ('__sigaction_u', __sigaction_u), - ('sa_mask', sigset_t), - ('sa_flags', c_int), -] -assert sizeof(sigaction) == 12, sizeof(sigaction) -assert alignment(sigaction) == 4, alignment(sigaction) -sig_t = CFUNCTYPE(None, c_int) -stack_t = __darwin_stack_t -class sigvec(Structure): - pass -sigvec._fields_ = [ - ('sv_handler', CFUNCTYPE(None, c_int)), - ('sv_mask', c_int), - ('sv_flags', c_int), -] -assert sizeof(sigvec) == 12, sizeof(sigvec) -assert alignment(sigvec) == 4, alignment(sigvec) -class sigstack(Structure): - pass -sigstack._fields_ = [ - ('ss_sp', STRING), - ('ss_onstack', c_int), -] -assert sizeof(sigstack) == 8, sizeof(sigstack) -assert alignment(sigstack) == 4, alignment(sigstack) -u_char = c_ubyte -u_short = c_ushort -u_int = c_uint -u_long = c_ulong -ushort = c_ushort -uint = c_uint -u_quad_t = u_int64_t -quad_t = int64_t -qaddr_t = POINTER(quad_t) -caddr_t = STRING -daddr_t = int32_t -fixpt_t = u_int32_t -blkcnt_t = __darwin_blkcnt_t -blksize_t = __darwin_blksize_t -gid_t = __darwin_gid_t -in_addr_t = __uint32_t -in_port_t = __uint16_t -ino_t = __darwin_ino_t -key_t = __int32_t -nlink_t = __uint16_t -off_t = __darwin_off_t -segsz_t = int32_t -swblk_t = int32_t -clock_t = __darwin_clock_t -ssize_t = __darwin_ssize_t -useconds_t = __darwin_useconds_t -suseconds_t = __darwin_suseconds_t -fd_mask = __int32_t -class fd_set(Structure): - pass -fd_set._fields_ = [ - ('fds_bits', __int32_t * 32), -] -assert sizeof(fd_set) == 128, sizeof(fd_set) -assert alignment(fd_set) == 4, alignment(fd_set) -pthread_cond_t = __darwin_pthread_cond_t -pthread_condattr_t = __darwin_pthread_condattr_t -pthread_mutex_t = __darwin_pthread_mutex_t -pthread_mutexattr_t = __darwin_pthread_mutexattr_t -pthread_once_t = __darwin_pthread_once_t -pthread_rwlock_t = __darwin_pthread_rwlock_t -pthread_rwlockattr_t = __darwin_pthread_rwlockattr_t -pthread_t = __darwin_pthread_t -pthread_key_t = __darwin_pthread_key_t -fsblkcnt_t = __darwin_fsblkcnt_t -fsfilcnt_t = __darwin_fsfilcnt_t - -# values for enumeration 'idtype_t' -idtype_t = c_int # enum -id_t = __darwin_id_t -class wait(Union): - pass -class N4wait3DOLLAR_3E(Structure): - pass -N4wait3DOLLAR_3E._fields_ = [ - ('w_Termsig', c_uint, 7), - ('w_Coredump', c_uint, 1), - ('w_Retcode', c_uint, 8), - ('w_Filler', c_uint, 16), -] -assert sizeof(N4wait3DOLLAR_3E) == 4, sizeof(N4wait3DOLLAR_3E) -assert alignment(N4wait3DOLLAR_3E) == 4, alignment(N4wait3DOLLAR_3E) -class N4wait3DOLLAR_4E(Structure): - pass -N4wait3DOLLAR_4E._fields_ = [ - ('w_Stopval', c_uint, 8), - ('w_Stopsig', c_uint, 8), - ('w_Filler', c_uint, 16), -] -assert sizeof(N4wait3DOLLAR_4E) == 4, sizeof(N4wait3DOLLAR_4E) -assert alignment(N4wait3DOLLAR_4E) == 4, alignment(N4wait3DOLLAR_4E) -wait._fields_ = [ - ('w_status', c_int), - ('w_T', N4wait3DOLLAR_3E), - ('w_S', N4wait3DOLLAR_4E), -] -assert sizeof(wait) == 4, sizeof(wait) -assert alignment(wait) == 4, alignment(wait) -class timespec(Structure): - pass -timespec._fields_ = [ - ('tv_sec', time_t), - ('tv_nsec', c_long), -] -assert sizeof(timespec) == 8, sizeof(timespec) -assert alignment(timespec) == 4, alignment(timespec) -class tm(Structure): - pass -tm._fields_ = [ - ('tm_sec', c_int), - ('tm_min', c_int), - ('tm_hour', c_int), - ('tm_mday', c_int), - ('tm_mon', c_int), - ('tm_year', c_int), - ('tm_wday', c_int), - ('tm_yday', c_int), - ('tm_isdst', c_int), - ('tm_gmtoff', c_long), - ('tm_zone', STRING), -] -assert sizeof(tm) == 44, sizeof(tm) -assert alignment(tm) == 4, alignment(tm) -__gnuc_va_list = STRING -ptrdiff_t = c_int -int8_t = c_byte -int16_t = c_short -uint8_t = c_ubyte -uint16_t = c_ushort -uint32_t = c_uint -uint64_t = c_ulonglong -int_least8_t = int8_t -int_least16_t = int16_t -int_least32_t = int32_t -int_least64_t = int64_t -uint_least8_t = uint8_t -uint_least16_t = uint16_t -uint_least32_t = uint32_t -uint_least64_t = uint64_t -int_fast8_t = int8_t -int_fast16_t = int16_t -int_fast32_t = int32_t -int_fast64_t = int64_t -uint_fast8_t = uint8_t -uint_fast16_t = uint16_t -uint_fast32_t = uint32_t -uint_fast64_t = uint64_t -intptr_t = c_long -uintptr_t = c_ulong -intmax_t = c_longlong -uintmax_t = c_ulonglong -__all__ = ['ENGINE', 'pkcs7_enc_content_st', '__int16_t', - 'X509_REVOKED', 'SSL_CTX', 'UIT_BOOLEAN', - '__darwin_time_t', 'ucontext64_t', 'int_fast32_t', - 'pem_ctx_st', 'uint8_t', 'fpos_t', 'X509', 'COMP_CTX', - 'tm', 'N10pem_ctx_st4DOLLAR_17E', 'swblk_t', - 'ASN1_TEMPLATE', '__darwin_pthread_t', 'fixpt_t', - 'BIO_METHOD', 'ASN1_PRINTABLESTRING', 'EVP_ENCODE_CTX', - 'dh_method', 'bio_f_buffer_ctx_struct', 'in_port_t', - 'X509_SIG', '__darwin_ssize_t', '__darwin_sigset_t', - 'wait', 'uint_fast16_t', 'N12asn1_type_st4DOLLAR_11E', - 'uint_least8_t', 'pthread_rwlock_t', 'ASN1_IA5STRING', - 'fsfilcnt_t', 'ucontext', '__uint64_t', 'timespec', - 'x509_cinf_st', 'COMP_METHOD', 'MD5_CTX', 'buf_mem_st', - 'ASN1_ENCODING_st', 'PBEPARAM', 'X509_NAME_ENTRY', - '__darwin_va_list', 'ucontext_t', 'lhash_st', - 'N4wait3DOLLAR_4E', '__darwin_uuid_t', - '_ossl_old_des_ks_struct', 'id_t', 'ASN1_BIT_STRING', - 'va_list', '__darwin_wchar_t', 'pthread_key_t', - 'pkcs7_signer_info_st', 'ASN1_METHOD', 'DSA_SIG', 'DSA', - 'UIT_NONE', 'pthread_t', '__darwin_useconds_t', - 'uint_fast8_t', 'UI_STRING', 'DES_cblock', - '__darwin_mcontext64_t', 'rlim_t', 'PEM_Encode_Seal_st', - 'SHAstate_st', 'u_quad_t', 'openssl_fptr', - '_opaque_pthread_rwlockattr_t', - 'N18x509_attributes_st4DOLLAR_13E', - '__darwin_pthread_rwlock_t', 'daddr_t', 'ui_string_st', - 'x509_file_st', 'X509_req_info_st', 'int_least64_t', - 'evp_Encode_Ctx_st', 'X509_OBJECTS', 'CRYPTO_EX_DATA', - '__int8_t', 'AUTHORITY_KEYID_st', '_opaque_pthread_attr_t', - 'sigstack', 'EVP_CIPHER_CTX', 'X509_extension_st', 'pid_t', - 'RSA_METHOD', 'PEM_USER', 'pem_recip_st', 'env_md_ctx_st', - 'rc5_key_st', 'ui_st', 'X509_PUBKEY', 'u_int8_t', - 'ASN1_ITEM_st', 'pkcs7_recip_info_st', 'ssl2_state_st', - 'off_t', 'N10ssl_ctx_st4DOLLAR_18E', 'crypto_ex_data_st', - 'ui_method_st', '__darwin_pthread_rwlockattr_t', - 'CRYPTO_EX_dup', '__darwin_ino_t', '__sFILE', - 'OSUnknownByteOrder', 'BN_MONT_CTX', 'ASN1_NULL', 'time_t', - 'CRYPTO_EX_new', 'asn1_type_st', 'CRYPTO_EX_DATA_FUNCS', - 'user_time_t', 'BIGNUM', 'pthread_rwlockattr_t', - 'ASN1_VALUE_st', 'DH_METHOD', '__darwin_off_t', - '_opaque_pthread_t', 'bn_blinding_st', 'RSA', 'ssize_t', - 'mcontext64_t', 'user_long_t', 'fsblkcnt_t', 'cert_st', - '__darwin_pthread_condattr_t', 'X509_PKEY', - '__darwin_id_t', '__darwin_nl_item', 'SSL2_STATE', 'FILE', - 'pthread_mutexattr_t', 'size_t', - '_ossl_old_des_key_schedule', 'pkcs7_issuer_and_serial_st', - 'sigval', 'CRYPTO_MEM_LEAK_CB', 'X509_NAME', 'blkcnt_t', - 'uint_least16_t', '__darwin_dev_t', 'evp_cipher_info_st', - 'BN_BLINDING', 'ssl3_state_st', 'uint_least64_t', - 'user_addr_t', 'DES_key_schedule', 'RIPEMD160_CTX', - 'u_char', 'X509_algor_st', 'uid_t', 'sess_cert_st', - 'u_int64_t', 'u_int16_t', 'sigset_t', '__darwin_ptrdiff_t', - 'ASN1_CTX', 'STACK', '__int32_t', 'UI_METHOD', - 'NETSCAPE_SPKI', 'UIT_PROMPT', 'st_CRYPTO_EX_DATA_IMPL', - 'cast_key_st', 'X509_HASH_DIR_CTX', 'sigevent', - 'user_ssize_t', 'clock_t', 'aes_key_st', - '__darwin_socklen_t', '__darwin_intptr_t', 'int_fast64_t', - 'asn1_string_table_st', 'uint_fast32_t', - 'ASN1_VISIBLESTRING', 'DSA_SIG_st', 'obj_name_st', - 'X509_LOOKUP_METHOD', 'u_int32_t', 'EVP_CIPHER_INFO', - '__gnuc_va_list', 'AES_KEY', 'PKCS7_ISSUER_AND_SERIAL', - 'BN_CTX', '__darwin_blkcnt_t', 'key_t', 'SHA_CTX', - 'pkcs7_signed_st', 'SSL', 'N10pem_ctx_st4DOLLAR_16E', - 'pthread_attr_t', 'EVP_MD', 'uint', 'ASN1_BOOLEAN', - 'ino_t', '__darwin_clock_t', 'ASN1_OCTET_STRING', - 'asn1_ctx_st', 'BIO_F_BUFFER_CTX', 'bn_mont_ctx_st', - 'X509_REQ_INFO', 'PEM_CTX', 'sigvec', - '__darwin_pthread_mutexattr_t', 'x509_attributes_st', - 'stack_t', '__darwin_mode_t', '__mbstate_t', - 'asn1_object_st', 'ASN1_ENCODING', '__uint8_t', - 'LHASH_NODE', 'PKCS7_SIGNER_INFO', 'asn1_method_st', - 'stack_st', 'bio_info_cb', 'div_t', 'UIT_VERIFY', - 'PBEPARAM_st', 'N4wait3DOLLAR_3E', 'quad_t', '__siginfo', - '__darwin_mbstate_t', 'rsa_st', 'ASN1_UNIVERSALSTRING', - 'uint64_t', 'ssl_comp_st', 'X509_OBJECT', 'pthread_cond_t', - 'DH', '__darwin_wctype_t', 'PKCS7_ENVELOPE', 'ASN1_TLC_st', - 'sig_atomic_t', 'BIO', 'nlink_t', 'BUF_MEM', 'SSL3_RECORD', - 'bio_method_st', 'timeval', 'UI_string_types', 'BIO_dummy', - 'ssl_ctx_st', 'NETSCAPE_CERT_SEQUENCE', - 'BIT_STRING_BITNAME_st', '__darwin_pthread_attr_t', - 'int8_t', '__darwin_wint_t', 'OBJ_NAME', - 'PKCS8_PRIV_KEY_INFO', 'PBE2PARAM_st', - 'LHASH_DOALL_FN_TYPE', 'x509_st', 'X509_VAL', 'dev_t', - 'ASN1_TEMPLATE_st', 'MD5state_st', '__uint16_t', - 'LHASH_DOALL_ARG_FN_TYPE', 'mdc2_ctx_st', 'SSL3_STATE', - 'ssl3_buffer_st', 'ASN1_ITEM_EXP', - '_opaque_pthread_condattr_t', 'mode_t', 'ASN1_VALUE', - 'qaddr_t', '__darwin_gid_t', 'EVP_PKEY', 'CRYPTO_EX_free', - '_ossl_old_des_cblock', 'X509_INFO', 'asn1_string_st', - 'intptr_t', 'UIT_INFO', 'int_fast8_t', 'sigaltstack', - 'env_md_st', 'LHASH', '__darwin_ucontext_t', - 'PKCS7_SIGN_ENVELOPE', '__darwin_mcontext_t', 'ct_rune_t', - 'MD2_CTX', 'pthread_once_t', 'SSL3_BUFFER', 'fd_mask', - 'ASN1_TYPE', 'PKCS7_SIGNED', 'ssl3_record_st', 'BF_KEY', - 'MD4state_st', 'MD4_CTX', 'int16_t', 'SSL_CIPHER', - 'rune_t', 'X509_TRUST', 'siginfo_t', 'X509_STORE', - '__sbuf', 'X509_STORE_CTX', '__darwin_blksize_t', 'ldiv_t', - 'ASN1_TIME', 'SSL_METHOD', 'X509_LOOKUP', - 'Netscape_spki_st', 'P_PID', 'sigaction', 'sig_t', - 'hostent', 'x509_cert_aux_st', '_opaque_pthread_cond_t', - 'segsz_t', 'ushort', '__darwin_ct_rune_t', 'fd_set', - 'BN_RECP_CTX', 'x509_lookup_st', 'uint16_t', 'pkcs7_st', - 'asn1_header_st', '__darwin_pthread_key_t', - 'x509_trust_st', '__darwin_pthread_handler_rec', 'int32_t', - 'X509_CRL_INFO', 'N11evp_pkey_st4DOLLAR_12E', 'MDC2_CTX', - 'N23_ossl_old_des_ks_struct4DOLLAR_10E', 'ASN1_HEADER', - 'X509_crl_info_st', 'LHASH_HASH_FN_TYPE', - '_opaque_pthread_mutexattr_t', 'ssl_st', - 'N8pkcs7_st4DOLLAR_15E', 'evp_pkey_st', - 'pkcs7_signedandenveloped_st', '__darwin_mach_port_t', - 'EVP_PBE_KEYGEN', '_opaque_pthread_mutex_t', - 'ASN1_UTCTIME', 'mcontext', 'crypto_ex_data_func_st', - 'u_long', 'PBKDF2PARAM_st', 'rc4_key_st', 'DSA_METHOD', - 'EVP_CIPHER', 'BIT_STRING_BITNAME', 'PKCS7_RECIP_INFO', - 'ssl3_enc_method', 'X509_CERT_AUX', 'uintmax_t', - 'int_fast16_t', 'RC5_32_KEY', 'ucontext64', 'ASN1_INTEGER', - 'u_short', 'N14x509_object_st4DOLLAR_14E', 'mcontext64', - 'X509_sig_st', 'ASN1_GENERALSTRING', 'PKCS7', '__sFILEX', - 'X509_name_entry_st', 'ssl_session_st', 'caddr_t', - 'bignum_st', 'X509_CINF', '__darwin_pthread_cond_t', - 'ASN1_TLC', 'PKCS7_ENCRYPT', 'NETSCAPE_SPKAC', - 'Netscape_spkac_st', 'idtype_t', 'UIT_ERROR', - 'uint_fast64_t', 'in_addr_t', 'pthread_mutex_t', - '__int64_t', 'ASN1_BMPSTRING', 'uint32_t', - 'PEM_ENCODE_SEAL_CTX', 'suseconds_t', 'ASN1_OBJECT', - 'X509_val_st', 'private_key_st', 'CRYPTO_dynlock', - 'X509_objects_st', 'CRYPTO_EX_DATA_IMPL', - 'pthread_condattr_t', 'PKCS7_DIGEST', 'uint_least32_t', - 'ASN1_STRING', '__uint32_t', 'P_PGID', 'rsa_meth_st', - 'X509_crl_st', 'RC2_KEY', '__darwin_fsfilcnt_t', - 'X509_revoked_st', 'PBE2PARAM', 'blksize_t', - 'Netscape_certificate_sequence', 'ssl_cipher_st', - 'bignum_ctx', 'register_t', 'ASN1_UTF8STRING', - 'pkcs7_encrypted_st', 'RC4_KEY', '__darwin_ucontext64_t', - 'N13ssl2_state_st4DOLLAR_19E', 'bn_recp_ctx_st', - 'CAST_KEY', 'X509_ATTRIBUTE', '__darwin_suseconds_t', - '__sigaction', 'user_ulong_t', 'syscall_arg_t', - 'evp_cipher_ctx_st', 'X509_ALGOR', 'mcontext_t', - 'const_DES_cblock', '__darwin_fsblkcnt_t', 'dsa_st', - 'int_least8_t', 'MD2state_st', 'X509_EXTENSION', - 'GEN_SESSION_CB', 'int_least16_t', '__darwin_wctrans_t', - 'PBKDF2PARAM', 'x509_lookup_method_st', 'pem_password_cb', - 'X509_info_st', 'x509_store_st', '__darwin_natural_t', - 'X509_pubkey_st', 'pkcs7_digest_st', '__darwin_size_t', - 'ASN1_STRING_TABLE', 'OSLittleEndian', 'RIPEMD160state_st', - 'pkcs7_enveloped_st', 'UI', 'ptrdiff_t', 'X509_REQ', - 'CRYPTO_dynlock_value', 'X509_req_st', 'x509_store_ctx_st', - 'N13ssl3_state_st4DOLLAR_20E', 'lhash_node_st', - '__darwin_pthread_mutex_t', 'LHASH_COMP_FN_TYPE', - '__darwin_rune_t', 'rlimit', '__darwin_pthread_once_t', - 'OSBigEndian', 'uintptr_t', '__darwin_uid_t', 'u_int', - 'ASN1_T61STRING', 'gid_t', 'ssl_method_st', 'ASN1_ITEM', - 'ASN1_ENUMERATED', '_opaque_pthread_rwlock_t', - 'pkcs8_priv_key_info_st', 'intmax_t', 'sigcontext', - 'X509_CRL', 'rc2_key_st', 'engine_st', 'x509_object_st', - '_opaque_pthread_once_t', 'DES_ks', 'SSL_COMP', - 'dsa_method', 'int64_t', 'bio_st', 'bf_key_st', - 'ASN1_GENERALIZEDTIME', 'PKCS7_ENC_CONTENT', - '__darwin_pid_t', 'lldiv_t', 'comp_method_st', - 'EVP_MD_CTX', 'evp_cipher_st', 'X509_name_st', - 'x509_hash_dir_st', '__darwin_mach_port_name_t', - 'useconds_t', 'user_size_t', 'SSL_SESSION', 'rusage', - 'ssl_crock_st', 'int_least32_t', '__sigaction_u', 'dh_st', - 'P_ALL', '__darwin_stack_t', 'N6DES_ks3DOLLAR_9E', - 'comp_ctx_st', 'X509_CERT_FILE_CTX'] diff --git a/Lib/test/test_lib2to3/data/py2_test_grammar.py b/Lib/test/test_lib2to3/data/py2_test_grammar.py deleted file mode 100644 index 1a631510f4dba7..00000000000000 --- a/Lib/test/test_lib2to3/data/py2_test_grammar.py +++ /dev/null @@ -1,971 +0,0 @@ -# Python test set -- part 1, grammar. -# This just tests whether the parser accepts them all. - -# NOTE: When you run this test as a script from the command line, you -# get warnings about certain hex/oct constants. Since those are -# issued by the parser, you can't suppress them by adding a -# filterwarnings() call to this module. Therefore, to shut up the -# regression test, the filterwarnings() call has been added to -# regrtest.py. - -from test.test_support import check_syntax_error -import unittest -import sys -# testing import * -from sys import * - -class TokenTests(unittest.TestCase): - - def testBackslash(self): - # Backslash means line continuation: - x = 1 \ - + 1 - self.assertEquals(x, 2, 'backslash for line continuation') - - # Backslash does not means continuation in comments :\ - x = 0 - self.assertEquals(x, 0, 'backslash ending comment') - - def testPlainIntegers(self): - self.assertEquals(0xff, 255) - self.assertEquals(0377, 255) - self.assertEquals(2147483647, 017777777777) - # "0x" is not a valid literal - self.assertRaises(SyntaxError, eval, "0x") - from sys import maxint - if maxint == 2147483647: - self.assertEquals(-2147483647-1, -020000000000) - # XXX -2147483648 - self.assert_(037777777777 > 0) - self.assert_(0xffffffff > 0) - for s in '2147483648', '040000000000', '0x100000000': - try: - x = eval(s) - except OverflowError: - self.fail("OverflowError on huge integer literal %r" % s) - elif maxint == 9223372036854775807: - self.assertEquals(-9223372036854775807-1, -01000000000000000000000) - self.assert_(01777777777777777777777 > 0) - self.assert_(0xffffffffffffffff > 0) - for s in '9223372036854775808', '02000000000000000000000', \ - '0x10000000000000000': - try: - x = eval(s) - except OverflowError: - self.fail("OverflowError on huge integer literal %r" % s) - else: - self.fail('Weird maxint value %r' % maxint) - - def testLongIntegers(self): - x = 0L - x = 0l - x = 0xffffffffffffffffL - x = 0xffffffffffffffffl - x = 077777777777777777L - x = 077777777777777777l - x = 123456789012345678901234567890L - x = 123456789012345678901234567890l - - def testFloats(self): - x = 3.14 - x = 314. - x = 0.314 - # XXX x = 000.314 - x = .314 - x = 3e14 - x = 3E14 - x = 3e-14 - x = 3e+14 - x = 3.e14 - x = .3e14 - x = 3.1e4 - - def testStringLiterals(self): - x = ''; y = ""; self.assert_(len(x) == 0 and x == y) - x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39) - x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34) - x = "doesn't \"shrink\" does it" - y = 'doesn\'t "shrink" does it' - self.assert_(len(x) == 24 and x == y) - x = "does \"shrink\" doesn't it" - y = 'does "shrink" doesn\'t it' - self.assert_(len(x) == 24 and x == y) - x = """ -The "quick" -brown fox -jumps over -the 'lazy' dog. -""" - y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n' - self.assertEquals(x, y) - y = ''' -The "quick" -brown fox -jumps over -the 'lazy' dog. -''' - self.assertEquals(x, y) - y = "\n\ -The \"quick\"\n\ -brown fox\n\ -jumps over\n\ -the 'lazy' dog.\n\ -" - self.assertEquals(x, y) - y = '\n\ -The \"quick\"\n\ -brown fox\n\ -jumps over\n\ -the \'lazy\' dog.\n\ -' - self.assertEquals(x, y) - - -class GrammarTests(unittest.TestCase): - - # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE - # XXX can't test in a script -- this rule is only used when interactive - - # file_input: (NEWLINE | stmt)* ENDMARKER - # Being tested as this very moment this very module - - # expr_input: testlist NEWLINE - # XXX Hard to test -- used only in calls to input() - - def testEvalInput(self): - # testlist ENDMARKER - x = eval('1, 0 or 1') - - def testFuncdef(self): - ### 'def' NAME parameters ':' suite - ### parameters: '(' [varargslist] ')' - ### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME] - ### | ('**'|'*' '*') NAME) - ### | fpdef ['=' test] (',' fpdef ['=' test])* [','] - ### fpdef: NAME | '(' fplist ')' - ### fplist: fpdef (',' fpdef)* [','] - ### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test) - ### argument: [test '='] test # Really [keyword '='] test - def f1(): pass - f1() - f1(*()) - f1(*(), **{}) - def f2(one_argument): pass - def f3(two, arguments): pass - def f4(two, (compound, (argument, list))): pass - def f5((compound, first), two): pass - self.assertEquals(f2.func_code.co_varnames, ('one_argument',)) - self.assertEquals(f3.func_code.co_varnames, ('two', 'arguments')) - if sys.platform.startswith('java'): - self.assertEquals(f4.func_code.co_varnames, - ('two', '(compound, (argument, list))', 'compound', 'argument', - 'list',)) - self.assertEquals(f5.func_code.co_varnames, - ('(compound, first)', 'two', 'compound', 'first')) - else: - self.assertEquals(f4.func_code.co_varnames, - ('two', '.1', 'compound', 'argument', 'list')) - self.assertEquals(f5.func_code.co_varnames, - ('.0', 'two', 'compound', 'first')) - def a1(one_arg,): pass - def a2(two, args,): pass - def v0(*rest): pass - def v1(a, *rest): pass - def v2(a, b, *rest): pass - def v3(a, (b, c), *rest): return a, b, c, rest - - f1() - f2(1) - f2(1,) - f3(1, 2) - f3(1, 2,) - f4(1, (2, (3, 4))) - v0() - v0(1) - v0(1,) - v0(1,2) - v0(1,2,3,4,5,6,7,8,9,0) - v1(1) - v1(1,) - v1(1,2) - v1(1,2,3) - v1(1,2,3,4,5,6,7,8,9,0) - v2(1,2) - v2(1,2,3) - v2(1,2,3,4) - v2(1,2,3,4,5,6,7,8,9,0) - v3(1,(2,3)) - v3(1,(2,3),4) - v3(1,(2,3),4,5,6,7,8,9,0) - - # ceval unpacks the formal arguments into the first argcount names; - # thus, the names nested inside tuples must appear after these names. - if sys.platform.startswith('java'): - self.assertEquals(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c')) - else: - self.assertEquals(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c')) - self.assertEquals(v3(1, (2, 3), 4), (1, 2, 3, (4,))) - def d01(a=1): pass - d01() - d01(1) - d01(*(1,)) - d01(**{'a':2}) - def d11(a, b=1): pass - d11(1) - d11(1, 2) - d11(1, **{'b':2}) - def d21(a, b, c=1): pass - d21(1, 2) - d21(1, 2, 3) - d21(*(1, 2, 3)) - d21(1, *(2, 3)) - d21(1, 2, *(3,)) - d21(1, 2, **{'c':3}) - def d02(a=1, b=2): pass - d02() - d02(1) - d02(1, 2) - d02(*(1, 2)) - d02(1, *(2,)) - d02(1, **{'b':2}) - d02(**{'a': 1, 'b': 2}) - def d12(a, b=1, c=2): pass - d12(1) - d12(1, 2) - d12(1, 2, 3) - def d22(a, b, c=1, d=2): pass - d22(1, 2) - d22(1, 2, 3) - d22(1, 2, 3, 4) - def d01v(a=1, *rest): pass - d01v() - d01v(1) - d01v(1, 2) - d01v(*(1, 2, 3, 4)) - d01v(*(1,)) - d01v(**{'a':2}) - def d11v(a, b=1, *rest): pass - d11v(1) - d11v(1, 2) - d11v(1, 2, 3) - def d21v(a, b, c=1, *rest): pass - d21v(1, 2) - d21v(1, 2, 3) - d21v(1, 2, 3, 4) - d21v(*(1, 2, 3, 4)) - d21v(1, 2, **{'c': 3}) - def d02v(a=1, b=2, *rest): pass - d02v() - d02v(1) - d02v(1, 2) - d02v(1, 2, 3) - d02v(1, *(2, 3, 4)) - d02v(**{'a': 1, 'b': 2}) - def d12v(a, b=1, c=2, *rest): pass - d12v(1) - d12v(1, 2) - d12v(1, 2, 3) - d12v(1, 2, 3, 4) - d12v(*(1, 2, 3, 4)) - d12v(1, 2, *(3, 4, 5)) - d12v(1, *(2,), **{'c': 3}) - def d22v(a, b, c=1, d=2, *rest): pass - d22v(1, 2) - d22v(1, 2, 3) - d22v(1, 2, 3, 4) - d22v(1, 2, 3, 4, 5) - d22v(*(1, 2, 3, 4)) - d22v(1, 2, *(3, 4, 5)) - d22v(1, *(2, 3), **{'d': 4}) - def d31v((x)): pass - d31v(1) - def d32v((x,)): pass - d32v((1,)) - - # keyword arguments after *arglist - def f(*args, **kwargs): - return args, kwargs - self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4), - {'x':2, 'y':5})) - self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)") - self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)") - - # Check ast errors in *args and *kwargs - check_syntax_error(self, "f(*g(1=2))") - check_syntax_error(self, "f(**g(1=2))") - - def testLambdef(self): - ### lambdef: 'lambda' [varargslist] ':' test - l1 = lambda : 0 - self.assertEquals(l1(), 0) - l2 = lambda : a[d] # XXX just testing the expression - l3 = lambda : [2 < x for x in [-1, 3, 0L]] - self.assertEquals(l3(), [0, 1, 0]) - l4 = lambda x = lambda y = lambda z=1 : z : y() : x() - self.assertEquals(l4(), 1) - l5 = lambda x, y, z=2: x + y + z - self.assertEquals(l5(1, 2), 5) - self.assertEquals(l5(1, 2, 3), 6) - check_syntax_error(self, "lambda x: x = 2") - check_syntax_error(self, "lambda (None,): None") - - ### stmt: simple_stmt | compound_stmt - # Tested below - - def testSimpleStmt(self): - ### simple_stmt: small_stmt (';' small_stmt)* [';'] - x = 1; pass; del x - def foo(): - # verify statements that end with semi-colons - x = 1; pass; del x; - foo() - - ### small_stmt: expr_stmt | print_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt - # Tested below - - def testExprStmt(self): - # (exprlist '=')* exprlist - 1 - 1, 2, 3 - x = 1 - x = 1, 2, 3 - x = y = z = 1, 2, 3 - x, y, z = 1, 2, 3 - abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4) - - check_syntax_error(self, "x + 1 = 1") - check_syntax_error(self, "a + 1 = b + 2") - - def testPrintStmt(self): - # 'print' (test ',')* [test] - import StringIO - - # Can't test printing to real stdout without comparing output - # which is not available in unittest. - save_stdout = sys.stdout - sys.stdout = StringIO.StringIO() - - print 1, 2, 3 - print 1, 2, 3, - print - print 0 or 1, 0 or 1, - print 0 or 1 - - # 'print' '>>' test ',' - print >> sys.stdout, 1, 2, 3 - print >> sys.stdout, 1, 2, 3, - print >> sys.stdout - print >> sys.stdout, 0 or 1, 0 or 1, - print >> sys.stdout, 0 or 1 - - # test printing to an instance - class Gulp: - def write(self, msg): pass - - gulp = Gulp() - print >> gulp, 1, 2, 3 - print >> gulp, 1, 2, 3, - print >> gulp - print >> gulp, 0 or 1, 0 or 1, - print >> gulp, 0 or 1 - - # test print >> None - def driver(): - oldstdout = sys.stdout - sys.stdout = Gulp() - try: - tellme(Gulp()) - tellme() - finally: - sys.stdout = oldstdout - - # we should see this once - def tellme(file=sys.stdout): - print >> file, 'hello world' - - driver() - - # we should not see this at all - def tellme(file=None): - print >> file, 'goodbye universe' - - driver() - - self.assertEqual(sys.stdout.getvalue(), '''\ -1 2 3 -1 2 3 -1 1 1 -1 2 3 -1 2 3 -1 1 1 -hello world -''') - sys.stdout = save_stdout - - # syntax errors - check_syntax_error(self, 'print ,') - check_syntax_error(self, 'print >> x,') - - def testDelStmt(self): - # 'del' exprlist - abc = [1,2,3] - x, y, z = abc - xyz = x, y, z - - del abc - del x, y, (z, xyz) - - def testPassStmt(self): - # 'pass' - pass - - # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt - # Tested below - - def testBreakStmt(self): - # 'break' - while 1: break - - def testContinueStmt(self): - # 'continue' - i = 1 - while i: i = 0; continue - - msg = "" - while not msg: - msg = "ok" - try: - continue - msg = "continue failed to continue inside try" - except: - msg = "continue inside try called except block" - if msg != "ok": - self.fail(msg) - - msg = "" - while not msg: - msg = "finally block not called" - try: - continue - finally: - msg = "ok" - if msg != "ok": - self.fail(msg) - - def test_break_continue_loop(self): - # This test warrants an explanation. It is a test specifically for SF bugs - # #463359 and #462937. The bug is that a 'break' statement executed or - # exception raised inside a try/except inside a loop, *after* a continue - # statement has been executed in that loop, will cause the wrong number of - # arguments to be popped off the stack and the instruction pointer reset to - # a very small number (usually 0.) Because of this, the following test - # *must* written as a function, and the tracking vars *must* be function - # arguments with default values. Otherwise, the test will loop and loop. - - def test_inner(extra_burning_oil = 1, count=0): - big_hippo = 2 - while big_hippo: - count += 1 - try: - if extra_burning_oil and big_hippo == 1: - extra_burning_oil -= 1 - break - big_hippo -= 1 - continue - except: - raise - if count > 2 or big_hippo <> 1: - self.fail("continue then break in try/except in loop broken!") - test_inner() - - def testReturn(self): - # 'return' [testlist] - def g1(): return - def g2(): return 1 - g1() - x = g2() - check_syntax_error(self, "class foo:return 1") - - def testYield(self): - check_syntax_error(self, "class foo:yield 1") - - def testRaise(self): - # 'raise' test [',' test] - try: raise RuntimeError, 'just testing' - except RuntimeError: pass - try: raise KeyboardInterrupt - except KeyboardInterrupt: pass - - def testImport(self): - # 'import' dotted_as_names - import sys - import time, sys - # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names) - from time import time - from time import (time) - # not testable inside a function, but already done at top of the module - # from sys import * - from sys import path, argv - from sys import (path, argv) - from sys import (path, argv,) - - def testGlobal(self): - # 'global' NAME (',' NAME)* - global a - global a, b - global one, two, three, four, five, six, seven, eight, nine, ten - - def testExec(self): - # 'exec' expr ['in' expr [',' expr]] - z = None - del z - exec 'z=1+1\n' - if z != 2: self.fail('exec \'z=1+1\'\\n') - del z - exec 'z=1+1' - if z != 2: self.fail('exec \'z=1+1\'') - z = None - del z - import types - if hasattr(types, "UnicodeType"): - exec r"""if 1: - exec u'z=1+1\n' - if z != 2: self.fail('exec u\'z=1+1\'\\n') - del z - exec u'z=1+1' - if z != 2: self.fail('exec u\'z=1+1\'')""" - g = {} - exec 'z = 1' in g - if g.has_key('__builtins__'): del g['__builtins__'] - if g != {'z': 1}: self.fail('exec \'z = 1\' in g') - g = {} - l = {} - - import warnings - warnings.filterwarnings("ignore", "global statement", module="") - exec 'global a; a = 1; b = 2' in g, l - if g.has_key('__builtins__'): del g['__builtins__'] - if l.has_key('__builtins__'): del l['__builtins__'] - if (g, l) != ({'a':1}, {'b':2}): - self.fail('exec ... in g (%s), l (%s)' %(g,l)) - - def testAssert(self): - # assert_stmt: 'assert' test [',' test] - assert 1 - assert 1, 1 - assert lambda x:x - assert 1, lambda x:x+1 - try: - assert 0, "msg" - except AssertionError, e: - self.assertEquals(e.args[0], "msg") - else: - if __debug__: - self.fail("AssertionError not raised by assert 0") - - ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef - # Tested below - - def testIf(self): - # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] - if 1: pass - if 1: pass - else: pass - if 0: pass - elif 0: pass - if 0: pass - elif 0: pass - elif 0: pass - elif 0: pass - else: pass - - def testWhile(self): - # 'while' test ':' suite ['else' ':' suite] - while 0: pass - while 0: pass - else: pass - - # Issue1920: "while 0" is optimized away, - # ensure that the "else" clause is still present. - x = 0 - while 0: - x = 1 - else: - x = 2 - self.assertEquals(x, 2) - - def testFor(self): - # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] - for i in 1, 2, 3: pass - for i, j, k in (): pass - else: pass - class Squares: - def __init__(self, max): - self.max = max - self.sofar = [] - def __len__(self): return len(self.sofar) - def __getitem__(self, i): - if not 0 <= i < self.max: raise IndexError - n = len(self.sofar) - while n <= i: - self.sofar.append(n*n) - n = n+1 - return self.sofar[i] - n = 0 - for x in Squares(10): n = n+x - if n != 285: - self.fail('for over growing sequence') - - result = [] - for x, in [(1,), (2,), (3,)]: - result.append(x) - self.assertEqual(result, [1, 2, 3]) - - def testTry(self): - ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] - ### | 'try' ':' suite 'finally' ':' suite - ### except_clause: 'except' [expr [('as' | ',') expr]] - try: - 1/0 - except ZeroDivisionError: - pass - else: - pass - try: 1/0 - except EOFError: pass - except TypeError as msg: pass - except RuntimeError, msg: pass - except: pass - else: pass - try: 1/0 - except (EOFError, TypeError, ZeroDivisionError): pass - try: 1/0 - except (EOFError, TypeError, ZeroDivisionError), msg: pass - try: pass - finally: pass - - def testSuite(self): - # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT - if 1: pass - if 1: - pass - if 1: - # - # - # - pass - pass - # - pass - # - - def testTest(self): - ### and_test ('or' and_test)* - ### and_test: not_test ('and' not_test)* - ### not_test: 'not' not_test | comparison - if not 1: pass - if 1 and 1: pass - if 1 or 1: pass - if not not not 1: pass - if not 1 and 1 and 1: pass - if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass - - def testComparison(self): - ### comparison: expr (comp_op expr)* - ### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' - if 1: pass - x = (1 == 1) - if 1 == 1: pass - if 1 != 1: pass - if 1 <> 1: pass - if 1 < 1: pass - if 1 > 1: pass - if 1 <= 1: pass - if 1 >= 1: pass - if 1 is 1: pass - if 1 is not 1: pass - if 1 in (): pass - if 1 not in (): pass - if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass - - def testBinaryMaskOps(self): - x = 1 & 1 - x = 1 ^ 1 - x = 1 | 1 - - def testShiftOps(self): - x = 1 << 1 - x = 1 >> 1 - x = 1 << 1 >> 1 - - def testAdditiveOps(self): - x = 1 - x = 1 + 1 - x = 1 - 1 - 1 - x = 1 - 1 + 1 - 1 + 1 - - def testMultiplicativeOps(self): - x = 1 * 1 - x = 1 / 1 - x = 1 % 1 - x = 1 / 1 * 1 % 1 - - def testUnaryOps(self): - x = +1 - x = -1 - x = ~1 - x = ~1 ^ 1 & 1 | 1 & 1 ^ -1 - x = -1*1/1 + 1*1 - ---1*1 - - def testSelectors(self): - ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME - ### subscript: expr | [expr] ':' [expr] - - import sys, time - c = sys.path[0] - x = time.time() - x = sys.modules['time'].time() - a = '01234' - c = a[0] - c = a[-1] - s = a[0:5] - s = a[:5] - s = a[0:] - s = a[:] - s = a[-5:] - s = a[:-1] - s = a[-4:-3] - # A rough test of SF bug 1333982. https://bugs.python.org/issue1333982 - # The testing here is fairly incomplete. - # Test cases should include: commas with 1 and 2 colons - d = {} - d[1] = 1 - d[1,] = 2 - d[1,2] = 3 - d[1,2,3] = 4 - L = list(d) - L.sort() - self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]') - - def testAtoms(self): - ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING - ### dictmaker: test ':' test (',' test ':' test)* [','] - - x = (1) - x = (1 or 2 or 3) - x = (1 or 2 or 3, 2, 3) - - x = [] - x = [1] - x = [1 or 2 or 3] - x = [1 or 2 or 3, 2, 3] - x = [] - - x = {} - x = {'one': 1} - x = {'one': 1,} - x = {'one' or 'two': 1 or 2} - x = {'one': 1, 'two': 2} - x = {'one': 1, 'two': 2,} - x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6} - - x = `x` - x = `1 or 2 or 3` - self.assertEqual(`1,2`, '(1, 2)') - - x = x - x = 'x' - x = 123 - - ### exprlist: expr (',' expr)* [','] - ### testlist: test (',' test)* [','] - # These have been exercised enough above - - def testClassdef(self): - # 'class' NAME ['(' [testlist] ')'] ':' suite - class B: pass - class B2(): pass - class C1(B): pass - class C2(B): pass - class D(C1, C2, B): pass - class C: - def meth1(self): pass - def meth2(self, arg): pass - def meth3(self, a1, a2): pass - # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE - # decorators: decorator+ - # decorated: decorators (classdef | funcdef) - def class_decorator(x): - x.decorated = True - return x - @class_decorator - class G: - pass - self.assertEqual(G.decorated, True) - - def testListcomps(self): - # list comprehension tests - nums = [1, 2, 3, 4, 5] - strs = ["Apple", "Banana", "Coconut"] - spcs = [" Apple", " Banana ", "Coco nut "] - - self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut']) - self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15]) - self.assertEqual([x for x in nums if x > 2], [3, 4, 5]) - self.assertEqual([(i, s) for i in nums for s in strs], - [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'), - (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'), - (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'), - (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'), - (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')]) - self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]], - [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'), - (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'), - (5, 'Banana'), (5, 'Coconut')]) - self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)], - [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]]) - - def test_in_func(l): - return [None < x < 3 for x in l if x > 2] - - self.assertEqual(test_in_func(nums), [False, False, False]) - - def test_nested_front(): - self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]], - [[1, 2], [3, 4], [5, 6]]) - - test_nested_front() - - check_syntax_error(self, "[i, s for i in nums for s in strs]") - check_syntax_error(self, "[x if y]") - - suppliers = [ - (1, "Boeing"), - (2, "Ford"), - (3, "Macdonalds") - ] - - parts = [ - (10, "Airliner"), - (20, "Engine"), - (30, "Cheeseburger") - ] - - suppart = [ - (1, 10), (1, 20), (2, 20), (3, 30) - ] - - x = [ - (sname, pname) - for (sno, sname) in suppliers - for (pno, pname) in parts - for (sp_sno, sp_pno) in suppart - if sno == sp_sno and pno == sp_pno - ] - - self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'), - ('Macdonalds', 'Cheeseburger')]) - - def testGenexps(self): - # generator expression tests - g = ([x for x in range(10)] for x in range(1)) - self.assertEqual(g.next(), [x for x in range(10)]) - try: - g.next() - self.fail('should produce StopIteration exception') - except StopIteration: - pass - - a = 1 - try: - g = (a for d in a) - g.next() - self.fail('should produce TypeError') - except TypeError: - pass - - self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd']) - self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy']) - - a = [x for x in range(10)] - b = (x for x in (y for y in a)) - self.assertEqual(sum(b), sum([x for x in range(10)])) - - self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)])) - self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2])) - self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0) - check_syntax_error(self, "foo(x for x in range(10), 100)") - check_syntax_error(self, "foo(100, x for x in range(10))") - - def testComprehensionSpecials(self): - # test for outmost iterable precomputation - x = 10; g = (i for i in range(x)); x = 5 - self.assertEqual(len(list(g)), 10) - - # This should hold, since we're only precomputing outmost iterable. - x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x)) - x = 5; t = True; - self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g)) - - # Grammar allows multiple adjacent 'if's in listcomps and genexps, - # even though it's silly. Make sure it works (ifelse broke this.) - self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7]) - self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7]) - - # verify unpacking single element tuples in listcomp/genexp. - self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6]) - self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9]) - - def test_with_statement(self): - class manager(object): - def __enter__(self): - return (1, 2) - def __exit__(self, *args): - pass - - with manager(): - pass - with manager() as x: - pass - with manager() as (x, y): - pass - with manager(), manager(): - pass - with manager() as x, manager() as y: - pass - with manager() as x, manager(): - pass - - def testIfElseExpr(self): - # Test ifelse expressions in various cases - def _checkeval(msg, ret): - "helper to check that evaluation of expressions is done correctly" - print x - return ret - - self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True]) - self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True]) - self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True]) - self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5) - self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5) - self.assertEqual((5 and 6 if 0 else 1), 1) - self.assertEqual(((5 and 6) if 0 else 1), 1) - self.assertEqual((5 and (6 if 1 else 1)), 6) - self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3) - self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1) - self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5) - self.assertEqual((not 5 if 1 else 1), False) - self.assertEqual((not 5 if 0 else 1), 1) - self.assertEqual((6 + 1 if 1 else 2), 7) - self.assertEqual((6 - 1 if 1 else 2), 5) - self.assertEqual((6 * 2 if 1 else 4), 12) - self.assertEqual((6 / 2 if 1 else 3), 3) - self.assertEqual((6 < 4 if 0 else 2), 2) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_lib2to3/data/py3_test_grammar.py b/Lib/test/test_lib2to3/data/py3_test_grammar.py deleted file mode 100644 index 774851f5bd7e85..00000000000000 --- a/Lib/test/test_lib2to3/data/py3_test_grammar.py +++ /dev/null @@ -1,956 +0,0 @@ -# Python test set -- part 1, grammar. -# This just tests whether the parser accepts them all. - -# NOTE: When you run this test as a script from the command line, you -# get warnings about certain hex/oct constants. Since those are -# issued by the parser, you can't suppress them by adding a -# filterwarnings() call to this module. Therefore, to shut up the -# regression test, the filterwarnings() call has been added to -# regrtest.py. - -from test.support import check_syntax_error -import unittest -import sys -# testing import * -from sys import * - -class TokenTests(unittest.TestCase): - - def testBackslash(self): - # Backslash means line continuation: - x = 1 \ - + 1 - self.assertEquals(x, 2, 'backslash for line continuation') - - # Backslash does not means continuation in comments :\ - x = 0 - self.assertEquals(x, 0, 'backslash ending comment') - - def testPlainIntegers(self): - self.assertEquals(type(000), type(0)) - self.assertEquals(0xff, 255) - self.assertEquals(0o377, 255) - self.assertEquals(2147483647, 0o17777777777) - self.assertEquals(0b1001, 9) - # "0x" is not a valid literal - self.assertRaises(SyntaxError, eval, "0x") - from sys import maxsize - if maxsize == 2147483647: - self.assertEquals(-2147483647-1, -0o20000000000) - # XXX -2147483648 - self.assert_(0o37777777777 > 0) - self.assert_(0xffffffff > 0) - self.assert_(0b1111111111111111111111111111111 > 0) - for s in ('2147483648', '0o40000000000', '0x100000000', - '0b10000000000000000000000000000000'): - try: - x = eval(s) - except OverflowError: - self.fail("OverflowError on huge integer literal %r" % s) - elif maxsize == 9223372036854775807: - self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000) - self.assert_(0o1777777777777777777777 > 0) - self.assert_(0xffffffffffffffff > 0) - self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0) - for s in '9223372036854775808', '0o2000000000000000000000', \ - '0x10000000000000000', \ - '0b100000000000000000000000000000000000000000000000000000000000000': - try: - x = eval(s) - except OverflowError: - self.fail("OverflowError on huge integer literal %r" % s) - else: - self.fail('Weird maxsize value %r' % maxsize) - - def testLongIntegers(self): - x = 0 - x = 0xffffffffffffffff - x = 0Xffffffffffffffff - x = 0o77777777777777777 - x = 0O77777777777777777 - x = 123456789012345678901234567890 - x = 0b100000000000000000000000000000000000000000000000000000000000000000000 - x = 0B111111111111111111111111111111111111111111111111111111111111111111111 - - def testUnderscoresInNumbers(self): - # Integers - x = 1_0 - x = 123_456_7_89 - x = 0xabc_123_4_5 - x = 0X_abc_123 - x = 0B11_01 - x = 0b_11_01 - x = 0o45_67 - x = 0O_45_67 - - # Floats - x = 3_1.4 - x = 03_1.4 - x = 3_1. - x = .3_1 - x = 3.1_4 - x = 0_3.1_4 - x = 3e1_4 - x = 3_1e+4_1 - x = 3_1E-4_1 - - def testFloats(self): - x = 3.14 - x = 314. - x = 0.314 - # XXX x = 000.314 - x = .314 - x = 3e14 - x = 3E14 - x = 3e-14 - x = 3e+14 - x = 3.e14 - x = .3e14 - x = 3.1e4 - - def testStringLiterals(self): - x = ''; y = ""; self.assert_(len(x) == 0 and x == y) - x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39) - x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34) - x = "doesn't \"shrink\" does it" - y = 'doesn\'t "shrink" does it' - self.assert_(len(x) == 24 and x == y) - x = "does \"shrink\" doesn't it" - y = 'does "shrink" doesn\'t it' - self.assert_(len(x) == 24 and x == y) - x = """ -The "quick" -brown fox -jumps over -the 'lazy' dog. -""" - y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n' - self.assertEquals(x, y) - y = ''' -The "quick" -brown fox -jumps over -the 'lazy' dog. -''' - self.assertEquals(x, y) - y = "\n\ -The \"quick\"\n\ -brown fox\n\ -jumps over\n\ -the 'lazy' dog.\n\ -" - self.assertEquals(x, y) - y = '\n\ -The \"quick\"\n\ -brown fox\n\ -jumps over\n\ -the \'lazy\' dog.\n\ -' - self.assertEquals(x, y) - x = rf"hello \{True}"; y = f"hello \\{True}" - self.assertEquals(x, y) - - def testEllipsis(self): - x = ... - self.assert_(x is Ellipsis) - self.assertRaises(SyntaxError, eval, ".. .") - -class GrammarTests(unittest.TestCase): - - # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE - # XXX can't test in a script -- this rule is only used when interactive - - # file_input: (NEWLINE | stmt)* ENDMARKER - # Being tested as this very moment this very module - - # expr_input: testlist NEWLINE - # XXX Hard to test -- used only in calls to input() - - def testEvalInput(self): - # testlist ENDMARKER - x = eval('1, 0 or 1') - - def testFuncdef(self): - ### [decorators] 'def' NAME parameters ['->' test] ':' suite - ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE - ### decorators: decorator+ - ### parameters: '(' [typedargslist] ')' - ### typedargslist: ((tfpdef ['=' test] ',')* - ### ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) - ### | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) - ### tfpdef: NAME [':' test] - ### varargslist: ((vfpdef ['=' test] ',')* - ### ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) - ### | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) - ### vfpdef: NAME - def f1(): pass - f1() - f1(*()) - f1(*(), **{}) - def f2(one_argument): pass - def f3(two, arguments): pass - self.assertEquals(f2.__code__.co_varnames, ('one_argument',)) - self.assertEquals(f3.__code__.co_varnames, ('two', 'arguments')) - def a1(one_arg,): pass - def a2(two, args,): pass - def v0(*rest): pass - def v1(a, *rest): pass - def v2(a, b, *rest): pass - - f1() - f2(1) - f2(1,) - f3(1, 2) - f3(1, 2,) - v0() - v0(1) - v0(1,) - v0(1,2) - v0(1,2,3,4,5,6,7,8,9,0) - v1(1) - v1(1,) - v1(1,2) - v1(1,2,3) - v1(1,2,3,4,5,6,7,8,9,0) - v2(1,2) - v2(1,2,3) - v2(1,2,3,4) - v2(1,2,3,4,5,6,7,8,9,0) - - def d01(a=1): pass - d01() - d01(1) - d01(*(1,)) - d01(**{'a':2}) - def d11(a, b=1): pass - d11(1) - d11(1, 2) - d11(1, **{'b':2}) - def d21(a, b, c=1): pass - d21(1, 2) - d21(1, 2, 3) - d21(*(1, 2, 3)) - d21(1, *(2, 3)) - d21(1, 2, *(3,)) - d21(1, 2, **{'c':3}) - def d02(a=1, b=2): pass - d02() - d02(1) - d02(1, 2) - d02(*(1, 2)) - d02(1, *(2,)) - d02(1, **{'b':2}) - d02(**{'a': 1, 'b': 2}) - def d12(a, b=1, c=2): pass - d12(1) - d12(1, 2) - d12(1, 2, 3) - def d22(a, b, c=1, d=2): pass - d22(1, 2) - d22(1, 2, 3) - d22(1, 2, 3, 4) - def d01v(a=1, *rest): pass - d01v() - d01v(1) - d01v(1, 2) - d01v(*(1, 2, 3, 4)) - d01v(*(1,)) - d01v(**{'a':2}) - def d11v(a, b=1, *rest): pass - d11v(1) - d11v(1, 2) - d11v(1, 2, 3) - def d21v(a, b, c=1, *rest): pass - d21v(1, 2) - d21v(1, 2, 3) - d21v(1, 2, 3, 4) - d21v(*(1, 2, 3, 4)) - d21v(1, 2, **{'c': 3}) - def d02v(a=1, b=2, *rest): pass - d02v() - d02v(1) - d02v(1, 2) - d02v(1, 2, 3) - d02v(1, *(2, 3, 4)) - d02v(**{'a': 1, 'b': 2}) - def d12v(a, b=1, c=2, *rest): pass - d12v(1) - d12v(1, 2) - d12v(1, 2, 3) - d12v(1, 2, 3, 4) - d12v(*(1, 2, 3, 4)) - d12v(1, 2, *(3, 4, 5)) - d12v(1, *(2,), **{'c': 3}) - def d22v(a, b, c=1, d=2, *rest): pass - d22v(1, 2) - d22v(1, 2, 3) - d22v(1, 2, 3, 4) - d22v(1, 2, 3, 4, 5) - d22v(*(1, 2, 3, 4)) - d22v(1, 2, *(3, 4, 5)) - d22v(1, *(2, 3), **{'d': 4}) - - # keyword argument type tests - try: - str('x', **{b'foo':1 }) - except TypeError: - pass - else: - self.fail('Bytes should not work as keyword argument names') - # keyword only argument tests - def pos0key1(*, key): return key - pos0key1(key=100) - def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2 - pos2key2(1, 2, k1=100) - pos2key2(1, 2, k1=100, k2=200) - pos2key2(1, 2, k2=100, k1=200) - def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg - pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200) - pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100) - - # keyword arguments after *arglist - def f(*args, **kwargs): - return args, kwargs - self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4), - {'x':2, 'y':5})) - self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)") - self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)") - - # argument annotation tests - def f(x) -> list: pass - self.assertEquals(f.__annotations__, {'return': list}) - def f(x:int): pass - self.assertEquals(f.__annotations__, {'x': int}) - def f(*x:str): pass - self.assertEquals(f.__annotations__, {'x': str}) - def f(**x:float): pass - self.assertEquals(f.__annotations__, {'x': float}) - def f(x, y:1+2): pass - self.assertEquals(f.__annotations__, {'y': 3}) - def f(a, b:1, c:2, d): pass - self.assertEquals(f.__annotations__, {'b': 1, 'c': 2}) - def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass - self.assertEquals(f.__annotations__, - {'b': 1, 'c': 2, 'e': 3, 'g': 6}) - def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10, - **k:11) -> 12: pass - self.assertEquals(f.__annotations__, - {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9, - 'k': 11, 'return': 12}) - # Check for SF Bug #1697248 - mixing decorators and a return annotation - def null(x): return x - @null - def f(x) -> list: pass - self.assertEquals(f.__annotations__, {'return': list}) - - # test closures with a variety of oparg's - closure = 1 - def f(): return closure - def f(x=1): return closure - def f(*, k=1): return closure - def f() -> int: return closure - - # Check ast errors in *args and *kwargs - check_syntax_error(self, "f(*g(1=2))") - check_syntax_error(self, "f(**g(1=2))") - - def testLambdef(self): - ### lambdef: 'lambda' [varargslist] ':' test - l1 = lambda : 0 - self.assertEquals(l1(), 0) - l2 = lambda : a[d] # XXX just testing the expression - l3 = lambda : [2 < x for x in [-1, 3, 0]] - self.assertEquals(l3(), [0, 1, 0]) - l4 = lambda x = lambda y = lambda z=1 : z : y() : x() - self.assertEquals(l4(), 1) - l5 = lambda x, y, z=2: x + y + z - self.assertEquals(l5(1, 2), 5) - self.assertEquals(l5(1, 2, 3), 6) - check_syntax_error(self, "lambda x: x = 2") - check_syntax_error(self, "lambda (None,): None") - l6 = lambda x, y, *, k=20: x+y+k - self.assertEquals(l6(1,2), 1+2+20) - self.assertEquals(l6(1,2,k=10), 1+2+10) - - - ### stmt: simple_stmt | compound_stmt - # Tested below - - def testSimpleStmt(self): - ### simple_stmt: small_stmt (';' small_stmt)* [';'] - x = 1; pass; del x - def foo(): - # verify statements that end with semi-colons - x = 1; pass; del x; - foo() - - ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt - # Tested below - - def testExprStmt(self): - # (exprlist '=')* exprlist - 1 - 1, 2, 3 - x = 1 - x = 1, 2, 3 - x = y = z = 1, 2, 3 - x, y, z = 1, 2, 3 - abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4) - - check_syntax_error(self, "x + 1 = 1") - check_syntax_error(self, "a + 1 = b + 2") - - def testDelStmt(self): - # 'del' exprlist - abc = [1,2,3] - x, y, z = abc - xyz = x, y, z - - del abc - del x, y, (z, xyz) - - def testPassStmt(self): - # 'pass' - pass - - # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt - # Tested below - - def testBreakStmt(self): - # 'break' - while 1: break - - def testContinueStmt(self): - # 'continue' - i = 1 - while i: i = 0; continue - - msg = "" - while not msg: - msg = "ok" - try: - continue - msg = "continue failed to continue inside try" - except: - msg = "continue inside try called except block" - if msg != "ok": - self.fail(msg) - - msg = "" - while not msg: - msg = "finally block not called" - try: - continue - finally: - msg = "ok" - if msg != "ok": - self.fail(msg) - - def test_break_continue_loop(self): - # This test warrants an explanation. It is a test specifically for SF bugs - # #463359 and #462937. The bug is that a 'break' statement executed or - # exception raised inside a try/except inside a loop, *after* a continue - # statement has been executed in that loop, will cause the wrong number of - # arguments to be popped off the stack and the instruction pointer reset to - # a very small number (usually 0.) Because of this, the following test - # *must* written as a function, and the tracking vars *must* be function - # arguments with default values. Otherwise, the test will loop and loop. - - def test_inner(extra_burning_oil = 1, count=0): - big_hippo = 2 - while big_hippo: - count += 1 - try: - if extra_burning_oil and big_hippo == 1: - extra_burning_oil -= 1 - break - big_hippo -= 1 - continue - except: - raise - if count > 2 or big_hippo != 1: - self.fail("continue then break in try/except in loop broken!") - test_inner() - - def testReturn(self): - # 'return' [testlist_star_expr] - def g1(): return - def g2(): return 1 - return_list = [2, 3] - def g3(): return 1, *return_list - g1() - x = g2() - x3 = g3() - check_syntax_error(self, "class foo:return 1") - - def testYield(self): - # 'yield' [yield_arg] - def g1(): yield 1 - yield_list = [2, 3] - def g2(): yield 1, *yield_list - def g3(): yield from iter(yield_list) - x1 = g1() - x2 = g2() - x3 = g3() - check_syntax_error(self, "class foo:yield 1") - check_syntax_error(self, "def g4(): yield from *a") - - def testRaise(self): - # 'raise' test [',' test] - try: raise RuntimeError('just testing') - except RuntimeError: pass - try: raise KeyboardInterrupt - except KeyboardInterrupt: pass - - def testImport(self): - # 'import' dotted_as_names - import sys - import time, sys - # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names) - from time import time - from time import (time) - # not testable inside a function, but already done at top of the module - # from sys import * - from sys import path, argv - from sys import (path, argv) - from sys import (path, argv,) - - def testGlobal(self): - # 'global' NAME (',' NAME)* - global a - global a, b - global one, two, three, four, five, six, seven, eight, nine, ten - - def testNonlocal(self): - # 'nonlocal' NAME (',' NAME)* - x = 0 - y = 0 - def f(): - nonlocal x - nonlocal x, y - - def testAssert(self): - # assert_stmt: 'assert' test [',' test] - assert 1 - assert 1, 1 - assert lambda x:x - assert 1, lambda x:x+1 - try: - assert 0, "msg" - except AssertionError as e: - self.assertEquals(e.args[0], "msg") - else: - if __debug__: - self.fail("AssertionError not raised by assert 0") - - ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef - # Tested below - - def testIf(self): - # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] - if 1: pass - if 1: pass - else: pass - if 0: pass - elif 0: pass - if 0: pass - elif 0: pass - elif 0: pass - elif 0: pass - else: pass - - def testWhile(self): - # 'while' test ':' suite ['else' ':' suite] - while 0: pass - while 0: pass - else: pass - - # Issue1920: "while 0" is optimized away, - # ensure that the "else" clause is still present. - x = 0 - while 0: - x = 1 - else: - x = 2 - self.assertEquals(x, 2) - - def testFor(self): - # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] - for i in 1, 2, 3: pass - for i, j, k in (): pass - else: pass - class Squares: - def __init__(self, max): - self.max = max - self.sofar = [] - def __len__(self): return len(self.sofar) - def __getitem__(self, i): - if not 0 <= i < self.max: raise IndexError - n = len(self.sofar) - while n <= i: - self.sofar.append(n*n) - n = n+1 - return self.sofar[i] - n = 0 - for x in Squares(10): n = n+x - if n != 285: - self.fail('for over growing sequence') - - result = [] - for x, in [(1,), (2,), (3,)]: - result.append(x) - self.assertEqual(result, [1, 2, 3]) - - def testTry(self): - ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] - ### | 'try' ':' suite 'finally' ':' suite - ### except_clause: 'except' [expr ['as' expr]] - try: - 1/0 - except ZeroDivisionError: - pass - else: - pass - try: 1/0 - except EOFError: pass - except TypeError as msg: pass - except RuntimeError as msg: pass - except: pass - else: pass - try: 1/0 - except (EOFError, TypeError, ZeroDivisionError): pass - try: 1/0 - except (EOFError, TypeError, ZeroDivisionError) as msg: pass - try: pass - finally: pass - - def testSuite(self): - # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT - if 1: pass - if 1: - pass - if 1: - # - # - # - pass - pass - # - pass - # - - def testTest(self): - ### and_test ('or' and_test)* - ### and_test: not_test ('and' not_test)* - ### not_test: 'not' not_test | comparison - if not 1: pass - if 1 and 1: pass - if 1 or 1: pass - if not not not 1: pass - if not 1 and 1 and 1: pass - if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass - - def testComparison(self): - ### comparison: expr (comp_op expr)* - ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not' - if 1: pass - x = (1 == 1) - if 1 == 1: pass - if 1 != 1: pass - if 1 < 1: pass - if 1 > 1: pass - if 1 <= 1: pass - if 1 >= 1: pass - if 1 is 1: pass - if 1 is not 1: pass - if 1 in (): pass - if 1 not in (): pass - if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass - - def testBinaryMaskOps(self): - x = 1 & 1 - x = 1 ^ 1 - x = 1 | 1 - - def testShiftOps(self): - x = 1 << 1 - x = 1 >> 1 - x = 1 << 1 >> 1 - - def testAdditiveOps(self): - x = 1 - x = 1 + 1 - x = 1 - 1 - 1 - x = 1 - 1 + 1 - 1 + 1 - - def testMultiplicativeOps(self): - x = 1 * 1 - x = 1 / 1 - x = 1 % 1 - x = 1 / 1 * 1 % 1 - - def testUnaryOps(self): - x = +1 - x = -1 - x = ~1 - x = ~1 ^ 1 & 1 | 1 & 1 ^ -1 - x = -1*1/1 + 1*1 - ---1*1 - - def testSelectors(self): - ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME - ### subscript: expr | [expr] ':' [expr] - - import sys, time - c = sys.path[0] - x = time.time() - x = sys.modules['time'].time() - a = '01234' - c = a[0] - c = a[-1] - s = a[0:5] - s = a[:5] - s = a[0:] - s = a[:] - s = a[-5:] - s = a[:-1] - s = a[-4:-3] - # A rough test of SF bug 1333982. https://bugs.python.org/issue1333982 - # The testing here is fairly incomplete. - # Test cases should include: commas with 1 and 2 colons - d = {} - d[1] = 1 - d[1,] = 2 - d[1,2] = 3 - d[1,2,3] = 4 - L = list(d) - L.sort(key=lambda x: x if isinstance(x, tuple) else ()) - self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]') - - def testAtoms(self): - ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING - ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [',']) - - x = (1) - x = (1 or 2 or 3) - x = (1 or 2 or 3, 2, 3) - - x = [] - x = [1] - x = [1 or 2 or 3] - x = [1 or 2 or 3, 2, 3] - x = [] - - x = {} - x = {'one': 1} - x = {'one': 1,} - x = {'one' or 'two': 1 or 2} - x = {'one': 1, 'two': 2} - x = {'one': 1, 'two': 2,} - x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6} - - x = {'one'} - x = {'one', 1,} - x = {'one', 'two', 'three'} - x = {2, 3, 4,} - - x = x - x = 'x' - x = 123 - - ### exprlist: expr (',' expr)* [','] - ### testlist: test (',' test)* [','] - # These have been exercised enough above - - def testClassdef(self): - # 'class' NAME ['(' [testlist] ')'] ':' suite - class B: pass - class B2(): pass - class C1(B): pass - class C2(B): pass - class D(C1, C2, B): pass - class C: - def meth1(self): pass - def meth2(self, arg): pass - def meth3(self, a1, a2): pass - - # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE - # decorators: decorator+ - # decorated: decorators (classdef | funcdef) - def class_decorator(x): return x - @class_decorator - class G: pass - - def testDictcomps(self): - # dictorsetmaker: ( (test ':' test (comp_for | - # (',' test ':' test)* [','])) | - # (test (comp_for | (',' test)* [','])) ) - nums = [1, 2, 3] - self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4}) - - def testListcomps(self): - # list comprehension tests - nums = [1, 2, 3, 4, 5] - strs = ["Apple", "Banana", "Coconut"] - spcs = [" Apple", " Banana ", "Coco nut "] - - self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut']) - self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15]) - self.assertEqual([x for x in nums if x > 2], [3, 4, 5]) - self.assertEqual([(i, s) for i in nums for s in strs], - [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'), - (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'), - (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'), - (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'), - (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')]) - self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]], - [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'), - (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'), - (5, 'Banana'), (5, 'Coconut')]) - self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)], - [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]]) - - def test_in_func(l): - return [0 < x < 3 for x in l if x > 2] - - self.assertEqual(test_in_func(nums), [False, False, False]) - - def test_nested_front(): - self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]], - [[1, 2], [3, 4], [5, 6]]) - - test_nested_front() - - check_syntax_error(self, "[i, s for i in nums for s in strs]") - check_syntax_error(self, "[x if y]") - - suppliers = [ - (1, "Boeing"), - (2, "Ford"), - (3, "Macdonalds") - ] - - parts = [ - (10, "Airliner"), - (20, "Engine"), - (30, "Cheeseburger") - ] - - suppart = [ - (1, 10), (1, 20), (2, 20), (3, 30) - ] - - x = [ - (sname, pname) - for (sno, sname) in suppliers - for (pno, pname) in parts - for (sp_sno, sp_pno) in suppart - if sno == sp_sno and pno == sp_pno - ] - - self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'), - ('Macdonalds', 'Cheeseburger')]) - - def testGenexps(self): - # generator expression tests - g = ([x for x in range(10)] for x in range(1)) - self.assertEqual(next(g), [x for x in range(10)]) - try: - next(g) - self.fail('should produce StopIteration exception') - except StopIteration: - pass - - a = 1 - try: - g = (a for d in a) - next(g) - self.fail('should produce TypeError') - except TypeError: - pass - - self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd']) - self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy']) - - a = [x for x in range(10)] - b = (x for x in (y for y in a)) - self.assertEqual(sum(b), sum([x for x in range(10)])) - - self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)])) - self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2])) - self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)])) - self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0) - check_syntax_error(self, "foo(x for x in range(10), 100)") - check_syntax_error(self, "foo(100, x for x in range(10))") - - def testComprehensionSpecials(self): - # test for outmost iterable precomputation - x = 10; g = (i for i in range(x)); x = 5 - self.assertEqual(len(list(g)), 10) - - # This should hold, since we're only precomputing outmost iterable. - x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x)) - x = 5; t = True; - self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g)) - - # Grammar allows multiple adjacent 'if's in listcomps and genexps, - # even though it's silly. Make sure it works (ifelse broke this.) - self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7]) - self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7]) - - # verify unpacking single element tuples in listcomp/genexp. - self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6]) - self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9]) - - def test_with_statement(self): - class manager(object): - def __enter__(self): - return (1, 2) - def __exit__(self, *args): - pass - - with manager(): - pass - with manager() as x: - pass - with manager() as (x, y): - pass - with manager(), manager(): - pass - with manager() as x, manager() as y: - pass - with manager() as x, manager(): - pass - - def testIfElseExpr(self): - # Test ifelse expressions in various cases - def _checkeval(msg, ret): - "helper to check that evaluation of expressions is done correctly" - print(x) - return ret - - # the next line is not allowed anymore - #self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True]) - self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True]) - self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True]) - self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5) - self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5) - self.assertEqual((5 and 6 if 0 else 1), 1) - self.assertEqual(((5 and 6) if 0 else 1), 1) - self.assertEqual((5 and (6 if 1 else 1)), 6) - self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3) - self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1) - self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5) - self.assertEqual((not 5 if 1 else 1), False) - self.assertEqual((not 5 if 0 else 1), 1) - self.assertEqual((6 + 1 if 1 else 2), 7) - self.assertEqual((6 - 1 if 1 else 2), 5) - self.assertEqual((6 * 2 if 1 else 4), 12) - self.assertEqual((6 / 2 if 1 else 3), 3) - self.assertEqual((6 < 4 if 0 else 2), 2) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_lib2to3/pytree_idempotency.py b/Lib/test/test_lib2to3/pytree_idempotency.py deleted file mode 100755 index eb2e2aa02ae0ed..00000000000000 --- a/Lib/test/test_lib2to3/pytree_idempotency.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Main program for testing the infrastructure.""" - -from __future__ import print_function - -__author__ = "Guido van Rossum " - -# Support imports (need to be imported first) -from . import support - -# Python imports -import os -import sys -import logging - -# Local imports -from lib2to3 import pytree -from lib2to3 import pgen2 -from lib2to3.pgen2 import driver - -logging.basicConfig() - -def main(): - gr = driver.load_grammar("Grammar.txt") - dr = driver.Driver(gr, convert=pytree.convert) - - fn = "example.py" - tree = dr.parse_file(fn, debug=True) - if not diff(fn, tree): - print("No diffs.") - if not sys.argv[1:]: - return # Pass a dummy argument to run the complete test suite below - - problems = [] - - # Process every imported module - for name in sys.modules: - mod = sys.modules[name] - if mod is None or not hasattr(mod, "__file__"): - continue - fn = mod.__file__ - if fn.endswith(".pyc"): - fn = fn[:-1] - if not fn.endswith(".py"): - continue - print("Parsing", fn, file=sys.stderr) - tree = dr.parse_file(fn, debug=True) - if diff(fn, tree): - problems.append(fn) - - # Process every single module on sys.path (but not in packages) - for dir in sys.path: - try: - names = os.listdir(dir) - except OSError: - continue - print("Scanning", dir, "...", file=sys.stderr) - for name in names: - if not name.endswith(".py"): - continue - print("Parsing", name, file=sys.stderr) - fn = os.path.join(dir, name) - try: - tree = dr.parse_file(fn, debug=True) - except pgen2.parse.ParseError as err: - print("ParseError:", err) - else: - if diff(fn, tree): - problems.append(fn) - - # Show summary of problem files - if not problems: - print("No problems. Congratulations!") - else: - print("Problems in following files:") - for fn in problems: - print("***", fn) - -def diff(fn, tree): - f = open("@", "w") - try: - f.write(str(tree)) - finally: - f.close() - try: - return os.system("diff -u %s @" % fn) - finally: - os.remove("@") - -if __name__ == "__main__": - main() diff --git a/Lib/test/test_lib2to3/support.py b/Lib/test/test_lib2to3/support.py deleted file mode 100644 index 9e56273e95992b..00000000000000 --- a/Lib/test/test_lib2to3/support.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Support code for test_*.py files""" -# Author: Collin Winter - -# Python imports -import unittest -import os -import os.path -from textwrap import dedent - -# Local imports -import lib2to3 -from lib2to3 import pytree, refactor -from lib2to3.pgen2 import driver as pgen2_driver - -lib2to3_dir = os.path.dirname(lib2to3.__file__) -test_dir = os.path.dirname(__file__) -proj_dir = os.path.normpath(os.path.join(test_dir, "..")) -grammar_path = os.path.join(lib2to3_dir, "Grammar.txt") -grammar = pgen2_driver.load_grammar(grammar_path) -grammar_no_print_statement = pgen2_driver.load_grammar(grammar_path) -del grammar_no_print_statement.keywords["print"] -driver = pgen2_driver.Driver(grammar, convert=pytree.convert) -driver_no_print_statement = pgen2_driver.Driver( - grammar_no_print_statement, - convert=pytree.convert -) - -def parse_string(string): - return driver.parse_string(reformat(string), debug=True) - -def run_all_tests(test_mod=None, tests=None): - if tests is None: - tests = unittest.TestLoader().loadTestsFromModule(test_mod) - unittest.TextTestRunner(verbosity=2).run(tests) - -def reformat(string): - return dedent(string) + "\n\n" - -def get_refactorer(fixer_pkg="lib2to3", fixers=None, options=None): - """ - A convenience function for creating a RefactoringTool for tests. - - fixers is a list of fixers for the RefactoringTool to use. By default - "lib2to3.fixes.*" is used. options is an optional dictionary of options to - be passed to the RefactoringTool. - """ - if fixers is not None: - fixers = [fixer_pkg + ".fixes.fix_" + fix for fix in fixers] - else: - fixers = refactor.get_fixers_from_package(fixer_pkg + ".fixes") - options = options or {} - return refactor.RefactoringTool(fixers, options, explicit=True) - -def _all_project_files(root, files): - for dirpath, dirnames, filenames in os.walk(root): - for filename in filenames: - if not filename.endswith(".py"): - continue - files.append(os.path.join(dirpath, filename)) - -def all_project_files(): - files = [] - _all_project_files(lib2to3_dir, files) - _all_project_files(test_dir, files) - # Sort to get more reproducible tests - files.sort() - return files - -TestCase = unittest.TestCase diff --git a/Lib/test/test_lib2to3/test_all_fixers.py b/Lib/test/test_lib2to3/test_all_fixers.py deleted file mode 100644 index d0fca7072482cb..00000000000000 --- a/Lib/test/test_lib2to3/test_all_fixers.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Tests that run all fixer modules over an input stream. - -This has been broken out into its own test module because of its -running time. -""" -# Author: Collin Winter - -# Python imports -import os.path -import test.support -import unittest - -# Local imports -from . import support - - -@test.support.requires_resource('cpu') -class Test_all(support.TestCase): - - def setUp(self): - self.refactor = support.get_refactorer() - - def refactor_file(self, filepath): - if test.support.verbose: - print(f"Refactor file: {filepath}") - if os.path.basename(filepath) == 'infinite_recursion.py': - # bpo-46542: Processing infinite_recursion.py can crash Python - # if Python is built in debug mode: lower the recursion limit - # to prevent a crash. - with test.support.infinite_recursion(150): - self.refactor.refactor_file(filepath) - else: - self.refactor.refactor_file(filepath) - - def test_all_project_files(self): - for filepath in support.all_project_files(): - with self.subTest(filepath=filepath): - self.refactor_file(filepath) - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_lib2to3/test_fixers.py b/Lib/test/test_lib2to3/test_fixers.py deleted file mode 100644 index 68efeee7abb4d5..00000000000000 --- a/Lib/test/test_lib2to3/test_fixers.py +++ /dev/null @@ -1,4649 +0,0 @@ -""" Test suite for the fixer modules """ - -# Python imports -import os -from itertools import chain -from operator import itemgetter - -# Local imports -from lib2to3 import pygram, fixer_util -from test.test_lib2to3 import support - - -class FixerTestCase(support.TestCase): - - # Other test cases can subclass this class and replace "fixer_pkg" with - # their own. - def setUp(self, fix_list=None, fixer_pkg="lib2to3", options=None): - if fix_list is None: - fix_list = [self.fixer] - self.refactor = support.get_refactorer(fixer_pkg, fix_list, options) - self.fixer_log = [] - self.filename = "" - - for fixer in chain(self.refactor.pre_order, - self.refactor.post_order): - fixer.log = self.fixer_log - - def _check(self, before, after): - before = support.reformat(before) - after = support.reformat(after) - tree = self.refactor.refactor_string(before, self.filename) - self.assertEqual(after, str(tree)) - return tree - - def check(self, before, after, ignore_warnings=False): - tree = self._check(before, after) - self.assertTrue(tree.was_changed) - if not ignore_warnings: - self.assertEqual(self.fixer_log, []) - - def warns(self, before, after, message, unchanged=False): - tree = self._check(before, after) - self.assertIn(message, "".join(self.fixer_log)) - if not unchanged: - self.assertTrue(tree.was_changed) - - def warns_unchanged(self, before, message): - self.warns(before, before, message, unchanged=True) - - def unchanged(self, before, ignore_warnings=False): - self._check(before, before) - if not ignore_warnings: - self.assertEqual(self.fixer_log, []) - - def assert_runs_after(self, *names): - fixes = [self.fixer] - fixes.extend(names) - r = support.get_refactorer("lib2to3", fixes) - (pre, post) = r.get_fixers() - n = "fix_" + self.fixer - if post and post[-1].__class__.__module__.endswith(n): - # We're the last fixer to run - return - if pre and pre[-1].__class__.__module__.endswith(n) and not post: - # We're the last in pre and post is empty - return - self.fail("Fixer run order (%s) is incorrect; %s should be last."\ - %(", ".join([x.__class__.__module__ for x in (pre+post)]), n)) - -class Test_ne(FixerTestCase): - fixer = "ne" - - def test_basic(self): - b = """if x <> y: - pass""" - - a = """if x != y: - pass""" - self.check(b, a) - - def test_no_spaces(self): - b = """if x<>y: - pass""" - - a = """if x!=y: - pass""" - self.check(b, a) - - def test_chained(self): - b = """if x<>y<>z: - pass""" - - a = """if x!=y!=z: - pass""" - self.check(b, a) - -class Test_has_key(FixerTestCase): - fixer = "has_key" - - def test_1(self): - b = """x = d.has_key("x") or d.has_key("y")""" - a = """x = "x" in d or "y" in d""" - self.check(b, a) - - def test_2(self): - b = """x = a.b.c.d.has_key("x") ** 3""" - a = """x = ("x" in a.b.c.d) ** 3""" - self.check(b, a) - - def test_3(self): - b = """x = a.b.has_key(1 + 2).__repr__()""" - a = """x = (1 + 2 in a.b).__repr__()""" - self.check(b, a) - - def test_4(self): - b = """x = a.b.has_key(1 + 2).__repr__() ** -3 ** 4""" - a = """x = (1 + 2 in a.b).__repr__() ** -3 ** 4""" - self.check(b, a) - - def test_5(self): - b = """x = a.has_key(f or g)""" - a = """x = (f or g) in a""" - self.check(b, a) - - def test_6(self): - b = """x = a + b.has_key(c)""" - a = """x = a + (c in b)""" - self.check(b, a) - - def test_7(self): - b = """x = a.has_key(lambda: 12)""" - a = """x = (lambda: 12) in a""" - self.check(b, a) - - def test_8(self): - b = """x = a.has_key(a for a in b)""" - a = """x = (a for a in b) in a""" - self.check(b, a) - - def test_9(self): - b = """if not a.has_key(b): pass""" - a = """if b not in a: pass""" - self.check(b, a) - - def test_10(self): - b = """if not a.has_key(b).__repr__(): pass""" - a = """if not (b in a).__repr__(): pass""" - self.check(b, a) - - def test_11(self): - b = """if not a.has_key(b) ** 2: pass""" - a = """if not (b in a) ** 2: pass""" - self.check(b, a) - -class Test_apply(FixerTestCase): - fixer = "apply" - - def test_1(self): - b = """x = apply(f, g + h)""" - a = """x = f(*g + h)""" - self.check(b, a) - - def test_2(self): - b = """y = apply(f, g, h)""" - a = """y = f(*g, **h)""" - self.check(b, a) - - def test_3(self): - b = """z = apply(fs[0], g or h, h or g)""" - a = """z = fs[0](*g or h, **h or g)""" - self.check(b, a) - - def test_4(self): - b = """apply(f, (x, y) + t)""" - a = """f(*(x, y) + t)""" - self.check(b, a) - - def test_5(self): - b = """apply(f, args,)""" - a = """f(*args)""" - self.check(b, a) - - def test_6(self): - b = """apply(f, args, kwds,)""" - a = """f(*args, **kwds)""" - self.check(b, a) - - # Test that complex functions are parenthesized - - def test_complex_1(self): - b = """x = apply(f+g, args)""" - a = """x = (f+g)(*args)""" - self.check(b, a) - - def test_complex_2(self): - b = """x = apply(f*g, args)""" - a = """x = (f*g)(*args)""" - self.check(b, a) - - def test_complex_3(self): - b = """x = apply(f**g, args)""" - a = """x = (f**g)(*args)""" - self.check(b, a) - - # But dotted names etc. not - - def test_dotted_name(self): - b = """x = apply(f.g, args)""" - a = """x = f.g(*args)""" - self.check(b, a) - - def test_subscript(self): - b = """x = apply(f[x], args)""" - a = """x = f[x](*args)""" - self.check(b, a) - - def test_call(self): - b = """x = apply(f(), args)""" - a = """x = f()(*args)""" - self.check(b, a) - - # Extreme case - def test_extreme(self): - b = """x = apply(a.b.c.d.e.f, args, kwds)""" - a = """x = a.b.c.d.e.f(*args, **kwds)""" - self.check(b, a) - - # XXX Comments in weird places still get lost - def test_weird_comments(self): - b = """apply( # foo - f, # bar - args)""" - a = """f(*args)""" - self.check(b, a) - - # These should *not* be touched - - def test_unchanged_1(self): - s = """apply()""" - self.unchanged(s) - - def test_unchanged_2(self): - s = """apply(f)""" - self.unchanged(s) - - def test_unchanged_3(self): - s = """apply(f,)""" - self.unchanged(s) - - def test_unchanged_4(self): - s = """apply(f, args, kwds, extras)""" - self.unchanged(s) - - def test_unchanged_5(self): - s = """apply(f, *args, **kwds)""" - self.unchanged(s) - - def test_unchanged_6(self): - s = """apply(f, *args)""" - self.unchanged(s) - - def test_unchanged_6b(self): - s = """apply(f, **kwds)""" - self.unchanged(s) - - def test_unchanged_7(self): - s = """apply(func=f, args=args, kwds=kwds)""" - self.unchanged(s) - - def test_unchanged_8(self): - s = """apply(f, args=args, kwds=kwds)""" - self.unchanged(s) - - def test_unchanged_9(self): - s = """apply(f, args, kwds=kwds)""" - self.unchanged(s) - - def test_space_1(self): - a = """apply( f, args, kwds)""" - b = """f(*args, **kwds)""" - self.check(a, b) - - def test_space_2(self): - a = """apply( f ,args,kwds )""" - b = """f(*args, **kwds)""" - self.check(a, b) - -class Test_reload(FixerTestCase): - fixer = "reload" - - def test(self): - b = """reload(a)""" - a = """import importlib\nimportlib.reload(a)""" - self.check(b, a) - - def test_comment(self): - b = """reload( a ) # comment""" - a = """import importlib\nimportlib.reload( a ) # comment""" - self.check(b, a) - - # PEP 8 comments - b = """reload( a ) # comment""" - a = """import importlib\nimportlib.reload( a ) # comment""" - self.check(b, a) - - def test_space(self): - b = """reload( a )""" - a = """import importlib\nimportlib.reload( a )""" - self.check(b, a) - - b = """reload( a)""" - a = """import importlib\nimportlib.reload( a)""" - self.check(b, a) - - b = """reload(a )""" - a = """import importlib\nimportlib.reload(a )""" - self.check(b, a) - - def test_unchanged(self): - s = """reload(a=1)""" - self.unchanged(s) - - s = """reload(f, g)""" - self.unchanged(s) - - s = """reload(f, *h)""" - self.unchanged(s) - - s = """reload(f, *h, **i)""" - self.unchanged(s) - - s = """reload(f, **i)""" - self.unchanged(s) - - s = """reload(*h, **i)""" - self.unchanged(s) - - s = """reload(*h)""" - self.unchanged(s) - - s = """reload(**i)""" - self.unchanged(s) - - s = """reload()""" - self.unchanged(s) - -class Test_intern(FixerTestCase): - fixer = "intern" - - def test_prefix_preservation(self): - b = """x = intern( a )""" - a = """import sys\nx = sys.intern( a )""" - self.check(b, a) - - b = """y = intern("b" # test - )""" - a = """import sys\ny = sys.intern("b" # test - )""" - self.check(b, a) - - b = """z = intern(a+b+c.d, )""" - a = """import sys\nz = sys.intern(a+b+c.d, )""" - self.check(b, a) - - def test(self): - b = """x = intern(a)""" - a = """import sys\nx = sys.intern(a)""" - self.check(b, a) - - b = """z = intern(a+b+c.d,)""" - a = """import sys\nz = sys.intern(a+b+c.d,)""" - self.check(b, a) - - b = """intern("y%s" % 5).replace("y", "")""" - a = """import sys\nsys.intern("y%s" % 5).replace("y", "")""" - self.check(b, a) - - # These should not be refactored - - def test_unchanged(self): - s = """intern(a=1)""" - self.unchanged(s) - - s = """intern(f, g)""" - self.unchanged(s) - - s = """intern(*h)""" - self.unchanged(s) - - s = """intern(**i)""" - self.unchanged(s) - - s = """intern()""" - self.unchanged(s) - -class Test_reduce(FixerTestCase): - fixer = "reduce" - - def test_simple_call(self): - b = "reduce(a, b, c)" - a = "from functools import reduce\nreduce(a, b, c)" - self.check(b, a) - - def test_bug_7253(self): - # fix_tuple_params was being bad and orphaning nodes in the tree. - b = "def x(arg): reduce(sum, [])" - a = "from functools import reduce\ndef x(arg): reduce(sum, [])" - self.check(b, a) - - def test_call_with_lambda(self): - b = "reduce(lambda x, y: x + y, seq)" - a = "from functools import reduce\nreduce(lambda x, y: x + y, seq)" - self.check(b, a) - - def test_unchanged(self): - s = "reduce(a)" - self.unchanged(s) - - s = "reduce(a, b=42)" - self.unchanged(s) - - s = "reduce(a, b, c, d)" - self.unchanged(s) - - s = "reduce(**c)" - self.unchanged(s) - - s = "reduce()" - self.unchanged(s) - -class Test_print(FixerTestCase): - fixer = "print" - - def test_prefix_preservation(self): - b = """print 1, 1+1, 1+1+1""" - a = """print(1, 1+1, 1+1+1)""" - self.check(b, a) - - def test_idempotency(self): - s = """print()""" - self.unchanged(s) - - s = """print('')""" - self.unchanged(s) - - def test_idempotency_print_as_function(self): - self.refactor.driver.grammar = pygram.python_grammar_no_print_statement - s = """print(1, 1+1, 1+1+1)""" - self.unchanged(s) - - s = """print()""" - self.unchanged(s) - - s = """print('')""" - self.unchanged(s) - - def test_1(self): - b = """print 1, 1+1, 1+1+1""" - a = """print(1, 1+1, 1+1+1)""" - self.check(b, a) - - def test_2(self): - b = """print 1, 2""" - a = """print(1, 2)""" - self.check(b, a) - - def test_3(self): - b = """print""" - a = """print()""" - self.check(b, a) - - def test_4(self): - # from bug 3000 - b = """print whatever; print""" - a = """print(whatever); print()""" - self.check(b, a) - - def test_5(self): - b = """print; print whatever;""" - a = """print(); print(whatever);""" - self.check(b, a) - - def test_tuple(self): - b = """print (a, b, c)""" - a = """print((a, b, c))""" - self.check(b, a) - - # trailing commas - - def test_trailing_comma_1(self): - b = """print 1, 2, 3,""" - a = """print(1, 2, 3, end=' ')""" - self.check(b, a) - - def test_trailing_comma_2(self): - b = """print 1, 2,""" - a = """print(1, 2, end=' ')""" - self.check(b, a) - - def test_trailing_comma_3(self): - b = """print 1,""" - a = """print(1, end=' ')""" - self.check(b, a) - - # >> stuff - - def test_vargs_without_trailing_comma(self): - b = """print >>sys.stderr, 1, 2, 3""" - a = """print(1, 2, 3, file=sys.stderr)""" - self.check(b, a) - - def test_with_trailing_comma(self): - b = """print >>sys.stderr, 1, 2,""" - a = """print(1, 2, end=' ', file=sys.stderr)""" - self.check(b, a) - - def test_no_trailing_comma(self): - b = """print >>sys.stderr, 1+1""" - a = """print(1+1, file=sys.stderr)""" - self.check(b, a) - - def test_spaces_before_file(self): - b = """print >> sys.stderr""" - a = """print(file=sys.stderr)""" - self.check(b, a) - - def test_with_future_print_function(self): - s = "from __future__ import print_function\n" \ - "print('Hai!', end=' ')" - self.unchanged(s) - - b = "print 'Hello, world!'" - a = "print('Hello, world!')" - self.check(b, a) - - -class Test_exec(FixerTestCase): - fixer = "exec" - - def test_prefix_preservation(self): - b = """ exec code in ns1, ns2""" - a = """ exec(code, ns1, ns2)""" - self.check(b, a) - - def test_basic(self): - b = """exec code""" - a = """exec(code)""" - self.check(b, a) - - def test_with_globals(self): - b = """exec code in ns""" - a = """exec(code, ns)""" - self.check(b, a) - - def test_with_globals_locals(self): - b = """exec code in ns1, ns2""" - a = """exec(code, ns1, ns2)""" - self.check(b, a) - - def test_complex_1(self): - b = """exec (a.b()) in ns""" - a = """exec((a.b()), ns)""" - self.check(b, a) - - def test_complex_2(self): - b = """exec a.b() + c in ns""" - a = """exec(a.b() + c, ns)""" - self.check(b, a) - - # These should not be touched - - def test_unchanged_1(self): - s = """exec(code)""" - self.unchanged(s) - - def test_unchanged_2(self): - s = """exec (code)""" - self.unchanged(s) - - def test_unchanged_3(self): - s = """exec(code, ns)""" - self.unchanged(s) - - def test_unchanged_4(self): - s = """exec(code, ns1, ns2)""" - self.unchanged(s) - -class Test_repr(FixerTestCase): - fixer = "repr" - - def test_prefix_preservation(self): - b = """x = `1 + 2`""" - a = """x = repr(1 + 2)""" - self.check(b, a) - - def test_simple_1(self): - b = """x = `1 + 2`""" - a = """x = repr(1 + 2)""" - self.check(b, a) - - def test_simple_2(self): - b = """y = `x`""" - a = """y = repr(x)""" - self.check(b, a) - - def test_complex(self): - b = """z = `y`.__repr__()""" - a = """z = repr(y).__repr__()""" - self.check(b, a) - - def test_tuple(self): - b = """x = `1, 2, 3`""" - a = """x = repr((1, 2, 3))""" - self.check(b, a) - - def test_nested(self): - b = """x = `1 + `2``""" - a = """x = repr(1 + repr(2))""" - self.check(b, a) - - def test_nested_tuples(self): - b = """x = `1, 2 + `3, 4``""" - a = """x = repr((1, 2 + repr((3, 4))))""" - self.check(b, a) - -class Test_except(FixerTestCase): - fixer = "except" - - def test_prefix_preservation(self): - b = """ - try: - pass - except (RuntimeError, ImportError), e: - pass""" - a = """ - try: - pass - except (RuntimeError, ImportError) as e: - pass""" - self.check(b, a) - - def test_simple(self): - b = """ - try: - pass - except Foo, e: - pass""" - a = """ - try: - pass - except Foo as e: - pass""" - self.check(b, a) - - def test_simple_no_space_before_target(self): - b = """ - try: - pass - except Foo,e: - pass""" - a = """ - try: - pass - except Foo as e: - pass""" - self.check(b, a) - - def test_tuple_unpack(self): - b = """ - def foo(): - try: - pass - except Exception, (f, e): - pass - except ImportError, e: - pass""" - - a = """ - def foo(): - try: - pass - except Exception as xxx_todo_changeme: - (f, e) = xxx_todo_changeme.args - pass - except ImportError as e: - pass""" - self.check(b, a) - - def test_multi_class(self): - b = """ - try: - pass - except (RuntimeError, ImportError), e: - pass""" - - a = """ - try: - pass - except (RuntimeError, ImportError) as e: - pass""" - self.check(b, a) - - def test_list_unpack(self): - b = """ - try: - pass - except Exception, [a, b]: - pass""" - - a = """ - try: - pass - except Exception as xxx_todo_changeme: - [a, b] = xxx_todo_changeme.args - pass""" - self.check(b, a) - - def test_weird_target_1(self): - b = """ - try: - pass - except Exception, d[5]: - pass""" - - a = """ - try: - pass - except Exception as xxx_todo_changeme: - d[5] = xxx_todo_changeme - pass""" - self.check(b, a) - - def test_weird_target_2(self): - b = """ - try: - pass - except Exception, a.foo: - pass""" - - a = """ - try: - pass - except Exception as xxx_todo_changeme: - a.foo = xxx_todo_changeme - pass""" - self.check(b, a) - - def test_weird_target_3(self): - b = """ - try: - pass - except Exception, a().foo: - pass""" - - a = """ - try: - pass - except Exception as xxx_todo_changeme: - a().foo = xxx_todo_changeme - pass""" - self.check(b, a) - - def test_bare_except(self): - b = """ - try: - pass - except Exception, a: - pass - except: - pass""" - - a = """ - try: - pass - except Exception as a: - pass - except: - pass""" - self.check(b, a) - - def test_bare_except_and_else_finally(self): - b = """ - try: - pass - except Exception, a: - pass - except: - pass - else: - pass - finally: - pass""" - - a = """ - try: - pass - except Exception as a: - pass - except: - pass - else: - pass - finally: - pass""" - self.check(b, a) - - def test_multi_fixed_excepts_before_bare_except(self): - b = """ - try: - pass - except TypeError, b: - pass - except Exception, a: - pass - except: - pass""" - - a = """ - try: - pass - except TypeError as b: - pass - except Exception as a: - pass - except: - pass""" - self.check(b, a) - - def test_one_line_suites(self): - b = """ - try: raise TypeError - except TypeError, e: - pass - """ - a = """ - try: raise TypeError - except TypeError as e: - pass - """ - self.check(b, a) - b = """ - try: - raise TypeError - except TypeError, e: pass - """ - a = """ - try: - raise TypeError - except TypeError as e: pass - """ - self.check(b, a) - b = """ - try: raise TypeError - except TypeError, e: pass - """ - a = """ - try: raise TypeError - except TypeError as e: pass - """ - self.check(b, a) - b = """ - try: raise TypeError - except TypeError, e: pass - else: function() - finally: done() - """ - a = """ - try: raise TypeError - except TypeError as e: pass - else: function() - finally: done() - """ - self.check(b, a) - - # These should not be touched: - - def test_unchanged_1(self): - s = """ - try: - pass - except: - pass""" - self.unchanged(s) - - def test_unchanged_2(self): - s = """ - try: - pass - except Exception: - pass""" - self.unchanged(s) - - def test_unchanged_3(self): - s = """ - try: - pass - except (Exception, SystemExit): - pass""" - self.unchanged(s) - -class Test_raise(FixerTestCase): - fixer = "raise" - - def test_basic(self): - b = """raise Exception, 5""" - a = """raise Exception(5)""" - self.check(b, a) - - def test_prefix_preservation(self): - b = """raise Exception,5""" - a = """raise Exception(5)""" - self.check(b, a) - - b = """raise Exception, 5""" - a = """raise Exception(5)""" - self.check(b, a) - - def test_with_comments(self): - b = """raise Exception, 5 # foo""" - a = """raise Exception(5) # foo""" - self.check(b, a) - - b = """raise E, (5, 6) % (a, b) # foo""" - a = """raise E((5, 6) % (a, b)) # foo""" - self.check(b, a) - - b = """def foo(): - raise Exception, 5, 6 # foo""" - a = """def foo(): - raise Exception(5).with_traceback(6) # foo""" - self.check(b, a) - - def test_None_value(self): - b = """raise Exception(5), None, tb""" - a = """raise Exception(5).with_traceback(tb)""" - self.check(b, a) - - def test_tuple_value(self): - b = """raise Exception, (5, 6, 7)""" - a = """raise Exception(5, 6, 7)""" - self.check(b, a) - - def test_tuple_detection(self): - b = """raise E, (5, 6) % (a, b)""" - a = """raise E((5, 6) % (a, b))""" - self.check(b, a) - - def test_tuple_exc_1(self): - b = """raise (((E1, E2), E3), E4), V""" - a = """raise E1(V)""" - self.check(b, a) - - def test_tuple_exc_2(self): - b = """raise (E1, (E2, E3), E4), V""" - a = """raise E1(V)""" - self.check(b, a) - - # These should produce a warning - - def test_string_exc(self): - s = """raise 'foo'""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - def test_string_exc_val(self): - s = """raise "foo", 5""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - def test_string_exc_val_tb(self): - s = """raise "foo", 5, 6""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - # These should result in traceback-assignment - - def test_tb_1(self): - b = """def foo(): - raise Exception, 5, 6""" - a = """def foo(): - raise Exception(5).with_traceback(6)""" - self.check(b, a) - - def test_tb_2(self): - b = """def foo(): - a = 5 - raise Exception, 5, 6 - b = 6""" - a = """def foo(): - a = 5 - raise Exception(5).with_traceback(6) - b = 6""" - self.check(b, a) - - def test_tb_3(self): - b = """def foo(): - raise Exception,5,6""" - a = """def foo(): - raise Exception(5).with_traceback(6)""" - self.check(b, a) - - def test_tb_4(self): - b = """def foo(): - a = 5 - raise Exception,5,6 - b = 6""" - a = """def foo(): - a = 5 - raise Exception(5).with_traceback(6) - b = 6""" - self.check(b, a) - - def test_tb_5(self): - b = """def foo(): - raise Exception, (5, 6, 7), 6""" - a = """def foo(): - raise Exception(5, 6, 7).with_traceback(6)""" - self.check(b, a) - - def test_tb_6(self): - b = """def foo(): - a = 5 - raise Exception, (5, 6, 7), 6 - b = 6""" - a = """def foo(): - a = 5 - raise Exception(5, 6, 7).with_traceback(6) - b = 6""" - self.check(b, a) - -class Test_throw(FixerTestCase): - fixer = "throw" - - def test_1(self): - b = """g.throw(Exception, 5)""" - a = """g.throw(Exception(5))""" - self.check(b, a) - - def test_2(self): - b = """g.throw(Exception,5)""" - a = """g.throw(Exception(5))""" - self.check(b, a) - - def test_3(self): - b = """g.throw(Exception, (5, 6, 7))""" - a = """g.throw(Exception(5, 6, 7))""" - self.check(b, a) - - def test_4(self): - b = """5 + g.throw(Exception, 5)""" - a = """5 + g.throw(Exception(5))""" - self.check(b, a) - - # These should produce warnings - - def test_warn_1(self): - s = """g.throw("foo")""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - def test_warn_2(self): - s = """g.throw("foo", 5)""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - def test_warn_3(self): - s = """g.throw("foo", 5, 6)""" - self.warns_unchanged(s, "Python 3 does not support string exceptions") - - # These should not be touched - - def test_untouched_1(self): - s = """g.throw(Exception)""" - self.unchanged(s) - - def test_untouched_2(self): - s = """g.throw(Exception(5, 6))""" - self.unchanged(s) - - def test_untouched_3(self): - s = """5 + g.throw(Exception(5, 6))""" - self.unchanged(s) - - # These should result in traceback-assignment - - def test_tb_1(self): - b = """def foo(): - g.throw(Exception, 5, 6)""" - a = """def foo(): - g.throw(Exception(5).with_traceback(6))""" - self.check(b, a) - - def test_tb_2(self): - b = """def foo(): - a = 5 - g.throw(Exception, 5, 6) - b = 6""" - a = """def foo(): - a = 5 - g.throw(Exception(5).with_traceback(6)) - b = 6""" - self.check(b, a) - - def test_tb_3(self): - b = """def foo(): - g.throw(Exception,5,6)""" - a = """def foo(): - g.throw(Exception(5).with_traceback(6))""" - self.check(b, a) - - def test_tb_4(self): - b = """def foo(): - a = 5 - g.throw(Exception,5,6) - b = 6""" - a = """def foo(): - a = 5 - g.throw(Exception(5).with_traceback(6)) - b = 6""" - self.check(b, a) - - def test_tb_5(self): - b = """def foo(): - g.throw(Exception, (5, 6, 7), 6)""" - a = """def foo(): - g.throw(Exception(5, 6, 7).with_traceback(6))""" - self.check(b, a) - - def test_tb_6(self): - b = """def foo(): - a = 5 - g.throw(Exception, (5, 6, 7), 6) - b = 6""" - a = """def foo(): - a = 5 - g.throw(Exception(5, 6, 7).with_traceback(6)) - b = 6""" - self.check(b, a) - - def test_tb_7(self): - b = """def foo(): - a + g.throw(Exception, 5, 6)""" - a = """def foo(): - a + g.throw(Exception(5).with_traceback(6))""" - self.check(b, a) - - def test_tb_8(self): - b = """def foo(): - a = 5 - a + g.throw(Exception, 5, 6) - b = 6""" - a = """def foo(): - a = 5 - a + g.throw(Exception(5).with_traceback(6)) - b = 6""" - self.check(b, a) - -class Test_long(FixerTestCase): - fixer = "long" - - def test_1(self): - b = """x = long(x)""" - a = """x = int(x)""" - self.check(b, a) - - def test_2(self): - b = """y = isinstance(x, long)""" - a = """y = isinstance(x, int)""" - self.check(b, a) - - def test_3(self): - b = """z = type(x) in (int, long)""" - a = """z = type(x) in (int, int)""" - self.check(b, a) - - def test_unchanged(self): - s = """long = True""" - self.unchanged(s) - - s = """s.long = True""" - self.unchanged(s) - - s = """def long(): pass""" - self.unchanged(s) - - s = """class long(): pass""" - self.unchanged(s) - - s = """def f(long): pass""" - self.unchanged(s) - - s = """def f(g, long): pass""" - self.unchanged(s) - - s = """def f(x, long=True): pass""" - self.unchanged(s) - - def test_prefix_preservation(self): - b = """x = long( x )""" - a = """x = int( x )""" - self.check(b, a) - - -class Test_execfile(FixerTestCase): - fixer = "execfile" - - def test_conversion(self): - b = """execfile("fn")""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'))""" - self.check(b, a) - - b = """execfile("fn", glob)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), glob)""" - self.check(b, a) - - b = """execfile("fn", glob, loc)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), glob, loc)""" - self.check(b, a) - - b = """execfile("fn", globals=glob)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), globals=glob)""" - self.check(b, a) - - b = """execfile("fn", locals=loc)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), locals=loc)""" - self.check(b, a) - - b = """execfile("fn", globals=glob, locals=loc)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), globals=glob, locals=loc)""" - self.check(b, a) - - def test_spacing(self): - b = """execfile( "fn" )""" - a = """exec(compile(open( "fn", "rb" ).read(), "fn", 'exec'))""" - self.check(b, a) - - b = """execfile("fn", globals = glob)""" - a = """exec(compile(open("fn", "rb").read(), "fn", 'exec'), globals = glob)""" - self.check(b, a) - - -class Test_isinstance(FixerTestCase): - fixer = "isinstance" - - def test_remove_multiple_items(self): - b = """isinstance(x, (int, int, int))""" - a = """isinstance(x, int)""" - self.check(b, a) - - b = """isinstance(x, (int, float, int, int, float))""" - a = """isinstance(x, (int, float))""" - self.check(b, a) - - b = """isinstance(x, (int, float, int, int, float, str))""" - a = """isinstance(x, (int, float, str))""" - self.check(b, a) - - b = """isinstance(foo() + bar(), (x(), y(), x(), int, int))""" - a = """isinstance(foo() + bar(), (x(), y(), x(), int))""" - self.check(b, a) - - def test_prefix_preservation(self): - b = """if isinstance( foo(), ( bar, bar, baz )) : pass""" - a = """if isinstance( foo(), ( bar, baz )) : pass""" - self.check(b, a) - - def test_unchanged(self): - self.unchanged("isinstance(x, (str, int))") - -class Test_dict(FixerTestCase): - fixer = "dict" - - def test_prefix_preservation(self): - b = "if d. keys ( ) : pass" - a = "if list(d. keys ( )) : pass" - self.check(b, a) - - b = "if d. items ( ) : pass" - a = "if list(d. items ( )) : pass" - self.check(b, a) - - b = "if d. iterkeys ( ) : pass" - a = "if iter(d. keys ( )) : pass" - self.check(b, a) - - b = "[i for i in d. iterkeys( ) ]" - a = "[i for i in d. keys( ) ]" - self.check(b, a) - - b = "if d. viewkeys ( ) : pass" - a = "if d. keys ( ) : pass" - self.check(b, a) - - b = "[i for i in d. viewkeys( ) ]" - a = "[i for i in d. keys( ) ]" - self.check(b, a) - - def test_trailing_comment(self): - b = "d.keys() # foo" - a = "list(d.keys()) # foo" - self.check(b, a) - - b = "d.items() # foo" - a = "list(d.items()) # foo" - self.check(b, a) - - b = "d.iterkeys() # foo" - a = "iter(d.keys()) # foo" - self.check(b, a) - - b = """[i for i in d.iterkeys() # foo - ]""" - a = """[i for i in d.keys() # foo - ]""" - self.check(b, a) - - b = """[i for i in d.iterkeys() # foo - ]""" - a = """[i for i in d.keys() # foo - ]""" - self.check(b, a) - - b = "d.viewitems() # foo" - a = "d.items() # foo" - self.check(b, a) - - def test_unchanged(self): - for wrapper in fixer_util.consuming_calls: - s = "s = %s(d.keys())" % wrapper - self.unchanged(s) - - s = "s = %s(d.values())" % wrapper - self.unchanged(s) - - s = "s = %s(d.items())" % wrapper - self.unchanged(s) - - def test_01(self): - b = "d.keys()" - a = "list(d.keys())" - self.check(b, a) - - b = "a[0].foo().keys()" - a = "list(a[0].foo().keys())" - self.check(b, a) - - def test_02(self): - b = "d.items()" - a = "list(d.items())" - self.check(b, a) - - def test_03(self): - b = "d.values()" - a = "list(d.values())" - self.check(b, a) - - def test_04(self): - b = "d.iterkeys()" - a = "iter(d.keys())" - self.check(b, a) - - def test_05(self): - b = "d.iteritems()" - a = "iter(d.items())" - self.check(b, a) - - def test_06(self): - b = "d.itervalues()" - a = "iter(d.values())" - self.check(b, a) - - def test_07(self): - s = "list(d.keys())" - self.unchanged(s) - - def test_08(self): - s = "sorted(d.keys())" - self.unchanged(s) - - def test_09(self): - b = "iter(d.keys())" - a = "iter(list(d.keys()))" - self.check(b, a) - - def test_10(self): - b = "foo(d.keys())" - a = "foo(list(d.keys()))" - self.check(b, a) - - def test_11(self): - b = "for i in d.keys(): print i" - a = "for i in list(d.keys()): print i" - self.check(b, a) - - def test_12(self): - b = "for i in d.iterkeys(): print i" - a = "for i in d.keys(): print i" - self.check(b, a) - - def test_13(self): - b = "[i for i in d.keys()]" - a = "[i for i in list(d.keys())]" - self.check(b, a) - - def test_14(self): - b = "[i for i in d.iterkeys()]" - a = "[i for i in d.keys()]" - self.check(b, a) - - def test_15(self): - b = "(i for i in d.keys())" - a = "(i for i in list(d.keys()))" - self.check(b, a) - - def test_16(self): - b = "(i for i in d.iterkeys())" - a = "(i for i in d.keys())" - self.check(b, a) - - def test_17(self): - b = "iter(d.iterkeys())" - a = "iter(d.keys())" - self.check(b, a) - - def test_18(self): - b = "list(d.iterkeys())" - a = "list(d.keys())" - self.check(b, a) - - def test_19(self): - b = "sorted(d.iterkeys())" - a = "sorted(d.keys())" - self.check(b, a) - - def test_20(self): - b = "foo(d.iterkeys())" - a = "foo(iter(d.keys()))" - self.check(b, a) - - def test_21(self): - b = "print h.iterkeys().next()" - a = "print iter(h.keys()).next()" - self.check(b, a) - - def test_22(self): - b = "print h.keys()[0]" - a = "print list(h.keys())[0]" - self.check(b, a) - - def test_23(self): - b = "print list(h.iterkeys().next())" - a = "print list(iter(h.keys()).next())" - self.check(b, a) - - def test_24(self): - b = "for x in h.keys()[0]: print x" - a = "for x in list(h.keys())[0]: print x" - self.check(b, a) - - def test_25(self): - b = "d.viewkeys()" - a = "d.keys()" - self.check(b, a) - - def test_26(self): - b = "d.viewitems()" - a = "d.items()" - self.check(b, a) - - def test_27(self): - b = "d.viewvalues()" - a = "d.values()" - self.check(b, a) - - def test_28(self): - b = "[i for i in d.viewkeys()]" - a = "[i for i in d.keys()]" - self.check(b, a) - - def test_29(self): - b = "(i for i in d.viewkeys())" - a = "(i for i in d.keys())" - self.check(b, a) - - def test_30(self): - b = "iter(d.viewkeys())" - a = "iter(d.keys())" - self.check(b, a) - - def test_31(self): - b = "list(d.viewkeys())" - a = "list(d.keys())" - self.check(b, a) - - def test_32(self): - b = "sorted(d.viewkeys())" - a = "sorted(d.keys())" - self.check(b, a) - -class Test_xrange(FixerTestCase): - fixer = "xrange" - - def test_prefix_preservation(self): - b = """x = xrange( 10 )""" - a = """x = range( 10 )""" - self.check(b, a) - - b = """x = xrange( 1 , 10 )""" - a = """x = range( 1 , 10 )""" - self.check(b, a) - - b = """x = xrange( 0 , 10 , 2 )""" - a = """x = range( 0 , 10 , 2 )""" - self.check(b, a) - - def test_single_arg(self): - b = """x = xrange(10)""" - a = """x = range(10)""" - self.check(b, a) - - def test_two_args(self): - b = """x = xrange(1, 10)""" - a = """x = range(1, 10)""" - self.check(b, a) - - def test_three_args(self): - b = """x = xrange(0, 10, 2)""" - a = """x = range(0, 10, 2)""" - self.check(b, a) - - def test_wrap_in_list(self): - b = """x = range(10, 3, 9)""" - a = """x = list(range(10, 3, 9))""" - self.check(b, a) - - b = """x = foo(range(10, 3, 9))""" - a = """x = foo(list(range(10, 3, 9)))""" - self.check(b, a) - - b = """x = range(10, 3, 9) + [4]""" - a = """x = list(range(10, 3, 9)) + [4]""" - self.check(b, a) - - b = """x = range(10)[::-1]""" - a = """x = list(range(10))[::-1]""" - self.check(b, a) - - b = """x = range(10) [3]""" - a = """x = list(range(10)) [3]""" - self.check(b, a) - - def test_xrange_in_for(self): - b = """for i in xrange(10):\n j=i""" - a = """for i in range(10):\n j=i""" - self.check(b, a) - - b = """[i for i in xrange(10)]""" - a = """[i for i in range(10)]""" - self.check(b, a) - - def test_range_in_for(self): - self.unchanged("for i in range(10): pass") - self.unchanged("[i for i in range(10)]") - - def test_in_contains_test(self): - self.unchanged("x in range(10, 3, 9)") - - def test_in_consuming_context(self): - for call in fixer_util.consuming_calls: - self.unchanged("a = %s(range(10))" % call) - -class Test_xrange_with_reduce(FixerTestCase): - - def setUp(self): - super(Test_xrange_with_reduce, self).setUp(["xrange", "reduce"]) - - def test_double_transform(self): - b = """reduce(x, xrange(5))""" - a = """from functools import reduce -reduce(x, range(5))""" - self.check(b, a) - -class Test_raw_input(FixerTestCase): - fixer = "raw_input" - - def test_prefix_preservation(self): - b = """x = raw_input( )""" - a = """x = input( )""" - self.check(b, a) - - b = """x = raw_input( '' )""" - a = """x = input( '' )""" - self.check(b, a) - - def test_1(self): - b = """x = raw_input()""" - a = """x = input()""" - self.check(b, a) - - def test_2(self): - b = """x = raw_input('')""" - a = """x = input('')""" - self.check(b, a) - - def test_3(self): - b = """x = raw_input('prompt')""" - a = """x = input('prompt')""" - self.check(b, a) - - def test_4(self): - b = """x = raw_input(foo(a) + 6)""" - a = """x = input(foo(a) + 6)""" - self.check(b, a) - - def test_5(self): - b = """x = raw_input(invite).split()""" - a = """x = input(invite).split()""" - self.check(b, a) - - def test_6(self): - b = """x = raw_input(invite) . split ()""" - a = """x = input(invite) . split ()""" - self.check(b, a) - - def test_8(self): - b = "x = int(raw_input())" - a = "x = int(input())" - self.check(b, a) - -class Test_funcattrs(FixerTestCase): - fixer = "funcattrs" - - attrs = ["closure", "doc", "name", "defaults", "code", "globals", "dict"] - - def test(self): - for attr in self.attrs: - b = "a.func_%s" % attr - a = "a.__%s__" % attr - self.check(b, a) - - b = "self.foo.func_%s.foo_bar" % attr - a = "self.foo.__%s__.foo_bar" % attr - self.check(b, a) - - def test_unchanged(self): - for attr in self.attrs: - s = "foo(func_%s + 5)" % attr - self.unchanged(s) - - s = "f(foo.__%s__)" % attr - self.unchanged(s) - - s = "f(foo.__%s__.foo)" % attr - self.unchanged(s) - -class Test_xreadlines(FixerTestCase): - fixer = "xreadlines" - - def test_call(self): - b = "for x in f.xreadlines(): pass" - a = "for x in f: pass" - self.check(b, a) - - b = "for x in foo().xreadlines(): pass" - a = "for x in foo(): pass" - self.check(b, a) - - b = "for x in (5 + foo()).xreadlines(): pass" - a = "for x in (5 + foo()): pass" - self.check(b, a) - - def test_attr_ref(self): - b = "foo(f.xreadlines + 5)" - a = "foo(f.__iter__ + 5)" - self.check(b, a) - - b = "foo(f().xreadlines + 5)" - a = "foo(f().__iter__ + 5)" - self.check(b, a) - - b = "foo((5 + f()).xreadlines + 5)" - a = "foo((5 + f()).__iter__ + 5)" - self.check(b, a) - - def test_unchanged(self): - s = "for x in f.xreadlines(5): pass" - self.unchanged(s) - - s = "for x in f.xreadlines(k=5): pass" - self.unchanged(s) - - s = "for x in f.xreadlines(*k, **v): pass" - self.unchanged(s) - - s = "foo(xreadlines)" - self.unchanged(s) - - -class ImportsFixerTests: - - def test_import_module(self): - for old, new in self.modules.items(): - b = "import %s" % old - a = "import %s" % new - self.check(b, a) - - b = "import foo, %s, bar" % old - a = "import foo, %s, bar" % new - self.check(b, a) - - def test_import_from(self): - for old, new in self.modules.items(): - b = "from %s import foo" % old - a = "from %s import foo" % new - self.check(b, a) - - b = "from %s import foo, bar" % old - a = "from %s import foo, bar" % new - self.check(b, a) - - b = "from %s import (yes, no)" % old - a = "from %s import (yes, no)" % new - self.check(b, a) - - def test_import_module_as(self): - for old, new in self.modules.items(): - b = "import %s as foo_bar" % old - a = "import %s as foo_bar" % new - self.check(b, a) - - b = "import %s as foo_bar" % old - a = "import %s as foo_bar" % new - self.check(b, a) - - def test_import_from_as(self): - for old, new in self.modules.items(): - b = "from %s import foo as bar" % old - a = "from %s import foo as bar" % new - self.check(b, a) - - def test_star(self): - for old, new in self.modules.items(): - b = "from %s import *" % old - a = "from %s import *" % new - self.check(b, a) - - def test_import_module_usage(self): - for old, new in self.modules.items(): - b = """ - import %s - foo(%s.bar) - """ % (old, old) - a = """ - import %s - foo(%s.bar) - """ % (new, new) - self.check(b, a) - - b = """ - from %s import x - %s = 23 - """ % (old, old) - a = """ - from %s import x - %s = 23 - """ % (new, old) - self.check(b, a) - - s = """ - def f(): - %s.method() - """ % (old,) - self.unchanged(s) - - # test nested usage - b = """ - import %s - %s.bar(%s.foo) - """ % (old, old, old) - a = """ - import %s - %s.bar(%s.foo) - """ % (new, new, new) - self.check(b, a) - - b = """ - import %s - x.%s - """ % (old, old) - a = """ - import %s - x.%s - """ % (new, old) - self.check(b, a) - - -class Test_imports(FixerTestCase, ImportsFixerTests): - fixer = "imports" - from lib2to3.fixes.fix_imports import MAPPING as modules - - def test_multiple_imports(self): - b = """import urlparse, cStringIO""" - a = """import urllib.parse, io""" - self.check(b, a) - - def test_multiple_imports_as(self): - b = """ - import copy_reg as bar, HTMLParser as foo, urlparse - s = urlparse.spam(bar.foo()) - """ - a = """ - import copyreg as bar, html.parser as foo, urllib.parse - s = urllib.parse.spam(bar.foo()) - """ - self.check(b, a) - - -class Test_imports2(FixerTestCase, ImportsFixerTests): - fixer = "imports2" - from lib2to3.fixes.fix_imports2 import MAPPING as modules - - -class Test_imports_fixer_order(FixerTestCase, ImportsFixerTests): - - def setUp(self): - super(Test_imports_fixer_order, self).setUp(['imports', 'imports2']) - from lib2to3.fixes.fix_imports2 import MAPPING as mapping2 - self.modules = mapping2.copy() - from lib2to3.fixes.fix_imports import MAPPING as mapping1 - for key in ('dbhash', 'dumbdbm', 'dbm', 'gdbm'): - self.modules[key] = mapping1[key] - - def test_after_local_imports_refactoring(self): - for fix in ("imports", "imports2"): - self.fixer = fix - self.assert_runs_after("import") - - -class Test_urllib(FixerTestCase): - fixer = "urllib" - from lib2to3.fixes.fix_urllib import MAPPING as modules - - def test_import_module(self): - for old, changes in self.modules.items(): - b = "import %s" % old - a = "import %s" % ", ".join(map(itemgetter(0), changes)) - self.check(b, a) - - def test_import_from(self): - for old, changes in self.modules.items(): - all_members = [] - for new, members in changes: - for member in members: - all_members.append(member) - b = "from %s import %s" % (old, member) - a = "from %s import %s" % (new, member) - self.check(b, a) - - s = "from foo import %s" % member - self.unchanged(s) - - b = "from %s import %s" % (old, ", ".join(members)) - a = "from %s import %s" % (new, ", ".join(members)) - self.check(b, a) - - s = "from foo import %s" % ", ".join(members) - self.unchanged(s) - - # test the breaking of a module into multiple replacements - b = "from %s import %s" % (old, ", ".join(all_members)) - a = "\n".join(["from %s import %s" % (new, ", ".join(members)) - for (new, members) in changes]) - self.check(b, a) - - def test_import_module_as(self): - for old in self.modules: - s = "import %s as foo" % old - self.warns_unchanged(s, "This module is now multiple modules") - - def test_import_from_as(self): - for old, changes in self.modules.items(): - for new, members in changes: - for member in members: - b = "from %s import %s as foo_bar" % (old, member) - a = "from %s import %s as foo_bar" % (new, member) - self.check(b, a) - b = "from %s import %s as blah, %s" % (old, member, member) - a = "from %s import %s as blah, %s" % (new, member, member) - self.check(b, a) - - def test_star(self): - for old in self.modules: - s = "from %s import *" % old - self.warns_unchanged(s, "Cannot handle star imports") - - def test_indented(self): - b = """ -def foo(): - from urllib import urlencode, urlopen -""" - a = """ -def foo(): - from urllib.parse import urlencode - from urllib.request import urlopen -""" - self.check(b, a) - - b = """ -def foo(): - other() - from urllib import urlencode, urlopen -""" - a = """ -def foo(): - other() - from urllib.parse import urlencode - from urllib.request import urlopen -""" - self.check(b, a) - - def test_single_import(self): - b = "from urllib import getproxies" - a = "from urllib.request import getproxies" - - self.check(b, a) - - def test_import_module_usage(self): - for old, changes in self.modules.items(): - for new, members in changes: - for member in members: - new_import = ", ".join([n for (n, mems) - in self.modules[old]]) - b = """ - import %s - foo(%s.%s) - """ % (old, old, member) - a = """ - import %s - foo(%s.%s) - """ % (new_import, new, member) - self.check(b, a) - b = """ - import %s - %s.%s(%s.%s) - """ % (old, old, member, old, member) - a = """ - import %s - %s.%s(%s.%s) - """ % (new_import, new, member, new, member) - self.check(b, a) - - -class Test_input(FixerTestCase): - fixer = "input" - - def test_prefix_preservation(self): - b = """x = input( )""" - a = """x = eval(input( ))""" - self.check(b, a) - - b = """x = input( '' )""" - a = """x = eval(input( '' ))""" - self.check(b, a) - - def test_trailing_comment(self): - b = """x = input() # foo""" - a = """x = eval(input()) # foo""" - self.check(b, a) - - def test_idempotency(self): - s = """x = eval(input())""" - self.unchanged(s) - - s = """x = eval(input(''))""" - self.unchanged(s) - - s = """x = eval(input(foo(5) + 9))""" - self.unchanged(s) - - def test_1(self): - b = """x = input()""" - a = """x = eval(input())""" - self.check(b, a) - - def test_2(self): - b = """x = input('')""" - a = """x = eval(input(''))""" - self.check(b, a) - - def test_3(self): - b = """x = input('prompt')""" - a = """x = eval(input('prompt'))""" - self.check(b, a) - - def test_4(self): - b = """x = input(foo(5) + 9)""" - a = """x = eval(input(foo(5) + 9))""" - self.check(b, a) - -class Test_tuple_params(FixerTestCase): - fixer = "tuple_params" - - def test_unchanged_1(self): - s = """def foo(): pass""" - self.unchanged(s) - - def test_unchanged_2(self): - s = """def foo(a, b, c): pass""" - self.unchanged(s) - - def test_unchanged_3(self): - s = """def foo(a=3, b=4, c=5): pass""" - self.unchanged(s) - - def test_1(self): - b = """ - def foo(((a, b), c)): - x = 5""" - - a = """ - def foo(xxx_todo_changeme): - ((a, b), c) = xxx_todo_changeme - x = 5""" - self.check(b, a) - - def test_2(self): - b = """ - def foo(((a, b), c), d): - x = 5""" - - a = """ - def foo(xxx_todo_changeme, d): - ((a, b), c) = xxx_todo_changeme - x = 5""" - self.check(b, a) - - def test_3(self): - b = """ - def foo(((a, b), c), d) -> e: - x = 5""" - - a = """ - def foo(xxx_todo_changeme, d) -> e: - ((a, b), c) = xxx_todo_changeme - x = 5""" - self.check(b, a) - - def test_semicolon(self): - b = """ - def foo(((a, b), c)): x = 5; y = 7""" - - a = """ - def foo(xxx_todo_changeme): ((a, b), c) = xxx_todo_changeme; x = 5; y = 7""" - self.check(b, a) - - def test_keywords(self): - b = """ - def foo(((a, b), c), d, e=5) -> z: - x = 5""" - - a = """ - def foo(xxx_todo_changeme, d, e=5) -> z: - ((a, b), c) = xxx_todo_changeme - x = 5""" - self.check(b, a) - - def test_varargs(self): - b = """ - def foo(((a, b), c), d, *vargs, **kwargs) -> z: - x = 5""" - - a = """ - def foo(xxx_todo_changeme, d, *vargs, **kwargs) -> z: - ((a, b), c) = xxx_todo_changeme - x = 5""" - self.check(b, a) - - def test_multi_1(self): - b = """ - def foo(((a, b), c), (d, e, f)) -> z: - x = 5""" - - a = """ - def foo(xxx_todo_changeme, xxx_todo_changeme1) -> z: - ((a, b), c) = xxx_todo_changeme - (d, e, f) = xxx_todo_changeme1 - x = 5""" - self.check(b, a) - - def test_multi_2(self): - b = """ - def foo(x, ((a, b), c), d, (e, f, g), y) -> z: - x = 5""" - - a = """ - def foo(x, xxx_todo_changeme, d, xxx_todo_changeme1, y) -> z: - ((a, b), c) = xxx_todo_changeme - (e, f, g) = xxx_todo_changeme1 - x = 5""" - self.check(b, a) - - def test_docstring(self): - b = """ - def foo(((a, b), c), (d, e, f)) -> z: - "foo foo foo foo" - x = 5""" - - a = """ - def foo(xxx_todo_changeme, xxx_todo_changeme1) -> z: - "foo foo foo foo" - ((a, b), c) = xxx_todo_changeme - (d, e, f) = xxx_todo_changeme1 - x = 5""" - self.check(b, a) - - def test_lambda_no_change(self): - s = """lambda x: x + 5""" - self.unchanged(s) - - def test_lambda_parens_single_arg(self): - b = """lambda (x): x + 5""" - a = """lambda x: x + 5""" - self.check(b, a) - - b = """lambda(x): x + 5""" - a = """lambda x: x + 5""" - self.check(b, a) - - b = """lambda ((((x)))): x + 5""" - a = """lambda x: x + 5""" - self.check(b, a) - - b = """lambda((((x)))): x + 5""" - a = """lambda x: x + 5""" - self.check(b, a) - - def test_lambda_simple(self): - b = """lambda (x, y): x + f(y)""" - a = """lambda x_y: x_y[0] + f(x_y[1])""" - self.check(b, a) - - b = """lambda(x, y): x + f(y)""" - a = """lambda x_y: x_y[0] + f(x_y[1])""" - self.check(b, a) - - b = """lambda (((x, y))): x + f(y)""" - a = """lambda x_y: x_y[0] + f(x_y[1])""" - self.check(b, a) - - b = """lambda(((x, y))): x + f(y)""" - a = """lambda x_y: x_y[0] + f(x_y[1])""" - self.check(b, a) - - def test_lambda_one_tuple(self): - b = """lambda (x,): x + f(x)""" - a = """lambda x1: x1[0] + f(x1[0])""" - self.check(b, a) - - b = """lambda (((x,))): x + f(x)""" - a = """lambda x1: x1[0] + f(x1[0])""" - self.check(b, a) - - def test_lambda_simple_multi_use(self): - b = """lambda (x, y): x + x + f(x) + x""" - a = """lambda x_y: x_y[0] + x_y[0] + f(x_y[0]) + x_y[0]""" - self.check(b, a) - - def test_lambda_simple_reverse(self): - b = """lambda (x, y): y + x""" - a = """lambda x_y: x_y[1] + x_y[0]""" - self.check(b, a) - - def test_lambda_nested(self): - b = """lambda (x, (y, z)): x + y + z""" - a = """lambda x_y_z: x_y_z[0] + x_y_z[1][0] + x_y_z[1][1]""" - self.check(b, a) - - b = """lambda (((x, (y, z)))): x + y + z""" - a = """lambda x_y_z: x_y_z[0] + x_y_z[1][0] + x_y_z[1][1]""" - self.check(b, a) - - def test_lambda_nested_multi_use(self): - b = """lambda (x, (y, z)): x + y + f(y)""" - a = """lambda x_y_z: x_y_z[0] + x_y_z[1][0] + f(x_y_z[1][0])""" - self.check(b, a) - -class Test_methodattrs(FixerTestCase): - fixer = "methodattrs" - - attrs = ["func", "self", "class"] - - def test(self): - for attr in self.attrs: - b = "a.im_%s" % attr - if attr == "class": - a = "a.__self__.__class__" - else: - a = "a.__%s__" % attr - self.check(b, a) - - b = "self.foo.im_%s.foo_bar" % attr - if attr == "class": - a = "self.foo.__self__.__class__.foo_bar" - else: - a = "self.foo.__%s__.foo_bar" % attr - self.check(b, a) - - def test_unchanged(self): - for attr in self.attrs: - s = "foo(im_%s + 5)" % attr - self.unchanged(s) - - s = "f(foo.__%s__)" % attr - self.unchanged(s) - - s = "f(foo.__%s__.foo)" % attr - self.unchanged(s) - -class Test_next(FixerTestCase): - fixer = "next" - - def test_1(self): - b = """it.next()""" - a = """next(it)""" - self.check(b, a) - - def test_2(self): - b = """a.b.c.d.next()""" - a = """next(a.b.c.d)""" - self.check(b, a) - - def test_3(self): - b = """(a + b).next()""" - a = """next((a + b))""" - self.check(b, a) - - def test_4(self): - b = """a().next()""" - a = """next(a())""" - self.check(b, a) - - def test_5(self): - b = """a().next() + b""" - a = """next(a()) + b""" - self.check(b, a) - - def test_6(self): - b = """c( a().next() + b)""" - a = """c( next(a()) + b)""" - self.check(b, a) - - def test_prefix_preservation_1(self): - b = """ - for a in b: - foo(a) - a.next() - """ - a = """ - for a in b: - foo(a) - next(a) - """ - self.check(b, a) - - def test_prefix_preservation_2(self): - b = """ - for a in b: - foo(a) # abc - # def - a.next() - """ - a = """ - for a in b: - foo(a) # abc - # def - next(a) - """ - self.check(b, a) - - def test_prefix_preservation_3(self): - b = """ - next = 5 - for a in b: - foo(a) - a.next() - """ - a = """ - next = 5 - for a in b: - foo(a) - a.__next__() - """ - self.check(b, a, ignore_warnings=True) - - def test_prefix_preservation_4(self): - b = """ - next = 5 - for a in b: - foo(a) # abc - # def - a.next() - """ - a = """ - next = 5 - for a in b: - foo(a) # abc - # def - a.__next__() - """ - self.check(b, a, ignore_warnings=True) - - def test_prefix_preservation_5(self): - b = """ - next = 5 - for a in b: - foo(foo(a), # abc - a.next()) - """ - a = """ - next = 5 - for a in b: - foo(foo(a), # abc - a.__next__()) - """ - self.check(b, a, ignore_warnings=True) - - def test_prefix_preservation_6(self): - b = """ - for a in b: - foo(foo(a), # abc - a.next()) - """ - a = """ - for a in b: - foo(foo(a), # abc - next(a)) - """ - self.check(b, a) - - def test_method_1(self): - b = """ - class A: - def next(self): - pass - """ - a = """ - class A: - def __next__(self): - pass - """ - self.check(b, a) - - def test_method_2(self): - b = """ - class A(object): - def next(self): - pass - """ - a = """ - class A(object): - def __next__(self): - pass - """ - self.check(b, a) - - def test_method_3(self): - b = """ - class A: - def next(x): - pass - """ - a = """ - class A: - def __next__(x): - pass - """ - self.check(b, a) - - def test_method_4(self): - b = """ - class A: - def __init__(self, foo): - self.foo = foo - - def next(self): - pass - - def __iter__(self): - return self - """ - a = """ - class A: - def __init__(self, foo): - self.foo = foo - - def __next__(self): - pass - - def __iter__(self): - return self - """ - self.check(b, a) - - def test_method_unchanged(self): - s = """ - class A: - def next(self, a, b): - pass - """ - self.unchanged(s) - - def test_shadowing_assign_simple(self): - s = """ - next = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_assign_tuple_1(self): - s = """ - (next, a) = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_assign_tuple_2(self): - s = """ - (a, (b, (next, c)), a) = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_assign_list_1(self): - s = """ - [next, a] = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_assign_list_2(self): - s = """ - [a, [b, [next, c]], a] = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_builtin_assign(self): - s = """ - def foo(): - __builtin__.next = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_builtin_assign_in_tuple(self): - s = """ - def foo(): - (a, __builtin__.next) = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_builtin_assign_in_list(self): - s = """ - def foo(): - [a, __builtin__.next] = foo - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_assign_to_next(self): - s = """ - def foo(): - A.next = foo - - class A: - def next(self, a, b): - pass - """ - self.unchanged(s) - - def test_assign_to_next_in_tuple(self): - s = """ - def foo(): - (a, A.next) = foo - - class A: - def next(self, a, b): - pass - """ - self.unchanged(s) - - def test_assign_to_next_in_list(self): - s = """ - def foo(): - [a, A.next] = foo - - class A: - def next(self, a, b): - pass - """ - self.unchanged(s) - - def test_shadowing_import_1(self): - s = """ - import foo.bar as next - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_2(self): - s = """ - import bar, bar.foo as next - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_3(self): - s = """ - import bar, bar.foo as next, baz - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_from_1(self): - s = """ - from x import next - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_from_2(self): - s = """ - from x.a import next - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_from_3(self): - s = """ - from x import a, next, b - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_import_from_4(self): - s = """ - from x.a import a, next, b - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_funcdef_1(self): - s = """ - def next(a): - pass - - class A: - def next(self, a, b): - pass - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_funcdef_2(self): - b = """ - def next(a): - pass - - class A: - def next(self): - pass - - it.next() - """ - a = """ - def next(a): - pass - - class A: - def __next__(self): - pass - - it.__next__() - """ - self.warns(b, a, "Calls to builtin next() possibly shadowed") - - def test_shadowing_global_1(self): - s = """ - def f(): - global next - next = 5 - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_global_2(self): - s = """ - def f(): - global a, next, b - next = 5 - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_for_simple(self): - s = """ - for next in it(): - pass - - b = 5 - c = 6 - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_for_tuple_1(self): - s = """ - for next, b in it(): - pass - - b = 5 - c = 6 - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_shadowing_for_tuple_2(self): - s = """ - for a, (next, c), b in it(): - pass - - b = 5 - c = 6 - """ - self.warns_unchanged(s, "Calls to builtin next() possibly shadowed") - - def test_noncall_access_1(self): - b = """gnext = g.next""" - a = """gnext = g.__next__""" - self.check(b, a) - - def test_noncall_access_2(self): - b = """f(g.next + 5)""" - a = """f(g.__next__ + 5)""" - self.check(b, a) - - def test_noncall_access_3(self): - b = """f(g().next + 5)""" - a = """f(g().__next__ + 5)""" - self.check(b, a) - -class Test_nonzero(FixerTestCase): - fixer = "nonzero" - - def test_1(self): - b = """ - class A: - def __nonzero__(self): - pass - """ - a = """ - class A: - def __bool__(self): - pass - """ - self.check(b, a) - - def test_2(self): - b = """ - class A(object): - def __nonzero__(self): - pass - """ - a = """ - class A(object): - def __bool__(self): - pass - """ - self.check(b, a) - - def test_unchanged_1(self): - s = """ - class A(object): - def __bool__(self): - pass - """ - self.unchanged(s) - - def test_unchanged_2(self): - s = """ - class A(object): - def __nonzero__(self, a): - pass - """ - self.unchanged(s) - - def test_unchanged_func(self): - s = """ - def __nonzero__(self): - pass - """ - self.unchanged(s) - -class Test_numliterals(FixerTestCase): - fixer = "numliterals" - - def test_octal_1(self): - b = """0755""" - a = """0o755""" - self.check(b, a) - - def test_long_int_1(self): - b = """a = 12L""" - a = """a = 12""" - self.check(b, a) - - def test_long_int_2(self): - b = """a = 12l""" - a = """a = 12""" - self.check(b, a) - - def test_long_hex(self): - b = """b = 0x12l""" - a = """b = 0x12""" - self.check(b, a) - - def test_comments_and_spacing(self): - b = """b = 0x12L""" - a = """b = 0x12""" - self.check(b, a) - - b = """b = 0755 # spam""" - a = """b = 0o755 # spam""" - self.check(b, a) - - def test_unchanged_int(self): - s = """5""" - self.unchanged(s) - - def test_unchanged_float(self): - s = """5.0""" - self.unchanged(s) - - def test_unchanged_octal(self): - s = """0o755""" - self.unchanged(s) - - def test_unchanged_hex(self): - s = """0xABC""" - self.unchanged(s) - - def test_unchanged_exp(self): - s = """5.0e10""" - self.unchanged(s) - - def test_unchanged_complex_int(self): - s = """5 + 4j""" - self.unchanged(s) - - def test_unchanged_complex_float(self): - s = """5.4 + 4.9j""" - self.unchanged(s) - - def test_unchanged_complex_bare(self): - s = """4j""" - self.unchanged(s) - s = """4.4j""" - self.unchanged(s) - -class Test_renames(FixerTestCase): - fixer = "renames" - - modules = {"sys": ("maxint", "maxsize"), - } - - def test_import_from(self): - for mod, (old, new) in list(self.modules.items()): - b = "from %s import %s" % (mod, old) - a = "from %s import %s" % (mod, new) - self.check(b, a) - - s = "from foo import %s" % old - self.unchanged(s) - - def test_import_from_as(self): - for mod, (old, new) in list(self.modules.items()): - b = "from %s import %s as foo_bar" % (mod, old) - a = "from %s import %s as foo_bar" % (mod, new) - self.check(b, a) - - def test_import_module_usage(self): - for mod, (old, new) in list(self.modules.items()): - b = """ - import %s - foo(%s, %s.%s) - """ % (mod, mod, mod, old) - a = """ - import %s - foo(%s, %s.%s) - """ % (mod, mod, mod, new) - self.check(b, a) - - def XXX_test_from_import_usage(self): - # not implemented yet - for mod, (old, new) in list(self.modules.items()): - b = """ - from %s import %s - foo(%s, %s) - """ % (mod, old, mod, old) - a = """ - from %s import %s - foo(%s, %s) - """ % (mod, new, mod, new) - self.check(b, a) - -class Test_unicode(FixerTestCase): - fixer = "unicode" - - def test_whitespace(self): - b = """unicode( x)""" - a = """str( x)""" - self.check(b, a) - - b = """ unicode(x )""" - a = """ str(x )""" - self.check(b, a) - - b = """ u'h'""" - a = """ 'h'""" - self.check(b, a) - - def test_unicode_call(self): - b = """unicode(x, y, z)""" - a = """str(x, y, z)""" - self.check(b, a) - - def test_unichr(self): - b = """unichr(u'h')""" - a = """chr('h')""" - self.check(b, a) - - def test_unicode_literal_1(self): - b = '''u"x"''' - a = '''"x"''' - self.check(b, a) - - def test_unicode_literal_2(self): - b = """ur'x'""" - a = """r'x'""" - self.check(b, a) - - def test_unicode_literal_3(self): - b = """UR'''x''' """ - a = """R'''x''' """ - self.check(b, a) - - def test_native_literal_escape_u(self): - b = r"""'\\\u20ac\U0001d121\\u20ac'""" - a = r"""'\\\\u20ac\\U0001d121\\u20ac'""" - self.check(b, a) - - b = r"""r'\\\u20ac\U0001d121\\u20ac'""" - a = r"""r'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - def test_bytes_literal_escape_u(self): - b = r"""b'\\\u20ac\U0001d121\\u20ac'""" - a = r"""b'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - b = r"""br'\\\u20ac\U0001d121\\u20ac'""" - a = r"""br'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - def test_unicode_literal_escape_u(self): - b = r"""u'\\\u20ac\U0001d121\\u20ac'""" - a = r"""'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - b = r"""ur'\\\u20ac\U0001d121\\u20ac'""" - a = r"""r'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - def test_native_unicode_literal_escape_u(self): - f = 'from __future__ import unicode_literals\n' - b = f + r"""'\\\u20ac\U0001d121\\u20ac'""" - a = f + r"""'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - b = f + r"""r'\\\u20ac\U0001d121\\u20ac'""" - a = f + r"""r'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - -class Test_filter(FixerTestCase): - fixer = "filter" - - def test_prefix_preservation(self): - b = """x = filter( foo, 'abc' )""" - a = """x = list(filter( foo, 'abc' ))""" - self.check(b, a) - - b = """x = filter( None , 'abc' )""" - a = """x = [_f for _f in 'abc' if _f]""" - self.check(b, a) - - def test_filter_basic(self): - b = """x = filter(None, 'abc')""" - a = """x = [_f for _f in 'abc' if _f]""" - self.check(b, a) - - b = """x = len(filter(f, 'abc'))""" - a = """x = len(list(filter(f, 'abc')))""" - self.check(b, a) - - b = """x = filter(lambda x: x%2 == 0, range(10))""" - a = """x = [x for x in range(10) if x%2 == 0]""" - self.check(b, a) - - # Note the parens around x - b = """x = filter(lambda (x): x%2 == 0, range(10))""" - a = """x = [x for x in range(10) if x%2 == 0]""" - self.check(b, a) - - # bpo-38871 - b = """filter(lambda x: True if x > 2 else False, [1, 2, 3])""" - a = """[x for x in [1, 2, 3] if (True if x > 2 else False)]""" - self.check(b, a) - - def test_filter_trailers(self): - b = """x = filter(None, 'abc')[0]""" - a = """x = [_f for _f in 'abc' if _f][0]""" - self.check(b, a) - - b = """x = len(filter(f, 'abc')[0])""" - a = """x = len(list(filter(f, 'abc'))[0])""" - self.check(b, a) - - b = """x = filter(lambda x: x%2 == 0, range(10))[0]""" - a = """x = [x for x in range(10) if x%2 == 0][0]""" - self.check(b, a) - - # Note the parens around x - b = """x = filter(lambda (x): x%2 == 0, range(10))[0]""" - a = """x = [x for x in range(10) if x%2 == 0][0]""" - self.check(b, a) - - def test_filter_nochange(self): - a = """b.join(filter(f, 'abc'))""" - self.unchanged(a) - a = """(a + foo(5)).join(filter(f, 'abc'))""" - self.unchanged(a) - a = """iter(filter(f, 'abc'))""" - self.unchanged(a) - a = """list(filter(f, 'abc'))""" - self.unchanged(a) - a = """list(filter(f, 'abc'))[0]""" - self.unchanged(a) - a = """set(filter(f, 'abc'))""" - self.unchanged(a) - a = """set(filter(f, 'abc')).pop()""" - self.unchanged(a) - a = """tuple(filter(f, 'abc'))""" - self.unchanged(a) - a = """any(filter(f, 'abc'))""" - self.unchanged(a) - a = """all(filter(f, 'abc'))""" - self.unchanged(a) - a = """sum(filter(f, 'abc'))""" - self.unchanged(a) - a = """sorted(filter(f, 'abc'))""" - self.unchanged(a) - a = """sorted(filter(f, 'abc'), key=blah)""" - self.unchanged(a) - a = """sorted(filter(f, 'abc'), key=blah)[0]""" - self.unchanged(a) - a = """enumerate(filter(f, 'abc'))""" - self.unchanged(a) - a = """enumerate(filter(f, 'abc'), start=1)""" - self.unchanged(a) - a = """for i in filter(f, 'abc'): pass""" - self.unchanged(a) - a = """[x for x in filter(f, 'abc')]""" - self.unchanged(a) - a = """(x for x in filter(f, 'abc'))""" - self.unchanged(a) - - def test_future_builtins(self): - a = "from future_builtins import spam, filter; filter(f, 'ham')" - self.unchanged(a) - - b = """from future_builtins import spam; x = filter(f, 'abc')""" - a = """from future_builtins import spam; x = list(filter(f, 'abc'))""" - self.check(b, a) - - a = "from future_builtins import *; filter(f, 'ham')" - self.unchanged(a) - -class Test_map(FixerTestCase): - fixer = "map" - - def check(self, b, a): - self.unchanged("from future_builtins import map; " + b, a) - super(Test_map, self).check(b, a) - - def test_prefix_preservation(self): - b = """x = map( f, 'abc' )""" - a = """x = list(map( f, 'abc' ))""" - self.check(b, a) - - def test_map_trailers(self): - b = """x = map(f, 'abc')[0]""" - a = """x = list(map(f, 'abc'))[0]""" - self.check(b, a) - - b = """x = map(None, l)[0]""" - a = """x = list(l)[0]""" - self.check(b, a) - - b = """x = map(lambda x:x, l)[0]""" - a = """x = [x for x in l][0]""" - self.check(b, a) - - b = """x = map(f, 'abc')[0][1]""" - a = """x = list(map(f, 'abc'))[0][1]""" - self.check(b, a) - - def test_trailing_comment(self): - b = """x = map(f, 'abc') # foo""" - a = """x = list(map(f, 'abc')) # foo""" - self.check(b, a) - - def test_None_with_multiple_arguments(self): - s = """x = map(None, a, b, c)""" - self.warns_unchanged(s, "cannot convert map(None, ...) with " - "multiple arguments") - - def test_map_basic(self): - b = """x = map(f, 'abc')""" - a = """x = list(map(f, 'abc'))""" - self.check(b, a) - - b = """x = len(map(f, 'abc', 'def'))""" - a = """x = len(list(map(f, 'abc', 'def')))""" - self.check(b, a) - - b = """x = map(None, 'abc')""" - a = """x = list('abc')""" - self.check(b, a) - - b = """x = map(lambda x: x+1, range(4))""" - a = """x = [x+1 for x in range(4)]""" - self.check(b, a) - - # Note the parens around x - b = """x = map(lambda (x): x+1, range(4))""" - a = """x = [x+1 for x in range(4)]""" - self.check(b, a) - - b = """ - foo() - # foo - map(f, x) - """ - a = """ - foo() - # foo - list(map(f, x)) - """ - self.warns(b, a, "You should use a for loop here") - - def test_map_nochange(self): - a = """b.join(map(f, 'abc'))""" - self.unchanged(a) - a = """(a + foo(5)).join(map(f, 'abc'))""" - self.unchanged(a) - a = """iter(map(f, 'abc'))""" - self.unchanged(a) - a = """list(map(f, 'abc'))""" - self.unchanged(a) - a = """list(map(f, 'abc'))[0]""" - self.unchanged(a) - a = """set(map(f, 'abc'))""" - self.unchanged(a) - a = """set(map(f, 'abc')).pop()""" - self.unchanged(a) - a = """tuple(map(f, 'abc'))""" - self.unchanged(a) - a = """any(map(f, 'abc'))""" - self.unchanged(a) - a = """all(map(f, 'abc'))""" - self.unchanged(a) - a = """sum(map(f, 'abc'))""" - self.unchanged(a) - a = """sorted(map(f, 'abc'))""" - self.unchanged(a) - a = """sorted(map(f, 'abc'), key=blah)""" - self.unchanged(a) - a = """sorted(map(f, 'abc'), key=blah)[0]""" - self.unchanged(a) - a = """enumerate(map(f, 'abc'))""" - self.unchanged(a) - a = """enumerate(map(f, 'abc'), start=1)""" - self.unchanged(a) - a = """for i in map(f, 'abc'): pass""" - self.unchanged(a) - a = """[x for x in map(f, 'abc')]""" - self.unchanged(a) - a = """(x for x in map(f, 'abc'))""" - self.unchanged(a) - - def test_future_builtins(self): - a = "from future_builtins import spam, map, eggs; map(f, 'ham')" - self.unchanged(a) - - b = """from future_builtins import spam, eggs; x = map(f, 'abc')""" - a = """from future_builtins import spam, eggs; x = list(map(f, 'abc'))""" - self.check(b, a) - - a = "from future_builtins import *; map(f, 'ham')" - self.unchanged(a) - -class Test_zip(FixerTestCase): - fixer = "zip" - - def check(self, b, a): - self.unchanged("from future_builtins import zip; " + b, a) - super(Test_zip, self).check(b, a) - - def test_zip_basic(self): - b = """x = zip()""" - a = """x = list(zip())""" - self.check(b, a) - - b = """x = zip(a, b, c)""" - a = """x = list(zip(a, b, c))""" - self.check(b, a) - - b = """x = len(zip(a, b))""" - a = """x = len(list(zip(a, b)))""" - self.check(b, a) - - def test_zip_trailers(self): - b = """x = zip(a, b, c)[0]""" - a = """x = list(zip(a, b, c))[0]""" - self.check(b, a) - - b = """x = zip(a, b, c)[0][1]""" - a = """x = list(zip(a, b, c))[0][1]""" - self.check(b, a) - - def test_zip_nochange(self): - a = """b.join(zip(a, b))""" - self.unchanged(a) - a = """(a + foo(5)).join(zip(a, b))""" - self.unchanged(a) - a = """iter(zip(a, b))""" - self.unchanged(a) - a = """list(zip(a, b))""" - self.unchanged(a) - a = """list(zip(a, b))[0]""" - self.unchanged(a) - a = """set(zip(a, b))""" - self.unchanged(a) - a = """set(zip(a, b)).pop()""" - self.unchanged(a) - a = """tuple(zip(a, b))""" - self.unchanged(a) - a = """any(zip(a, b))""" - self.unchanged(a) - a = """all(zip(a, b))""" - self.unchanged(a) - a = """sum(zip(a, b))""" - self.unchanged(a) - a = """sorted(zip(a, b))""" - self.unchanged(a) - a = """sorted(zip(a, b), key=blah)""" - self.unchanged(a) - a = """sorted(zip(a, b), key=blah)[0]""" - self.unchanged(a) - a = """enumerate(zip(a, b))""" - self.unchanged(a) - a = """enumerate(zip(a, b), start=1)""" - self.unchanged(a) - a = """for i in zip(a, b): pass""" - self.unchanged(a) - a = """[x for x in zip(a, b)]""" - self.unchanged(a) - a = """(x for x in zip(a, b))""" - self.unchanged(a) - - def test_future_builtins(self): - a = "from future_builtins import spam, zip, eggs; zip(a, b)" - self.unchanged(a) - - b = """from future_builtins import spam, eggs; x = zip(a, b)""" - a = """from future_builtins import spam, eggs; x = list(zip(a, b))""" - self.check(b, a) - - a = "from future_builtins import *; zip(a, b)" - self.unchanged(a) - -class Test_standarderror(FixerTestCase): - fixer = "standarderror" - - def test(self): - b = """x = StandardError()""" - a = """x = Exception()""" - self.check(b, a) - - b = """x = StandardError(a, b, c)""" - a = """x = Exception(a, b, c)""" - self.check(b, a) - - b = """f(2 + StandardError(a, b, c))""" - a = """f(2 + Exception(a, b, c))""" - self.check(b, a) - -class Test_types(FixerTestCase): - fixer = "types" - - def test_basic_types_convert(self): - b = """types.StringType""" - a = """bytes""" - self.check(b, a) - - b = """types.DictType""" - a = """dict""" - self.check(b, a) - - b = """types . IntType""" - a = """int""" - self.check(b, a) - - b = """types.ListType""" - a = """list""" - self.check(b, a) - - b = """types.LongType""" - a = """int""" - self.check(b, a) - - b = """types.NoneType""" - a = """type(None)""" - self.check(b, a) - - b = "types.StringTypes" - a = "(str,)" - self.check(b, a) - -class Test_idioms(FixerTestCase): - fixer = "idioms" - - def test_while(self): - b = """while 1: foo()""" - a = """while True: foo()""" - self.check(b, a) - - b = """while 1: foo()""" - a = """while True: foo()""" - self.check(b, a) - - b = """ - while 1: - foo() - """ - a = """ - while True: - foo() - """ - self.check(b, a) - - def test_while_unchanged(self): - s = """while 11: foo()""" - self.unchanged(s) - - s = """while 0: foo()""" - self.unchanged(s) - - s = """while foo(): foo()""" - self.unchanged(s) - - s = """while []: foo()""" - self.unchanged(s) - - def test_eq_simple(self): - b = """type(x) == T""" - a = """isinstance(x, T)""" - self.check(b, a) - - b = """if type(x) == T: pass""" - a = """if isinstance(x, T): pass""" - self.check(b, a) - - def test_eq_reverse(self): - b = """T == type(x)""" - a = """isinstance(x, T)""" - self.check(b, a) - - b = """if T == type(x): pass""" - a = """if isinstance(x, T): pass""" - self.check(b, a) - - def test_eq_expression(self): - b = """type(x+y) == d.get('T')""" - a = """isinstance(x+y, d.get('T'))""" - self.check(b, a) - - b = """type( x + y) == d.get('T')""" - a = """isinstance(x + y, d.get('T'))""" - self.check(b, a) - - def test_is_simple(self): - b = """type(x) is T""" - a = """isinstance(x, T)""" - self.check(b, a) - - b = """if type(x) is T: pass""" - a = """if isinstance(x, T): pass""" - self.check(b, a) - - def test_is_reverse(self): - b = """T is type(x)""" - a = """isinstance(x, T)""" - self.check(b, a) - - b = """if T is type(x): pass""" - a = """if isinstance(x, T): pass""" - self.check(b, a) - - def test_is_expression(self): - b = """type(x+y) is d.get('T')""" - a = """isinstance(x+y, d.get('T'))""" - self.check(b, a) - - b = """type( x + y) is d.get('T')""" - a = """isinstance(x + y, d.get('T'))""" - self.check(b, a) - - def test_is_not_simple(self): - b = """type(x) is not T""" - a = """not isinstance(x, T)""" - self.check(b, a) - - b = """if type(x) is not T: pass""" - a = """if not isinstance(x, T): pass""" - self.check(b, a) - - def test_is_not_reverse(self): - b = """T is not type(x)""" - a = """not isinstance(x, T)""" - self.check(b, a) - - b = """if T is not type(x): pass""" - a = """if not isinstance(x, T): pass""" - self.check(b, a) - - def test_is_not_expression(self): - b = """type(x+y) is not d.get('T')""" - a = """not isinstance(x+y, d.get('T'))""" - self.check(b, a) - - b = """type( x + y) is not d.get('T')""" - a = """not isinstance(x + y, d.get('T'))""" - self.check(b, a) - - def test_ne_simple(self): - b = """type(x) != T""" - a = """not isinstance(x, T)""" - self.check(b, a) - - b = """if type(x) != T: pass""" - a = """if not isinstance(x, T): pass""" - self.check(b, a) - - def test_ne_reverse(self): - b = """T != type(x)""" - a = """not isinstance(x, T)""" - self.check(b, a) - - b = """if T != type(x): pass""" - a = """if not isinstance(x, T): pass""" - self.check(b, a) - - def test_ne_expression(self): - b = """type(x+y) != d.get('T')""" - a = """not isinstance(x+y, d.get('T'))""" - self.check(b, a) - - b = """type( x + y) != d.get('T')""" - a = """not isinstance(x + y, d.get('T'))""" - self.check(b, a) - - def test_type_unchanged(self): - a = """type(x).__name__""" - self.unchanged(a) - - def test_sort_list_call(self): - b = """ - v = list(t) - v.sort() - foo(v) - """ - a = """ - v = sorted(t) - foo(v) - """ - self.check(b, a) - - b = """ - v = list(foo(b) + d) - v.sort() - foo(v) - """ - a = """ - v = sorted(foo(b) + d) - foo(v) - """ - self.check(b, a) - - b = """ - while x: - v = list(t) - v.sort() - foo(v) - """ - a = """ - while x: - v = sorted(t) - foo(v) - """ - self.check(b, a) - - b = """ - v = list(t) - # foo - v.sort() - foo(v) - """ - a = """ - v = sorted(t) - # foo - foo(v) - """ - self.check(b, a) - - b = r""" - v = list( t) - v.sort() - foo(v) - """ - a = r""" - v = sorted( t) - foo(v) - """ - self.check(b, a) - - b = r""" - try: - m = list(s) - m.sort() - except: pass - """ - - a = r""" - try: - m = sorted(s) - except: pass - """ - self.check(b, a) - - b = r""" - try: - m = list(s) - # foo - m.sort() - except: pass - """ - - a = r""" - try: - m = sorted(s) - # foo - except: pass - """ - self.check(b, a) - - b = r""" - m = list(s) - # more comments - m.sort()""" - - a = r""" - m = sorted(s) - # more comments""" - self.check(b, a) - - def test_sort_simple_expr(self): - b = """ - v = t - v.sort() - foo(v) - """ - a = """ - v = sorted(t) - foo(v) - """ - self.check(b, a) - - b = """ - v = foo(b) - v.sort() - foo(v) - """ - a = """ - v = sorted(foo(b)) - foo(v) - """ - self.check(b, a) - - b = """ - v = b.keys() - v.sort() - foo(v) - """ - a = """ - v = sorted(b.keys()) - foo(v) - """ - self.check(b, a) - - b = """ - v = foo(b) + d - v.sort() - foo(v) - """ - a = """ - v = sorted(foo(b) + d) - foo(v) - """ - self.check(b, a) - - b = """ - while x: - v = t - v.sort() - foo(v) - """ - a = """ - while x: - v = sorted(t) - foo(v) - """ - self.check(b, a) - - b = """ - v = t - # foo - v.sort() - foo(v) - """ - a = """ - v = sorted(t) - # foo - foo(v) - """ - self.check(b, a) - - b = r""" - v = t - v.sort() - foo(v) - """ - a = r""" - v = sorted(t) - foo(v) - """ - self.check(b, a) - - def test_sort_unchanged(self): - s = """ - v = list(t) - w.sort() - foo(w) - """ - self.unchanged(s) - - s = """ - v = list(t) - v.sort(u) - foo(v) - """ - self.unchanged(s) - -class Test_basestring(FixerTestCase): - fixer = "basestring" - - def test_basestring(self): - b = """isinstance(x, basestring)""" - a = """isinstance(x, str)""" - self.check(b, a) - -class Test_buffer(FixerTestCase): - fixer = "buffer" - - def test_buffer(self): - b = """x = buffer(y)""" - a = """x = memoryview(y)""" - self.check(b, a) - - def test_slicing(self): - b = """buffer(y)[4:5]""" - a = """memoryview(y)[4:5]""" - self.check(b, a) - -class Test_future(FixerTestCase): - fixer = "future" - - def test_future(self): - b = """from __future__ import braces""" - a = """""" - self.check(b, a) - - b = """# comment\nfrom __future__ import braces""" - a = """# comment\n""" - self.check(b, a) - - b = """from __future__ import braces\n# comment""" - a = """\n# comment""" - self.check(b, a) - - def test_run_order(self): - self.assert_runs_after('print') - -class Test_itertools(FixerTestCase): - fixer = "itertools" - - def checkall(self, before, after): - # Because we need to check with and without the itertools prefix - # and on each of the three functions, these loops make it all - # much easier - for i in ('itertools.', ''): - for f in ('map', 'filter', 'zip'): - b = before %(i+'i'+f) - a = after %(f) - self.check(b, a) - - def test_0(self): - # A simple example -- test_1 covers exactly the same thing, - # but it's not quite as clear. - b = "itertools.izip(a, b)" - a = "zip(a, b)" - self.check(b, a) - - def test_1(self): - b = """%s(f, a)""" - a = """%s(f, a)""" - self.checkall(b, a) - - def test_qualified(self): - b = """itertools.ifilterfalse(a, b)""" - a = """itertools.filterfalse(a, b)""" - self.check(b, a) - - b = """itertools.izip_longest(a, b)""" - a = """itertools.zip_longest(a, b)""" - self.check(b, a) - - def test_2(self): - b = """ifilterfalse(a, b)""" - a = """filterfalse(a, b)""" - self.check(b, a) - - b = """izip_longest(a, b)""" - a = """zip_longest(a, b)""" - self.check(b, a) - - def test_space_1(self): - b = """ %s(f, a)""" - a = """ %s(f, a)""" - self.checkall(b, a) - - def test_space_2(self): - b = """ itertools.ifilterfalse(a, b)""" - a = """ itertools.filterfalse(a, b)""" - self.check(b, a) - - b = """ itertools.izip_longest(a, b)""" - a = """ itertools.zip_longest(a, b)""" - self.check(b, a) - - def test_run_order(self): - self.assert_runs_after('map', 'zip', 'filter') - - -class Test_itertools_imports(FixerTestCase): - fixer = 'itertools_imports' - - def test_reduced(self): - b = "from itertools import imap, izip, foo" - a = "from itertools import foo" - self.check(b, a) - - b = "from itertools import bar, imap, izip, foo" - a = "from itertools import bar, foo" - self.check(b, a) - - b = "from itertools import chain, imap, izip" - a = "from itertools import chain" - self.check(b, a) - - def test_comments(self): - b = "#foo\nfrom itertools import imap, izip" - a = "#foo\n" - self.check(b, a) - - def test_none(self): - b = "from itertools import imap, izip" - a = "" - self.check(b, a) - - b = "from itertools import izip" - a = "" - self.check(b, a) - - def test_import_as(self): - b = "from itertools import izip, bar as bang, imap" - a = "from itertools import bar as bang" - self.check(b, a) - - b = "from itertools import izip as _zip, imap, bar" - a = "from itertools import bar" - self.check(b, a) - - b = "from itertools import imap as _map" - a = "" - self.check(b, a) - - b = "from itertools import imap as _map, izip as _zip" - a = "" - self.check(b, a) - - s = "from itertools import bar as bang" - self.unchanged(s) - - def test_ifilter_and_zip_longest(self): - for name in "filterfalse", "zip_longest": - b = "from itertools import i%s" % (name,) - a = "from itertools import %s" % (name,) - self.check(b, a) - - b = "from itertools import imap, i%s, foo" % (name,) - a = "from itertools import %s, foo" % (name,) - self.check(b, a) - - b = "from itertools import bar, i%s, foo" % (name,) - a = "from itertools import bar, %s, foo" % (name,) - self.check(b, a) - - def test_import_star(self): - s = "from itertools import *" - self.unchanged(s) - - - def test_unchanged(self): - s = "from itertools import foo" - self.unchanged(s) - - -class Test_import(FixerTestCase): - fixer = "import" - - def setUp(self): - super(Test_import, self).setUp() - # Need to replace fix_import's exists method - # so we can check that it's doing the right thing - self.files_checked = [] - self.present_files = set() - self.always_exists = True - def fake_exists(name): - self.files_checked.append(name) - return self.always_exists or (name in self.present_files) - - from lib2to3.fixes import fix_import - fix_import.exists = fake_exists - - def tearDown(self): - from lib2to3.fixes import fix_import - fix_import.exists = os.path.exists - - def check_both(self, b, a): - self.always_exists = True - super(Test_import, self).check(b, a) - self.always_exists = False - super(Test_import, self).unchanged(b) - - def test_files_checked(self): - def p(path): - # Takes a unix path and returns a path with correct separators - return os.path.pathsep.join(path.split("/")) - - self.always_exists = False - self.present_files = set(['__init__.py']) - expected_extensions = ('.py', os.path.sep, '.pyc', '.so', '.sl', '.pyd') - names_to_test = (p("/spam/eggs.py"), "ni.py", p("../../shrubbery.py")) - - for name in names_to_test: - self.files_checked = [] - self.filename = name - self.unchanged("import jam") - - if os.path.dirname(name): - name = os.path.dirname(name) + '/jam' - else: - name = 'jam' - expected_checks = set(name + ext for ext in expected_extensions) - expected_checks.add("__init__.py") - - self.assertEqual(set(self.files_checked), expected_checks) - - def test_not_in_package(self): - s = "import bar" - self.always_exists = False - self.present_files = set(["bar.py"]) - self.unchanged(s) - - def test_with_absolute_import_enabled(self): - s = "from __future__ import absolute_import\nimport bar" - self.always_exists = False - self.present_files = set(["__init__.py", "bar.py"]) - self.unchanged(s) - - def test_in_package(self): - b = "import bar" - a = "from . import bar" - self.always_exists = False - self.present_files = set(["__init__.py", "bar.py"]) - self.check(b, a) - - def test_import_from_package(self): - b = "import bar" - a = "from . import bar" - self.always_exists = False - self.present_files = set(["__init__.py", "bar" + os.path.sep]) - self.check(b, a) - - def test_already_relative_import(self): - s = "from . import bar" - self.unchanged(s) - - def test_comments_and_indent(self): - b = "import bar # Foo" - a = "from . import bar # Foo" - self.check(b, a) - - def test_from(self): - b = "from foo import bar, baz" - a = "from .foo import bar, baz" - self.check_both(b, a) - - b = "from foo import bar" - a = "from .foo import bar" - self.check_both(b, a) - - b = "from foo import (bar, baz)" - a = "from .foo import (bar, baz)" - self.check_both(b, a) - - def test_dotted_from(self): - b = "from green.eggs import ham" - a = "from .green.eggs import ham" - self.check_both(b, a) - - def test_from_as(self): - b = "from green.eggs import ham as spam" - a = "from .green.eggs import ham as spam" - self.check_both(b, a) - - def test_import(self): - b = "import foo" - a = "from . import foo" - self.check_both(b, a) - - b = "import foo, bar" - a = "from . import foo, bar" - self.check_both(b, a) - - b = "import foo, bar, x" - a = "from . import foo, bar, x" - self.check_both(b, a) - - b = "import x, y, z" - a = "from . import x, y, z" - self.check_both(b, a) - - def test_import_as(self): - b = "import foo as x" - a = "from . import foo as x" - self.check_both(b, a) - - b = "import a as b, b as c, c as d" - a = "from . import a as b, b as c, c as d" - self.check_both(b, a) - - def test_local_and_absolute(self): - self.always_exists = False - self.present_files = set(["foo.py", "__init__.py"]) - - s = "import foo, bar" - self.warns_unchanged(s, "absolute and local imports together") - - def test_dotted_import(self): - b = "import foo.bar" - a = "from . import foo.bar" - self.check_both(b, a) - - def test_dotted_import_as(self): - b = "import foo.bar as bang" - a = "from . import foo.bar as bang" - self.check_both(b, a) - - def test_prefix(self): - b = """ - # prefix - import foo.bar - """ - a = """ - # prefix - from . import foo.bar - """ - self.check_both(b, a) - - -class Test_set_literal(FixerTestCase): - - fixer = "set_literal" - - def test_basic(self): - b = """set([1, 2, 3])""" - a = """{1, 2, 3}""" - self.check(b, a) - - b = """set((1, 2, 3))""" - a = """{1, 2, 3}""" - self.check(b, a) - - b = """set((1,))""" - a = """{1}""" - self.check(b, a) - - b = """set([1])""" - self.check(b, a) - - b = """set((a, b))""" - a = """{a, b}""" - self.check(b, a) - - b = """set([a, b])""" - self.check(b, a) - - b = """set((a*234, f(args=23)))""" - a = """{a*234, f(args=23)}""" - self.check(b, a) - - b = """set([a*23, f(23)])""" - a = """{a*23, f(23)}""" - self.check(b, a) - - b = """set([a-234**23])""" - a = """{a-234**23}""" - self.check(b, a) - - def test_listcomps(self): - b = """set([x for x in y])""" - a = """{x for x in y}""" - self.check(b, a) - - b = """set([x for x in y if x == m])""" - a = """{x for x in y if x == m}""" - self.check(b, a) - - b = """set([x for x in y for a in b])""" - a = """{x for x in y for a in b}""" - self.check(b, a) - - b = """set([f(x) - 23 for x in y])""" - a = """{f(x) - 23 for x in y}""" - self.check(b, a) - - def test_whitespace(self): - b = """set( [1, 2])""" - a = """{1, 2}""" - self.check(b, a) - - b = """set([1 , 2])""" - a = """{1 , 2}""" - self.check(b, a) - - b = """set([ 1 ])""" - a = """{ 1 }""" - self.check(b, a) - - b = """set( [1] )""" - a = """{1}""" - self.check(b, a) - - b = """set([ 1, 2 ])""" - a = """{ 1, 2 }""" - self.check(b, a) - - b = """set([x for x in y ])""" - a = """{x for x in y }""" - self.check(b, a) - - b = """set( - [1, 2] - ) - """ - a = """{1, 2}\n""" - self.check(b, a) - - def test_comments(self): - b = """set((1, 2)) # Hi""" - a = """{1, 2} # Hi""" - self.check(b, a) - - # This isn't optimal behavior, but the fixer is optional. - b = """ - # Foo - set( # Bar - (1, 2) - ) - """ - a = """ - # Foo - {1, 2} - """ - self.check(b, a) - - def test_unchanged(self): - s = """set()""" - self.unchanged(s) - - s = """set(a)""" - self.unchanged(s) - - s = """set(a, b, c)""" - self.unchanged(s) - - # Don't transform generators because they might have to be lazy. - s = """set(x for x in y)""" - self.unchanged(s) - - s = """set(x for x in y if z)""" - self.unchanged(s) - - s = """set(a*823-23**2 + f(23))""" - self.unchanged(s) - - -class Test_sys_exc(FixerTestCase): - fixer = "sys_exc" - - def test_0(self): - b = "sys.exc_type" - a = "sys.exc_info()[0]" - self.check(b, a) - - def test_1(self): - b = "sys.exc_value" - a = "sys.exc_info()[1]" - self.check(b, a) - - def test_2(self): - b = "sys.exc_traceback" - a = "sys.exc_info()[2]" - self.check(b, a) - - def test_3(self): - b = "sys.exc_type # Foo" - a = "sys.exc_info()[0] # Foo" - self.check(b, a) - - def test_4(self): - b = "sys. exc_type" - a = "sys. exc_info()[0]" - self.check(b, a) - - def test_5(self): - b = "sys .exc_type" - a = "sys .exc_info()[0]" - self.check(b, a) - - -class Test_paren(FixerTestCase): - fixer = "paren" - - def test_0(self): - b = """[i for i in 1, 2 ]""" - a = """[i for i in (1, 2) ]""" - self.check(b, a) - - def test_1(self): - b = """[i for i in 1, 2, ]""" - a = """[i for i in (1, 2,) ]""" - self.check(b, a) - - def test_2(self): - b = """[i for i in 1, 2 ]""" - a = """[i for i in (1, 2) ]""" - self.check(b, a) - - def test_3(self): - b = """[i for i in 1, 2 if i]""" - a = """[i for i in (1, 2) if i]""" - self.check(b, a) - - def test_4(self): - b = """[i for i in 1, 2 ]""" - a = """[i for i in (1, 2) ]""" - self.check(b, a) - - def test_5(self): - b = """(i for i in 1, 2)""" - a = """(i for i in (1, 2))""" - self.check(b, a) - - def test_6(self): - b = """(i for i in 1 ,2 if i)""" - a = """(i for i in (1 ,2) if i)""" - self.check(b, a) - - def test_unchanged_0(self): - s = """[i for i in (1, 2)]""" - self.unchanged(s) - - def test_unchanged_1(self): - s = """[i for i in foo()]""" - self.unchanged(s) - - def test_unchanged_2(self): - s = """[i for i in (1, 2) if nothing]""" - self.unchanged(s) - - def test_unchanged_3(self): - s = """(i for i in (1, 2))""" - self.unchanged(s) - - def test_unchanged_4(self): - s = """[i for i in m]""" - self.unchanged(s) - -class Test_metaclass(FixerTestCase): - - fixer = 'metaclass' - - def test_unchanged(self): - self.unchanged("class X(): pass") - self.unchanged("class X(object): pass") - self.unchanged("class X(object1, object2): pass") - self.unchanged("class X(object1, object2, object3): pass") - self.unchanged("class X(metaclass=Meta): pass") - self.unchanged("class X(b, arg=23, metclass=Meta): pass") - self.unchanged("class X(b, arg=23, metaclass=Meta, other=42): pass") - - s = """ - class X: - def __metaclass__(self): pass - """ - self.unchanged(s) - - s = """ - class X: - a[23] = 74 - """ - self.unchanged(s) - - def test_comments(self): - b = """ - class X: - # hi - __metaclass__ = AppleMeta - """ - a = """ - class X(metaclass=AppleMeta): - # hi - pass - """ - self.check(b, a) - - b = """ - class X: - __metaclass__ = Meta - # Bedtime! - """ - a = """ - class X(metaclass=Meta): - pass - # Bedtime! - """ - self.check(b, a) - - def test_meta(self): - # no-parent class, odd body - b = """ - class X(): - __metaclass__ = Q - pass - """ - a = """ - class X(metaclass=Q): - pass - """ - self.check(b, a) - - # one parent class, no body - b = """class X(object): __metaclass__ = Q""" - a = """class X(object, metaclass=Q): pass""" - self.check(b, a) - - - # one parent, simple body - b = """ - class X(object): - __metaclass__ = Meta - bar = 7 - """ - a = """ - class X(object, metaclass=Meta): - bar = 7 - """ - self.check(b, a) - - b = """ - class X: - __metaclass__ = Meta; x = 4; g = 23 - """ - a = """ - class X(metaclass=Meta): - x = 4; g = 23 - """ - self.check(b, a) - - # one parent, simple body, __metaclass__ last - b = """ - class X(object): - bar = 7 - __metaclass__ = Meta - """ - a = """ - class X(object, metaclass=Meta): - bar = 7 - """ - self.check(b, a) - - # redefining __metaclass__ - b = """ - class X(): - __metaclass__ = A - __metaclass__ = B - bar = 7 - """ - a = """ - class X(metaclass=B): - bar = 7 - """ - self.check(b, a) - - # multiple inheritance, simple body - b = """ - class X(clsA, clsB): - __metaclass__ = Meta - bar = 7 - """ - a = """ - class X(clsA, clsB, metaclass=Meta): - bar = 7 - """ - self.check(b, a) - - # keywords in the class statement - b = """class m(a, arg=23): __metaclass__ = Meta""" - a = """class m(a, arg=23, metaclass=Meta): pass""" - self.check(b, a) - - b = """ - class X(expression(2 + 4)): - __metaclass__ = Meta - """ - a = """ - class X(expression(2 + 4), metaclass=Meta): - pass - """ - self.check(b, a) - - b = """ - class X(expression(2 + 4), x**4): - __metaclass__ = Meta - """ - a = """ - class X(expression(2 + 4), x**4, metaclass=Meta): - pass - """ - self.check(b, a) - - b = """ - class X: - __metaclass__ = Meta - save.py = 23 - """ - a = """ - class X(metaclass=Meta): - save.py = 23 - """ - self.check(b, a) - - -class Test_getcwdu(FixerTestCase): - - fixer = 'getcwdu' - - def test_basic(self): - b = """os.getcwdu""" - a = """os.getcwd""" - self.check(b, a) - - b = """os.getcwdu()""" - a = """os.getcwd()""" - self.check(b, a) - - b = """meth = os.getcwdu""" - a = """meth = os.getcwd""" - self.check(b, a) - - b = """os.getcwdu(args)""" - a = """os.getcwd(args)""" - self.check(b, a) - - def test_comment(self): - b = """os.getcwdu() # Foo""" - a = """os.getcwd() # Foo""" - self.check(b, a) - - def test_unchanged(self): - s = """os.getcwd()""" - self.unchanged(s) - - s = """getcwdu()""" - self.unchanged(s) - - s = """os.getcwdb()""" - self.unchanged(s) - - def test_indentation(self): - b = """ - if 1: - os.getcwdu() - """ - a = """ - if 1: - os.getcwd() - """ - self.check(b, a) - - def test_multilation(self): - b = """os .getcwdu()""" - a = """os .getcwd()""" - self.check(b, a) - - b = """os. getcwdu""" - a = """os. getcwd""" - self.check(b, a) - - b = """os.getcwdu ( )""" - a = """os.getcwd ( )""" - self.check(b, a) - - -class Test_operator(FixerTestCase): - - fixer = "operator" - - def test_operator_isCallable(self): - b = "operator.isCallable(x)" - a = "callable(x)" - self.check(b, a) - - def test_operator_sequenceIncludes(self): - b = "operator.sequenceIncludes(x, y)" - a = "operator.contains(x, y)" - self.check(b, a) - - b = "operator .sequenceIncludes(x, y)" - a = "operator .contains(x, y)" - self.check(b, a) - - b = "operator. sequenceIncludes(x, y)" - a = "operator. contains(x, y)" - self.check(b, a) - - def test_operator_isSequenceType(self): - b = "operator.isSequenceType(x)" - a = "import collections.abc\nisinstance(x, collections.abc.Sequence)" - self.check(b, a) - - def test_operator_isMappingType(self): - b = "operator.isMappingType(x)" - a = "import collections.abc\nisinstance(x, collections.abc.Mapping)" - self.check(b, a) - - def test_operator_isNumberType(self): - b = "operator.isNumberType(x)" - a = "import numbers\nisinstance(x, numbers.Number)" - self.check(b, a) - - def test_operator_repeat(self): - b = "operator.repeat(x, n)" - a = "operator.mul(x, n)" - self.check(b, a) - - b = "operator .repeat(x, n)" - a = "operator .mul(x, n)" - self.check(b, a) - - b = "operator. repeat(x, n)" - a = "operator. mul(x, n)" - self.check(b, a) - - def test_operator_irepeat(self): - b = "operator.irepeat(x, n)" - a = "operator.imul(x, n)" - self.check(b, a) - - b = "operator .irepeat(x, n)" - a = "operator .imul(x, n)" - self.check(b, a) - - b = "operator. irepeat(x, n)" - a = "operator. imul(x, n)" - self.check(b, a) - - def test_bare_isCallable(self): - s = "isCallable(x)" - t = "You should use 'callable(x)' here." - self.warns_unchanged(s, t) - - def test_bare_sequenceIncludes(self): - s = "sequenceIncludes(x, y)" - t = "You should use 'operator.contains(x, y)' here." - self.warns_unchanged(s, t) - - def test_bare_operator_isSequenceType(self): - s = "isSequenceType(z)" - t = "You should use 'isinstance(z, collections.abc.Sequence)' here." - self.warns_unchanged(s, t) - - def test_bare_operator_isMappingType(self): - s = "isMappingType(x)" - t = "You should use 'isinstance(x, collections.abc.Mapping)' here." - self.warns_unchanged(s, t) - - def test_bare_operator_isNumberType(self): - s = "isNumberType(y)" - t = "You should use 'isinstance(y, numbers.Number)' here." - self.warns_unchanged(s, t) - - def test_bare_operator_repeat(self): - s = "repeat(x, n)" - t = "You should use 'operator.mul(x, n)' here." - self.warns_unchanged(s, t) - - def test_bare_operator_irepeat(self): - s = "irepeat(y, 187)" - t = "You should use 'operator.imul(y, 187)' here." - self.warns_unchanged(s, t) - - -class Test_exitfunc(FixerTestCase): - - fixer = "exitfunc" - - def test_simple(self): - b = """ - import sys - sys.exitfunc = my_atexit - """ - a = """ - import sys - import atexit - atexit.register(my_atexit) - """ - self.check(b, a) - - def test_names_import(self): - b = """ - import sys, crumbs - sys.exitfunc = my_func - """ - a = """ - import sys, crumbs, atexit - atexit.register(my_func) - """ - self.check(b, a) - - def test_complex_expression(self): - b = """ - import sys - sys.exitfunc = do(d)/a()+complex(f=23, g=23)*expression - """ - a = """ - import sys - import atexit - atexit.register(do(d)/a()+complex(f=23, g=23)*expression) - """ - self.check(b, a) - - def test_comments(self): - b = """ - import sys # Foo - sys.exitfunc = f # Blah - """ - a = """ - import sys - import atexit # Foo - atexit.register(f) # Blah - """ - self.check(b, a) - - b = """ - import apples, sys, crumbs, larry # Pleasant comments - sys.exitfunc = func - """ - a = """ - import apples, sys, crumbs, larry, atexit # Pleasant comments - atexit.register(func) - """ - self.check(b, a) - - def test_in_a_function(self): - b = """ - import sys - def f(): - sys.exitfunc = func - """ - a = """ - import sys - import atexit - def f(): - atexit.register(func) - """ - self.check(b, a) - - def test_no_sys_import(self): - b = """sys.exitfunc = f""" - a = """atexit.register(f)""" - msg = ("Can't find sys import; Please add an atexit import at the " - "top of your file.") - self.warns(b, a, msg) - - - def test_unchanged(self): - s = """f(sys.exitfunc)""" - self.unchanged(s) - - -class Test_asserts(FixerTestCase): - - fixer = "asserts" - - def test_deprecated_names(self): - tests = [ - ('self.assert_(True)', 'self.assertTrue(True)'), - ('self.assertEquals(2, 2)', 'self.assertEqual(2, 2)'), - ('self.assertNotEquals(2, 3)', 'self.assertNotEqual(2, 3)'), - ('self.assertAlmostEquals(2, 3)', 'self.assertAlmostEqual(2, 3)'), - ('self.assertNotAlmostEquals(2, 8)', 'self.assertNotAlmostEqual(2, 8)'), - ('self.failUnlessEqual(2, 2)', 'self.assertEqual(2, 2)'), - ('self.failIfEqual(2, 3)', 'self.assertNotEqual(2, 3)'), - ('self.failUnlessAlmostEqual(2, 3)', 'self.assertAlmostEqual(2, 3)'), - ('self.failIfAlmostEqual(2, 8)', 'self.assertNotAlmostEqual(2, 8)'), - ('self.failUnless(True)', 'self.assertTrue(True)'), - ('self.failUnlessRaises(foo)', 'self.assertRaises(foo)'), - ('self.failIf(False)', 'self.assertFalse(False)'), - ] - for b, a in tests: - self.check(b, a) - - def test_variants(self): - b = 'eq = self.assertEquals' - a = 'eq = self.assertEqual' - self.check(b, a) - b = 'self.assertEquals(2, 3, msg="fail")' - a = 'self.assertEqual(2, 3, msg="fail")' - self.check(b, a) - b = 'self.assertEquals(2, 3, msg="fail") # foo' - a = 'self.assertEqual(2, 3, msg="fail") # foo' - self.check(b, a) - b = 'self.assertEquals (2, 3)' - a = 'self.assertEqual (2, 3)' - self.check(b, a) - b = ' self.assertEquals (2, 3)' - a = ' self.assertEqual (2, 3)' - self.check(b, a) - b = 'with self.failUnlessRaises(Explosion): explode()' - a = 'with self.assertRaises(Explosion): explode()' - self.check(b, a) - b = 'with self.failUnlessRaises(Explosion) as cm: explode()' - a = 'with self.assertRaises(Explosion) as cm: explode()' - self.check(b, a) - - def test_unchanged(self): - self.unchanged('self.assertEqualsOnSaturday') - self.unchanged('self.assertEqualsOnSaturday(3, 5)') diff --git a/Lib/test/test_lib2to3/test_main.py b/Lib/test/test_lib2to3/test_main.py deleted file mode 100644 index a33c45c50a0de2..00000000000000 --- a/Lib/test/test_lib2to3/test_main.py +++ /dev/null @@ -1,139 +0,0 @@ -# -*- coding: utf-8 -*- -import codecs -import io -import logging -import os -import re -import shutil -import sys -import tempfile -import unittest - -from lib2to3 import main - - -TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") -PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py") - - -class TestMain(unittest.TestCase): - - def setUp(self): - self.temp_dir = None # tearDown() will rmtree this directory if set. - - def tearDown(self): - # Clean up logging configuration down by main. - del logging.root.handlers[:] - if self.temp_dir: - shutil.rmtree(self.temp_dir) - - def run_2to3_capture(self, args, in_capture, out_capture, err_capture): - save_stdin = sys.stdin - save_stdout = sys.stdout - save_stderr = sys.stderr - sys.stdin = in_capture - sys.stdout = out_capture - sys.stderr = err_capture - try: - return main.main("lib2to3.fixes", args) - finally: - sys.stdin = save_stdin - sys.stdout = save_stdout - sys.stderr = save_stderr - - def test_unencodable_diff(self): - input_stream = io.StringIO("print 'nothing'\nprint u'über'\n") - out = io.BytesIO() - out_enc = codecs.getwriter("ascii")(out) - err = io.StringIO() - ret = self.run_2to3_capture(["-"], input_stream, out_enc, err) - self.assertEqual(ret, 0) - output = out.getvalue().decode("ascii") - self.assertIn("-print 'nothing'", output) - self.assertIn("WARNING: couldn't encode 's diff for " - "your terminal", err.getvalue()) - - def setup_test_source_trees(self): - """Setup a test source tree and output destination tree.""" - self.temp_dir = tempfile.mkdtemp() # tearDown() cleans this up. - self.py2_src_dir = os.path.join(self.temp_dir, "python2_project") - self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project") - os.mkdir(self.py2_src_dir) - os.mkdir(self.py3_dest_dir) - # Turn it into a package with a few files. - self.setup_files = [] - open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close() - self.setup_files.append("__init__.py") - shutil.copy(PY2_TEST_MODULE, self.py2_src_dir) - self.setup_files.append(os.path.basename(PY2_TEST_MODULE)) - self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py") - self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py") - with open(self.trivial_py2_file, "w") as trivial: - trivial.write("print 'I need a simple conversion.'") - self.setup_files.append("trivial.py") - - def test_filename_changing_on_output_single_dir(self): - """2to3 a single directory with a new output dir and suffix.""" - self.setup_test_source_trees() - out = io.StringIO() - err = io.StringIO() - suffix = "TEST" - ret = self.run_2to3_capture( - ["-n", "--add-suffix", suffix, "--write-unchanged-files", - "--no-diffs", "--output-dir", - self.py3_dest_dir, self.py2_src_dir], - io.StringIO(""), out, err) - self.assertEqual(ret, 0) - stderr = err.getvalue() - self.assertIn(" implies -w.", stderr) - self.assertIn( - "Output in %r will mirror the input directory %r layout" % ( - self.py3_dest_dir, self.py2_src_dir), stderr) - self.assertEqual(set(name+suffix for name in self.setup_files), - set(os.listdir(self.py3_dest_dir))) - for name in self.setup_files: - self.assertIn("Writing converted %s to %s" % ( - os.path.join(self.py2_src_dir, name), - os.path.join(self.py3_dest_dir, name+suffix)), stderr) - sep = re.escape(os.sep) - self.assertRegex( - stderr, r"No changes to .*/__init__\.py".replace("/", sep)) - self.assertNotRegex( - stderr, r"No changes to .*/trivial\.py".replace("/", sep)) - - def test_filename_changing_on_output_two_files(self): - """2to3 two files in one directory with a new output dir.""" - self.setup_test_source_trees() - err = io.StringIO() - py2_files = [self.trivial_py2_file, self.init_py2_file] - expected_files = set(os.path.basename(name) for name in py2_files) - ret = self.run_2to3_capture( - ["-n", "-w", "--write-unchanged-files", - "--no-diffs", "--output-dir", self.py3_dest_dir] + py2_files, - io.StringIO(""), io.StringIO(), err) - self.assertEqual(ret, 0) - stderr = err.getvalue() - self.assertIn( - "Output in %r will mirror the input directory %r layout" % ( - self.py3_dest_dir, self.py2_src_dir), stderr) - self.assertEqual(expected_files, set(os.listdir(self.py3_dest_dir))) - - def test_filename_changing_on_output_single_file(self): - """2to3 a single file with a new output dir.""" - self.setup_test_source_trees() - err = io.StringIO() - ret = self.run_2to3_capture( - ["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir, - self.trivial_py2_file], - io.StringIO(""), io.StringIO(), err) - self.assertEqual(ret, 0) - stderr = err.getvalue() - self.assertIn( - "Output in %r will mirror the input directory %r layout" % ( - self.py3_dest_dir, self.py2_src_dir), stderr) - self.assertEqual(set([os.path.basename(self.trivial_py2_file)]), - set(os.listdir(self.py3_dest_dir))) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_lib2to3/test_parser.py b/Lib/test/test_lib2to3/test_parser.py deleted file mode 100644 index 2c798b181fdbda..00000000000000 --- a/Lib/test/test_lib2to3/test_parser.py +++ /dev/null @@ -1,718 +0,0 @@ -"""Test suite for 2to3's parser and grammar files. - -This is the place to add tests for changes to 2to3's grammar, such as those -merging the grammars for Python 2 and 3. In addition to specific tests for -parts of the grammar we've changed, we also make sure we can parse the -test_grammar.py files from both Python 2 and Python 3. -""" - -# Testing imports -from . import support -from .support import driver, driver_no_print_statement - -# Python imports -import difflib -import importlib -import operator -import os -import pickle -import shutil -import subprocess -import sys -import tempfile -import test.support -import unittest - -# Local imports -from lib2to3.pgen2 import driver as pgen2_driver -from lib2to3.pgen2 import tokenize -from lib2to3.pgen2.parse import ParseError -from lib2to3.pygram import python_symbols as syms - - -class TestDriver(support.TestCase): - - def test_formfeed(self): - s = """print 1\n\x0Cprint 2\n""" - t = driver.parse_string(s) - self.assertEqual(t.children[0].children[0].type, syms.print_stmt) - self.assertEqual(t.children[1].children[0].type, syms.print_stmt) - - -class TestPgen2Caching(support.TestCase): - def test_load_grammar_from_txt_file(self): - pgen2_driver.load_grammar(support.grammar_path, save=False, force=True) - - def test_load_grammar_from_pickle(self): - # Make a copy of the grammar file in a temp directory we are - # guaranteed to be able to write to. - tmpdir = tempfile.mkdtemp() - try: - grammar_copy = os.path.join( - tmpdir, os.path.basename(support.grammar_path)) - shutil.copy(support.grammar_path, grammar_copy) - pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) - - pgen2_driver.load_grammar(grammar_copy, save=True, force=True) - self.assertTrue(os.path.exists(pickle_name)) - - os.unlink(grammar_copy) # Only the pickle remains... - pgen2_driver.load_grammar(grammar_copy, save=False, force=False) - finally: - shutil.rmtree(tmpdir) - - @unittest.skipIf(sys.executable is None, 'sys.executable required') - @unittest.skipIf( - sys.platform in {'emscripten', 'wasi'}, 'requires working subprocess' - ) - def test_load_grammar_from_subprocess(self): - tmpdir = tempfile.mkdtemp() - tmpsubdir = os.path.join(tmpdir, 'subdir') - try: - os.mkdir(tmpsubdir) - grammar_base = os.path.basename(support.grammar_path) - grammar_copy = os.path.join(tmpdir, grammar_base) - grammar_sub_copy = os.path.join(tmpsubdir, grammar_base) - shutil.copy(support.grammar_path, grammar_copy) - shutil.copy(support.grammar_path, grammar_sub_copy) - pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) - pickle_sub_name = pgen2_driver._generate_pickle_name( - grammar_sub_copy) - self.assertNotEqual(pickle_name, pickle_sub_name) - - # Generate a pickle file from this process. - pgen2_driver.load_grammar(grammar_copy, save=True, force=True) - self.assertTrue(os.path.exists(pickle_name)) - - # Generate a new pickle file in a subprocess with a most likely - # different hash randomization seed. - sub_env = dict(os.environ) - sub_env['PYTHONHASHSEED'] = 'random' - code = """ -from lib2to3.pgen2 import driver as pgen2_driver -pgen2_driver.load_grammar(%r, save=True, force=True) - """ % (grammar_sub_copy,) - cmd = [sys.executable, - '-Wignore:lib2to3:DeprecationWarning', - '-c', code] - subprocess.check_call( cmd, env=sub_env) - self.assertTrue(os.path.exists(pickle_sub_name)) - - with open(pickle_name, 'rb') as pickle_f_1, \ - open(pickle_sub_name, 'rb') as pickle_f_2: - self.assertEqual( - pickle_f_1.read(), pickle_f_2.read(), - msg='Grammar caches generated using different hash seeds' - ' were not identical.') - finally: - shutil.rmtree(tmpdir) - - def test_load_packaged_grammar(self): - modname = __name__ + '.load_test' - class MyLoader: - def get_data(self, where): - return pickle.dumps({'elephant': 19}) - class MyModule: - __file__ = 'parsertestmodule' - __spec__ = importlib.util.spec_from_loader(modname, MyLoader()) - sys.modules[modname] = MyModule() - self.addCleanup(operator.delitem, sys.modules, modname) - g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt') - self.assertEqual(g.elephant, 19) - - -class GrammarTest(support.TestCase): - def validate(self, code): - support.parse_string(code) - - def invalid_syntax(self, code): - try: - self.validate(code) - except ParseError: - pass - else: - raise AssertionError("Syntax shouldn't have been valid") - - -class TestMatrixMultiplication(GrammarTest): - def test_matrix_multiplication_operator(self): - self.validate("a @ b") - self.validate("a @= b") - - -class TestYieldFrom(GrammarTest): - def test_yield_from(self): - self.validate("yield from x") - self.validate("(yield from x) + y") - self.invalid_syntax("yield from") - - -class TestAsyncAwait(GrammarTest): - def test_await_expr(self): - self.validate("""async def foo(): - await x - """) - - self.validate("""async def foo(): - [i async for i in b] - """) - - self.validate("""async def foo(): - {i for i in b - async for i in a if await i - for b in i} - """) - - self.validate("""async def foo(): - [await i for i in b if await c] - """) - - self.validate("""async def foo(): - [ i for i in b if c] - """) - - self.validate("""async def foo(): - - def foo(): pass - - def foo(): pass - - await x - """) - - self.validate("""async def foo(): return await a""") - - self.validate("""def foo(): - def foo(): pass - async def foo(): await x - """) - - self.invalid_syntax("await x") - self.invalid_syntax("""def foo(): - await x""") - - self.invalid_syntax("""def foo(): - def foo(): pass - async def foo(): pass - await x - """) - - def test_async_var(self): - self.validate("""async = 1""") - self.validate("""await = 1""") - self.validate("""def async(): pass""") - - def test_async_for(self): - self.validate("""async def foo(): - async for a in b: pass""") - - def test_async_with(self): - self.validate("""async def foo(): - async with a: pass""") - - self.invalid_syntax("""def foo(): - async with a: pass""") - - def test_async_generator(self): - self.validate( - """async def foo(): - return (i * 2 async for i in arange(42))""" - ) - self.validate( - """def foo(): - return (i * 2 async for i in arange(42))""" - ) - - -class TestRaiseChanges(GrammarTest): - def test_2x_style_1(self): - self.validate("raise") - - def test_2x_style_2(self): - self.validate("raise E, V") - - def test_2x_style_3(self): - self.validate("raise E, V, T") - - def test_2x_style_invalid_1(self): - self.invalid_syntax("raise E, V, T, Z") - - def test_3x_style(self): - self.validate("raise E1 from E2") - - def test_3x_style_invalid_1(self): - self.invalid_syntax("raise E, V from E1") - - def test_3x_style_invalid_2(self): - self.invalid_syntax("raise E from E1, E2") - - def test_3x_style_invalid_3(self): - self.invalid_syntax("raise from E1, E2") - - def test_3x_style_invalid_4(self): - self.invalid_syntax("raise E from") - - -# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292 -# and Lib/test/text_parser.py test_list_displays, test_set_displays, -# test_dict_displays, test_argument_unpacking, ... changes. -class TestUnpackingGeneralizations(GrammarTest): - def test_mid_positional_star(self): - self.validate("""func(1, *(2, 3), 4)""") - - def test_double_star_dict_literal(self): - self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""") - - def test_double_star_dict_literal_after_keywords(self): - self.validate("""func(spam='fried', **{'eggs':'scrambled'})""") - - def test_double_star_expression(self): - self.validate("""func(**{'a':2} or {})""") - self.validate("""func(**() or {})""") - - def test_star_expression(self): - self.validate("""func(*[] or [2])""") - - def test_list_display(self): - self.validate("""[*{2}, 3, *[4]]""") - - def test_set_display(self): - self.validate("""{*{2}, 3, *[4]}""") - - def test_dict_display_1(self): - self.validate("""{**{}}""") - - def test_dict_display_2(self): - self.validate("""{**{}, 3:4, **{5:6, 7:8}}""") - - def test_complex_star_expression(self): - self.validate("func(* [] or [1])") - - def test_complex_double_star_expression(self): - self.validate("func(**{1: 3} if False else {x: x for x in range(3)})") - - def test_argument_unpacking_1(self): - self.validate("""f(a, *b, *c, d)""") - - def test_argument_unpacking_2(self): - self.validate("""f(**a, **b)""") - - def test_argument_unpacking_3(self): - self.validate("""f(2, *a, *b, **b, **c, **d)""") - - def test_trailing_commas_1(self): - self.validate("def f(a, b): call(a, b)") - self.validate("def f(a, b,): call(a, b,)") - - def test_trailing_commas_2(self): - self.validate("def f(a, *b): call(a, *b)") - self.validate("def f(a, *b,): call(a, *b,)") - - def test_trailing_commas_3(self): - self.validate("def f(a, b=1): call(a, b=1)") - self.validate("def f(a, b=1,): call(a, b=1,)") - - def test_trailing_commas_4(self): - self.validate("def f(a, **b): call(a, **b)") - self.validate("def f(a, **b,): call(a, **b,)") - - def test_trailing_commas_5(self): - self.validate("def f(*a, b=1): call(*a, b=1)") - self.validate("def f(*a, b=1,): call(*a, b=1,)") - - def test_trailing_commas_6(self): - self.validate("def f(*a, **b): call(*a, **b)") - self.validate("def f(*a, **b,): call(*a, **b,)") - - def test_trailing_commas_7(self): - self.validate("def f(*, b=1): call(*b)") - self.validate("def f(*, b=1,): call(*b,)") - - def test_trailing_commas_8(self): - self.validate("def f(a=1, b=2): call(a=1, b=2)") - self.validate("def f(a=1, b=2,): call(a=1, b=2,)") - - def test_trailing_commas_9(self): - self.validate("def f(a=1, **b): call(a=1, **b)") - self.validate("def f(a=1, **b,): call(a=1, **b,)") - - def test_trailing_commas_lambda_1(self): - self.validate("f = lambda a, b: call(a, b)") - self.validate("f = lambda a, b,: call(a, b,)") - - def test_trailing_commas_lambda_2(self): - self.validate("f = lambda a, *b: call(a, *b)") - self.validate("f = lambda a, *b,: call(a, *b,)") - - def test_trailing_commas_lambda_3(self): - self.validate("f = lambda a, b=1: call(a, b=1)") - self.validate("f = lambda a, b=1,: call(a, b=1,)") - - def test_trailing_commas_lambda_4(self): - self.validate("f = lambda a, **b: call(a, **b)") - self.validate("f = lambda a, **b,: call(a, **b,)") - - def test_trailing_commas_lambda_5(self): - self.validate("f = lambda *a, b=1: call(*a, b=1)") - self.validate("f = lambda *a, b=1,: call(*a, b=1,)") - - def test_trailing_commas_lambda_6(self): - self.validate("f = lambda *a, **b: call(*a, **b)") - self.validate("f = lambda *a, **b,: call(*a, **b,)") - - def test_trailing_commas_lambda_7(self): - self.validate("f = lambda *, b=1: call(*b)") - self.validate("f = lambda *, b=1,: call(*b,)") - - def test_trailing_commas_lambda_8(self): - self.validate("f = lambda a=1, b=2: call(a=1, b=2)") - self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)") - - def test_trailing_commas_lambda_9(self): - self.validate("f = lambda a=1, **b: call(a=1, **b)") - self.validate("f = lambda a=1, **b,: call(a=1, **b,)") - - -# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef -class TestFunctionAnnotations(GrammarTest): - def test_1(self): - self.validate("""def f(x) -> list: pass""") - - def test_2(self): - self.validate("""def f(x:int): pass""") - - def test_3(self): - self.validate("""def f(*x:str): pass""") - - def test_4(self): - self.validate("""def f(**x:float): pass""") - - def test_5(self): - self.validate("""def f(x, y:1+2): pass""") - - def test_6(self): - self.validate("""def f(a, (b:1, c:2, d)): pass""") - - def test_7(self): - self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""") - - def test_8(self): - s = """def f(a, (b:1, c:2, d), e:3=4, f=5, - *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" - self.validate(s) - - def test_9(self): - s = """def f( - a: str, - b: int, - *, - c: bool = False, - **kwargs, - ) -> None: - call(c=c, **kwargs,)""" - self.validate(s) - - def test_10(self): - s = """def f( - a: str, - ) -> None: - call(a,)""" - self.validate(s) - - def test_11(self): - s = """def f( - a: str = '', - ) -> None: - call(a=a,)""" - self.validate(s) - - def test_12(self): - s = """def f( - *args: str, - ) -> None: - call(*args,)""" - self.validate(s) - - def test_13(self): - self.validate("def f(a: str, b: int) -> None: call(a, b)") - self.validate("def f(a: str, b: int,) -> None: call(a, b,)") - - def test_14(self): - self.validate("def f(a: str, *b: int) -> None: call(a, *b)") - self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)") - - def test_15(self): - self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)") - self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)") - - def test_16(self): - self.validate("def f(a: str, **b: int) -> None: call(a, **b)") - self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)") - - def test_17(self): - self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)") - self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)") - - def test_18(self): - self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)") - self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)") - - def test_19(self): - self.validate("def f(*, b: int=1) -> None: call(*b)") - self.validate("def f(*, b: int=1,) -> None: call(*b,)") - - def test_20(self): - self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)") - self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)") - - def test_21(self): - self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)") - self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)") - - -# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot -class TestVarAnnotations(GrammarTest): - def test_1(self): - self.validate("var1: int = 5") - - def test_2(self): - self.validate("var2: [int, str]") - - def test_3(self): - self.validate("def f():\n" - " st: str = 'Hello'\n" - " a.b: int = (1, 2)\n" - " return st\n") - - def test_4(self): - self.validate("def fbad():\n" - " x: int\n" - " print(x)\n") - - def test_5(self): - self.validate("class C:\n" - " x: int\n" - " s: str = 'attr'\n" - " z = 2\n" - " def __init__(self, x):\n" - " self.x: int = x\n") - - def test_6(self): - self.validate("lst: List[int] = []") - - -class TestExcept(GrammarTest): - def test_new(self): - s = """ - try: - x - except E as N: - y""" - self.validate(s) - - def test_old(self): - s = """ - try: - x - except E, N: - y""" - self.validate(s) - - -class TestStringLiterals(GrammarTest): - prefixes = ("'", '"', - "r'", 'r"', "R'", 'R"', - "u'", 'u"', "U'", 'U"', - "b'", 'b"', "B'", 'B"', - "f'", 'f"', "F'", 'F"', - "ur'", 'ur"', "Ur'", 'Ur"', - "uR'", 'uR"', "UR'", 'UR"', - "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"', - "rb'", 'rb"', "Rb'", 'Rb"', - "rB'", 'rB"', "RB'", 'RB"',) - - def test_lit(self): - for pre in self.prefixes: - single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1]) - self.validate(single) - triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1]) - self.validate(triple) - - -# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms -class TestSetLiteral(GrammarTest): - def test_1(self): - self.validate("""x = {'one'}""") - - def test_2(self): - self.validate("""x = {'one', 1,}""") - - def test_3(self): - self.validate("""x = {'one', 'two', 'three'}""") - - def test_4(self): - self.validate("""x = {2, 3, 4,}""") - - -# Adapted from Python 3's Lib/test/test_unicode_identifiers.py and -# Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers -class TestIdentifier(GrammarTest): - def test_non_ascii_identifiers(self): - self.validate("Örter = 'places'\ngrün = 'green'") - self.validate("蟒 = a蟒 = 锦蛇 = 1") - self.validate("µ = aµ = µµ = 1") - self.validate("𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = a_𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1") - - -class TestNumericLiterals(GrammarTest): - def test_new_octal_notation(self): - self.validate("""0o7777777777777""") - self.invalid_syntax("""0o7324528887""") - - def test_new_binary_notation(self): - self.validate("""0b101010""") - self.invalid_syntax("""0b0101021""") - - -class TestClassDef(GrammarTest): - def test_new_syntax(self): - self.validate("class B(t=7): pass") - self.validate("class B(t, *args): pass") - self.validate("class B(t, **kwargs): pass") - self.validate("class B(t, *args, **kwargs): pass") - self.validate("class B(t, y=9, *args, **kwargs,): pass") - - -class TestParserIdempotency(support.TestCase): - - """A cut-down version of pytree_idempotency.py.""" - - def parse_file(self, filepath): - if test.support.verbose: - print(f"Parse file: {filepath}") - with open(filepath, "rb") as fp: - encoding = tokenize.detect_encoding(fp.readline)[0] - self.assertIsNotNone(encoding, - "can't detect encoding for %s" % filepath) - with open(filepath, "r", encoding=encoding) as fp: - source = fp.read() - try: - tree = driver.parse_string(source) - except ParseError: - try: - tree = driver_no_print_statement.parse_string(source) - except ParseError as err: - self.fail('ParseError on file %s (%s)' % (filepath, err)) - new = str(tree) - if new != source: - print(diff_texts(source, new, filepath)) - self.fail("Idempotency failed: %s" % filepath) - - def test_all_project_files(self): - for filepath in support.all_project_files(): - with self.subTest(filepath=filepath): - self.parse_file(filepath) - - def test_extended_unpacking(self): - driver.parse_string("a, *b, c = x\n") - driver.parse_string("[*a, b] = x\n") - driver.parse_string("(z, *y, w) = m\n") - driver.parse_string("for *z, m in d: pass\n") - - -class TestLiterals(GrammarTest): - - def validate(self, s): - driver.parse_string(support.dedent(s) + "\n\n") - - def test_multiline_bytes_literals(self): - s = """ - md5test(b"\xaa" * 80, - (b"Test Using Larger Than Block-Size Key " - b"and Larger Than One Block-Size Data"), - "6f630fad67cda0ee1fb1f562db3aa53e") - """ - self.validate(s) - - def test_multiline_bytes_tripquote_literals(self): - s = ''' - b""" - - - """ - ''' - self.validate(s) - - def test_multiline_str_literals(self): - s = """ - md5test("\xaa" * 80, - ("Test Using Larger Than Block-Size Key " - "and Larger Than One Block-Size Data"), - "6f630fad67cda0ee1fb1f562db3aa53e") - """ - self.validate(s) - - -class TestNamedAssignments(GrammarTest): - """Also known as the walrus operator.""" - - def test_named_assignment_if(self): - driver.parse_string("if f := x(): pass\n") - - def test_named_assignment_while(self): - driver.parse_string("while f := x(): pass\n") - - def test_named_assignment_generator(self): - driver.parse_string("any((lastNum := num) == 1 for num in [1, 2, 3])\n") - - def test_named_assignment_listcomp(self): - driver.parse_string("[(lastNum := num) == 1 for num in [1, 2, 3]]\n") - - -class TestPositionalOnlyArgs(GrammarTest): - - def test_one_pos_only_arg(self): - driver.parse_string("def one_pos_only_arg(a, /): pass\n") - - def test_all_markers(self): - driver.parse_string( - "def all_markers(a, b=2, /, c, d=4, *, e=5, f): pass\n") - - def test_all_with_args_and_kwargs(self): - driver.parse_string( - """def all_markers_with_args_and_kwargs( - aa, b, /, _cc, d, *args, e, f_f, **kwargs, - ): - pass\n""") - - def test_lambda_soup(self): - driver.parse_string( - "lambda a, b, /, c, d, *args, e, f, **kw: kw\n") - - def test_only_positional_or_keyword(self): - driver.parse_string("def func(a,b,/,*,g,e=3): pass\n") - - -class TestPickleableException(unittest.TestCase): - def test_ParseError(self): - err = ParseError('msg', 2, None, (1, 'context')) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - err2 = pickle.loads(pickle.dumps(err, protocol=proto)) - self.assertEqual(err.args, err2.args) - self.assertEqual(err.msg, err2.msg) - self.assertEqual(err.type, err2.type) - self.assertEqual(err.value, err2.value) - self.assertEqual(err.context, err2.context) - - -def diff_texts(a, b, filename): - a = a.splitlines() - b = b.splitlines() - return difflib.unified_diff(a, b, filename, filename, - "(original)", "(reserialized)", - lineterm="") - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_lib2to3/test_pytree.py b/Lib/test/test_lib2to3/test_pytree.py deleted file mode 100644 index 177126d5452910..00000000000000 --- a/Lib/test/test_lib2to3/test_pytree.py +++ /dev/null @@ -1,472 +0,0 @@ -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Unit tests for pytree.py. - -NOTE: Please *don't* add doc strings to individual test methods! -In verbose mode, printing of the module, class and method name is much -more helpful than printing of (the first line of) the docstring, -especially when debugging a test. -""" - -# Testing imports -from . import support - -from lib2to3 import pytree - -try: - sorted -except NameError: - def sorted(lst): - l = list(lst) - l.sort() - return l - -class TestNodes(support.TestCase): - - """Unit tests for nodes (Base, Leaf, Node).""" - - def test_instantiate_base(self): - if __debug__: - # Test that instantiating Base() raises an AssertionError - self.assertRaises(AssertionError, pytree.Base) - - def test_leaf(self): - l1 = pytree.Leaf(100, "foo") - self.assertEqual(l1.type, 100) - self.assertEqual(l1.value, "foo") - - def test_leaf_repr(self): - l1 = pytree.Leaf(100, "foo") - self.assertEqual(repr(l1), "Leaf(100, 'foo')") - - def test_leaf_str(self): - l1 = pytree.Leaf(100, "foo") - self.assertEqual(str(l1), "foo") - l2 = pytree.Leaf(100, "foo", context=(" ", (10, 1))) - self.assertEqual(str(l2), " foo") - - def test_leaf_str_numeric_value(self): - # Make sure that the Leaf's value is stringified. Failing to - # do this can cause a TypeError in certain situations. - l1 = pytree.Leaf(2, 5) - l1.prefix = "foo_" - self.assertEqual(str(l1), "foo_5") - - def test_leaf_equality(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "foo", context=(" ", (1, 0))) - self.assertEqual(l1, l2) - l3 = pytree.Leaf(101, "foo") - l4 = pytree.Leaf(100, "bar") - self.assertNotEqual(l1, l3) - self.assertNotEqual(l1, l4) - - def test_leaf_prefix(self): - l1 = pytree.Leaf(100, "foo") - self.assertEqual(l1.prefix, "") - self.assertFalse(l1.was_changed) - l1.prefix = " ##\n\n" - self.assertEqual(l1.prefix, " ##\n\n") - self.assertTrue(l1.was_changed) - - def test_node(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(200, "bar") - n1 = pytree.Node(1000, [l1, l2]) - self.assertEqual(n1.type, 1000) - self.assertEqual(n1.children, [l1, l2]) - - def test_node_repr(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar", context=(" ", (1, 0))) - n1 = pytree.Node(1000, [l1, l2]) - self.assertEqual(repr(n1), - "Node(1000, [%s, %s])" % (repr(l1), repr(l2))) - - def test_node_str(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar", context=(" ", (1, 0))) - n1 = pytree.Node(1000, [l1, l2]) - self.assertEqual(str(n1), "foo bar") - - def test_node_prefix(self): - l1 = pytree.Leaf(100, "foo") - self.assertEqual(l1.prefix, "") - n1 = pytree.Node(1000, [l1]) - self.assertEqual(n1.prefix, "") - n1.prefix = " " - self.assertEqual(n1.prefix, " ") - self.assertEqual(l1.prefix, " ") - - def test_get_suffix(self): - l1 = pytree.Leaf(100, "foo", prefix="a") - l2 = pytree.Leaf(100, "bar", prefix="b") - n1 = pytree.Node(1000, [l1, l2]) - - self.assertEqual(l1.get_suffix(), l2.prefix) - self.assertEqual(l2.get_suffix(), "") - self.assertEqual(n1.get_suffix(), "") - - l3 = pytree.Leaf(100, "bar", prefix="c") - n2 = pytree.Node(1000, [n1, l3]) - - self.assertEqual(n1.get_suffix(), l3.prefix) - self.assertEqual(l3.get_suffix(), "") - self.assertEqual(n2.get_suffix(), "") - - def test_node_equality(self): - n1 = pytree.Node(1000, ()) - n2 = pytree.Node(1000, [], context=(" ", (1, 0))) - self.assertEqual(n1, n2) - n3 = pytree.Node(1001, ()) - self.assertNotEqual(n1, n3) - - def test_node_recursive_equality(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1]) - n2 = pytree.Node(1000, [l2]) - self.assertEqual(n1, n2) - l3 = pytree.Leaf(100, "bar") - n3 = pytree.Node(1000, [l3]) - self.assertNotEqual(n1, n3) - - def test_replace(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "+") - l3 = pytree.Leaf(100, "bar") - n1 = pytree.Node(1000, [l1, l2, l3]) - self.assertEqual(n1.children, [l1, l2, l3]) - self.assertIsInstance(n1.children, list) - self.assertFalse(n1.was_changed) - l2new = pytree.Leaf(100, "-") - l2.replace(l2new) - self.assertEqual(n1.children, [l1, l2new, l3]) - self.assertIsInstance(n1.children, list) - self.assertTrue(n1.was_changed) - - def test_replace_with_list(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "+") - l3 = pytree.Leaf(100, "bar") - n1 = pytree.Node(1000, [l1, l2, l3]) - - l2.replace([pytree.Leaf(100, "*"), pytree.Leaf(100, "*")]) - self.assertEqual(str(n1), "foo**bar") - self.assertIsInstance(n1.children, list) - - def test_leaves(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - l3 = pytree.Leaf(100, "fooey") - n2 = pytree.Node(1000, [l1, l2]) - n3 = pytree.Node(1000, [l3]) - n1 = pytree.Node(1000, [n2, n3]) - - self.assertEqual(list(n1.leaves()), [l1, l2, l3]) - - def test_depth(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - n2 = pytree.Node(1000, [l1, l2]) - n3 = pytree.Node(1000, []) - n1 = pytree.Node(1000, [n2, n3]) - - self.assertEqual(l1.depth(), 2) - self.assertEqual(n3.depth(), 1) - self.assertEqual(n1.depth(), 0) - - def test_post_order(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - l3 = pytree.Leaf(100, "fooey") - c1 = pytree.Node(1000, [l1, l2]) - n1 = pytree.Node(1000, [c1, l3]) - self.assertEqual(list(n1.post_order()), [l1, l2, c1, l3, n1]) - - def test_pre_order(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - l3 = pytree.Leaf(100, "fooey") - c1 = pytree.Node(1000, [l1, l2]) - n1 = pytree.Node(1000, [c1, l3]) - self.assertEqual(list(n1.pre_order()), [n1, c1, l1, l2, l3]) - - def test_changed(self): - l1 = pytree.Leaf(100, "f") - self.assertFalse(l1.was_changed) - l1.changed() - self.assertTrue(l1.was_changed) - - l1 = pytree.Leaf(100, "f") - n1 = pytree.Node(1000, [l1]) - self.assertFalse(n1.was_changed) - n1.changed() - self.assertTrue(n1.was_changed) - - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "+") - l3 = pytree.Leaf(100, "bar") - n1 = pytree.Node(1000, [l1, l2, l3]) - n2 = pytree.Node(1000, [n1]) - self.assertFalse(l1.was_changed) - self.assertFalse(n1.was_changed) - self.assertFalse(n2.was_changed) - - n1.changed() - self.assertTrue(n1.was_changed) - self.assertTrue(n2.was_changed) - self.assertFalse(l1.was_changed) - - def test_leaf_constructor_prefix(self): - for prefix in ("xyz_", ""): - l1 = pytree.Leaf(100, "self", prefix=prefix) - self.assertTrue(str(l1), prefix + "self") - self.assertEqual(l1.prefix, prefix) - - def test_node_constructor_prefix(self): - for prefix in ("xyz_", ""): - l1 = pytree.Leaf(100, "self") - l2 = pytree.Leaf(100, "foo", prefix="_") - n1 = pytree.Node(1000, [l1, l2], prefix=prefix) - self.assertTrue(str(n1), prefix + "self_foo") - self.assertEqual(n1.prefix, prefix) - self.assertEqual(l1.prefix, prefix) - self.assertEqual(l2.prefix, "_") - - def test_remove(self): - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1, l2]) - n2 = pytree.Node(1000, [n1]) - - self.assertEqual(n1.remove(), 0) - self.assertEqual(n2.children, []) - self.assertEqual(l1.parent, n1) - self.assertEqual(n1.parent, None) - self.assertEqual(n2.parent, None) - self.assertFalse(n1.was_changed) - self.assertTrue(n2.was_changed) - - self.assertEqual(l2.remove(), 1) - self.assertEqual(l1.remove(), 0) - self.assertEqual(n1.children, []) - self.assertEqual(l1.parent, None) - self.assertEqual(n1.parent, None) - self.assertEqual(n2.parent, None) - self.assertTrue(n1.was_changed) - self.assertTrue(n2.was_changed) - - def test_remove_parentless(self): - n1 = pytree.Node(1000, []) - n1.remove() - self.assertEqual(n1.parent, None) - - l1 = pytree.Leaf(100, "foo") - l1.remove() - self.assertEqual(l1.parent, None) - - def test_node_set_child(self): - l1 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1]) - - l2 = pytree.Leaf(100, "bar") - n1.set_child(0, l2) - self.assertEqual(l1.parent, None) - self.assertEqual(l2.parent, n1) - self.assertEqual(n1.children, [l2]) - - n2 = pytree.Node(1000, [l1]) - n2.set_child(0, n1) - self.assertEqual(l1.parent, None) - self.assertEqual(n1.parent, n2) - self.assertEqual(n2.parent, None) - self.assertEqual(n2.children, [n1]) - - self.assertRaises(IndexError, n1.set_child, 4, l2) - # I don't care what it raises, so long as it's an exception - self.assertRaises(Exception, n1.set_child, 0, list) - - def test_node_insert_child(self): - l1 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1]) - - l2 = pytree.Leaf(100, "bar") - n1.insert_child(0, l2) - self.assertEqual(l2.parent, n1) - self.assertEqual(n1.children, [l2, l1]) - - l3 = pytree.Leaf(100, "abc") - n1.insert_child(2, l3) - self.assertEqual(n1.children, [l2, l1, l3]) - - # I don't care what it raises, so long as it's an exception - self.assertRaises(Exception, n1.insert_child, 0, list) - - def test_node_append_child(self): - n1 = pytree.Node(1000, []) - - l1 = pytree.Leaf(100, "foo") - n1.append_child(l1) - self.assertEqual(l1.parent, n1) - self.assertEqual(n1.children, [l1]) - - l2 = pytree.Leaf(100, "bar") - n1.append_child(l2) - self.assertEqual(l2.parent, n1) - self.assertEqual(n1.children, [l1, l2]) - - # I don't care what it raises, so long as it's an exception - self.assertRaises(Exception, n1.append_child, list) - - def test_node_next_sibling(self): - n1 = pytree.Node(1000, []) - n2 = pytree.Node(1000, []) - p1 = pytree.Node(1000, [n1, n2]) - - self.assertIs(n1.next_sibling, n2) - self.assertEqual(n2.next_sibling, None) - self.assertEqual(p1.next_sibling, None) - - def test_leaf_next_sibling(self): - l1 = pytree.Leaf(100, "a") - l2 = pytree.Leaf(100, "b") - p1 = pytree.Node(1000, [l1, l2]) - - self.assertIs(l1.next_sibling, l2) - self.assertEqual(l2.next_sibling, None) - self.assertEqual(p1.next_sibling, None) - - def test_node_prev_sibling(self): - n1 = pytree.Node(1000, []) - n2 = pytree.Node(1000, []) - p1 = pytree.Node(1000, [n1, n2]) - - self.assertIs(n2.prev_sibling, n1) - self.assertEqual(n1.prev_sibling, None) - self.assertEqual(p1.prev_sibling, None) - - def test_leaf_prev_sibling(self): - l1 = pytree.Leaf(100, "a") - l2 = pytree.Leaf(100, "b") - p1 = pytree.Node(1000, [l1, l2]) - - self.assertIs(l2.prev_sibling, l1) - self.assertEqual(l1.prev_sibling, None) - self.assertEqual(p1.prev_sibling, None) - - -class TestPatterns(support.TestCase): - - """Unit tests for tree matching patterns.""" - - def test_basic_patterns(self): - # Build a tree - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - l3 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1, l2]) - n2 = pytree.Node(1000, [l3]) - root = pytree.Node(1000, [n1, n2]) - # Build a pattern matching a leaf - pl = pytree.LeafPattern(100, "foo", name="pl") - r = {} - self.assertFalse(pl.match(root, results=r)) - self.assertEqual(r, {}) - self.assertFalse(pl.match(n1, results=r)) - self.assertEqual(r, {}) - self.assertFalse(pl.match(n2, results=r)) - self.assertEqual(r, {}) - self.assertTrue(pl.match(l1, results=r)) - self.assertEqual(r, {"pl": l1}) - r = {} - self.assertFalse(pl.match(l2, results=r)) - self.assertEqual(r, {}) - # Build a pattern matching a node - pn = pytree.NodePattern(1000, [pl], name="pn") - self.assertFalse(pn.match(root, results=r)) - self.assertEqual(r, {}) - self.assertFalse(pn.match(n1, results=r)) - self.assertEqual(r, {}) - self.assertTrue(pn.match(n2, results=r)) - self.assertEqual(r, {"pn": n2, "pl": l3}) - r = {} - self.assertFalse(pn.match(l1, results=r)) - self.assertEqual(r, {}) - self.assertFalse(pn.match(l2, results=r)) - self.assertEqual(r, {}) - - def test_wildcard(self): - # Build a tree for testing - l1 = pytree.Leaf(100, "foo") - l2 = pytree.Leaf(100, "bar") - l3 = pytree.Leaf(100, "foo") - n1 = pytree.Node(1000, [l1, l2]) - n2 = pytree.Node(1000, [l3]) - root = pytree.Node(1000, [n1, n2]) - # Build a pattern - pl = pytree.LeafPattern(100, "foo", name="pl") - pn = pytree.NodePattern(1000, [pl], name="pn") - pw = pytree.WildcardPattern([[pn], [pl, pl]], name="pw") - r = {} - self.assertFalse(pw.match_seq([root], r)) - self.assertEqual(r, {}) - self.assertFalse(pw.match_seq([n1], r)) - self.assertEqual(r, {}) - self.assertTrue(pw.match_seq([n2], r)) - # These are easier to debug - self.assertEqual(sorted(r.keys()), ["pl", "pn", "pw"]) - self.assertEqual(r["pl"], l1) - self.assertEqual(r["pn"], n2) - self.assertEqual(r["pw"], [n2]) - # But this is equivalent - self.assertEqual(r, {"pl": l1, "pn": n2, "pw": [n2]}) - r = {} - self.assertTrue(pw.match_seq([l1, l3], r)) - self.assertEqual(r, {"pl": l3, "pw": [l1, l3]}) - self.assertIs(r["pl"], l3) - r = {} - - def test_generate_matches(self): - la = pytree.Leaf(1, "a") - lb = pytree.Leaf(1, "b") - lc = pytree.Leaf(1, "c") - ld = pytree.Leaf(1, "d") - le = pytree.Leaf(1, "e") - lf = pytree.Leaf(1, "f") - leaves = [la, lb, lc, ld, le, lf] - root = pytree.Node(1000, leaves) - pa = pytree.LeafPattern(1, "a", "pa") - pb = pytree.LeafPattern(1, "b", "pb") - pc = pytree.LeafPattern(1, "c", "pc") - pd = pytree.LeafPattern(1, "d", "pd") - pe = pytree.LeafPattern(1, "e", "pe") - pf = pytree.LeafPattern(1, "f", "pf") - pw = pytree.WildcardPattern([[pa, pb, pc], [pd, pe], - [pa, pb], [pc, pd], [pe, pf]], - min=1, max=4, name="pw") - self.assertEqual([x[0] for x in pw.generate_matches(leaves)], - [3, 5, 2, 4, 6]) - pr = pytree.NodePattern(type=1000, content=[pw], name="pr") - matches = list(pytree.generate_matches([pr], [root])) - self.assertEqual(len(matches), 1) - c, r = matches[0] - self.assertEqual(c, 1) - self.assertEqual(str(r["pr"]), "abcdef") - self.assertEqual(r["pw"], [la, lb, lc, ld, le, lf]) - for c in "abcdef": - self.assertEqual(r["p" + c], pytree.Leaf(1, c)) - - def test_has_key_example(self): - pattern = pytree.NodePattern(331, - (pytree.LeafPattern(7), - pytree.WildcardPattern(name="args"), - pytree.LeafPattern(8))) - l1 = pytree.Leaf(7, "(") - l2 = pytree.Leaf(3, "x") - l3 = pytree.Leaf(8, ")") - node = pytree.Node(331, [l1, l2, l3]) - r = {} - self.assertTrue(pattern.match(node, r)) - self.assertEqual(r["args"], [l2]) diff --git a/Lib/test/test_lib2to3/test_refactor.py b/Lib/test/test_lib2to3/test_refactor.py deleted file mode 100644 index be705679f06dba..00000000000000 --- a/Lib/test/test_lib2to3/test_refactor.py +++ /dev/null @@ -1,337 +0,0 @@ -""" -Unit tests for refactor.py. -""" - -import sys -import os -import codecs -import io -import re -import tempfile -import shutil -import unittest - -from lib2to3 import refactor, pygram, fixer_base -from lib2to3.pgen2 import token - - -TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") -FIXER_DIR = os.path.join(TEST_DATA_DIR, "fixers") - -sys.path.append(FIXER_DIR) -try: - _DEFAULT_FIXERS = refactor.get_fixers_from_package("myfixes") -finally: - sys.path.pop() - -_2TO3_FIXERS = refactor.get_fixers_from_package("lib2to3.fixes") - -class TestRefactoringTool(unittest.TestCase): - - def setUp(self): - sys.path.append(FIXER_DIR) - - def tearDown(self): - sys.path.pop() - - def check_instances(self, instances, classes): - for inst, cls in zip(instances, classes): - if not isinstance(inst, cls): - self.fail("%s are not instances of %s" % instances, classes) - - def rt(self, options=None, fixers=_DEFAULT_FIXERS, explicit=None): - return refactor.RefactoringTool(fixers, options, explicit) - - def test_print_function_option(self): - rt = self.rt({"print_function" : True}) - self.assertNotIn("print", rt.grammar.keywords) - self.assertNotIn("print", rt.driver.grammar.keywords) - - def test_exec_function_option(self): - rt = self.rt({"exec_function" : True}) - self.assertNotIn("exec", rt.grammar.keywords) - self.assertNotIn("exec", rt.driver.grammar.keywords) - - def test_write_unchanged_files_option(self): - rt = self.rt() - self.assertFalse(rt.write_unchanged_files) - rt = self.rt({"write_unchanged_files" : True}) - self.assertTrue(rt.write_unchanged_files) - - def test_fixer_loading_helpers(self): - contents = ["explicit", "first", "last", "parrot", "preorder"] - non_prefixed = refactor.get_all_fix_names("myfixes") - prefixed = refactor.get_all_fix_names("myfixes", False) - full_names = refactor.get_fixers_from_package("myfixes") - self.assertEqual(prefixed, ["fix_" + name for name in contents]) - self.assertEqual(non_prefixed, contents) - self.assertEqual(full_names, - ["myfixes.fix_" + name for name in contents]) - - def test_detect_future_features(self): - run = refactor._detect_future_features - fs = frozenset - empty = fs() - self.assertEqual(run(""), empty) - self.assertEqual(run("from __future__ import print_function"), - fs(("print_function",))) - self.assertEqual(run("from __future__ import generators"), - fs(("generators",))) - self.assertEqual(run("from __future__ import generators, feature"), - fs(("generators", "feature"))) - inp = "from __future__ import generators, print_function" - self.assertEqual(run(inp), fs(("generators", "print_function"))) - inp ="from __future__ import print_function, generators" - self.assertEqual(run(inp), fs(("print_function", "generators"))) - inp = "from __future__ import (print_function,)" - self.assertEqual(run(inp), fs(("print_function",))) - inp = "from __future__ import (generators, print_function)" - self.assertEqual(run(inp), fs(("generators", "print_function"))) - inp = "from __future__ import (generators, nested_scopes)" - self.assertEqual(run(inp), fs(("generators", "nested_scopes"))) - inp = """from __future__ import generators -from __future__ import print_function""" - self.assertEqual(run(inp), fs(("generators", "print_function"))) - invalid = ("from", - "from 4", - "from x", - "from x 5", - "from x im", - "from x import", - "from x import 4", - ) - for inp in invalid: - self.assertEqual(run(inp), empty) - inp = "'docstring'\nfrom __future__ import print_function" - self.assertEqual(run(inp), fs(("print_function",))) - inp = "'docstring'\n'somng'\nfrom __future__ import print_function" - self.assertEqual(run(inp), empty) - inp = "# comment\nfrom __future__ import print_function" - self.assertEqual(run(inp), fs(("print_function",))) - inp = "# comment\n'doc'\nfrom __future__ import print_function" - self.assertEqual(run(inp), fs(("print_function",))) - inp = "class x: pass\nfrom __future__ import print_function" - self.assertEqual(run(inp), empty) - - def test_get_headnode_dict(self): - class NoneFix(fixer_base.BaseFix): - pass - - class FileInputFix(fixer_base.BaseFix): - PATTERN = "file_input< any * >" - - class SimpleFix(fixer_base.BaseFix): - PATTERN = "'name'" - - no_head = NoneFix({}, []) - with_head = FileInputFix({}, []) - simple = SimpleFix({}, []) - d = refactor._get_headnode_dict([no_head, with_head, simple]) - top_fixes = d.pop(pygram.python_symbols.file_input) - self.assertEqual(top_fixes, [with_head, no_head]) - name_fixes = d.pop(token.NAME) - self.assertEqual(name_fixes, [simple, no_head]) - for fixes in d.values(): - self.assertEqual(fixes, [no_head]) - - def test_fixer_loading(self): - from myfixes.fix_first import FixFirst - from myfixes.fix_last import FixLast - from myfixes.fix_parrot import FixParrot - from myfixes.fix_preorder import FixPreorder - - rt = self.rt() - pre, post = rt.get_fixers() - - self.check_instances(pre, [FixPreorder]) - self.check_instances(post, [FixFirst, FixParrot, FixLast]) - - def test_naughty_fixers(self): - self.assertRaises(ImportError, self.rt, fixers=["not_here"]) - self.assertRaises(refactor.FixerError, self.rt, fixers=["no_fixer_cls"]) - self.assertRaises(refactor.FixerError, self.rt, fixers=["bad_order"]) - - def test_refactor_string(self): - rt = self.rt() - input = "def parrot(): pass\n\n" - tree = rt.refactor_string(input, "") - self.assertNotEqual(str(tree), input) - - input = "def f(): pass\n\n" - tree = rt.refactor_string(input, "") - self.assertEqual(str(tree), input) - - def test_refactor_stdin(self): - - class MyRT(refactor.RefactoringTool): - - def print_output(self, old_text, new_text, filename, equal): - results.extend([old_text, new_text, filename, equal]) - - results = [] - rt = MyRT(_DEFAULT_FIXERS) - save = sys.stdin - sys.stdin = io.StringIO("def parrot(): pass\n\n") - try: - rt.refactor_stdin() - finally: - sys.stdin = save - expected = ["def parrot(): pass\n\n", - "def cheese(): pass\n\n", - "", False] - self.assertEqual(results, expected) - - def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS, - options=None, mock_log_debug=None, - actually_write=True): - test_file = self.init_test_file(test_file) - old_contents = self.read_file(test_file) - rt = self.rt(fixers=fixers, options=options) - if mock_log_debug: - rt.log_debug = mock_log_debug - - rt.refactor_file(test_file) - self.assertEqual(old_contents, self.read_file(test_file)) - - if not actually_write: - return - rt.refactor_file(test_file, True) - new_contents = self.read_file(test_file) - self.assertNotEqual(old_contents, new_contents) - return new_contents - - def init_test_file(self, test_file): - tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor") - self.addCleanup(shutil.rmtree, tmpdir) - shutil.copy(test_file, tmpdir) - test_file = os.path.join(tmpdir, os.path.basename(test_file)) - os.chmod(test_file, 0o644) - return test_file - - def read_file(self, test_file): - with open(test_file, "rb") as fp: - return fp.read() - - def refactor_file(self, test_file, fixers=_2TO3_FIXERS): - test_file = self.init_test_file(test_file) - old_contents = self.read_file(test_file) - rt = self.rt(fixers=fixers) - rt.refactor_file(test_file, True) - new_contents = self.read_file(test_file) - return old_contents, new_contents - - def test_refactor_file(self): - test_file = os.path.join(FIXER_DIR, "parrot_example.py") - self.check_file_refactoring(test_file, _DEFAULT_FIXERS) - - def test_refactor_file_write_unchanged_file(self): - test_file = os.path.join(FIXER_DIR, "parrot_example.py") - debug_messages = [] - def recording_log_debug(msg, *args): - debug_messages.append(msg % args) - self.check_file_refactoring(test_file, fixers=(), - options={"write_unchanged_files": True}, - mock_log_debug=recording_log_debug, - actually_write=False) - # Testing that it logged this message when write=False was passed is - # sufficient to see that it did not bail early after "No changes". - message_regex = r"Not writing changes to .*%s" % \ - re.escape(os.sep + os.path.basename(test_file)) - for message in debug_messages: - if "Not writing changes" in message: - self.assertRegex(message, message_regex) - break - else: - self.fail("%r not matched in %r" % (message_regex, debug_messages)) - - def test_refactor_dir(self): - def check(structure, expected): - def mock_refactor_file(self, f, *args): - got.append(f) - save_func = refactor.RefactoringTool.refactor_file - refactor.RefactoringTool.refactor_file = mock_refactor_file - rt = self.rt() - got = [] - dir = tempfile.mkdtemp(prefix="2to3-test_refactor") - try: - os.mkdir(os.path.join(dir, "a_dir")) - for fn in structure: - open(os.path.join(dir, fn), "wb").close() - rt.refactor_dir(dir) - finally: - refactor.RefactoringTool.refactor_file = save_func - shutil.rmtree(dir) - self.assertEqual(got, - [os.path.join(dir, path) for path in expected]) - check([], []) - tree = ["nothing", - "hi.py", - ".dumb", - ".after.py", - "notpy.npy", - "sappy"] - expected = ["hi.py"] - check(tree, expected) - tree = ["hi.py", - os.path.join("a_dir", "stuff.py")] - check(tree, tree) - - def test_file_encoding(self): - fn = os.path.join(TEST_DATA_DIR, "different_encoding.py") - self.check_file_refactoring(fn) - - def test_false_file_encoding(self): - fn = os.path.join(TEST_DATA_DIR, "false_encoding.py") - data = self.check_file_refactoring(fn) - - def test_bom(self): - fn = os.path.join(TEST_DATA_DIR, "bom.py") - data = self.check_file_refactoring(fn) - self.assertTrue(data.startswith(codecs.BOM_UTF8)) - - def test_crlf_newlines(self): - old_sep = os.linesep - os.linesep = "\r\n" - try: - fn = os.path.join(TEST_DATA_DIR, "crlf.py") - fixes = refactor.get_fixers_from_package("lib2to3.fixes") - self.check_file_refactoring(fn, fixes) - finally: - os.linesep = old_sep - - def test_crlf_unchanged(self): - fn = os.path.join(TEST_DATA_DIR, "crlf.py") - old, new = self.refactor_file(fn) - self.assertIn(b"\r\n", old) - self.assertIn(b"\r\n", new) - self.assertNotIn(b"\r\r\n", new) - - def test_refactor_docstring(self): - rt = self.rt() - - doc = """ ->>> example() -42 -""" - out = rt.refactor_docstring(doc, "") - self.assertEqual(out, doc) - - doc = """ ->>> def parrot(): -... return 43 -""" - out = rt.refactor_docstring(doc, "") - self.assertNotEqual(out, doc) - - def test_explicit(self): - from myfixes.fix_explicit import FixExplicit - - rt = self.rt(fixers=["myfixes.fix_explicit"]) - self.assertEqual(len(rt.post_order), 0) - - rt = self.rt(explicit=["myfixes.fix_explicit"]) - for fix in rt.post_order: - if isinstance(fix, FixExplicit): - break - else: - self.fail("explicit fixer not loaded") diff --git a/Lib/test/test_lib2to3/test_util.py b/Lib/test/test_lib2to3/test_util.py deleted file mode 100644 index c6c613972dac7e..00000000000000 --- a/Lib/test/test_lib2to3/test_util.py +++ /dev/null @@ -1,591 +0,0 @@ -""" Test suite for the code in fixer_util """ - -# Testing imports -from . import support - -# Local imports -from lib2to3.pytree import Node, Leaf -from lib2to3 import fixer_util -from lib2to3.fixer_util import Attr, Name, Call, Comma -from lib2to3.pgen2 import token - -def parse(code, strip_levels=0): - # The topmost node is file_input, which we don't care about. - # The next-topmost node is a *_stmt node, which we also don't care about - tree = support.parse_string(code) - for i in range(strip_levels): - tree = tree.children[0] - tree.parent = None - return tree - -class MacroTestCase(support.TestCase): - def assertStr(self, node, string): - if isinstance(node, (tuple, list)): - node = Node(fixer_util.syms.simple_stmt, node) - self.assertEqual(str(node), string) - - -class Test_is_tuple(support.TestCase): - def is_tuple(self, string): - return fixer_util.is_tuple(parse(string, strip_levels=2)) - - def test_valid(self): - self.assertTrue(self.is_tuple("(a, b)")) - self.assertTrue(self.is_tuple("(a, (b, c))")) - self.assertTrue(self.is_tuple("((a, (b, c)),)")) - self.assertTrue(self.is_tuple("(a,)")) - self.assertTrue(self.is_tuple("()")) - - def test_invalid(self): - self.assertFalse(self.is_tuple("(a)")) - self.assertFalse(self.is_tuple("('foo') % (b, c)")) - - -class Test_is_list(support.TestCase): - def is_list(self, string): - return fixer_util.is_list(parse(string, strip_levels=2)) - - def test_valid(self): - self.assertTrue(self.is_list("[]")) - self.assertTrue(self.is_list("[a]")) - self.assertTrue(self.is_list("[a, b]")) - self.assertTrue(self.is_list("[a, [b, c]]")) - self.assertTrue(self.is_list("[[a, [b, c]],]")) - - def test_invalid(self): - self.assertFalse(self.is_list("[]+[]")) - - -class Test_Attr(MacroTestCase): - def test(self): - call = parse("foo()", strip_levels=2) - - self.assertStr(Attr(Name("a"), Name("b")), "a.b") - self.assertStr(Attr(call, Name("b")), "foo().b") - - def test_returns(self): - attr = Attr(Name("a"), Name("b")) - self.assertEqual(type(attr), list) - - -class Test_Name(MacroTestCase): - def test(self): - self.assertStr(Name("a"), "a") - self.assertStr(Name("foo.foo().bar"), "foo.foo().bar") - self.assertStr(Name("a", prefix="b"), "ba") - - -class Test_Call(MacroTestCase): - def _Call(self, name, args=None, prefix=None): - """Help the next test""" - children = [] - if isinstance(args, list): - for arg in args: - children.append(arg) - children.append(Comma()) - children.pop() - return Call(Name(name), children, prefix) - - def test(self): - kids = [None, - [Leaf(token.NUMBER, 1), Leaf(token.NUMBER, 2), - Leaf(token.NUMBER, 3)], - [Leaf(token.NUMBER, 1), Leaf(token.NUMBER, 3), - Leaf(token.NUMBER, 2), Leaf(token.NUMBER, 4)], - [Leaf(token.STRING, "b"), Leaf(token.STRING, "j", prefix=" ")] - ] - self.assertStr(self._Call("A"), "A()") - self.assertStr(self._Call("b", kids[1]), "b(1,2,3)") - self.assertStr(self._Call("a.b().c", kids[2]), "a.b().c(1,3,2,4)") - self.assertStr(self._Call("d", kids[3], prefix=" "), " d(b, j)") - - -class Test_does_tree_import(support.TestCase): - def _find_bind_rec(self, name, node): - # Search a tree for a binding -- used to find the starting - # point for these tests. - c = fixer_util.find_binding(name, node) - if c: return c - for child in node.children: - c = self._find_bind_rec(name, child) - if c: return c - - def does_tree_import(self, package, name, string): - node = parse(string) - # Find the binding of start -- that's what we'll go from - node = self._find_bind_rec('start', node) - return fixer_util.does_tree_import(package, name, node) - - def try_with(self, string): - failing_tests = (("a", "a", "from a import b"), - ("a.d", "a", "from a.d import b"), - ("d.a", "a", "from d.a import b"), - (None, "a", "import b"), - (None, "a", "import b, c, d")) - for package, name, import_ in failing_tests: - n = self.does_tree_import(package, name, import_ + "\n" + string) - self.assertFalse(n) - n = self.does_tree_import(package, name, string + "\n" + import_) - self.assertFalse(n) - - passing_tests = (("a", "a", "from a import a"), - ("x", "a", "from x import a"), - ("x", "a", "from x import b, c, a, d"), - ("x.b", "a", "from x.b import a"), - ("x.b", "a", "from x.b import b, c, a, d"), - (None, "a", "import a"), - (None, "a", "import b, c, a, d")) - for package, name, import_ in passing_tests: - n = self.does_tree_import(package, name, import_ + "\n" + string) - self.assertTrue(n) - n = self.does_tree_import(package, name, string + "\n" + import_) - self.assertTrue(n) - - def test_in_function(self): - self.try_with("def foo():\n\tbar.baz()\n\tstart=3") - -class Test_find_binding(support.TestCase): - def find_binding(self, name, string, package=None): - return fixer_util.find_binding(name, parse(string), package) - - def test_simple_assignment(self): - self.assertTrue(self.find_binding("a", "a = b")) - self.assertTrue(self.find_binding("a", "a = [b, c, d]")) - self.assertTrue(self.find_binding("a", "a = foo()")) - self.assertTrue(self.find_binding("a", "a = foo().foo.foo[6][foo]")) - self.assertFalse(self.find_binding("a", "foo = a")) - self.assertFalse(self.find_binding("a", "foo = (a, b, c)")) - - def test_tuple_assignment(self): - self.assertTrue(self.find_binding("a", "(a,) = b")) - self.assertTrue(self.find_binding("a", "(a, b, c) = [b, c, d]")) - self.assertTrue(self.find_binding("a", "(c, (d, a), b) = foo()")) - self.assertTrue(self.find_binding("a", "(a, b) = foo().foo[6][foo]")) - self.assertFalse(self.find_binding("a", "(foo, b) = (b, a)")) - self.assertFalse(self.find_binding("a", "(foo, (b, c)) = (a, b, c)")) - - def test_list_assignment(self): - self.assertTrue(self.find_binding("a", "[a] = b")) - self.assertTrue(self.find_binding("a", "[a, b, c] = [b, c, d]")) - self.assertTrue(self.find_binding("a", "[c, [d, a], b] = foo()")) - self.assertTrue(self.find_binding("a", "[a, b] = foo().foo[a][foo]")) - self.assertFalse(self.find_binding("a", "[foo, b] = (b, a)")) - self.assertFalse(self.find_binding("a", "[foo, [b, c]] = (a, b, c)")) - - def test_invalid_assignments(self): - self.assertFalse(self.find_binding("a", "foo.a = 5")) - self.assertFalse(self.find_binding("a", "foo[a] = 5")) - self.assertFalse(self.find_binding("a", "foo(a) = 5")) - self.assertFalse(self.find_binding("a", "foo(a, b) = 5")) - - def test_simple_import(self): - self.assertTrue(self.find_binding("a", "import a")) - self.assertTrue(self.find_binding("a", "import b, c, a, d")) - self.assertFalse(self.find_binding("a", "import b")) - self.assertFalse(self.find_binding("a", "import b, c, d")) - - def test_from_import(self): - self.assertTrue(self.find_binding("a", "from x import a")) - self.assertTrue(self.find_binding("a", "from a import a")) - self.assertTrue(self.find_binding("a", "from x import b, c, a, d")) - self.assertTrue(self.find_binding("a", "from x.b import a")) - self.assertTrue(self.find_binding("a", "from x.b import b, c, a, d")) - self.assertFalse(self.find_binding("a", "from a import b")) - self.assertFalse(self.find_binding("a", "from a.d import b")) - self.assertFalse(self.find_binding("a", "from d.a import b")) - - def test_import_as(self): - self.assertTrue(self.find_binding("a", "import b as a")) - self.assertTrue(self.find_binding("a", "import b as a, c, a as f, d")) - self.assertFalse(self.find_binding("a", "import a as f")) - self.assertFalse(self.find_binding("a", "import b, c as f, d as e")) - - def test_from_import_as(self): - self.assertTrue(self.find_binding("a", "from x import b as a")) - self.assertTrue(self.find_binding("a", "from x import g as a, d as b")) - self.assertTrue(self.find_binding("a", "from x.b import t as a")) - self.assertTrue(self.find_binding("a", "from x.b import g as a, d")) - self.assertFalse(self.find_binding("a", "from a import b as t")) - self.assertFalse(self.find_binding("a", "from a.d import b as t")) - self.assertFalse(self.find_binding("a", "from d.a import b as t")) - - def test_simple_import_with_package(self): - self.assertTrue(self.find_binding("b", "import b")) - self.assertTrue(self.find_binding("b", "import b, c, d")) - self.assertFalse(self.find_binding("b", "import b", "b")) - self.assertFalse(self.find_binding("b", "import b, c, d", "c")) - - def test_from_import_with_package(self): - self.assertTrue(self.find_binding("a", "from x import a", "x")) - self.assertTrue(self.find_binding("a", "from a import a", "a")) - self.assertTrue(self.find_binding("a", "from x import *", "x")) - self.assertTrue(self.find_binding("a", "from x import b, c, a, d", "x")) - self.assertTrue(self.find_binding("a", "from x.b import a", "x.b")) - self.assertTrue(self.find_binding("a", "from x.b import *", "x.b")) - self.assertTrue(self.find_binding("a", "from x.b import b, c, a, d", "x.b")) - self.assertFalse(self.find_binding("a", "from a import b", "a")) - self.assertFalse(self.find_binding("a", "from a.d import b", "a.d")) - self.assertFalse(self.find_binding("a", "from d.a import b", "a.d")) - self.assertFalse(self.find_binding("a", "from x.y import *", "a.b")) - - def test_import_as_with_package(self): - self.assertFalse(self.find_binding("a", "import b.c as a", "b.c")) - self.assertFalse(self.find_binding("a", "import a as f", "f")) - self.assertFalse(self.find_binding("a", "import a as f", "a")) - - def test_from_import_as_with_package(self): - # Because it would take a lot of special-case code in the fixers - # to deal with from foo import bar as baz, we'll simply always - # fail if there is an "from ... import ... as ..." - self.assertFalse(self.find_binding("a", "from x import b as a", "x")) - self.assertFalse(self.find_binding("a", "from x import g as a, d as b", "x")) - self.assertFalse(self.find_binding("a", "from x.b import t as a", "x.b")) - self.assertFalse(self.find_binding("a", "from x.b import g as a, d", "x.b")) - self.assertFalse(self.find_binding("a", "from a import b as t", "a")) - self.assertFalse(self.find_binding("a", "from a import b as t", "b")) - self.assertFalse(self.find_binding("a", "from a import b as t", "t")) - - def test_function_def(self): - self.assertTrue(self.find_binding("a", "def a(): pass")) - self.assertTrue(self.find_binding("a", "def a(b, c, d): pass")) - self.assertTrue(self.find_binding("a", "def a(): b = 7")) - self.assertFalse(self.find_binding("a", "def d(b, (c, a), e): pass")) - self.assertFalse(self.find_binding("a", "def d(a=7): pass")) - self.assertFalse(self.find_binding("a", "def d(a): pass")) - self.assertFalse(self.find_binding("a", "def d(): a = 7")) - - s = """ - def d(): - def a(): - pass""" - self.assertFalse(self.find_binding("a", s)) - - def test_class_def(self): - self.assertTrue(self.find_binding("a", "class a: pass")) - self.assertTrue(self.find_binding("a", "class a(): pass")) - self.assertTrue(self.find_binding("a", "class a(b): pass")) - self.assertTrue(self.find_binding("a", "class a(b, c=8): pass")) - self.assertFalse(self.find_binding("a", "class d: pass")) - self.assertFalse(self.find_binding("a", "class d(a): pass")) - self.assertFalse(self.find_binding("a", "class d(b, a=7): pass")) - self.assertFalse(self.find_binding("a", "class d(b, *a): pass")) - self.assertFalse(self.find_binding("a", "class d(b, **a): pass")) - self.assertFalse(self.find_binding("a", "class d: a = 7")) - - s = """ - class d(): - class a(): - pass""" - self.assertFalse(self.find_binding("a", s)) - - def test_for(self): - self.assertTrue(self.find_binding("a", "for a in r: pass")) - self.assertTrue(self.find_binding("a", "for a, b in r: pass")) - self.assertTrue(self.find_binding("a", "for (a, b) in r: pass")) - self.assertTrue(self.find_binding("a", "for c, (a,) in r: pass")) - self.assertTrue(self.find_binding("a", "for c, (a, b) in r: pass")) - self.assertTrue(self.find_binding("a", "for c in r: a = c")) - self.assertFalse(self.find_binding("a", "for c in a: pass")) - - def test_for_nested(self): - s = """ - for b in r: - for a in b: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for a, c in b: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for (a, c) in b: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for (a,) in b: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for c, (a, d) in b: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for c in b: - a = 7""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - for b in r: - for c in b: - d = a""" - self.assertFalse(self.find_binding("a", s)) - - s = """ - for b in r: - for c in a: - d = 7""" - self.assertFalse(self.find_binding("a", s)) - - def test_if(self): - self.assertTrue(self.find_binding("a", "if b in r: a = c")) - self.assertFalse(self.find_binding("a", "if a in r: d = e")) - - def test_if_nested(self): - s = """ - if b in r: - if c in d: - a = c""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - if b in r: - if c in d: - c = a""" - self.assertFalse(self.find_binding("a", s)) - - def test_while(self): - self.assertTrue(self.find_binding("a", "while b in r: a = c")) - self.assertFalse(self.find_binding("a", "while a in r: d = e")) - - def test_while_nested(self): - s = """ - while b in r: - while c in d: - a = c""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - while b in r: - while c in d: - c = a""" - self.assertFalse(self.find_binding("a", s)) - - def test_try_except(self): - s = """ - try: - a = 6 - except: - b = 8""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except KeyError: - pass - except: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - b = 6""" - self.assertFalse(self.find_binding("a", s)) - - def test_try_except_nested(self): - s = """ - try: - try: - a = 6 - except: - pass - except: - b = 8""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - try: - a = 6 - except: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - try: - pass - except: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - try: - b = 8 - except KeyError: - pass - except: - a = 6 - except: - pass""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - pass - except: - try: - b = 8 - except KeyError: - pass - except: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - b = 6""" - self.assertFalse(self.find_binding("a", s)) - - s = """ - try: - try: - b = 8 - except: - c = d - except: - try: - b = 6 - except: - t = 8 - except: - o = y""" - self.assertFalse(self.find_binding("a", s)) - - def test_try_except_finally(self): - s = """ - try: - c = 6 - except: - b = 8 - finally: - a = 9""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - finally: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - finally: - b = 6""" - self.assertFalse(self.find_binding("a", s)) - - s = """ - try: - b = 8 - except: - b = 9 - finally: - b = 6""" - self.assertFalse(self.find_binding("a", s)) - - def test_try_except_finally_nested(self): - s = """ - try: - c = 6 - except: - b = 8 - finally: - try: - a = 9 - except: - b = 9 - finally: - c = 9""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - finally: - try: - pass - finally: - a = 6""" - self.assertTrue(self.find_binding("a", s)) - - s = """ - try: - b = 8 - finally: - try: - b = 6 - finally: - b = 7""" - self.assertFalse(self.find_binding("a", s)) - -class Test_touch_import(support.TestCase): - - def test_after_docstring(self): - node = parse('"""foo"""\nbar()') - fixer_util.touch_import(None, "foo", node) - self.assertEqual(str(node), '"""foo"""\nimport foo\nbar()\n\n') - - def test_after_imports(self): - node = parse('"""foo"""\nimport bar\nbar()') - fixer_util.touch_import(None, "foo", node) - self.assertEqual(str(node), '"""foo"""\nimport bar\nimport foo\nbar()\n\n') - - def test_beginning(self): - node = parse('bar()') - fixer_util.touch_import(None, "foo", node) - self.assertEqual(str(node), 'import foo\nbar()\n\n') - - def test_from_import(self): - node = parse('bar()') - fixer_util.touch_import("html", "escape", node) - self.assertEqual(str(node), 'from html import escape\nbar()\n\n') - - def test_name_import(self): - node = parse('bar()') - fixer_util.touch_import(None, "cgi", node) - self.assertEqual(str(node), 'import cgi\nbar()\n\n') - -class Test_find_indentation(support.TestCase): - - def test_nothing(self): - fi = fixer_util.find_indentation - node = parse("node()") - self.assertEqual(fi(node), "") - node = parse("") - self.assertEqual(fi(node), "") - - def test_simple(self): - fi = fixer_util.find_indentation - node = parse("def f():\n x()") - self.assertEqual(fi(node), "") - self.assertEqual(fi(node.children[0].children[4].children[2]), " ") - node = parse("def f():\n x()\n y()") - self.assertEqual(fi(node.children[0].children[4].children[4]), " ") diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index ab2c2b232a0411..8b68cdc9b7d003 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -2080,26 +2080,6 @@ def test_resolve_nonexist_relative_issue38671(self): finally: os.chdir(old_cwd) - def test_with(self): - p = self.cls(BASE) - it = p.iterdir() - it2 = p.iterdir() - next(it2) - # bpo-46556: path context managers are deprecated in Python 3.11. - with self.assertWarns(DeprecationWarning): - with p: - pass - # Using a path as a context manager is a no-op, thus the following - # operations should still succeed after the context manage exits. - next(it) - next(it2) - p.exists() - p.resolve() - p.absolute() - with self.assertWarns(DeprecationWarning): - with p: - pass - @os_helper.skip_unless_working_chmod def test_chmod(self): p = self.cls(BASE) / 'fileA' diff --git a/Lib/test/test_unicode.py b/Lib/test/test_str.py similarity index 99% rename from Lib/test/test_unicode.py rename to Lib/test/test_str.py index 4ebbb9d32a3d75..a6bcc2455de15e 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_str.py @@ -55,8 +55,7 @@ def duplicate_string(text): class StrSubclass(str): pass -class UnicodeTest(string_tests.CommonTest, - string_tests.MixinStrUnicodeUserStringTest, +class StrTest(string_tests.StringLikeTest, string_tests.MixinStrUnicodeTest, unittest.TestCase): @@ -213,7 +212,7 @@ def test_pickle_iterator(self): self.assertEqual(case, pickled) def test_count(self): - string_tests.CommonTest.test_count(self) + string_tests.StringLikeTest.test_count(self) # check mixed argument types self.checkequalnofix(3, 'aaa', 'count', 'a') self.checkequalnofix(0, 'aaa', 'count', 'b') @@ -243,7 +242,7 @@ class MyStr(str): self.checkequal(3, MyStr('aaa'), 'count', 'a') def test_find(self): - string_tests.CommonTest.test_find(self) + string_tests.StringLikeTest.test_find(self) # test implementation details of the memchr fast path self.checkequal(100, 'a' * 100 + '\u0102', 'find', '\u0102') self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0201') @@ -288,7 +287,7 @@ def test_find(self): self.checkequal(-1, '\u0102' * 100, 'find', '\u0102\U00100304') def test_rfind(self): - string_tests.CommonTest.test_rfind(self) + string_tests.StringLikeTest.test_rfind(self) # test implementation details of the memrchr fast path self.checkequal(0, '\u0102' + 'a' * 100 , 'rfind', '\u0102') self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0201') @@ -329,7 +328,7 @@ def test_rfind(self): self.checkequal(-1, '\u0102' * 100, 'rfind', '\U00100304\u0102') def test_index(self): - string_tests.CommonTest.test_index(self) + string_tests.StringLikeTest.test_index(self) self.checkequalnofix(0, 'abcdefghiabc', 'index', '') self.checkequalnofix(3, 'abcdefghiabc', 'index', 'def') self.checkequalnofix(0, 'abcdefghiabc', 'index', 'abc') @@ -353,7 +352,7 @@ def test_index(self): self.assertRaises(ValueError, ('\u0102' * 100).index, '\u0102\U00100304') def test_rindex(self): - string_tests.CommonTest.test_rindex(self) + string_tests.StringLikeTest.test_rindex(self) self.checkequalnofix(12, 'abcdefghiabc', 'rindex', '') self.checkequalnofix(3, 'abcdefghiabc', 'rindex', 'def') self.checkequalnofix(9, 'abcdefghiabc', 'rindex', 'abc') @@ -449,7 +448,7 @@ def test_maketrans_translate(self): self.assertRaises(TypeError, 'abababc'.translate, 'abc', 'xyz') def test_split(self): - string_tests.CommonTest.test_split(self) + string_tests.StringLikeTest.test_split(self) # test mixed kinds for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): @@ -466,7 +465,7 @@ def test_split(self): left + delim * 2 + right, 'split', delim *2) def test_rsplit(self): - string_tests.CommonTest.test_rsplit(self) + string_tests.StringLikeTest.test_rsplit(self) # test mixed kinds for left, right in ('ba', 'юё', '\u0101\u0100', '\U00010301\U00010300'): left *= 9 @@ -486,7 +485,7 @@ def test_rsplit(self): left + right, 'rsplit', None) def test_partition(self): - string_tests.MixinStrUnicodeUserStringTest.test_partition(self) + string_tests.StringLikeTest.test_partition(self) # test mixed kinds self.checkequal(('ABCDEFGH', '', ''), 'ABCDEFGH', 'partition', '\u4200') for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): @@ -503,7 +502,7 @@ def test_partition(self): left + delim * 2 + right, 'partition', delim * 2) def test_rpartition(self): - string_tests.MixinStrUnicodeUserStringTest.test_rpartition(self) + string_tests.StringLikeTest.test_rpartition(self) # test mixed kinds self.checkequal(('', '', 'ABCDEFGH'), 'ABCDEFGH', 'rpartition', '\u4200') for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): @@ -520,7 +519,7 @@ def test_rpartition(self): left + delim * 2 + right, 'rpartition', delim * 2) def test_join(self): - string_tests.MixinStrUnicodeUserStringTest.test_join(self) + string_tests.StringLikeTest.test_join(self) class MyWrapper: def __init__(self, sval): self.sval = sval @@ -547,7 +546,7 @@ def test_join_overflow(self): self.assertRaises(OverflowError, ''.join, seq) def test_replace(self): - string_tests.CommonTest.test_replace(self) + string_tests.StringLikeTest.test_replace(self) # method call forwarded from str implementation because of unicode argument self.checkequalnofix('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1) @@ -866,7 +865,7 @@ def test_surrogates(self): def test_lower(self): - string_tests.CommonTest.test_lower(self) + string_tests.StringLikeTest.test_lower(self) self.assertEqual('\U00010427'.lower(), '\U0001044F') self.assertEqual('\U00010427\U00010427'.lower(), '\U0001044F\U0001044F') @@ -897,7 +896,7 @@ def test_casefold(self): self.assertEqual('\u00b5'.casefold(), '\u03bc') def test_upper(self): - string_tests.CommonTest.test_upper(self) + string_tests.StringLikeTest.test_upper(self) self.assertEqual('\U0001044F'.upper(), '\U00010427') self.assertEqual('\U0001044F\U0001044F'.upper(), '\U00010427\U00010427') @@ -914,7 +913,7 @@ def test_upper(self): self.assertEqual('\u2177'.upper(), '\u2167') def test_capitalize(self): - string_tests.CommonTest.test_capitalize(self) + string_tests.StringLikeTest.test_capitalize(self) self.assertEqual('\U0001044F'.capitalize(), '\U00010427') self.assertEqual('\U0001044F\U0001044F'.capitalize(), '\U00010427\U0001044F') @@ -948,7 +947,7 @@ def test_title(self): self.assertEqual('A\u03a3A'.title(), 'A\u03c3a') def test_swapcase(self): - string_tests.CommonTest.test_swapcase(self) + string_tests.StringLikeTest.test_swapcase(self) self.assertEqual('\U0001044F'.swapcase(), '\U00010427') self.assertEqual('\U00010427'.swapcase(), '\U0001044F') self.assertEqual('\U0001044F\U0001044F'.swapcase(), @@ -974,7 +973,7 @@ def test_swapcase(self): self.assertEqual('\u1fd2'.swapcase(), '\u0399\u0308\u0300') def test_center(self): - string_tests.CommonTest.test_center(self) + string_tests.StringLikeTest.test_center(self) self.assertEqual('x'.center(2, '\U0010FFFF'), 'x\U0010FFFF') self.assertEqual('x'.center(3, '\U0010FFFF'), @@ -1475,7 +1474,7 @@ def __format__(self, spec): self.assertEqual('{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g') def test_formatting(self): - string_tests.MixinStrUnicodeUserStringTest.test_formatting(self) + string_tests.StringLikeTest.test_formatting(self) # Testing Unicode formatting strings... self.assertEqual("%s, %s" % ("abc", "abc"), 'abc, abc') self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, 2, 3), 'abc, abc, 1, 2.000000, 3.00') diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 0571919066bb73..5603c3cdbf3c5e 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1524,6 +1524,52 @@ def __init__(self): (3, 'return'), (1, 'return')]) + def test_class_creation_with_decorator(self): + def func(): + def decorator(arg): + def _dec(c): + return c + return _dec + + @decorator(6) + @decorator( + len([8]), + ) + class MyObject: + pass + + self.run_and_compare(func, [ + (0, 'call'), + (1, 'line'), + (6, 'line'), + (1, 'call'), + (2, 'line'), + (4, 'line'), + (4, 'return'), + (7, 'line'), + (8, 'line'), + (7, 'line'), + (1, 'call'), + (2, 'line'), + (4, 'line'), + (4, 'return'), + (10, 'line'), + (6, 'call'), + (6, 'line'), + (11, 'line'), + (11, 'return'), + (7, 'line'), + (2, 'call'), + (3, 'line'), + (3, 'return'), + (6, 'line'), + (2, 'call'), + (3, 'line'), + (3, 'return'), + (10, 'line'), + (10, 'return'), + ]) + @support.cpython_only def test_no_line_event_after_creating_generator(self): # Spurious line events before call events only show up with C tracer diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index dac47318011d9d..aa700118f735d9 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -317,7 +317,7 @@ def test_with_errored_file(self): with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: stderr = f"{file_path!r}: Token Error: " stderr += ('unindent does not match any outer indentation level' - ' (, line 3)') + ' (, line 3)') self.validate_cmd(file_path, stderr=stderr, expect_failure=True) def test_with_error_free_file(self): diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index e8d322d20a5a8e..2eda7fc4ceac71 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -100,6 +100,14 @@ def setUp(self): def tearDown(self): self.tar.close() +class StreamModeTest(ReadTest): + + # Only needs to change how the tarfile is opened to set + # stream mode + def setUp(self): + self.tar = tarfile.open(self.tarname, mode=self.mode, + encoding="iso8859-1", + stream=True) class UstarReadTest(ReadTest, unittest.TestCase): @@ -852,6 +860,21 @@ class Bz2StreamReadTest(Bz2Test, StreamReadTest): class LzmaStreamReadTest(LzmaTest, StreamReadTest): pass +class TarStreamModeReadTest(StreamModeTest, unittest.TestCase): + + def test_stream_mode_no_cache(self): + for _ in self.tar: + pass + self.assertEqual(self.tar.members, []) + +class GzipStreamModeReadTest(GzipTest, TarStreamModeReadTest): + pass + +class Bz2StreamModeReadTest(Bz2Test, TarStreamModeReadTest): + pass + +class LzmaStreamModeReadTest(LzmaTest, TarStreamModeReadTest): + pass class DetectReadTest(TarTest, unittest.TestCase): def _testfunc_file(self, name, mode): diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py deleted file mode 100644 index a9cade2ee466f9..00000000000000 --- a/Lib/test/test_telnetlib.py +++ /dev/null @@ -1,405 +0,0 @@ -import socket -import selectors -import threading -import contextlib - -from test import support -from test.support import socket_helper, warnings_helper -import unittest - -support.requires_working_socket(module=True) - -telnetlib = warnings_helper.import_deprecated('telnetlib') - -HOST = socket_helper.HOST - -def server(evt, serv): - serv.listen() - evt.set() - try: - conn, addr = serv.accept() - conn.close() - except TimeoutError: - pass - finally: - serv.close() - -class GeneralTests(unittest.TestCase): - - def setUp(self): - self.evt = threading.Event() - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.settimeout(60) # Safety net. Look issue 11812 - self.port = socket_helper.bind_port(self.sock) - self.thread = threading.Thread(target=server, args=(self.evt,self.sock)) - self.thread.daemon = True - self.thread.start() - self.evt.wait() - - def tearDown(self): - self.thread.join() - del self.thread # Clear out any dangling Thread objects. - - def testBasic(self): - # connects - telnet = telnetlib.Telnet(HOST, self.port) - telnet.sock.close() - - def testContextManager(self): - with telnetlib.Telnet(HOST, self.port) as tn: - self.assertIsNotNone(tn.get_socket()) - self.assertIsNone(tn.get_socket()) - - def testTimeoutDefault(self): - self.assertTrue(socket.getdefaulttimeout() is None) - socket.setdefaulttimeout(30) - try: - telnet = telnetlib.Telnet(HOST, self.port) - finally: - socket.setdefaulttimeout(None) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testTimeoutNone(self): - # None, having other default - self.assertTrue(socket.getdefaulttimeout() is None) - socket.setdefaulttimeout(30) - try: - telnet = telnetlib.Telnet(HOST, self.port, timeout=None) - finally: - socket.setdefaulttimeout(None) - self.assertTrue(telnet.sock.gettimeout() is None) - telnet.sock.close() - - def testTimeoutValue(self): - telnet = telnetlib.Telnet(HOST, self.port, timeout=30) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testTimeoutOpen(self): - telnet = telnetlib.Telnet() - telnet.open(HOST, self.port, timeout=30) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testGetters(self): - # Test telnet getter methods - telnet = telnetlib.Telnet(HOST, self.port, timeout=30) - t_sock = telnet.sock - self.assertEqual(telnet.get_socket(), t_sock) - self.assertEqual(telnet.fileno(), t_sock.fileno()) - telnet.sock.close() - -class SocketStub(object): - ''' a socket proxy that re-defines sendall() ''' - def __init__(self, reads=()): - self.reads = list(reads) # Intentionally make a copy. - self.writes = [] - self.block = False - def sendall(self, data): - self.writes.append(data) - def recv(self, size): - out = b'' - while self.reads and len(out) < size: - out += self.reads.pop(0) - if len(out) > size: - self.reads.insert(0, out[size:]) - out = out[:size] - return out - -class TelnetAlike(telnetlib.Telnet): - def fileno(self): - raise NotImplementedError() - def close(self): pass - def sock_avail(self): - return (not self.sock.block) - def msg(self, msg, *args): - with support.captured_stdout() as out: - telnetlib.Telnet.msg(self, msg, *args) - self._messages += out.getvalue() - return - -class MockSelector(selectors.BaseSelector): - - def __init__(self): - self.keys = {} - - @property - def resolution(self): - return 1e-3 - - def register(self, fileobj, events, data=None): - key = selectors.SelectorKey(fileobj, 0, events, data) - self.keys[fileobj] = key - return key - - def unregister(self, fileobj): - return self.keys.pop(fileobj) - - def select(self, timeout=None): - block = False - for fileobj in self.keys: - if isinstance(fileobj, TelnetAlike): - block = fileobj.sock.block - break - if block: - return [] - else: - return [(key, key.events) for key in self.keys.values()] - - def get_map(self): - return self.keys - - -@contextlib.contextmanager -def test_socket(reads): - def new_conn(*ignored): - return SocketStub(reads) - try: - old_conn = socket.create_connection - socket.create_connection = new_conn - yield None - finally: - socket.create_connection = old_conn - return - -def test_telnet(reads=(), cls=TelnetAlike): - ''' return a telnetlib.Telnet object that uses a SocketStub with - reads queued up to be read ''' - for x in reads: - assert type(x) is bytes, x - with test_socket(reads): - telnet = cls('dummy', 0) - telnet._messages = '' # debuglevel output - return telnet - -class ExpectAndReadTestCase(unittest.TestCase): - def setUp(self): - self.old_selector = telnetlib._TelnetSelector - telnetlib._TelnetSelector = MockSelector - def tearDown(self): - telnetlib._TelnetSelector = self.old_selector - -class ReadTests(ExpectAndReadTestCase): - def test_read_until(self): - """ - read_until(expected, timeout=None) - test the blocking version of read_util - """ - want = [b'xxxmatchyyy'] - telnet = test_telnet(want) - data = telnet.read_until(b'match') - self.assertEqual(data, b'xxxmatch', msg=(telnet.cookedq, telnet.rawq, telnet.sock.reads)) - - reads = [b'x' * 50, b'match', b'y' * 50] - expect = b''.join(reads[:-1]) - telnet = test_telnet(reads) - data = telnet.read_until(b'match') - self.assertEqual(data, expect) - - - def test_read_all(self): - """ - read_all() - Read all data until EOF; may block. - """ - reads = [b'x' * 500, b'y' * 500, b'z' * 500] - expect = b''.join(reads) - telnet = test_telnet(reads) - data = telnet.read_all() - self.assertEqual(data, expect) - return - - def test_read_some(self): - """ - read_some() - Read at least one byte or EOF; may block. - """ - # test 'at least one byte' - telnet = test_telnet([b'x' * 500]) - data = telnet.read_some() - self.assertTrue(len(data) >= 1) - # test EOF - telnet = test_telnet() - data = telnet.read_some() - self.assertEqual(b'', data) - - def _read_eager(self, func_name): - """ - read_*_eager() - Read all data available already queued or on the socket, - without blocking. - """ - want = b'x' * 100 - telnet = test_telnet([want]) - func = getattr(telnet, func_name) - telnet.sock.block = True - self.assertEqual(b'', func()) - telnet.sock.block = False - data = b'' - while True: - try: - data += func() - except EOFError: - break - self.assertEqual(data, want) - - def test_read_eager(self): - # read_eager and read_very_eager make the same guarantees - # (they behave differently but we only test the guarantees) - self._read_eager('read_eager') - self._read_eager('read_very_eager') - # NB -- we need to test the IAC block which is mentioned in the - # docstring but not in the module docs - - def read_very_lazy(self): - want = b'x' * 100 - telnet = test_telnet([want]) - self.assertEqual(b'', telnet.read_very_lazy()) - while telnet.sock.reads: - telnet.fill_rawq() - data = telnet.read_very_lazy() - self.assertEqual(want, data) - self.assertRaises(EOFError, telnet.read_very_lazy) - - def test_read_lazy(self): - want = b'x' * 100 - telnet = test_telnet([want]) - self.assertEqual(b'', telnet.read_lazy()) - data = b'' - while True: - try: - read_data = telnet.read_lazy() - data += read_data - if not read_data: - telnet.fill_rawq() - except EOFError: - break - self.assertTrue(want.startswith(data)) - self.assertEqual(data, want) - -class nego_collector(object): - def __init__(self, sb_getter=None): - self.seen = b'' - self.sb_getter = sb_getter - self.sb_seen = b'' - - def do_nego(self, sock, cmd, opt): - self.seen += cmd + opt - if cmd == tl.SE and self.sb_getter: - sb_data = self.sb_getter() - self.sb_seen += sb_data - -tl = telnetlib - -class WriteTests(unittest.TestCase): - '''The only thing that write does is replace each tl.IAC for - tl.IAC+tl.IAC''' - - def test_write(self): - data_sample = [b'data sample without IAC', - b'data sample with' + tl.IAC + b' one IAC', - b'a few' + tl.IAC + tl.IAC + b' iacs' + tl.IAC, - tl.IAC, - b''] - for data in data_sample: - telnet = test_telnet() - telnet.write(data) - written = b''.join(telnet.sock.writes) - self.assertEqual(data.replace(tl.IAC,tl.IAC+tl.IAC), written) - -class OptionTests(unittest.TestCase): - # RFC 854 commands - cmds = [tl.AO, tl.AYT, tl.BRK, tl.EC, tl.EL, tl.GA, tl.IP, tl.NOP] - - def _test_command(self, data): - """ helper for testing IAC + cmd """ - telnet = test_telnet(data) - data_len = len(b''.join(data)) - nego = nego_collector() - telnet.set_option_negotiation_callback(nego.do_nego) - txt = telnet.read_all() - cmd = nego.seen - self.assertTrue(len(cmd) > 0) # we expect at least one command - self.assertIn(cmd[:1], self.cmds) - self.assertEqual(cmd[1:2], tl.NOOPT) - self.assertEqual(data_len, len(txt + cmd)) - nego.sb_getter = None # break the nego => telnet cycle - - def test_IAC_commands(self): - for cmd in self.cmds: - self._test_command([tl.IAC, cmd]) - self._test_command([b'x' * 100, tl.IAC, cmd, b'y'*100]) - self._test_command([b'x' * 10, tl.IAC, cmd, b'y'*10]) - # all at once - self._test_command([tl.IAC + cmd for (cmd) in self.cmds]) - - def test_SB_commands(self): - # RFC 855, subnegotiations portion - send = [tl.IAC + tl.SB + tl.IAC + tl.SE, - tl.IAC + tl.SB + tl.IAC + tl.IAC + tl.IAC + tl.SE, - tl.IAC + tl.SB + tl.IAC + tl.IAC + b'aa' + tl.IAC + tl.SE, - tl.IAC + tl.SB + b'bb' + tl.IAC + tl.IAC + tl.IAC + tl.SE, - tl.IAC + tl.SB + b'cc' + tl.IAC + tl.IAC + b'dd' + tl.IAC + tl.SE, - ] - telnet = test_telnet(send) - nego = nego_collector(telnet.read_sb_data) - telnet.set_option_negotiation_callback(nego.do_nego) - txt = telnet.read_all() - self.assertEqual(txt, b'') - want_sb_data = tl.IAC + tl.IAC + b'aabb' + tl.IAC + b'cc' + tl.IAC + b'dd' - self.assertEqual(nego.sb_seen, want_sb_data) - self.assertEqual(b'', telnet.read_sb_data()) - nego.sb_getter = None # break the nego => telnet cycle - - def test_debuglevel_reads(self): - # test all the various places that self.msg(...) is called - given_a_expect_b = [ - # Telnet.fill_rawq - (b'a', ": recv b''\n"), - # Telnet.process_rawq - (tl.IAC + bytes([88]), ": IAC 88 not recognized\n"), - (tl.IAC + tl.DO + bytes([1]), ": IAC DO 1\n"), - (tl.IAC + tl.DONT + bytes([1]), ": IAC DONT 1\n"), - (tl.IAC + tl.WILL + bytes([1]), ": IAC WILL 1\n"), - (tl.IAC + tl.WONT + bytes([1]), ": IAC WONT 1\n"), - ] - for a, b in given_a_expect_b: - telnet = test_telnet([a]) - telnet.set_debuglevel(1) - txt = telnet.read_all() - self.assertIn(b, telnet._messages) - return - - def test_debuglevel_write(self): - telnet = test_telnet() - telnet.set_debuglevel(1) - telnet.write(b'xxx') - expected = "send b'xxx'\n" - self.assertIn(expected, telnet._messages) - - def test_debug_accepts_str_port(self): - # Issue 10695 - with test_socket([]): - telnet = TelnetAlike('dummy', '0') - telnet._messages = '' - telnet.set_debuglevel(1) - telnet.msg('test') - self.assertRegex(telnet._messages, r'0.*test') - - -class ExpectTests(ExpectAndReadTestCase): - def test_expect(self): - """ - expect(expected, [timeout]) - Read until the expected string has been seen, or a timeout is - hit (default is no timeout); may block. - """ - want = [b'x' * 10, b'match', b'y' * 10] - telnet = test_telnet(want) - (_,_,data) = telnet.expect([b'match']) - self.assertEqual(data, b''.join(want[:-1])) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 97165264b34bbe..648533923dcd81 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -747,7 +747,7 @@ def f(): rc, out, err = assert_python_ok("-c", code) self.assertEqual(err, b"") - def test_tstate_lock(self): + def test_running_lock(self): # Test an implementation detail of Thread objects. started = _thread.allocate_lock() finish = _thread.allocate_lock() @@ -757,29 +757,29 @@ def f(): started.release() finish.acquire() time.sleep(0.01) - # The tstate lock is None until the thread is started + # The running lock is None until the thread is started t = threading.Thread(target=f) - self.assertIs(t._tstate_lock, None) + self.assertIs(t._running_lock, None) t.start() started.acquire() self.assertTrue(t.is_alive()) - # The tstate lock can't be acquired when the thread is running + # The running lock can't be acquired when the thread is running # (or suspended). - tstate_lock = t._tstate_lock - self.assertFalse(tstate_lock.acquire(timeout=0), False) + running_lock = t._running_lock + self.assertFalse(running_lock.acquire(timeout=0), False) finish.release() # When the thread ends, the state_lock can be successfully # acquired. - self.assertTrue(tstate_lock.acquire(timeout=support.SHORT_TIMEOUT), False) - # But is_alive() is still True: we hold _tstate_lock now, which - # prevents is_alive() from knowing the thread's end-of-life C code + self.assertTrue(running_lock.acquire(timeout=support.SHORT_TIMEOUT), False) + # But is_alive() is still True: we hold _running_lock now, which + # prevents is_alive() from knowing the thread's Python code # is done. self.assertTrue(t.is_alive()) # Let is_alive() find out the C code is done. - tstate_lock.release() + running_lock.release() self.assertFalse(t.is_alive()) - # And verify the thread disposed of _tstate_lock. - self.assertIsNone(t._tstate_lock) + # And verify the thread disposed of _running_lock. + self.assertIsNone(t._running_lock) t.join() def test_repr_stopped(self): diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index b45245162f2446..76cc16e5b977de 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -901,6 +901,12 @@ def test_coords(self): c.coords(i, [21, 31, 41, 51, 61, 11]) self.assertEqual(c.coords(i), [21.0, 31.0, 41.0, 51.0, 61.0, 11.0]) + c.coords(i, (22, 32), (42, 52), (62, 12)) + self.assertEqual(c.coords(i), [22.0, 32.0, 42.0, 52.0, 62.0, 12.0]) + + c.coords(i, [(23, 33), (43, 53), (63, 13)]) + self.assertEqual(c.coords(i), [23.0, 33.0, 43.0, 53.0, 63.0, 13.0]) + c.coords(i, 20, 30, 60, 10) self.assertEqual(c.coords(i), [20.0, 30.0, 60.0, 10.0]) self.assertEqual(c.bbox(i), (18, 8, 62, 32)) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index dda7243bfa19fe..8e7ab3d4b7b578 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -92,9 +92,18 @@ def k(x): readline = BytesIO(indent_error_file).readline with self.assertRaisesRegex(IndentationError, "unindent does not match any " - "outer indentation level"): + "outer indentation level") as e: for tok in tokenize(readline): pass + self.assertEqual(e.exception.lineno, 3) + self.assertEqual(e.exception.filename, '') + self.assertEqual(e.exception.end_lineno, None) + self.assertEqual(e.exception.end_offset, None) + self.assertEqual( + e.exception.msg, + 'unindent does not match any outer indentation level') + self.assertEqual(e.exception.offset, 9) + self.assertEqual(e.exception.text, ' x += 5\n') def test_int(self): # Ordinary integers and binary operators diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py index 3177fafb84a65b..2f8ba272164d32 100644 --- a/Lib/test/test_tools/test_sundry.py +++ b/Lib/test/test_tools/test_sundry.py @@ -14,11 +14,6 @@ skip_if_missing() class TestSundryScripts(unittest.TestCase): - # At least make sure the rest don't have syntax errors. When tests are - # added for a script it should be added to the allowlist below. - - skiplist = ['2to3'] - # import logging registers "atfork" functions which keep indirectly the # logging module dictionary alive. Mock the function to be able to unload # cleanly the logging module. @@ -31,9 +26,6 @@ def test_sundry(self, mock_os): continue name = fn[:-3] - if name in self.skiplist: - continue - import_tool(name) finally: # Unload all modules loaded in this test diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9a3e64289ee877..b4a5a68a2f7e4e 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7529,17 +7529,6 @@ def stuff(a: BinaryIO) -> bytes: a = stuff.__annotations__['a'] self.assertEqual(a.__parameters__, ()) - def test_io_submodule(self): - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings("default", category=DeprecationWarning) - from typing.io import IO, TextIO, BinaryIO, __all__, __name__ - self.assertIs(IO, typing.IO) - self.assertIs(TextIO, typing.TextIO) - self.assertIs(BinaryIO, typing.BinaryIO) - self.assertEqual(set(__all__), set(['IO', 'TextIO', 'BinaryIO'])) - self.assertEqual(__name__, 'typing.io') - self.assertEqual(len(w), 1) - class RETests(BaseTestCase): # Much of this is really testing _TypeAlias. @@ -7584,16 +7573,6 @@ def test_repr(self): self.assertEqual(repr(Match[str]), 'typing.Match[str]') self.assertEqual(repr(Match[bytes]), 'typing.Match[bytes]') - def test_re_submodule(self): - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings("default", category=DeprecationWarning) - from typing.re import Match, Pattern, __all__, __name__ - self.assertIs(Match, typing.Match) - self.assertIs(Pattern, typing.Pattern) - self.assertEqual(set(__all__), set(['Match', 'Pattern'])) - self.assertEqual(__name__, 'typing.re') - self.assertEqual(len(w), 1) - def test_cannot_subclass(self): with self.assertRaisesRegex( TypeError, @@ -8765,7 +8744,7 @@ def test_all(self): # Context managers. self.assertIn('ContextManager', a) self.assertIn('AsyncContextManager', a) - # Check that io and re are not exported. + # Check that former namespaces io and re are not exported. self.assertNotIn('io', a) self.assertNotIn('re', a) # Spot-check that stdlib modules aren't exported. @@ -8785,7 +8764,6 @@ def test_all_exported_names(self): if k in actual_all or ( # avoid private names not k.startswith('_') and - k not in {'io', 're'} and # there's a few types and metaclasses that aren't exported not k.endswith(('Meta', '_contra', '_co')) and not k.upper() == k and diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index d1cae47a40eed4..c7895e73ad9a8c 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -38,6 +38,17 @@ def cmeth(cls, a, b, c, d=None): pass def smeth(a, b, c, d=None): pass +class SomethingElse(object): + def __init__(self): + self._instance = None + + @property + def instance(self): + if not self._instance: + self._instance = 'object' + return self._instance + + class Typos(): autospect = None auto_spec = None @@ -2293,6 +2304,26 @@ class Foo(): f'{__name__}.Typos', autospect=True, set_spec=True, auto_spec=True): pass + def test_property_not_called_with_spec_mock(self): + obj = SomethingElse() + self.assertIsNone(obj._instance, msg='before mock') + mock = Mock(spec=obj) + self.assertIsNone(obj._instance, msg='after mock') + self.assertEqual('object', obj.instance) + + def test_decorated_async_methods_with_spec_mock(self): + class Foo(): + @classmethod + async def class_method(cls): + pass + @staticmethod + async def static_method(): + pass + async def method(self): + pass + mock = Mock(spec=Foo) + for m in (mock.method, mock.class_method, mock.static_method): + self.assertIsInstance(m, AsyncMock) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index 51b4f6041e49bf..74df52f5412af0 100644 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -7,8 +7,7 @@ from collections import UserString class UserStringTest( - string_tests.CommonTest, - string_tests.MixinStrUnicodeUserStringTest, + string_tests.StringLikeTest, unittest.TestCase ): diff --git a/Lib/threading.py b/Lib/threading.py index df273870fa4273..69b4f54c050adf 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -908,6 +908,7 @@ class is implemented. self._ident = None if _HAVE_THREAD_NATIVE_ID: self._native_id = None + self._running_lock = None self._tstate_lock = None self._started = Event() self._is_stopped = False @@ -926,6 +927,9 @@ def _reset_internal_locks(self, is_alive): # bpo-42350: If the fork happens when the thread is already stopped # (ex: after threading._shutdown() has been called), _tstate_lock # is None. Do nothing in this case. + if self._running_lock is not None: + self._running_lock._at_fork_reinit() + self._running_lock.acquire() if self._tstate_lock is not None: self._tstate_lock._at_fork_reinit() self._tstate_lock.acquire() @@ -933,6 +937,7 @@ def _reset_internal_locks(self, is_alive): # The thread isn't alive after fork: it doesn't have a tstate # anymore. self._is_stopped = True + self._running_lock = None self._tstate_lock = None def __repr__(self): @@ -1019,6 +1024,14 @@ def _set_ident(self): def _set_native_id(self): self._native_id = get_native_id() + def _set_running_lock(self): + """ + Set a lock object which will be released by the interpreter when + the target func has finished running. + """ + self._running_lock = _allocate_lock() + self._running_lock.acquire() + def _set_tstate_lock(self): """ Set a lock object which will be released by the interpreter when @@ -1035,6 +1048,7 @@ def _set_tstate_lock(self): def _bootstrap_inner(self): try: self._set_ident() + self._set_running_lock() self._set_tstate_lock() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() @@ -1054,29 +1068,29 @@ def _bootstrap_inner(self): self._invoke_excepthook(self) finally: self._delete() + self._running_lock.release() def _stop(self): # After calling ._stop(), .is_alive() returns False and .join() returns - # immediately. ._tstate_lock must be released before calling ._stop(). + # immediately. ._running_lock must be released before calling ._stop(). # - # Normal case: C code at the end of the thread's life - # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and - # that's detected by our ._wait_for_tstate_lock(), called by .join() + # Normal case: ._bootstrap_inner() releases ._running_lock, and + # that's detected by our ._wait_for_running_lock(), called by .join() # and .is_alive(). Any number of threads _may_ call ._stop() # simultaneously (for example, if multiple threads are blocked in # .join() calls), and they're not serialized. That's harmless - # they'll just make redundant rebindings of ._is_stopped and - # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the - # "assert self._is_stopped" in ._wait_for_tstate_lock() always works - # (the assert is executed only if ._tstate_lock is None). + # ._running_lock. Obscure: we rebind ._running_lock last so that the + # "assert self._is_stopped" in ._wait_for_running_lock() always works + # (the assert is executed only if ._running_lock is None). # - # Special case: _main_thread releases ._tstate_lock via this + # Special case: _main_thread releases ._running_lock via this # module's _shutdown() function. - lock = self._tstate_lock + lock = self._running_lock if lock is not None: assert not lock.locked() self._is_stopped = True - self._tstate_lock = None + self._running_lock = None if not self.daemon: with _shutdown_locks_lock: # Remove our lock and other released locks from _shutdown_locks @@ -1123,20 +1137,17 @@ def join(self, timeout=None): raise RuntimeError("cannot join current thread") if timeout is None: - self._wait_for_tstate_lock() + self._wait_for_running_lock() else: # the behavior of a negative timeout isn't documented, but # historically .join(timeout=x) for x<0 has acted as if timeout=0 - self._wait_for_tstate_lock(timeout=max(timeout, 0)) - - def _wait_for_tstate_lock(self, block=True, timeout=-1): - # Issue #18808: wait for the thread state to be gone. - # At the end of the thread's life, after all knowledge of the thread - # is removed from C data structures, C code releases our _tstate_lock. - # This method passes its arguments to _tstate_lock.acquire(). - # If the lock is acquired, the C code is done, and self._stop() is - # called. That sets ._is_stopped to True, and ._tstate_lock to None. - lock = self._tstate_lock + self._wait_for_running_lock(timeout=max(timeout, 0)) + + def _wait_for_running_lock(self, block=True, timeout=-1): + # This method passes its arguments to _running_lock.acquire(). + # If the lock is acquired, the python code is done, and self._stop() is + # called. That sets ._is_stopped to True, and ._running_lock to None. + lock = self._running_lock if lock is None: # already determined that the C code is done assert self._is_stopped @@ -1207,7 +1218,7 @@ def is_alive(self): assert self._initialized, "Thread.__init__() not called" if self._is_stopped or not self._started.is_set(): return False - self._wait_for_tstate_lock(False) + self._wait_for_running_lock(False) return not self._is_stopped @property @@ -1417,7 +1428,7 @@ class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) - self._set_tstate_lock() + self._set_running_lock() self._started.set() self._set_ident() if _HAVE_THREAD_NATIVE_ID: @@ -1558,7 +1569,7 @@ def _shutdown(): # dubious, but some code does it. We can't wait for C code to release # the main thread's tstate_lock - that won't happen until the interpreter # is nearly dead. So we release it here. Note that just calling _stop() - # isn't enough: other threads may already be waiting on _tstate_lock. + # isn't enough: other threads may already be waiting on _running_lock. if _main_thread._is_stopped: # _shutdown() was already called return @@ -1573,12 +1584,13 @@ def _shutdown(): # Main thread if _main_thread.ident == get_ident(): - tlock = _main_thread._tstate_lock - # The main thread isn't finished yet, so its thread state lock can't + assert _main_thread._tstate_lock is None + running_lock = _main_thread._running_lock + # The main thread isn't finished yet, so its running lock can't # have been released. - assert tlock is not None - assert tlock.locked() - tlock.release() + assert running_lock is not None + assert running_lock.locked() + running_lock.release() _main_thread._stop() else: # bpo-1596321: _shutdown() must be called in the main thread. diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index bf0b3b92155938..c675c511e04533 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -2817,7 +2817,7 @@ def canvasy(self, screeny, gridspacing=None): def coords(self, *args): """Return a list of coordinates for the item given in ARGS.""" - # XXX Should use _flatten on args + args = _flatten(args) return [self.tk.getdouble(x) for x in self.tk.splitlist( self.tk.call((self._w, 'coords') + args))] diff --git a/Lib/tokenize.py b/Lib/tokenize.py index bfe40c627fde57..911f0f12f9bb7e 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -56,6 +56,107 @@ def exact_type(self): else: return self.type +def group(*choices): return '(' + '|'.join(choices) + ')' +def any(*choices): return group(*choices) + '*' +def maybe(*choices): return group(*choices) + '?' + +# Note: we use unicode matching for names ("\w") but ascii matching for +# number literals. +Whitespace = r'[ \f\t]*' +Comment = r'#[^\r\n]*' +Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) +Name = r'\w+' + +Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+' +Binnumber = r'0[bB](?:_?[01])+' +Octnumber = r'0[oO](?:_?[0-7])+' +Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)' +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) +Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*' +Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?', + r'\.[0-9](?:_?[0-9])*') + maybe(Exponent) +Expfloat = r'[0-9](?:_?[0-9])*' + Exponent +Floatnumber = group(Pointfloat, Expfloat) +Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]') +Number = group(Imagnumber, Floatnumber, Intnumber) + +# Return the empty string, plus all of the valid string prefixes. +def _all_string_prefixes(): + # The valid string prefixes. Only contain the lower case versions, + # and don't contain any permutations (include 'fr', but not + # 'rf'). The various permutations will be generated. + _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + # if we add binary f-strings, add: ['fb', 'fbr'] + result = {''} + for prefix in _valid_string_prefixes: + for t in _itertools.permutations(prefix): + # create a list with upper and lower versions of each + # character + for u in _itertools.product(*[(c, c.upper()) for c in t]): + result.add(''.join(u)) + return result + +@functools.lru_cache +def _compile(expr): + return re.compile(expr, re.UNICODE) + +# Note that since _all_string_prefixes includes the empty string, +# StringPrefix can be the empty string (making it optional). +StringPrefix = group(*_all_string_prefixes()) + +# Tail end of ' string. +Single = r"[^'\\]*(?:\\.[^'\\]*)*'" +# Tail end of " string. +Double = r'[^"\\]*(?:\\.[^"\\]*)*"' +# Tail end of ''' string. +Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" +# Tail end of """ string. +Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' +Triple = group(StringPrefix + "'''", StringPrefix + '"""') +# Single-line ' or " string. +String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') + +# Sorting in reverse order puts the long operators before their prefixes. +# Otherwise if = came before ==, == would get recognized as two instances +# of =. +Special = group(*map(re.escape, sorted(EXACT_TOKEN_TYPES, reverse=True))) +Funny = group(r'\r?\n', Special) + +PlainToken = group(Number, Funny, String, Name) +Token = Ignore + PlainToken + +# First (or only) line of ' or " string. +ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + + group("'", r'\\\r?\n'), + StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + group('"', r'\\\r?\n')) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) +PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) + +# For a given string prefix plus quotes, endpats maps it to a regex +# to match the remainder of that string. _prefix can be empty, for +# a normal single or triple quoted string (with no prefix). +endpats = {} +for _prefix in _all_string_prefixes(): + endpats[_prefix + "'"] = Single + endpats[_prefix + '"'] = Double + endpats[_prefix + "'''"] = Single3 + endpats[_prefix + '"""'] = Double3 +del _prefix + +# A set of all of the single and triple quoted string prefixes, +# including the opening quotes. +single_quoted = set() +triple_quoted = set() +for t in _all_string_prefixes(): + for u in (t + '"', t + "'"): + single_quoted.add(u) + for u in (t + '"""', t + "'''"): + triple_quoted.add(u) +del t, u + +tabsize = 8 class TokenError(Exception): pass @@ -348,16 +449,6 @@ def _tokenize(rl_gen, encoding): source = b"".join(rl_gen).decode(encoding) token = None for token in _generate_tokens_from_c_tokenizer(source, extra_tokens=True): - # TODO: Marta -> limpiar esto - if 6 < token.type <= 54: - token = token._replace(type=OP) - if token.type in {ASYNC, AWAIT}: - token = token._replace(type=NAME) - if token.type == NEWLINE: - l_start, c_start = token.start - l_end, c_end = token.end - token = token._replace(string='\n', start=(l_start, c_start), end=(l_end, c_end+1)) - yield token if token is not None: last_line, _ = token.start @@ -449,8 +540,7 @@ def _generate_tokens_from_c_tokenizer(source, extra_tokens=False): """Tokenize a source reading Python code as unicode strings using the internal C tokenizer""" import _tokenize as c_tokenizer for info in c_tokenizer.TokenizerIter(source, extra_tokens=extra_tokens): - tok, type, lineno, end_lineno, col_off, end_col_off, line = info - yield TokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line) + yield TokenInfo._make(info) if __name__ == "__main__": diff --git a/Lib/typing.py b/Lib/typing.py index 96393d6a02815b..95dbc0b85bcddf 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -16,7 +16,6 @@ no_type_check_decorator. * Generic aliases for collections.abc ABCs and few additional protocols. * Special types: NewType, NamedTuple, TypedDict. -* Wrapper submodules for re and io related types. """ from abc import abstractmethod, ABCMeta @@ -27,7 +26,7 @@ import contextlib import functools import operator -import re as stdlib_re # Avoid confusion with the re we export. +import re as stdlib_re # Avoid confusion with the typing.re namespace on <=3.11 import sys import types import warnings @@ -158,10 +157,6 @@ 'Unpack', ] -# The pseudo-submodules 're' and 'io' are part of the public -# namespace, but excluded from __all__ because they might stomp on -# legitimate imports of those modules. - def _type_convert(arg, module=None, *, allow_special_forms=False): """For converting None to type(None), and strings to ForwardRef.""" @@ -3150,45 +3145,9 @@ def __enter__(self) -> 'TextIO': pass -class _DeprecatedType(type): - def __getattribute__(cls, name): - if name not in ("__dict__", "__module__") and name in cls.__dict__: - warnings.warn( - f"{cls.__name__} is deprecated, import directly " - f"from typing instead. {cls.__name__} will be removed " - "in Python 3.12.", - DeprecationWarning, - stacklevel=2, - ) - return super().__getattribute__(name) - - -class io(metaclass=_DeprecatedType): - """Wrapper namespace for IO generic classes.""" - - __all__ = ['IO', 'TextIO', 'BinaryIO'] - IO = IO - TextIO = TextIO - BinaryIO = BinaryIO - - -io.__name__ = __name__ + '.io' -sys.modules[io.__name__] = io - Pattern = _alias(stdlib_re.Pattern, 1) Match = _alias(stdlib_re.Match, 1) -class re(metaclass=_DeprecatedType): - """Wrapper namespace for re type aliases.""" - - __all__ = ['Pattern', 'Match'] - Pattern = Pattern - Match = Match - - -re.__name__ = __name__ + '.re' -sys.modules[re.__name__] = re - def reveal_type[T](obj: T, /) -> T: """Reveal the inferred type of a variable. diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 7ca085760650af..22f81e55b567f2 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -526,7 +526,13 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, spec_list = dir(spec) for attr in spec_list: - if iscoroutinefunction(getattr(spec, attr, None)): + static_attr = inspect.getattr_static(spec, attr, None) + unwrapped_attr = static_attr + try: + unwrapped_attr = inspect.unwrap(unwrapped_attr) + except ValueError: + pass + if iscoroutinefunction(unwrapped_attr): _spec_asyncs.append(attr) spec = spec_list diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 5474f87fd6cb62..177fc2044f694b 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -264,10 +264,10 @@ def library_recipes(): tk_patches = ['tk868_on_10_8_10_9.patch'] else: - tcl_tk_ver='8.6.12' - tcl_checksum='87ea890821d2221f2ab5157bc5eb885f' + tcl_tk_ver='8.6.13' + tcl_checksum='43a1fae7412f61ff11de2cfd05d28cfc3a73762f354a417c62370a54e2caf066' - tk_checksum='1d6dcf6120356e3d211e056dff5e462a' + tk_checksum='2e65fa069a23365440a3c56c556b8673b5e32a283800d8d9b257e3f584ce0675' tk_patches = [ ] diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index dbe8c520a7c78d..5bc356d5267045 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2639 +{\rtf1\ansi\ansicpg1252\cocoartf2709 \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fswiss\fcharset0 Helvetica-Oblique; \f3\fmodern\fcharset0 CourierNewPSMT;\f4\fmodern\fcharset0 Courier;} {\colortbl;\red255\green255\blue255;} @@ -11,7 +11,7 @@ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f1\b \cf0 NOTE: -\f0\b0 This is an alpha preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \cf0 \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 1f109ee9f13e0d..83b7aa9d883a16 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2708 +{\rtf1\ansi\ansicpg1252\cocoartf2709 \cocoascreenfonts1\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT; } {\colortbl;\red255\green255\blue255;} @@ -12,8 +12,9 @@ \f1\b macOS $MACOSX_DEPLOYMENT_TARGET \f0\b0 .\ \ +\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 -\f1\b Python for macOS +\f1\b \cf0 Python for macOS \f0\b0 consists of the {\field{\*\fldinst{HYPERLINK "https://www.python.org"}}{\fldrslt Python}} programming language interpreter and its batteries-included standard library to allow easy access to macOS features. It also includes the Python integrated development environment, \f1\b IDLE \f0\b0 . You can also use the included @@ -25,28 +26,6 @@ At the end of this install, click on \f0 to install a set of current SSL root certificates.\ \ -\f1\b macOS 13 Ventura users -\f0\b0 : Due to an issue with the macOS -\f1\b Installer -\f0\b0 app, installation of some third-party packages including this Python package may fail with a vague -\f1\b "The installer encountered an error" -\f0\b0 message if the -\f1\b Installer -\f0\b0 app does not have permission to access the folder containing the downloaded installer file, typically in the -\f1\b Downloads -\f0\b0 folder. Go to -\f1\b System Settings -\f0\b0 -> -\f1\b Privacy & Security -\f0\b0 -> -\f1\b Files and Folders -\f0\b0 , then click the mark in front of -\f1\b Installer -\f0\b0 to expand, and enable -\f1\b Downloads Folder -\f0\b0 by moving the toggle to the right. See {\field{\*\fldinst{HYPERLINK "https://github.com/python/cpython/issues/103207"}}{\fldrslt https://github.com/python/cpython/issues/103207}} for up-to-date information on this issue.\ -\ - \f1\b NOTE: -\f0\b0 This is an alpha test preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta test preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ } \ No newline at end of file diff --git a/Mac/BuildScript/scripts/postflight.framework b/Mac/BuildScript/scripts/postflight.framework index a63909358f1719..411fda13b89559 100755 --- a/Mac/BuildScript/scripts/postflight.framework +++ b/Mac/BuildScript/scripts/postflight.framework @@ -8,12 +8,12 @@ FWK="/Library/Frameworks/Python.framework/Versions/@PYVER@" "${FWK}/bin/python@PYVER@" -E -s -Wi \ "${FWK}/lib/python${PYVER}/compileall.py" -q -j0 \ - -f -x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \ + -f -x 'bad_coding|badsyntax|site-packages' \ "${FWK}/lib/python${PYVER}" "${FWK}/bin/python@PYVER@" -E -s -Wi -O \ "${FWK}/lib/python${PYVER}/compileall.py" -q -j0 \ - -f -x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \ + -f -x 'bad_coding|badsyntax|site-packages' \ "${FWK}/lib/python${PYVER}" "${FWK}/bin/python@PYVER@" -E -s -Wi \ diff --git a/Mac/Makefile.in b/Mac/Makefile.in index 69ab4198988570..78b4499cca986a 100644 --- a/Mac/Makefile.in +++ b/Mac/Makefile.in @@ -63,7 +63,6 @@ installunixtools: fi cd "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin" && \ for fn in \ - 2to3 \ idle3 \ pydoc3 \ python3 \ @@ -123,7 +122,6 @@ altinstallunixtools: fi cd "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin" && \ for fn in \ - 2to3-$(VERSION) \ idle$(VERSION) \ pydoc$(VERSION) \ python$(VERSION) \ diff --git a/Makefile.pre.in b/Makefile.pre.in index da3a8f6c13f90b..091df3e4c0b814 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -672,21 +672,55 @@ profile-opt: profile-run-stamp -rm -f profile-clean-stamp $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_USE_FLAG)" LDFLAGS_NODIST="$(LDFLAGS_NODIST)" -.PHONY: bolt-opt -bolt-opt: @PREBOLT_RULE@ +# List of binaries that BOLT runs on. +BOLT_BINARIES := @BOLT_BINARIES@ + +BOLT_INSTRUMENT_FLAGS := @BOLT_INSTRUMENT_FLAGS@ +BOLT_APPLY_FLAGS := @BOLT_APPLY_FLAGS@ + +.PHONY: clean-bolt +clean-bolt: + # Profile data. rm -f *.fdata - @if $(READELF) -p .note.bolt_info $(BUILDPYTHON) | grep BOLT > /dev/null; then\ - echo "skip: $(BUILDPYTHON) is already BOLTed."; \ - else \ - @LLVM_BOLT@ ./$(BUILDPYTHON) -instrument -instrumentation-file-append-pid -instrumentation-file=$(abspath $(BUILDPYTHON).bolt) -o $(BUILDPYTHON).bolt_inst; \ - ./$(BUILDPYTHON).bolt_inst $(PROFILE_TASK) || true; \ - @MERGE_FDATA@ $(BUILDPYTHON).*.fdata > $(BUILDPYTHON).fdata; \ - @LLVM_BOLT@ ./$(BUILDPYTHON) -o $(BUILDPYTHON).bolt -data=$(BUILDPYTHON).fdata -update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot; \ - rm -f *.fdata; \ - rm -f $(BUILDPYTHON).bolt_inst; \ - mv $(BUILDPYTHON).bolt $(BUILDPYTHON); \ - fi + # Pristine binaries before BOLT optimization. + rm -f *.prebolt + # BOLT instrumented binaries. + rm -f *.bolt_inst + +profile-bolt-stamp: $(BUILDPYTHON) + # Ensure a pristine, pre-BOLT copy of the binary and no profile data from last run. + for bin in $(BOLT_BINARIES); do \ + prebolt="$${bin}.prebolt"; \ + if [ -e "$${prebolt}" ]; then \ + echo "Restoring pre-BOLT binary $${prebolt}"; \ + mv "$${bin}.prebolt" "$${bin}"; \ + fi; \ + cp "$${bin}" "$${prebolt}"; \ + rm -f $${bin}.bolt.*.fdata $${bin}.fdata; \ + done + # Instrument each binary. + for bin in $(BOLT_BINARIES); do \ + @LLVM_BOLT@ "$${bin}" -instrument -instrumentation-file-append-pid -instrumentation-file=$(abspath $${bin}.bolt) -o $${bin}.bolt_inst $(BOLT_INSTRUMENT_FLAGS); \ + mv "$${bin}.bolt_inst" "$${bin}"; \ + done + # Run instrumented binaries to collect data. + $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true + # Merge all the data files together. + for bin in $(BOLT_BINARIES); do \ + @MERGE_FDATA@ $${bin}.*.fdata > "$${bin}.fdata"; \ + rm -f $${bin}.*.fdata; \ + done + # Run bolt against the merged data to produce an optimized binary. + for bin in $(BOLT_BINARIES); do \ + @LLVM_BOLT@ "$${bin}.prebolt" -o "$${bin}.bolt" -data="$${bin}.fdata" $(BOLT_APPLY_FLAGS); \ + mv "$${bin}.bolt" "$${bin}"; \ + done + touch $@ +.PHONY: bolt-opt +bolt-opt: + $(MAKE) @PREBOLT_RULE@ + $(MAKE) profile-bolt-stamp # Compile and run with gcov .PHONY: coverage @@ -1835,7 +1869,7 @@ hostrunnertest: all pythoninfo: all $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo -QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ +QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io \ test_multibytecodec test_urllib2_localnet test_itertools \ test_multiprocessing_fork test_multiprocessing_spawn \ test_multiprocessing_forkserver \ @@ -2008,8 +2042,6 @@ bininstall: altbininstall (cd $(DESTDIR)$(BINDIR); $(LN) -s idle$(VERSION) idle3) -rm -f $(DESTDIR)$(BINDIR)/pydoc3 (cd $(DESTDIR)$(BINDIR); $(LN) -s pydoc$(VERSION) pydoc3) - -rm -f $(DESTDIR)$(BINDIR)/2to3 - (cd $(DESTDIR)$(BINDIR); $(LN) -s 2to3-$(VERSION) 2to3) if test "x$(LIPO_32BIT_FLAGS)" != "x" ; then \ rm -f $(DESTDIR)$(BINDIR)/python3-32$(EXE); \ (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-32$(EXE) python3-32$(EXE)) \ @@ -2056,7 +2088,6 @@ LIBSUBDIRS= asyncio \ idlelib idlelib/Icons \ importlib importlib/resources importlib/metadata \ json \ - lib2to3 lib2to3/fixes lib2to3/pgen2 \ logging \ multiprocessing multiprocessing/dummy \ pydoc_data \ @@ -2150,10 +2181,6 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/resources/zipdata02 \ test/test_importlib/source \ test/test_json \ - test/test_lib2to3 \ - test/test_lib2to3/data \ - test/test_lib2to3/data/fixers \ - test/test_lib2to3/data/fixers/myfixes \ test/test_peg_generator \ test/test_sqlite3 \ test/test_tkinter \ @@ -2267,16 +2294,12 @@ libinstall: all $(srcdir)/Modules/xxmodule.c -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ -o 0 -o 1 -o 2 $(COMPILEALL_OPTS) -d $(LIBDEST) -f \ - -x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \ + -x 'bad_coding|badsyntax|site-packages' \ $(DESTDIR)$(LIBDEST) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ -o 0 -o 1 -o 2 $(COMPILEALL_OPTS) -d $(LIBDEST)/site-packages -f \ -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ - $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/Grammar.txt - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ - $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/PatternGrammar.txt # bpo-21536: Misc/python-config.sh is generated in the build directory # from $(srcdir)Misc/python-config.sh.in. @@ -2296,15 +2319,9 @@ python-config: $(srcdir)/Misc/python-config.in Misc/python-config.sh # macOS' make seems to ignore a dependency on a # "$(BUILD_SCRIPTS_DIR): $(MKDIR_P) $@" rule. BUILD_SCRIPTS_DIR=build/scripts-$(VERSION) -SCRIPT_2TO3=$(BUILD_SCRIPTS_DIR)/2to3-$(VERSION) SCRIPT_IDLE=$(BUILD_SCRIPTS_DIR)/idle$(VERSION) SCRIPT_PYDOC=$(BUILD_SCRIPTS_DIR)/pydoc$(VERSION) -$(SCRIPT_2TO3): $(srcdir)/Tools/scripts/2to3 - @$(MKDIR_P) $(BUILD_SCRIPTS_DIR) - sed -e "s,/usr/bin/env python3,$(EXENAME)," < $(srcdir)/Tools/scripts/2to3 > $@ - @chmod +x $@ - $(SCRIPT_IDLE): $(srcdir)/Tools/scripts/idle3 @$(MKDIR_P) $(BUILD_SCRIPTS_DIR) sed -e "s,/usr/bin/env python3,$(EXENAME)," < $(srcdir)/Tools/scripts/idle3 > $@ @@ -2316,7 +2333,7 @@ $(SCRIPT_PYDOC): $(srcdir)/Tools/scripts/pydoc3 @chmod +x $@ .PHONY: scripts -scripts: $(SCRIPT_2TO3) $(SCRIPT_IDLE) $(SCRIPT_PYDOC) python-config +scripts: $(SCRIPT_IDLE) $(SCRIPT_PYDOC) python-config # Install the include files INCLDIRSTOMAKE=$(INCLUDEDIR) $(CONFINCLUDEDIR) $(INCLUDEPY) $(CONFINCLUDEPY) @@ -2402,7 +2419,6 @@ libainstall: all scripts $(INSTALL_SCRIPT) $(srcdir)/install-sh $(DESTDIR)$(LIBPL)/install-sh $(INSTALL_SCRIPT) python-config.py $(DESTDIR)$(LIBPL)/python-config.py $(INSTALL_SCRIPT) python-config $(DESTDIR)$(BINDIR)/python$(LDVERSION)-config - $(INSTALL_SCRIPT) $(SCRIPT_2TO3) $(DESTDIR)$(BINDIR)/2to3-$(VERSION) $(INSTALL_SCRIPT) $(SCRIPT_IDLE) $(DESTDIR)$(BINDIR)/idle$(VERSION) $(INSTALL_SCRIPT) $(SCRIPT_PYDOC) $(DESTDIR)$(BINDIR)/pydoc$(VERSION) @if [ -s Modules/python.exp -a \ @@ -2603,7 +2619,6 @@ clean-retain-profile: pycremoval find build -name '*.py' -exec rm -f {} ';' || true find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt - -rm -f Lib/lib2to3/*Grammar*.pickle -rm -f _bootstrap_python -rm -f python.html python*.js python.data python*.symbols python*.map -rm -f $(WASM_STDLIB) @@ -2623,10 +2638,11 @@ profile-removal: rm -f $(COVERAGE_INFO) rm -rf $(COVERAGE_REPORT) rm -f profile-run-stamp + rm -f profile-bolt-stamp .PHONY: clean -clean: clean-retain-profile - @if test @DEF_MAKE_ALL_RULE@ = profile-opt; then \ +clean: clean-retain-profile clean-bolt + @if test @DEF_MAKE_ALL_RULE@ = profile-opt -o @DEF_MAKE_ALL_RULE@ = bolt-opt; then \ rm -f profile-gen-stamp profile-clean-stamp; \ $(MAKE) profile-removal; \ fi diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 7670e482ede5b6..10e123e325c863 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -1825,7 +1825,7 @@ Patch by Victor Stinner. .. nonce: 9nmMtB .. section: Library -The :mod:`lib2to3` package is now deprecated and may not be able to parse +The :mod:`!lib2to3` package is now deprecated and may not be able to parse Python 3.10 or newer. See the :pep:`617` (New PEG parser for CPython). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst new file mode 100644 index 00000000000000..a1ea082b3a2119 --- /dev/null +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -0,0 +1,2402 @@ +.. date: 2023-05-02-17-56-32 +.. gh-issue: 99889 +.. nonce: l664SU +.. release date: 2023-05-22 +.. section: Security + +Fixed a security in flaw in :func:`uu.decode` that could allow for directory +traversal based on the input if no ``out_file`` was specified. + +.. + +.. date: 2023-05-01-15-03-25 +.. gh-issue: 104049 +.. nonce: b01Y3g +.. section: Security + +Do not expose the local on-disk location in directory indexes produced by +:class:`http.client.SimpleHTTPRequestHandler`. + +.. + +.. date: 2023-04-17-14-38-12 +.. gh-issue: 99108 +.. nonce: 720lG8 +.. section: Security + +Upgrade built-in :mod:`hashlib` SHA3 implementation to a verified +implementation from the ``HACL*`` project. Used when OpenSSL is not present +or lacks SHA3. + +.. + +.. date: 2023-03-07-20-59-17 +.. gh-issue: 102153 +.. nonce: 14CLSZ +.. section: Security + +:func:`urllib.parse.urlsplit` now strips leading C0 control and space +characters following the specification for URLs defined by WHATWG in +response to CVE-2023-24329. Patch by Illia Volochii. + +.. + +.. date: 2023-05-20-23-08-48 +.. gh-issue: 102856 +.. nonce: Knv9WT +.. section: Core and Builtins + +Implement PEP 701 changes in the :mod:`tokenize` module. Patch by Marta +Gómez Macías and Pablo Galindo Salgado + +.. + +.. date: 2023-05-18-13-00-21 +.. gh-issue: 104615 +.. nonce: h_rtw2 +.. section: Core and Builtins + +Fix wrong ordering of assignments in code like ``a, a = x, y``. Contributed +by Carl Meyer. + +.. + +.. date: 2023-05-16-19-17-48 +.. gh-issue: 104572 +.. nonce: eBZQYS +.. section: Core and Builtins + +Improve syntax error message for invalid constructs in :pep:`695` contexts +and in annotations when ``from __future__ import annotations`` is active. + +.. + +.. date: 2023-05-14-18-56-54 +.. gh-issue: 104482 +.. nonce: yaQsv8 +.. section: Core and Builtins + +Fix three error handling bugs in ast.c's validation of pattern matching +statements. + +.. + +.. date: 2023-05-12-13-30-04 +.. gh-issue: 102818 +.. nonce: rnv1mH +.. section: Core and Builtins + +Do not add a frame to the traceback in the ``sys.setprofile`` and +``sys.settrace`` trampoline functions. This ensures that frames are not +duplicated if an exception is raised in the callback function, and ensures +that frames are not omitted if a C callback is used and that does not add +the frame. + +.. + +.. date: 2023-05-11-15-56-07 +.. gh-issue: 104405 +.. nonce: tXV5fn +.. section: Core and Builtins + +Fix an issue where some :term:`bytecode` instructions could ignore +:pep:`523` when "inlining" calls. + +.. + +.. date: 2023-05-10-20-52-29 +.. gh-issue: 103082 +.. nonce: y3LG5Q +.. section: Core and Builtins + +Change behavior of ``sys.monitoring.events.LINE`` events in +``sys.monitoring``: Line events now occur when a new line is reached +dynamically, instead of using a static approximation, as before. This makes +the behavior very similar to that of "line" events in ``sys.settrace``. This +should ease porting of tools from 3.11 to 3.12. + +.. + +.. date: 2023-05-08-10-34-55 +.. gh-issue: 104263 +.. nonce: ctHWI8 +.. section: Core and Builtins + +Fix ``float("nan")`` to produce a quiet NaN on platforms (like MIPS) where +the meaning of the signalling / quiet bit is inverted from its usual +meaning. Also introduce a new macro ``Py_INFINITY`` matching C99's +``INFINITY``, and refactor internals to rely on C99's ``NAN`` and +``INFINITY`` macros instead of hard-coding bit patterns for infinities and +NaNs. Thanks Sebastian Berg. + +.. + +.. date: 2023-05-05-13-18-56 +.. gh-issue: 99113 +.. nonce: hT1ajK +.. section: Core and Builtins + +Multi-phase init extension modules may now indicate that they support +running in subinterpreters that have their own GIL. This is done by using +``Py_MOD_PER_INTERPRETER_GIL_SUPPORTED`` as the value for the +``Py_mod_multiple_interpreters`` module def slot. Otherwise the module, by +default, cannot be imported in such subinterpreters. (This does not affect +the main interpreter or subinterpreters that do not have their own GIL.) In +addition to the isolation that multi-phase init already normally requires, +support for per-interpreter GIL involves one additional constraint: +thread-safety. If the module has external (linked) dependencies and those +libraries have any state that isn't thread-safe then the module must do the +additional work to add thread-safety. This should be an uncommon case. + +.. + +.. date: 2023-05-05-12-14-47 +.. gh-issue: 99113 +.. nonce: -RAdnv +.. section: Core and Builtins + +The GIL is now (optionally) per-interpreter. This is the fundamental change +for PEP 684. This is all made possible by virtue of the isolated state of +each interpreter in the process. The behavior of the main interpreter +remains unchanged. Likewise, interpreters created using +``Py_NewInterpreter()`` are not affected. To get an interpreter with its +own GIL, call ``Py_NewInterpreterFromConfig()``. + +.. + +.. date: 2023-05-03-17-46-47 +.. gh-issue: 104108 +.. nonce: GOxAYt +.. section: Core and Builtins + +Multi-phase init extension modules may now indicate whether or not they +actually support multiple interpreters. By default such modules are +expected to support use in multiple interpreters. In the uncommon case that +one does not, it may use the new ``Py_mod_multiple_interpreters`` module def +slot. A value of ``0`` means the module does not support them. ``1`` means +it does. The default is ``1``. + +.. + +.. date: 2023-05-02-18-29-49 +.. gh-issue: 104142 +.. nonce: _5Et6I +.. section: Core and Builtins + +Fix an issue where :class:`list` or :class:`tuple` repetition could fail to +respect :pep:`683`. + +.. + +.. date: 2023-05-01-21-05-47 +.. gh-issue: 104078 +.. nonce: vRaBsU +.. section: Core and Builtins + +Improve the performance of :c:func:`PyObject_HasAttrString` + +.. + +.. date: 2023-05-01-14-48-29 +.. gh-issue: 104066 +.. nonce: pzoUZQ +.. section: Core and Builtins + +Improve the performance of :func:`hasattr` for module objects with a missing +attribute. + +.. + +.. date: 2023-05-01-14-10-38 +.. gh-issue: 104028 +.. nonce: dxfh13 +.. section: Core and Builtins + +Reduce object creation while calling callback function from gc. Patch by +Dong-hee Na. + +.. + +.. date: 2023-05-01-12-03-52 +.. gh-issue: 104018 +.. nonce: PFxGS4 +.. section: Core and Builtins + +Disallow the "z" format specifier in %-format of bytes objects. + +.. + +.. date: 2023-05-01-08-08-05 +.. gh-issue: 102213 +.. nonce: nfH-4C +.. section: Core and Builtins + +Fix performance loss when accessing an object's attributes with +``__getattr__`` defined. + +.. + +.. date: 2023-04-26-17-56-18 +.. gh-issue: 103895 +.. nonce: ESB6tn +.. section: Core and Builtins + +Improve handling of edge cases in showing ``Exception.__notes__``. Ensures +that the messages always end with a newline and that string/bytes are not +exploded over multiple lines. Patch by Carey Metcalfe. + +.. + +.. date: 2023-04-26-16-26-35 +.. gh-issue: 103907 +.. nonce: kiONZQ +.. section: Core and Builtins + +Don't modify the refcounts of known immortal objects (:const:`True`, +:const:`False`, and :const:`None`) in the main interpreter loop. + +.. + +.. date: 2023-04-26-15-14-23 +.. gh-issue: 103899 +.. nonce: 1pqKPF +.. section: Core and Builtins + +Provide a helpful hint in the :exc:`TypeError` message when accidentally +calling a :term:`module` object that has a callable attribute of the same +name (such as :func:`dis.dis` or :class:`datetime.datetime`). + +.. + +.. date: 2023-04-25-20-56-01 +.. gh-issue: 103845 +.. nonce: V7NYFn +.. section: Core and Builtins + +Remove both line and instruction instrumentation before adding new ones for +monitoring, to avoid newly added instrumentation being removed immediately. + +.. + +.. date: 2023-04-25-08-43-11 +.. gh-issue: 103763 +.. nonce: ZLBZk1 +.. section: Core and Builtins + +Implement :pep:`695`, adding syntactic support for generic classes, generic +functions, and type aliases. + +A new ``type X = ...`` syntax is added for type aliases, which resolves at +runtime to an instance of the new class ``typing.TypeAliasType``. The value +is lazily evaluated and is accessible through the ``.__value__`` attribute. +This is implemented as a new AST node ``ast.TypeAlias``. + +New syntax (``class X[T]: ...``, ``def func[T](): ...``) is added for +defining generic functions and classes. This is implemented as a new +``type_params`` attribute on the AST nodes for classes and functions. This +node holds instances of the new AST classes ``ast.TypeVar``, +``ast.ParamSpec``, and ``ast.TypeVarTuple``. + +``typing.TypeVar``, ``typing.ParamSpec``, ``typing.ParamSpecArgs``, +``typing.ParamSpecKwargs``, ``typing.TypeVarTuple``, and ``typing.Generic`` +are now implemented in C rather than Python. + +There are new bytecode instructions ``LOAD_LOCALS``, +``LOAD_CLASSDICT_OR_GLOBAL``, and ``LOAD_CLASSDICT_OR_DEREF`` to support +correct resolution of names in class namespaces. + +Patch by Eric Traut, Larry Hastings, and Jelle Zijlstra. + +.. + +.. date: 2023-04-24-21-47-38 +.. gh-issue: 103801 +.. nonce: WaBanq +.. section: Core and Builtins + +Adds three minor linting fixes to the wasm module caught that were caught by +ruff. + +.. + +.. date: 2023-04-24-14-38-16 +.. gh-issue: 103793 +.. nonce: kqoH6Q +.. section: Core and Builtins + +Optimized asyncio Task creation by deferring expensive string formatting +(task name generation) from Task creation to the first time ``get_name`` is +called. This makes asyncio benchmarks up to 5% faster. + +.. + +.. date: 2023-04-21-17-03-14 +.. gh-issue: 102310 +.. nonce: anLjDx +.. section: Core and Builtins + +Change the error range for invalid bytes literals. + +.. + +.. date: 2023-04-21-16-12-41 +.. gh-issue: 103590 +.. nonce: 7DHDOE +.. section: Core and Builtins + +Do not wrap a single exception raised from a ``try-except*`` construct in an +:exc:`ExceptionGroup`. + +.. + +.. date: 2023-04-20-16-17-51 +.. gh-issue: 103650 +.. nonce: K1MFXR +.. section: Core and Builtins + +Change the perf map format to remove the '0x' prefix from the addresses + +.. + +.. date: 2023-04-17-16-00-32 +.. gh-issue: 102856 +.. nonce: UunJ7y +.. section: Core and Builtins + +Implement the required C tokenizer changes for PEP 701. Patch by Pablo +Galindo Salgado, Lysandros Nikolaou, Batuhan Taskaya, Marta Gómez Macías and +sunmy2019. + +.. + +.. date: 2023-04-16-14-38-39 +.. gh-issue: 100530 +.. nonce: OR6-sn +.. section: Core and Builtins + +Clarify the error message raised when the called part of a class pattern +isn't actually a class. + +.. + +.. date: 2023-04-14-22-35-23 +.. gh-issue: 101517 +.. nonce: 5EqM-S +.. section: Core and Builtins + +Fix bug in line numbers of instructions emitted for :keyword:`except* +`. + +.. + +.. date: 2023-04-13-00-58-55 +.. gh-issue: 103492 +.. nonce: P4k0Ay +.. section: Core and Builtins + +Clarify :exc:`SyntaxWarning` with literal ``is`` comparison by specifying +which literal is problematic, since comparisons using ``is`` with e.g. None +and bool literals are idiomatic. + +.. + +.. date: 2023-04-12-20-22-03 +.. gh-issue: 87729 +.. nonce: 99A7UO +.. section: Core and Builtins + +Add :opcode:`LOAD_SUPER_ATTR` (and a specialization for +``super().method()``) to speed up ``super().method()`` and ``super().attr``. +This makes ``super().method()`` roughly 2.3x faster and brings it within 20% +of the performance of a simple method call. Patch by Vladimir Matveev and +Carl Meyer. + +.. + +.. date: 2023-04-12-20-18-51 +.. gh-issue: 103488 +.. nonce: vYvlHD +.. section: Core and Builtins + +Change the internal offset distinguishing yield and return target addresses, +so that the instruction pointer is correct for exception handling and other +stack unwinding. + +.. + +.. date: 2023-04-12-19-55-24 +.. gh-issue: 82012 +.. nonce: FlcJAh +.. section: Core and Builtins + +The bitwise inversion operator (``~``) on bool is deprecated. It returns the +bitwise inversion of the underlying ``int`` representation such that +``bool(~True) == True``, which can be confusing. Use ``not`` for logical +negation of bools. In the rare case that you really need the bitwise +inversion of the underlying ``int``, convert to int explicitly ``~int(x)``. + +.. + +.. date: 2023-04-09-22-21-57 +.. gh-issue: 77757 +.. nonce: _Ow-u2 +.. section: Core and Builtins + +Exceptions raised in a typeobject's ``__set_name__`` method are no longer +wrapped by a :exc:`RuntimeError`. Context information is added to the +exception as a :pep:`678` note. + +.. + +.. date: 2023-04-09-04-30-02 +.. gh-issue: 103333 +.. nonce: gKOetS +.. section: Core and Builtins + +:exc:`AttributeError` now retains the ``name`` attribute when pickled and +unpickled. + +.. + +.. date: 2023-04-08-17-13-07 +.. gh-issue: 103242 +.. nonce: ysI1b3 +.. section: Core and Builtins + +Migrate :meth:`~ssl.SSLContext.set_ecdh_curve` method not to use deprecated +OpenSSL APIs. Patch by Dong-hee Na. + +.. + +.. date: 2023-04-07-12-18-41 +.. gh-issue: 103323 +.. nonce: 9802br +.. section: Core and Builtins + +We've replaced our use of ``_PyRuntime.tstate_current`` with a thread-local +variable. This is a fairly low-level implementation detail, and there +should be no change in behavior. + +.. + +.. date: 2023-04-02-22-14-57 +.. gh-issue: 84436 +.. nonce: hvMgwF +.. section: Core and Builtins + +The implementation of PEP-683 which adds Immortal Objects by using a fixed +reference count that skips reference counting to make objects truly +immutable. + +.. + +.. date: 2023-04-01-00-46-31 +.. gh-issue: 102700 +.. nonce: 493NB4 +.. section: Core and Builtins + +Allow built-in modules to be submodules. This allows submodules to be +statically linked into a CPython binary. + +.. + +.. date: 2023-03-31-17-24-03 +.. gh-issue: 103082 +.. nonce: isRUcV +.. section: Core and Builtins + +Implement :pep:`669` Low Impact Monitoring for CPython. + +.. + +.. date: 2023-03-25-23-24-38 +.. gh-issue: 88691 +.. nonce: 2SWBd1 +.. section: Core and Builtins + +Reduce the number of inline :opcode:`CACHE` entries for :opcode:`CALL`. + +.. + +.. date: 2023-03-07-17-37-00 +.. gh-issue: 102500 +.. nonce: RUSQhz +.. section: Core and Builtins + +Make the buffer protocol accessible in Python code using the new +``__buffer__`` and ``__release_buffer__`` magic methods. See :pep:`688` for +details. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-01-30-15-40-29 +.. gh-issue: 97933 +.. nonce: nUlp3r +.. section: Core and Builtins + +:pep:`709`: inline list, dict and set comprehensions to improve performance +and reduce bytecode size. + +.. + +.. date: 2022-11-08-12-36-25 +.. gh-issue: 99184 +.. nonce: KIaqzz +.. section: Core and Builtins + +Bypass instance attribute access of ``__name__`` in ``repr`` of +:class:`weakref.ref`. + +.. + +.. date: 2022-10-06-23-32-11 +.. gh-issue: 98003 +.. nonce: xWE0Yu +.. section: Core and Builtins + +Complex function calls are now faster and consume no C stack space. + +.. + +.. bpo: 39610 +.. date: 2020-02-11-15-54-40 +.. nonce: fvgsCl +.. section: Core and Builtins + +``len()`` for 0-dimensional :class:`memoryview`` objects (such as +``memoryview(ctypes.c_uint8(42))``) now raises a :exc:`TypeError`. +Previously this returned ``1``, which was not consistent with ``mem_0d[0]`` +raising an :exc:`IndexError``. + +.. + +.. bpo: 31821 +.. date: 2019-12-01-12-58-31 +.. nonce: 1FNmwk +.. section: Core and Builtins + +Fix :func:`!pause_reading` to work when called from :func:`!connection_made` +in :mod:`asyncio`. + +.. + +.. date: 2023-05-17-21-01-48 +.. gh-issue: 104600 +.. nonce: E6CK35 +.. section: Library + +:func:`functools.update_wrapper` now sets the ``__type_params__`` attribute +(added by :pep:`695`). + +.. + +.. date: 2023-05-17-20-03-01 +.. gh-issue: 104340 +.. nonce: kp_XmX +.. section: Library + +When an ``asyncio`` pipe protocol loses its connection due to an error, and +the caller doesn't await ``wait_closed()`` on the corresponding +``StreamWriter``, don't log a warning about an exception that was never +retrieved. After all, according to the ``StreamWriter.close()`` docs, the +``wait_closed()`` call is optional ("not mandatory"). + +.. + +.. date: 2023-05-17-16-58-23 +.. gh-issue: 104555 +.. nonce: 5rb5oM +.. section: Library + +Fix issue where an :func:`issubclass` check comparing a class ``X`` against +a :func:`runtime-checkable protocol ` ``Y`` with +non-callable members would not cause :exc:`TypeError` to be raised if an +:func:`isinstance` call had previously been made comparing an instance of +``X`` to ``Y``. This issue was present in edge cases on Python 3.11, but +became more prominent in 3.12 due to some unrelated changes that were made +to runtime-checkable protocols. Patch by Alex Waygood. + +.. + +.. date: 2023-05-17-08-01-36 +.. gh-issue: 104372 +.. nonce: jpoWs6 +.. section: Library + +Refactored the ``_posixsubprocess`` internals to avoid Python C API usage +between fork and exec when marking ``pass_fds=`` file descriptors +inheritable. + +.. + +.. date: 2023-05-17-03-14-07 +.. gh-issue: 104484 +.. nonce: y6KxL6 +.. section: Library + +Added *case_sensitive* argument to :meth:`pathlib.PurePath.match` + +.. + +.. date: 2023-05-16-11-02-44 +.. gh-issue: 75367 +.. nonce: qLWR35 +.. section: Library + +Fix data descriptor detection in :func:`inspect.getattr_static`. + +.. + +.. date: 2023-05-16-10-07-16 +.. gh-issue: 104536 +.. nonce: hFWD8f +.. section: Library + +Fix a race condition in the internal :mod:`multiprocessing.process` cleanup +logic that could manifest as an unintended ``AttributeError`` when calling +``process.close()``. + +.. + +.. date: 2023-05-12-19-29-28 +.. gh-issue: 103857 +.. nonce: 0IzSxr +.. section: Library + +Update datetime deprecations' stracktrace to point to the calling line + +.. + +.. date: 2023-05-11-21-32-18 +.. gh-issue: 101520 +.. nonce: l9MjRE +.. section: Library + +Move the core functionality of the ``tracemalloc`` module in the ``Python/`` +folder, leaving just the module wrapper in ``Modules/``. + +.. + +.. date: 2023-05-11-07-50-00 +.. gh-issue: 104392 +.. nonce: YSllzt +.. section: Library + +Remove undocumented and unused ``_paramspec_tvars`` attribute from some +classes in :mod:`typing`. + +.. + +.. date: 2023-05-11-01-07-42 +.. gh-issue: 102613 +.. nonce: uMsokt +.. section: Library + +Fix issue where :meth:`pathlib.Path.glob` raised :exc:`RecursionError` when +walking deep directory trees. + +.. + +.. date: 2023-05-10-19-33-36 +.. gh-issue: 103000 +.. nonce: j0KSfD +.. section: Library + +Improve performance of :func:`dataclasses.asdict` for the common case where +*dict_factory* is ``dict``. Patch by David C Ellis. + +.. + +.. date: 2023-05-09-18-46-24 +.. gh-issue: 104301 +.. nonce: gNnbId +.. section: Library + +Allow leading whitespace in disambiguated statements in :mod:`pdb`. + +.. + +.. date: 2023-05-08-23-01-59 +.. gh-issue: 104139 +.. nonce: 83Tnt- +.. section: Library + +Teach :func:`urllib.parse.unsplit` to retain the ``"//"`` when assembling +``itms-services://?action=generate-bugs`` style `Apple Platform Deployment +`_ URLs. + +.. + +.. date: 2023-05-08-20-57-17 +.. gh-issue: 104307 +.. nonce: DSB93G +.. section: Library + +:func:`socket.getnameinfo` now releases the GIL while contacting the DNS +server + +.. + +.. date: 2023-05-08-15-50-59 +.. gh-issue: 104310 +.. nonce: fXVSPY +.. section: Library + +Users may now use ``importlib.util.allowing_all_extensions()`` (a context +manager) to temporarily disable the strict compatibility checks for +importing extension modules in subinterpreters. + +.. + +.. date: 2023-05-08-15-39-00 +.. gh-issue: 87695 +.. nonce: f6iO7v +.. section: Library + +Fix issue where :meth:`pathlib.Path.glob` raised :exc:`OSError` when it +encountered a symlink to an overly long path. + +.. + +.. date: 2023-05-07-19-56-45 +.. gh-issue: 104265 +.. nonce: fVblry +.. section: Library + +Prevent possible crash by disallowing instantiation of the +:class:`!_csv.Reader` and :class:`!_csv.Writer` types. The regression was +introduced in 3.10.0a4 with PR 23224 (:issue:`14935`). Patch by Radislav +Chugunov. + +.. + +.. date: 2023-05-06-20-37-46 +.. gh-issue: 102613 +.. nonce: QZG9iX +.. section: Library + +Improve performance of :meth:`pathlib.Path.glob` when expanding recursive +wildcards ("``**``") by merging adjacent wildcards and de-duplicating +results only when necessary. + +.. + +.. date: 2023-05-05-18-52-22 +.. gh-issue: 65772 +.. nonce: w5P5Wv +.. section: Library + +Remove unneeded comments and code in turtle.py. + +.. + +.. date: 2023-05-03-19-22-24 +.. gh-issue: 90208 +.. nonce: tI00da +.. section: Library + +Fixed issue where :meth:`pathlib.Path.glob` returned incomplete results when +it encountered a :exc:`PermissionError`. This method now suppresses all +:exc:`OSError` exceptions, except those raised from calling +:meth:`~pathlib.Path.is_dir` on the top-level path. + +.. + +.. date: 2023-05-03-16-51-53 +.. gh-issue: 104144 +.. nonce: 653Q0P +.. section: Library + +Optimize :class:`asyncio.TaskGroup` when using +:func:`asyncio.eager_task_factory`. Skip scheduling a done callback if a +TaskGroup task completes eagerly. + +.. + +.. date: 2023-05-03-16-50-24 +.. gh-issue: 104144 +.. nonce: yNkjL8 +.. section: Library + +Optimize :func:`asyncio.gather` when using +:func:`asyncio.eager_task_factory` to complete eagerly if all fututres +completed eagerly. Avoid scheduling done callbacks for futures that complete +eagerly. + +.. + +.. date: 2023-05-03-03-14-33 +.. gh-issue: 104114 +.. nonce: RG26RD +.. section: Library + +Fix issue where :meth:`pathlib.Path.glob` returns paths using the case of +non-wildcard segments for corresponding path segments, rather than the real +filesystem case. + +.. + +.. date: 2023-05-02-21-05-30 +.. gh-issue: 104104 +.. nonce: 9tjplT +.. section: Library + +Improve performance of :meth:`pathlib.Path.glob` by using +:data:`re.IGNORECASE` to implement case-insensitive matching. + +.. + +.. date: 2023-05-02-20-43-03 +.. gh-issue: 104102 +.. nonce: vgSdEJ +.. section: Library + +Improve performance of :meth:`pathlib.Path.glob` when evaluating patterns +that contain ``'../'`` segments. + +.. + +.. date: 2023-05-02-04-49-45 +.. gh-issue: 103822 +.. nonce: m0QdAO +.. section: Library + +Update the return type of ``weekday`` to the newly added Day attribute + +.. + +.. date: 2023-05-01-19-10-05 +.. gh-issue: 103629 +.. nonce: 81bpZz +.. section: Library + +Update the ``repr`` of :class:`typing.Unpack` according to :pep:`692`. + +.. + +.. date: 2023-05-01-17-58-28 +.. gh-issue: 103963 +.. nonce: XWlHx7 +.. section: Library + +Make :mod:`dis` display the names of the args for +:opcode:`CALL_INTRINSIC_*`. + +.. + +.. date: 2023-05-01-16-43-28 +.. gh-issue: 104035 +.. nonce: MrJBw8 +.. section: Library + +Do not ignore user-defined ``__getstate__`` and ``__setstate__`` methods for +slotted frozen dataclasses. + +.. + +.. date: 2023-04-29-18-23-16 +.. gh-issue: 103987 +.. nonce: sRgALL +.. section: Library + +In :mod:`mmap`, fix several bugs that could lead to access to memory-mapped +files after they have been invalidated. + +.. + +.. date: 2023-04-28-19-08-50 +.. gh-issue: 103977 +.. nonce: msF70A +.. section: Library + +Improve import time of :mod:`platform` module. + +.. + +.. date: 2023-04-28-18-04-23 +.. gh-issue: 88773 +.. nonce: xXCNJw +.. section: Library + +Added :func:`turtle.teleport` to the :mod:`turtle` module to move a turtle +to a new point without tracing a line, visible or invisible. Patch by Liam +Gersten. + +.. + +.. date: 2023-04-27-20-03-08 +.. gh-issue: 103935 +.. nonce: Uaf2M0 +.. section: Library + +Use :func:`io.open_code` for files to be executed instead of raw +:func:`open` + +.. + +.. date: 2023-04-27-18-46-31 +.. gh-issue: 68968 +.. nonce: E3tnhy +.. section: Library + +Fixed garbled output of :meth:`~unittest.TestCase.assertEqual` when an input +lacks final newline. + +.. + +.. date: 2023-04-27-00-45-41 +.. gh-issue: 100370 +.. nonce: MgZ3KY +.. section: Library + +Fix potential :exc:`OverflowError` in :meth:`sqlite3.Connection.blobopen` +for 32-bit builds. Patch by Erlend E. Aasland. + +.. + +.. date: 2023-04-27-00-05-32 +.. gh-issue: 102628 +.. nonce: X230E- +.. section: Library + +Substitute CTRL-D with CTRL-Z in :mod:`sqlite3` CLI banner when running on +Windows. + +.. + +.. date: 2023-04-26-18-12-13 +.. gh-issue: 103636 +.. nonce: -KvCgO +.. section: Library + +Module-level attributes ``January`` and ``February`` are deprecated from +:mod:`calendar`. + +.. + +.. date: 2023-04-26-15-14-36 +.. gh-issue: 103583 +.. nonce: iCMDFt +.. section: Library + +Isolate :mod:`!_multibytecodec` and codecs extension modules. Patches by +Erlend E. Aasland. + +.. + +.. date: 2023-04-26-09-54-25 +.. gh-issue: 103848 +.. nonce: aDSnpR +.. section: Library + +Add checks to ensure that ``[`` bracketed ``]`` hosts found by +:func:`urllib.parse.urlsplit` are of IPv6 or IPvFuture format. + +.. + +.. date: 2023-04-26-09-38-47 +.. gh-issue: 103872 +.. nonce: 8LBsDz +.. section: Library + +Update the bundled copy of pip to version 23.1.2. + +.. + +.. date: 2023-04-25-22-59-06 +.. gh-issue: 99944 +.. nonce: pst8iT +.. section: Library + +Make :mod:`dis` display the value of oparg of :opcode:`KW_NAMES`. + +.. + +.. date: 2023-04-25-22-06-00 +.. gh-issue: 74940 +.. nonce: TOacQ9 +.. section: Library + +The C.UTF-8 locale is no longer converted to en_US.UTF-8, enabling the use +of UTF-8 encoding on systems which have no locales installed. + +.. + +.. date: 2023-04-25-19-58-13 +.. gh-issue: 103861 +.. nonce: JeozgD +.. section: Library + +Fix ``zipfile.Zipfile`` creating invalid zip files when ``force_zip64`` was +used to add files to them. Patch by Carey Metcalfe. + +.. + +.. date: 2023-04-25-17-03-18 +.. gh-issue: 103857 +.. nonce: Mr2Cak +.. section: Library + +Deprecated :meth:`datetime.datetime.utcnow` and +:meth:`datetime.datetime.utcfromtimestamp`. (Patch by Paul Ganssle) + +.. + +.. date: 2023-04-25-16-31-00 +.. gh-issue: 103839 +.. nonce: tpyLhI +.. section: Library + +Avoid compilation error due to tommath.h not being found when building +Tkinter against Tcl 8.7 built with bundled libtommath. + +.. + +.. date: 2023-04-24-23-07-56 +.. gh-issue: 103791 +.. nonce: bBPWdS +.. section: Library + +:class:`contextlib.suppress` now supports suppressing exceptions raised as +part of an :exc:`ExceptionGroup`. If other exceptions exist on the group, +they are re-raised in a group that does not contain the suppressed +exceptions. + +.. + +.. date: 2023-04-24-16-00-28 +.. gh-issue: 90750 +.. nonce: da0Xi8 +.. section: Library + +Use :meth:`datetime.datetime.fromisocalendar` in the implementation of +:meth:`datetime.datetime.strptime`, which should now accept only valid ISO +dates. (Patch by Paul Ganssle) + +.. + +.. date: 2023-04-24-00-34-23 +.. gh-issue: 103685 +.. nonce: U14jBM +.. section: Library + +Prepare :meth:`tkinter.Menu.index` for Tk 8.7 so that it does not raise +``TclError: expected integer but got ""`` when it should return ``None``. + +.. + +.. date: 2023-04-23-15-39-17 +.. gh-issue: 81403 +.. nonce: zVz9Td +.. section: Library + +:class:`urllib.request.CacheFTPHandler` no longer raises :class:`URLError` +if a cached FTP instance is reused. ftplib's endtransfer method calls +voidresp to drain the connection to handle FTP instance reuse properly. + +.. + +.. date: 2023-04-22-22-37-39 +.. gh-issue: 103699 +.. nonce: NizCjc +.. section: Library + +Add ``__orig_bases__`` to non-generic TypedDicts, call-based TypedDicts, and +call-based NamedTuples. Other TypedDicts and NamedTuples already had the +attribute. + +.. + +.. date: 2023-04-22-21-34-13 +.. gh-issue: 103693 +.. nonce: SBtuLQ +.. section: Library + +Add convenience variable feature to :mod:`pdb` + +.. + +.. date: 2023-04-22-12-30-10 +.. gh-issue: 92248 +.. nonce: NcVTKR +.. section: Library + +Deprecate ``type``, ``choices``, and ``metavar`` parameters of +``argparse.BooleanOptionalAction``. + +.. + +.. date: 2023-04-22-11-20-27 +.. gh-issue: 89415 +.. nonce: YHk760 +.. section: Library + +Add :mod:`socket` constants for source-specific multicast. Patch by Reese +Hyde. + +.. + +.. date: 2023-04-22-02-41-06 +.. gh-issue: 103673 +.. nonce: oE7S_k +.. section: Library + +:mod:`socketserver` gains ``ForkingUnixStreamServer`` and +``ForkingUnixDatagramServer`` classes. Patch by Jay Berry. + +.. + +.. date: 2023-04-21-10-25-39 +.. gh-issue: 103636 +.. nonce: YK6NEa +.. section: Library + +Added Enum for months and days in the calendar module. + +.. + +.. date: 2023-04-19-16-08-53 +.. gh-issue: 84976 +.. nonce: HwbzlD +.. section: Library + +Create a new ``Lib/_pydatetime.py`` file that defines the Python version of +the ``datetime`` module, and make ``datetime`` import the contents of the +new library only if the C implementation is missing. Currently, the full +Python implementation is defined and then deleted if the C implementation is +not available, slowing down ``import datetime`` unnecessarily. + +.. + +.. date: 2023-04-17-14-47-28 +.. gh-issue: 103596 +.. nonce: ME1y3_ +.. section: Library + +Attributes/methods are no longer shadowed by same-named enum members, +although they may be shadowed by enum.property's. + +.. + +.. date: 2023-04-16-19-48-21 +.. gh-issue: 103584 +.. nonce: 3mBTuM +.. section: Library + +Updated ``importlib.metadata`` with changes from ``importlib_metadata`` 5.2 +through 6.5.0, including: Support ``installed-files.txt`` for +``Distribution.files`` when present. ``PackageMetadata`` now stipulates an +additional ``get`` method allowing for easy querying of metadata keys that +may not be present. ``packages_distributions`` now honors packages and +modules with Python modules that not ``.py`` sources (e.g. ``.pyc``, +``.so``). Expand protocol for ``PackageMetadata.get_all`` to match the +upstream implementation of ``email.message.Message.get_all`` in +python/typeshed#9620. Deprecated use of ``Distribution`` without defining +abstract methods. Deprecated expectation that +``PackageMetadata.__getitem__`` will return ``None`` for missing keys. In +the future, it will raise a ``KeyError``. + +.. + +.. date: 2023-04-16-18-29-04 +.. gh-issue: 103578 +.. nonce: fly1wc +.. section: Library + +Fixed a bug where :mod:`pdb` crashes when reading source file with different +encoding by replacing :func:`io.open` with :func:`io.open_code`. The new +method would also call into the hook set by :func:`PyFile_SetOpenCodeHook`. + +.. + +.. date: 2023-04-15-12-19-14 +.. gh-issue: 103556 +.. nonce: TEf-2m +.. section: Library + +Now creating :class:`inspect.Signature` objects with positional-only +parameter with a default followed by a positional-or-keyword parameter +without one is impossible. + +.. + +.. date: 2023-04-15-11-21-38 +.. gh-issue: 103559 +.. nonce: a9rYHG +.. section: Library + +Update the bundled copy of pip to version 23.1.1. + +.. + +.. date: 2023-04-14-21-16-05 +.. gh-issue: 103548 +.. nonce: lagdpp +.. section: Library + +Improve performance of :meth:`pathlib.Path.absolute` and +:meth:`~pathlib.Path.cwd` by joining paths only when necessary. Also improve +performance of :meth:`pathlib.PurePath.is_absolute` on Posix by skipping +path parsing and normalization. + +.. + +.. date: 2023-04-14-21-12-32 +.. gh-issue: 103538 +.. nonce: M4FK_v +.. section: Library + +Remove ``_tkinter`` module code guarded by definition of the ``TK_AQUA`` +macro which was only needed for Tk 8.4.7 or earlier and was never actually +defined by any build system or documented for manual use. + +.. + +.. date: 2023-04-14-06-32-54 +.. gh-issue: 103533 +.. nonce: n_AfcS +.. section: Library + +Update :mod:`cProfile` to use PEP 669 API + +.. + +.. date: 2023-04-13-19-43-15 +.. gh-issue: 103525 +.. nonce: uY4VYg +.. section: Library + +Fix misleading exception message when mixed ``str`` and ``bytes`` arguments +are supplied to :class:`pathlib.PurePath` and :class:`~pathlib.Path`. + +.. + +.. date: 2023-04-13-13-17-47 +.. gh-issue: 103489 +.. nonce: ZSZgmu +.. section: Library + +Add :meth:`~sqlite3.Connection.getconfig` and +:meth:`~sqlite3.Connection.setconfig` to :class:`~sqlite3.Connection` to +make configuration changes to a database connection. Patch by Erlend E. +Aasland. + +.. + +.. date: 2023-04-12-17-59-55 +.. gh-issue: 103365 +.. nonce: UBEE0U +.. section: Library + +Set default Flag boundary to ``STRICT`` and fix bitwise operations. + +.. + +.. date: 2023-04-12-13-04-16 +.. gh-issue: 103472 +.. nonce: C6bOHv +.. section: Library + +Avoid a potential :exc:`ResourceWarning` in +:class:`http.client.HTTPConnection` by closing the proxy / tunnel's CONNECT +response explicitly. + +.. + +.. date: 2023-04-12-06-00-02 +.. gh-issue: 103462 +.. nonce: w6yBlM +.. section: Library + +Fixed an issue with using :meth:`~asyncio.WriteTransport.writelines` in +:mod:`asyncio` to send very large payloads that exceed the amount of data +that can be written in one call to :meth:`socket.socket.send` or +:meth:`socket.socket.sendmsg`, resulting in the remaining buffer being left +unwritten. + +.. + +.. date: 2023-04-11-21-38-39 +.. gh-issue: 103449 +.. nonce: -nxmhb +.. section: Library + +Fix a bug in doc string generation in :func:`dataclasses.dataclass`. + +.. + +.. date: 2023-04-09-06-59-36 +.. gh-issue: 103092 +.. nonce: vskbro +.. section: Library + +Isolate :mod:`!_collections` (apply :pep:`687`). Patch by Erlend E. Aasland. + +.. + +.. date: 2023-04-08-01-33-12 +.. gh-issue: 103357 +.. nonce: vjin28 +.. section: Library + +Added support for :class:`logging.Formatter` ``defaults`` parameter to +:func:`logging.config.dictConfig` and :func:`logging.config.fileConfig`. +Patch by Bar Harel. + +.. + +.. date: 2023-04-08-00-48-40 +.. gh-issue: 103092 +.. nonce: 5EFts0 +.. section: Library + +Adapt the :mod:`winreg` extension module to :pep:`687`. + +.. + +.. date: 2023-04-07-15-15-40 +.. gh-issue: 74690 +.. nonce: un84hh +.. section: Library + +The performance of :func:`isinstance` checks against +:func:`runtime-checkable protocols ` has been +considerably improved for protocols that only have a few members. To achieve +this improvement, several internal implementation details of the +:mod:`typing` module have been refactored, including +``typing._ProtocolMeta.__instancecheck__``, +``typing._is_callable_members_only``, and ``typing._get_protocol_attrs``. +Patches by Alex Waygood. + +.. + +.. date: 2023-04-07-15-09-26 +.. gh-issue: 74690 +.. nonce: 0f886b +.. section: Library + +The members of a runtime-checkable protocol are now considered "frozen" at +runtime as soon as the class has been created. See :ref:`"What's new in +Python 3.12" ` for more details. + +.. + +.. date: 2023-04-06-17-28-36 +.. gh-issue: 103256 +.. nonce: 1syxfs +.. section: Library + +Fixed a bug that caused :mod:`hmac` to raise an exception when the requested +hash algorithm was not available in OpenSSL despite being available +separately as part of ``hashlib`` itself. It now falls back properly to the +built-in. This could happen when, for example, your OpenSSL does not include +SHA3 support and you want to compute ``hmac.digest(b'K', b'M', +'sha3_256')``. + +.. + +.. date: 2023-04-06-16-55-51 +.. gh-issue: 102778 +.. nonce: BWeAmE +.. section: Library + +Support ``sys.last_exc`` in :mod:`idlelib`. + +.. + +.. date: 2023-04-06-04-35-59 +.. gh-issue: 103285 +.. nonce: rCZ9-G +.. section: Library + +Improve performance of :func:`ast.get_source_segment`. + +.. + +.. date: 2023-04-05-01-28-53 +.. gh-issue: 103225 +.. nonce: QD3JVU +.. section: Library + +Fix a bug in :mod:`pdb` when displaying line numbers of module-level source +code. + +.. + +.. date: 2023-04-04-21-44-25 +.. gh-issue: 103092 +.. nonce: Dz0_Xn +.. section: Library + +Adapt the :mod:`msvcrt` extension module to :pep:`687`. + +.. + +.. date: 2023-04-04-21-27-51 +.. gh-issue: 103092 +.. nonce: 7s7Bzf +.. section: Library + +Adapt the :mod:`winsound` extension module to :pep:`687`. + +.. + +.. date: 2023-04-04-12-43-38 +.. gh-issue: 93910 +.. nonce: jurMzv +.. section: Library + +Remove deprecation of enum ``memmber.member`` access. + +.. + +.. date: 2023-04-03-23-44-34 +.. gh-issue: 102978 +.. nonce: gy9eVk +.. section: Library + +Fixes :func:`unittest.mock.patch` not enforcing function signatures for +methods decorated with ``@classmethod`` or ``@staticmethod`` when patch is +called with ``autospec=True``. + +.. + +.. date: 2023-04-03-23-43-12 +.. gh-issue: 103092 +.. nonce: 3xqk4y +.. section: Library + +Isolate :mod:`!_socket` (apply :pep:`687`). Patch by Erlend E. Aasland. + +.. + +.. date: 2023-04-03-22-02-35 +.. gh-issue: 100479 +.. nonce: kNBjQm +.. section: Library + +Add :meth:`pathlib.PurePath.with_segments`, which creates a path object from +arguments. This method is called whenever a derivative path is created, such +as from :attr:`pathlib.PurePath.parent`. Subclasses may override this method +to share information between path objects. + +.. + +.. date: 2023-04-03-21-08-53 +.. gh-issue: 103220 +.. nonce: OW_Bj5 +.. section: Library + +Fix issue where :func:`os.path.join` added a slash when joining onto an +incomplete UNC drive with a trailing slash on Windows. + +.. + +.. date: 2023-04-02-23-05-22 +.. gh-issue: 103204 +.. nonce: bbDmu0 +.. section: Library + +Fixes :mod:`http.server` accepting HTTP requests with HTTP version numbers +preceded by '+', or '-', or with digit-separating '_' characters. The +length of the version numbers is also constrained. + +.. + +.. date: 2023-04-02-22-04-26 +.. gh-issue: 75586 +.. nonce: 526iJm +.. section: Library + +Fix various Windows-specific issues with ``shutil.which``. + +.. + +.. date: 2023-04-02-17-51-08 +.. gh-issue: 103193 +.. nonce: xrZbM1 +.. section: Library + +Improve performance of :func:`inspect.getattr_static`. Patch by Alex +Waygood. + +.. + +.. date: 2023-04-01-23-01-31 +.. gh-issue: 103176 +.. nonce: FBsdxa +.. section: Library + +:func:`sys._current_exceptions` now returns a mapping from thread-id to an +exception instance, rather than to a ``(typ, exc, tb)`` tuple. + +.. + +.. date: 2023-03-31-01-13-00 +.. gh-issue: 103143 +.. nonce: 6eMluy +.. section: Library + +Polish the help messages and docstrings of :mod:`pdb`. + +.. + +.. date: 2023-03-28-09-13-31 +.. gh-issue: 103015 +.. nonce: ETTfNf +.. section: Library + +Add *entrypoint* keyword-only parameter to +:meth:`sqlite3.Connection.load_extension`, for overriding the SQLite +extension entry point. Patch by Erlend E. Aasland. + +.. + +.. date: 2023-03-24-20-49-48 +.. gh-issue: 103000 +.. nonce: 6eVNZI +.. section: Library + +Improve performance of :func:`dataclasses.astuple` and +:func:`dataclasses.asdict` in cases where the contents are common Python +types. + +.. + +.. date: 2023-03-23-15-24-38 +.. gh-issue: 102953 +.. nonce: YR4KaK +.. section: Library + +The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, +have a new a *filter* argument that allows limiting tar features than may be +surprising or dangerous, such as creating files outside the destination +directory. See :ref:`tarfile-extraction-filter` for details. + +.. + +.. date: 2023-03-15-12-18-07 +.. gh-issue: 97696 +.. nonce: DtnpIC +.. section: Library + +Implemented an eager task factory in asyncio. When used as a task factory on +an event loop, it performs eager execution of coroutines. Coroutines that +are able to complete synchronously (e.g. return or raise without blocking) +are returned immediately as a finished task, and the task is never scheduled +to the event loop. If the coroutine blocks, the (pending) task is scheduled +and returned. + +.. + +.. date: 2023-03-15-00-37-43 +.. gh-issue: 81079 +.. nonce: heTAod +.. section: Library + +Add *case_sensitive* keyword-only argument to :meth:`pathlib.Path.glob` and +:meth:`~pathlib.Path.rglob`. + +.. + +.. date: 2023-03-14-11-20-19 +.. gh-issue: 101819 +.. nonce: 0-h0it +.. section: Library + +Isolate the :mod:`io` extension module by applying :pep:`687`. Patch by +Kumar Aditya, Victor Stinner, and Erlend E. Aasland. + +.. + +.. date: 2023-03-08-02-45-46 +.. gh-issue: 91896 +.. nonce: kgON_a +.. section: Library + +Deprecate :class:`collections.abc.ByteString` + +.. + +.. date: 2023-03-06-18-49-57 +.. gh-issue: 101362 +.. nonce: eSSy6L +.. section: Library + +Speed up :class:`pathlib.Path` construction by omitting the path anchor from +the internal list of path parts. + +.. + +.. date: 2023-02-21-14-57-34 +.. gh-issue: 102114 +.. nonce: uUDQzb +.. section: Library + +Functions in the :mod:`dis` module that accept a source code string as +argument now print a more concise traceback when the string contains a +syntax or indentation error. + +.. + +.. date: 2023-02-19-12-37-08 +.. gh-issue: 62432 +.. nonce: GnBFIB +.. section: Library + +The :mod:`unittest` runner will now exit with status code 5 if no tests were +run. It is common for test runner misconfiguration to fail to find any +tests, this should be an error. + +.. + +.. date: 2023-02-17-21-14-40 +.. gh-issue: 78079 +.. nonce: z3Szr6 +.. section: Library + +Fix incorrect normalization of UNC device path roots, and partial UNC share +path roots, in :class:`pathlib.PurePath`. Pathlib no longer appends a +trailing slash to such paths. + +.. + +.. date: 2023-02-11-21-18-10 +.. gh-issue: 85984 +.. nonce: nvzOD0 +.. section: Library + +Add :func:`tty.cfmakeraw` and :func:`tty.cfmakecbreak` to :mod:`tty` and +modernize, the behavior of :func:`tty.setraw` and :func:`tty.setcbreak` to +use POSIX.1-2017 Chapter 11 "General Terminal Interface" flag masks by +default. + +.. + +.. date: 2023-02-11-15-01-32 +.. gh-issue: 101688 +.. nonce: kwXmfM +.. section: Library + +Implement :func:`types.get_original_bases` to provide further introspection +for types. + +.. + +.. date: 2023-02-09-22-24-34 +.. gh-issue: 101640 +.. nonce: oFuEpB +.. section: Library + +:class:`argparse.ArgumentParser` now catches errors when writing messages, +such as when :data:`sys.stderr` is ``None``. Patch by Oleg Iarygin. + +.. + +.. date: 2023-02-06-16-45-18 +.. gh-issue: 83861 +.. nonce: mMbIU3 +.. section: Library + +Fix datetime.astimezone method return value when invoked on a naive datetime +instance that represents local time falling in a timezone transition gap. +PEP 495 requires that instances with fold=1 produce earlier times than those +with fold=0 in this case. + +.. + +.. date: 2023-01-22-14-53-12 +.. gh-issue: 89550 +.. nonce: c1U23f +.. section: Library + +Decrease execution time of some :mod:`gzip` file writes by 15% by adding +more appropriate buffering. + +.. + +.. date: 2023-01-14-17-54-56 +.. gh-issue: 95299 +.. nonce: vUhpKz +.. section: Library + +Remove the bundled setuptools wheel from ``ensurepip``, and stop installing +setuptools in environments created by ``venv``. + +.. + +.. date: 2022-11-10-16-26-47 +.. gh-issue: 99353 +.. nonce: DQFjnt +.. section: Library + +Respect the :class:`http.client.HTTPConnection` ``.debuglevel`` flag in +:class:`urllib.request.AbstractHTTPHandler` when its constructor parameter +``debuglevel`` is not set. And do the same for ``*HTTPS*``. + +.. + +.. date: 2022-10-21-17-20-57 +.. gh-issue: 98040 +.. nonce: 3btbmA +.. section: Library + +Remove the long-deprecated ``imp`` module. + +.. + +.. date: 2022-10-21-16-23-31 +.. gh-issue: 97850 +.. nonce: N46coo +.. section: Library + +Deprecate :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` in +favor of :func:`importlib.util.find_spec`. + +.. + +.. date: 2022-10-20-14-03-58 +.. gh-issue: 94473 +.. nonce: pzGX73 +.. section: Library + +Flatten arguments in :meth:`tkinter.Canvas.coords`. It now accepts not only +``x1, y1, x2, y2, ...`` and ``[x1, y1, x2, y2, ...]``, but also ``(x1, y1), +(x2, y2), ...`` and ``[(x1, y1), (x2, y2), ...]``. + +.. + +.. date: 2022-10-09-14-47-42 +.. gh-issue: 98040 +.. nonce: IN3qab +.. section: Library + +Remove more deprecated importlib APIs: ``find_loader()``, ``find_module()``, +``importlib.abc.Finder``, ``pkgutil.ImpImporter``, ``pkgutil.ImpLoader``. + +.. + +.. date: 2022-09-07-09-32-07 +.. gh-issue: 96522 +.. nonce: t73oqp +.. section: Library + +Fix potential deadlock in pty.spawn() + +.. + +.. date: 2022-09-03-09-24-02 +.. gh-issue: 96534 +.. nonce: EU4Oxv +.. section: Library + +Support divert(4) added in FreeBSD 14. + +.. + +.. date: 2022-08-27-21-41-41 +.. gh-issue: 87474 +.. nonce: 9X-kxt +.. section: Library + +Fix potential file descriptor leaks in :class:`subprocess.Popen`. + +.. + +.. date: 2022-07-16-17-15-29 +.. gh-issue: 94906 +.. nonce: C4G8DG +.. section: Library + +Support multiple steps in :func:`math.nextafter`. Patch by Shantanu Jain and +Matthias Gorgens. + +.. + +.. date: 2022-07-06-11-10-37 +.. gh-issue: 51574 +.. nonce: sveUeD +.. section: Library + +Make :func:`tempfile.mkdtemp` return absolute paths when its *dir* parameter +is relative. + +.. + +.. date: 2022-07-03-23-13-28 +.. gh-issue: 94518 +.. nonce: 511Tbh +.. section: Library + +Convert private :meth:`_posixsubprocess.fork_exec` to use Argument Clinic. + +.. + +.. date: 2022-05-02-16-21-05 +.. gh-issue: 92184 +.. nonce: hneGVW +.. section: Library + +When creating zip files using :mod:`zipfile`, ``os.altsep``, if not +``None``, will always be treated as a path separator even when it is not +``/``. Patch by Carey Metcalfe. + +.. + +.. bpo: 46797 +.. date: 2022-02-19-14-19-34 +.. nonce: 6BXZX4 +.. section: Library + +Deprecation warnings are now emitted for :class:`!ast.Num`, +:class:`!ast.Bytes`, :class:`!ast.Str`, :class:`!ast.NameConstant` and +:class:`!ast.Ellipsis`. These have been documented as deprecated since +Python 3.8, and will be removed in Python 3.14. + +.. + +.. bpo: 44844 +.. date: 2021-12-03-23-00-56 +.. nonce: tvg2VY +.. section: Library + +Enables :mod:`webbrowser` to detect and launch Microsoft Edge browser. + +.. + +.. bpo: 45606 +.. date: 2021-11-19-23-37-18 +.. nonce: UW5XE1 +.. section: Library + +Fixed the bug in :meth:`pathlib.Path.glob` -- previously a dangling symlink +would not be found by this method when the pattern is an exact match, but +would be found when the pattern contains a wildcard or the recursive +wildcard (``**``). With this change, a dangling symlink will be found in +both cases. + +.. + +.. bpo: 23041 +.. date: 2021-11-07-15-31-25 +.. nonce: 564i32 +.. section: Library + +Add :data:`~csv.QUOTE_STRINGS` and :data:`~csv.QUOTE_NOTNULL` to the suite +of :mod:`csv` module quoting styles. + +.. + +.. bpo: 24964 +.. date: 2021-05-16-14-28-30 +.. nonce: Oa5Ie_ +.. section: Library + +Added :meth:`http.client.HTTPConnection.get_proxy_response_headers` that +provides access to the HTTP headers on a proxy server response to the +``CONNECT`` request. + +.. + +.. bpo: 17258 +.. date: 2020-05-25-12-42-36 +.. nonce: lf2554 +.. section: Library + +:mod:`multiprocessing` now supports stronger HMAC algorithms for +inter-process connection authentication rather than only HMAC-MD5. + +.. + +.. bpo: 39744 +.. date: 2020-02-25-00-43-22 +.. nonce: hgK689 +.. section: Library + +Make :func:`asyncio.subprocess.Process.communicate` close the subprocess's +stdin even when called with ``input=None``. + +.. + +.. bpo: 22708 +.. date: 2018-07-16-14-10-29 +.. nonce: 592iRR +.. section: Library + +http.client CONNECT method tunnel improvements: Use HTTP 1.1 protocol; send +a matching Host: header with CONNECT, if one is not provided; convert IDN +domain names to Punycode. Patch by Michael Handler. + +.. + +.. date: 2023-05-14-12-11-28 +.. gh-issue: 67056 +.. nonce: nVC2Rf +.. section: Documentation + +Document that the effect of registering or unregistering an :mod:`atexit` +cleanup function from within a registered cleanup function is undefined. + +.. + +.. date: 2023-04-26-23-55-31 +.. gh-issue: 103629 +.. nonce: -0reqn +.. section: Documentation + +Mention the new way of typing ``**kwargs`` with ``Unpack`` and ``TypedDict`` +introduced in :pep:`692`. + +.. + +.. date: 2023-04-25-22-58-08 +.. gh-issue: 48241 +.. nonce: l1Gxxh +.. section: Documentation + +Clarifying documentation about the url parameter to urllib.request.urlopen +and urllib.request.Requst needing to be encoded properly. + +.. + +.. date: 2023-03-10-04-59-35 +.. gh-issue: 86094 +.. nonce: zOYdy8 +.. section: Documentation + +Add support for Unicode Path Extra Field in ZipFile. Patch by Yeojin Kim and +Andrea Giudiceandrea + +.. + +.. date: 2023-03-07-23-30-29 +.. gh-issue: 99202 +.. nonce: hhiAJF +.. section: Documentation + +Fix extension type from documentation for compiling in C++20 mode + +.. + +.. date: 2023-05-15-02-22-44 +.. gh-issue: 104494 +.. nonce: Bkrbfn +.. section: Tests + +Update ``test_pack_configure_in`` and ``test_place_configure_in`` for +changes to error message formatting in Tk 8.7. + +.. + +.. date: 2023-05-14-03-00-00 +.. gh-issue: 104461 +.. nonce: Rmex11 +.. section: Tests + +Run test_configure_screen on X11 only, since the ``DISPLAY`` environment +variable and ``-screen`` option for toplevels are not useful on Tk for Win32 +or Aqua. + +.. + +.. date: 2023-04-25-12-19-37 +.. gh-issue: 86275 +.. nonce: -RoLIt +.. section: Tests + +Added property-based tests to the :mod:`zoneinfo` tests, along with stubs +for the ``hypothesis`` interface. (Patch by Paul Ganssle) + +.. + +.. date: 2023-04-08-00-50-23 +.. gh-issue: 103329 +.. nonce: M38tqF +.. section: Tests + +Regression tests for the behaviour of ``unittest.mock.PropertyMock`` were +added. + +.. + +.. date: 2023-03-17-22-00-47 +.. gh-issue: 102795 +.. nonce: z21EoC +.. section: Tests + +fix use of poll in test_epoll's test_control_and_wait + +.. + +.. date: 2022-11-06-18-42-38 +.. gh-issue: 75729 +.. nonce: uGYJrv +.. section: Tests + +Fix the :func:`os.spawn* ` tests failing on Windows when the +working directory or interpreter path contains spaces. + +.. + +.. date: 2023-05-20-16-09-59 +.. gh-issue: 101282 +.. nonce: FvRARb +.. section: Build + +BOLT optimization is now applied to the libpython shared library if building +a shared library. BOLT instrumentation and application settings can now be +influenced via the ``BOLT_INSTRUMENT_FLAGS`` and ``BOLT_APPLY_FLAGS`` +configure variables. + +.. + +.. date: 2023-05-15-09-34-08 +.. gh-issue: 99017 +.. nonce: nToOQu +.. section: Build + +``PYTHON_FOR_REGEN`` now require Python 3.10 or newer. + +.. + +.. date: 2023-05-14-19-00-19 +.. gh-issue: 104490 +.. nonce: 1tA4AF +.. section: Build + +Define ``.PHONY`` / virtual make targets consistently and properly. + +.. + +.. date: 2023-05-04-10-56-14 +.. gh-issue: 104106 +.. nonce: -W9BJS +.. section: Build + +Add gcc fallback of mkfifoat/mknodat for macOS. Patch by Dong-hee Na. + +.. + +.. date: 2023-04-14-10-24-37 +.. gh-issue: 103532 +.. nonce: H1djkd +.. section: Build + +The ``TKINTER_PROTECT_LOADTK`` macro is no longer defined or used in the +``_tkinter`` module. It was previously only defined when building against +Tk 8.4.13 and older, but Tk older than 8.5.12 has been unsupported since +gh-issue-91152. + +.. + +.. date: 2023-02-11-05-31-05 +.. gh-issue: 99069 +.. nonce: X4LDvY +.. section: Build + +Extended workaround defining ``static_assert`` when missing from the libc +headers to all clang and gcc builds. In particular, this fixes building on +macOS <= 10.10. + +.. + +.. date: 2022-12-18-07-24-44 +.. gh-issue: 100220 +.. nonce: BgSV7C +.. section: Build + +Changed the default value of the ``SHELL`` Makefile variable from +``/bin/sh`` to ``/bin/sh -e`` to ensure that complex recipes correctly fail +after an error. Previously, ``make install`` could fail to install some +files and yet return a successful result. + +.. + +.. date: 2022-06-20-15-15-11 +.. gh-issue: 90656 +.. nonce: kFBbKe +.. section: Build + +Add platform triplets for 64-bit LoongArch: + +* loongarch64-linux-gnusf +* loongarch64-linux-gnuf32 +* loongarch64-linux-gnu + +Patch by Zhang Na. + +.. + +.. date: 2023-05-18-22-46-03 +.. gh-issue: 104623 +.. nonce: HJZhm1 +.. section: Windows + +Update Windows installer to use SQLite 3.42.0. + +.. + +.. date: 2023-04-24-15-51-11 +.. gh-issue: 82814 +.. nonce: GI3UkZ +.. section: Windows + +Fix a potential ``[Errno 13] Permission denied`` when using +:func:`shutil.copystat` within Windows Subsystem for Linux (WSL) on a +mounted filesystem by adding ``errno.EACCES`` to the list of ignored errors +within the internal implementation. + +.. + +.. date: 2023-04-12-10-49-21 +.. gh-issue: 103088 +.. nonce: Yjj-qJ +.. section: Windows + +Fix virtual environment :file:`activate` script having incorrect line +endings for Cygwin. + +.. + +.. date: 2023-04-11-09-22-22 +.. gh-issue: 103088 +.. nonce: 6AJEuR +.. section: Windows + +Fixes venvs not working in bash on Windows across different disks + +.. + +.. date: 2023-03-24-11-25-28 +.. gh-issue: 102997 +.. nonce: dredy2 +.. section: Windows + +Update Windows installer to use SQLite 3.41.2. + +.. + +.. date: 2023-03-18-21-38-00 +.. gh-issue: 88013 +.. nonce: Z3loxC +.. section: Windows + +Fixed a bug where :exc:`TypeError` was raised when calling +:func:`ntpath.realpath` with a bytes parameter in some cases. + +.. + +.. date: 2023-05-21-23-54-52 +.. gh-issue: 99834 +.. nonce: 6ANPts +.. section: macOS + +Update macOS installer to Tcl/Tk 8.6.13. + +.. + +.. date: 2023-05-18-22-31-49 +.. gh-issue: 104623 +.. nonce: 6h7Xfx +.. section: macOS + +Update macOS installer to SQLite 3.42.0. + +.. + +.. date: 2023-05-18-08-52-04 +.. gh-issue: 103545 +.. nonce: pi5k2N +.. section: macOS + +Add ``os.PRIO_DARWIN_THREAD``, ``os.PRIO_DARWIN_PROCESS``, +``os.PRIO_DARWIN_BG`` and ``os.PRIO_DARWIN_NONUI``. These can be used with +``os.setpriority`` to run the process at a lower priority and make use of +the efficiency cores on Apple Silicon systems. + +.. + +.. date: 2023-05-04-21-47-59 +.. gh-issue: 104180 +.. nonce: lEJCwd +.. section: macOS + +Support reading SOCKS proxy configuration from macOS System Configuration. +Patch by Sam Schott. + +.. + +.. date: 2023-04-24-18-37-48 +.. gh-issue: 60436 +.. nonce: in-IyF +.. section: macOS + +update curses textbox to additionally handle backspace using the +``curses.ascii.DEL`` key press. + +.. + +.. date: 2023-03-24-11-20-47 +.. gh-issue: 102997 +.. nonce: ZgQkbq +.. section: macOS + +Update macOS installer to SQLite 3.41.2. + +.. + +.. date: 2023-05-17-17-32-21 +.. gh-issue: 104499 +.. nonce: hNeqV4 +.. section: IDLE + +Fix completions for Tk Aqua 8.7 (currently blank). + +.. + +.. date: 2023-05-17-15-11-11 +.. gh-issue: 104496 +.. nonce: wjav-y +.. section: IDLE + +About prints both tcl and tk versions if different (expected someday). + +.. + +.. date: 2023-04-30-20-01-18 +.. gh-issue: 88496 +.. nonce: y65vUb +.. section: IDLE + +Fix IDLE test hang on macOS. + +.. + +.. date: 2023-05-11-15-12-11 +.. gh-issue: 104389 +.. nonce: EiOhB3 +.. section: Tools/Demos + +Argument Clinic C converters now accept the ``unused`` keyword, for wrapping +a parameter with :c:macro:`Py_UNUSED`. Patch by Erlend E. Aasland. + +.. + +.. date: 2023-05-18-20-53-05 +.. gh-issue: 101291 +.. nonce: ZBh9aR +.. section: C API + +Added unstable C API for extracting the value of "compact" integers: +:c:func:`PyUnstable_Long_IsCompact` and +:c:func:`PyUnstable_Long_CompactValue`. + +.. + +.. date: 2023-05-02-21-05-54 +.. gh-issue: 104109 +.. nonce: 0tnDZV +.. section: C API + +We've added ``Py_NewInterpreterFromConfig()`` and ``PyInterpreterConfig`` to +the public C-API (but not the stable ABI; not yet at least). The new +function may be used to create a new interpreter with various features +configured. The function was added to support PEP 684 (per-interpreter +GIL). + +.. + +.. date: 2023-04-28-18-04-38 +.. gh-issue: 103968 +.. nonce: EnVvOx +.. section: C API + +:c:func:`PyType_FromSpec` and its variants now allow creating classes whose +metaclass overrides :c:member:`~PyTypeObject.tp_new`. The ``tp_new`` is +ignored. This behavior is deprecated and will be disallowed in 3.14+. The +new :c:func:`PyType_FromMetaclass` already disallows it. + +.. + +.. date: 2023-04-24-10-31-59 +.. gh-issue: 103743 +.. nonce: 2xYA1K +.. section: C API + +Add :c:func:`PyUnstable_Object_GC_NewWithExtraData` function that can be +used to allocate additional memory after an object for data not managed by +Python. + +.. + +.. date: 2023-04-14-23-05-52 +.. gh-issue: 103295 +.. nonce: GRHY1Z +.. section: C API + +Introduced :c:func:`PyUnstable_WritePerfMapEntry`, +:c:func:`PyUnstable_PerfMapState_Init` and +:c:func:`PyUnstable_PerfMapState_Fini`. These allow extension modules (JIT +compilers in particular) to write to perf-map files in a thread safe manner. +The :doc:`../howto/perf_profiling` also uses these APIs to write entries in +the perf-map file. + +.. + +.. date: 2023-04-13-16-54-00 +.. gh-issue: 103509 +.. nonce: A26Qu8 +.. section: C API + +Added C API for extending types whose instance memory layout is opaque: +:c:member:`PyType_Spec.basicsize` can now be zero or negative, +:c:func:`PyObject_GetTypeData` can be used to get subclass-specific data, +and :c:macro:`Py_TPFLAGS_ITEMS_AT_END` can be used to safely extend +variable-size objects. See :pep:`697` for details. + +.. + +.. date: 2023-03-28-12-31-51 +.. gh-issue: 103091 +.. nonce: CzZyaZ +.. section: C API + +Add a new C-API function to eagerly assign a version tag to a PyTypeObject: +``PyUnstable_Type_AssignVersionTag()``. + +.. + +.. date: 2023-02-09-23-09-29 +.. gh-issue: 101408 +.. nonce: _paFIF +.. section: C API + +:c:func:`PyObject_GC_Resize` should calculate preheader size if needed. +Patch by Dong-hee Na. + +.. + +.. date: 2022-10-29-10-13-20 +.. gh-issue: 98836 +.. nonce: Cy5h_z +.. section: C API + +Add support of more formatting options (left aligning, octals, uppercase +hexadecimals, :c:expr:`intmax_t`, :c:expr:`ptrdiff_t`, :c:expr:`wchar_t` C +strings, variable width and precision) in :c:func:`PyUnicode_FromFormat` and +:c:func:`PyUnicode_FromFormatV`. + +.. + +.. date: 2022-09-15-15-21-34 +.. gh-issue: 96803 +.. nonce: ynBKIS +.. section: C API + +Add unstable C-API functions to get the code object, lasti and line number +from the internal ``_PyInterpreterFrame`` in the limited API. The functions +are: + +* ``PyCodeObject * PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame)`` diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 854458f2d1a994..2564167cdf0895 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -2747,7 +2747,7 @@ smaller file). Also, change the default compression level to 6 (tradeoff). .. nonce: xo5LAr .. section: Library -The :term:`2to3` :2to3fixer:`execfile` fixer now opens the file with mode +The 2to3 ``execfile`` fixer now opens the file with mode ``'rb'``. Patch by Zackery Spytz. .. @@ -3817,7 +3817,7 @@ user. .. nonce: w6g7tn .. section: Library -The :2to3fixer:`reload` fixer now uses :func:`importlib.reload` instead of +The ``reload`` fixer now uses :func:`importlib.reload` instead of deprecated :func:`!imp.reload`. .. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 0888a5c43087b5..7da4385973185a 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -99,7 +99,7 @@ after whitespace, e.g. '127.0.0.1 whatever'. Adds audit events for :mod:`ensurepip`, :mod:`ftplib`, :mod:`glob`, :mod:`imaplib`, :mod:`nntplib`, :mod:`pdb`, :mod:`poplib`, :mod:`shutil`, -:mod:`smtplib`, :mod:`sqlite3`, :mod:`subprocess`, :mod:`telnetlib`, +:mod:`smtplib`, :mod:`sqlite3`, :mod:`subprocess`, :mod:`!telnetlib`, :mod:`tempfile` and :mod:`webbrowser`, as well as :func:`os.listdir`, :func:`os.scandir` and :func:`breakpoint`. diff --git a/Misc/NEWS.d/3.9.0a3.rst b/Misc/NEWS.d/3.9.0a3.rst index 54b61ca3b7785f..b05b4c3d7204c0 100644 --- a/Misc/NEWS.d/3.9.0a3.rst +++ b/Misc/NEWS.d/3.9.0a3.rst @@ -571,7 +571,7 @@ new task spawning before exception raising. .. section: Library Correctly parenthesize filter-based statements that contain lambda -expressions in mod:`lib2to3`. Patch by Dong-hee Na. +expressions in mod:`!lib2to3`. Patch by Dong-hee Na. .. diff --git a/Misc/NEWS.d/3.9.0a6.rst b/Misc/NEWS.d/3.9.0a6.rst index af2cc7c3e97889..9594964917f390 100644 --- a/Misc/NEWS.d/3.9.0a6.rst +++ b/Misc/NEWS.d/3.9.0a6.rst @@ -343,7 +343,7 @@ https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=242274 .. nonce: Er8sv- .. section: Library -The :mod:`lib2to3` module is pending deprecation due to :pep:`617`. +The :mod:`!lib2to3` module is pending deprecation due to :pep:`617`. .. diff --git a/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst b/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst deleted file mode 100644 index dfe71a5552070d..00000000000000 --- a/Misc/NEWS.d/next/Build/2022-06-20-15-15-11.gh-issue-90656.kFBbKe.rst +++ /dev/null @@ -1,7 +0,0 @@ -Add platform triplets for 64-bit LoongArch: - -* loongarch64-linux-gnusf -* loongarch64-linux-gnuf32 -* loongarch64-linux-gnu - -Patch by Zhang Na. diff --git a/Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst b/Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst deleted file mode 100644 index 7135317cd06fa2..00000000000000 --- a/Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst +++ /dev/null @@ -1,4 +0,0 @@ -Changed the default value of the ``SHELL`` Makefile variable from ``/bin/sh`` -to ``/bin/sh -e`` to ensure that complex recipes correctly fail after an error. -Previously, ``make install`` could fail to install some files and yet return -a successful result. diff --git a/Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst b/Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst deleted file mode 100644 index ae9b4d59ca8cec..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst +++ /dev/null @@ -1 +0,0 @@ -Extended workaround defining ``static_assert`` when missing from the libc headers to all clang and gcc builds. In particular, this fixes building on macOS <= 10.10. diff --git a/Misc/NEWS.d/next/Build/2023-04-14-10-24-37.gh-issue-103532.H1djkd.rst b/Misc/NEWS.d/next/Build/2023-04-14-10-24-37.gh-issue-103532.H1djkd.rst deleted file mode 100644 index 255c9833282c2f..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-04-14-10-24-37.gh-issue-103532.H1djkd.rst +++ /dev/null @@ -1,4 +0,0 @@ -The ``TKINTER_PROTECT_LOADTK`` macro is no longer defined or used in the -``_tkinter`` module. It was previously only defined when building against -Tk 8.4.13 and older, but Tk older than 8.5.12 has been unsupported since -gh-issue-91152. diff --git a/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst b/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst deleted file mode 100644 index 900e5bd61d6033..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst +++ /dev/null @@ -1 +0,0 @@ -Add gcc fallback of mkfifoat/mknodat for macOS. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst b/Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst deleted file mode 100644 index 1315b5cb3c1969..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-05-14-19-00-19.gh-issue-104490.1tA4AF.rst +++ /dev/null @@ -1 +0,0 @@ -Define ``.PHONY`` / virtual make targets consistently and properly. diff --git a/Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst b/Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst deleted file mode 100644 index a3517ac0204b1d..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-05-15-09-34-08.gh-issue-99017.nToOQu.rst +++ /dev/null @@ -1 +0,0 @@ -``PYTHON_FOR_REGEN`` now require Python 3.10 or newer. diff --git a/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst b/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst deleted file mode 100644 index 6fc56d2249f581..00000000000000 --- a/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst +++ /dev/null @@ -1,6 +0,0 @@ -Add unstable C-API functions to get the code object, lasti and line number from -the internal ``_PyInterpreterFrame`` in the limited API. The functions are: - -* ``PyCodeObject * PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame)`` -* ``int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame)`` -* ``int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame)`` diff --git a/Misc/NEWS.d/next/C API/2023-02-09-23-09-29.gh-issue-101408._paFIF.rst b/Misc/NEWS.d/next/C API/2023-02-09-23-09-29.gh-issue-101408._paFIF.rst deleted file mode 100644 index 172d66163d42e6..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-09-23-09-29.gh-issue-101408._paFIF.rst +++ /dev/null @@ -1,2 +0,0 @@ -:c:func:`PyObject_GC_Resize` should calculate preheader size if needed. -Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/C API/2023-03-28-12-31-51.gh-issue-103091.CzZyaZ.rst b/Misc/NEWS.d/next/C API/2023-03-28-12-31-51.gh-issue-103091.CzZyaZ.rst deleted file mode 100644 index 28c77b6816af87..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-03-28-12-31-51.gh-issue-103091.CzZyaZ.rst +++ /dev/null @@ -1 +0,0 @@ -Add a new C-API function to eagerly assign a version tag to a PyTypeObject: ``PyUnstable_Type_AssignVersionTag()``. diff --git a/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst b/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst deleted file mode 100644 index af630c3aafa940..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst +++ /dev/null @@ -1,5 +0,0 @@ -Added C API for extending types whose instance memory layout is opaque: -:c:member:`PyType_Spec.basicsize` can now be zero or negative, -:c:func:`PyObject_GetTypeData` can be used to get subclass-specific data, -and :c:macro:`Py_TPFLAGS_ITEMS_AT_END` can be used to safely extend -variable-size objects. See :pep:`697` for details. diff --git a/Misc/NEWS.d/next/C API/2023-04-14-23-05-52.gh-issue-103295.GRHY1Z.rst b/Misc/NEWS.d/next/C API/2023-04-14-23-05-52.gh-issue-103295.GRHY1Z.rst deleted file mode 100644 index 88222239858bc3..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-04-14-23-05-52.gh-issue-103295.GRHY1Z.rst +++ /dev/null @@ -1,5 +0,0 @@ -Introduced :c:func:`PyUnstable_WritePerfMapEntry`, :c:func:`PyUnstable_PerfMapState_Init` and -:c:func:`PyUnstable_PerfMapState_Fini`. These allow extension modules (JIT compilers in -particular) to write to perf-map files in a thread safe manner. The -:doc:`../howto/perf_profiling` also uses these APIs to write -entries in the perf-map file. diff --git a/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst b/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst deleted file mode 100644 index d074350ed3ebbe..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :c:func:`PyUnstable_Object_GC_NewWithExtraData` function that can be used to -allocate additional memory after an object for data not managed by Python. diff --git a/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst b/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst deleted file mode 100644 index 5e4270f82afd84..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst +++ /dev/null @@ -1,4 +0,0 @@ -:c:func:`PyType_FromSpec` and its variants now allow creating classes whose -metaclass overrides :c:member:`~PyTypeObject.tp_new`. The ``tp_new`` is -ignored. This behavior is deprecated and will be disallowed in 3.14+. The -new :c:func:`PyType_FromMetaclass` already disallows it. diff --git a/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst b/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst deleted file mode 100644 index 2ffc0fa81c014a..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst +++ /dev/null @@ -1,5 +0,0 @@ -We've added ``Py_NewInterpreterFromConfig()`` and ``PyInterpreterConfig`` to -the public C-API (but not the stable ABI; not yet at least). The new -function may be used to create a new interpreter with various features -configured. The function was added to support PEP 684 (per-interpreter -GIL). diff --git a/Misc/NEWS.d/next/C API/2023-05-18-20-53-05.gh-issue-101291.ZBh9aR.rst b/Misc/NEWS.d/next/C API/2023-05-18-20-53-05.gh-issue-101291.ZBh9aR.rst deleted file mode 100644 index 465af3b209a0fc..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-05-18-20-53-05.gh-issue-101291.ZBh9aR.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added unstable C API for extracting the value of "compact" integers: -:c:func:`PyUnstable_Long_IsCompact` and -:c:func:`PyUnstable_Long_CompactValue`. diff --git a/Misc/NEWS.d/next/C API/2023-05-19-10-22-34.gh-issue-104668.MLX1g9.rst b/Misc/NEWS.d/next/C API/2023-05-19-10-22-34.gh-issue-104668.MLX1g9.rst new file mode 100644 index 00000000000000..7b882afd7f81a0 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-05-19-10-22-34.gh-issue-104668.MLX1g9.rst @@ -0,0 +1,5 @@ +Don't call :c:var:`PyOS_InputHook` or :c:var:`PyOS_ReadlineFunctionPointer` +in subinterpreters, since it's generally difficult to avoid using global +state in their registered callbacks. This also avoids situations where +extensions may find themselves running in a subinterpreter they don't +support (or haven't yet been loaded in). diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-12-01-12-58-31.bpo-31821.1FNmwk.rst b/Misc/NEWS.d/next/Core and Builtins/2019-12-01-12-58-31.bpo-31821.1FNmwk.rst deleted file mode 100644 index 13c054fdd68276..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-12-01-12-58-31.bpo-31821.1FNmwk.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`!pause_reading` to work when called from :func:`!connection_made` in :mod:`asyncio`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-11-15-54-40.bpo-39610.fvgsCl.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-11-15-54-40.bpo-39610.fvgsCl.rst deleted file mode 100644 index d65e0f3db9d6f5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-02-11-15-54-40.bpo-39610.fvgsCl.rst +++ /dev/null @@ -1,2 +0,0 @@ -``len()`` for 0-dimensional :class:`memoryview`` objects (such as ``memoryview(ctypes.c_uint8(42))``) now raises a :exc:`TypeError`. -Previously this returned ``1``, which was not consistent with ``mem_0d[0]`` raising an :exc:`IndexError``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst deleted file mode 100644 index f9e71bc1344bb3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst +++ /dev/null @@ -1,3 +0,0 @@ -Complex function calls are now faster and consume no C stack -space. - diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-08-12-36-25.gh-issue-99184.KIaqzz.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-08-12-36-25.gh-issue-99184.KIaqzz.rst deleted file mode 100644 index 80076831badfea..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-08-12-36-25.gh-issue-99184.KIaqzz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Bypass instance attribute access of ``__name__`` in ``repr`` of -:class:`weakref.ref`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst deleted file mode 100644 index 2eec05cb3ace5c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst +++ /dev/null @@ -1,2 +0,0 @@ -:pep:`709`: inline list, dict and set comprehensions to improve performance -and reduce bytecode size. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst deleted file mode 100644 index e03113ba05cd7d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make the buffer protocol accessible in Python code using the new -``__buffer__`` and ``__release_buffer__`` magic methods. See :pep:`688` for -details. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst deleted file mode 100644 index 761d45b0a3a84f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst +++ /dev/null @@ -1 +0,0 @@ -Reduce the number of inline :opcode:`CACHE` entries for :opcode:`CALL`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst deleted file mode 100644 index 631ef4c7890450..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst +++ /dev/null @@ -1 +0,0 @@ -Implement :pep:`669` Low Impact Monitoring for CPython. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst deleted file mode 100644 index 46951486e4f9c9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst +++ /dev/null @@ -1 +0,0 @@ -Allow built-in modules to be submodules. This allows submodules to be statically linked into a CPython binary. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-02-22-14-57.gh-issue-84436.hvMgwF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-02-22-14-57.gh-issue-84436.hvMgwF.rst deleted file mode 100644 index c4d8ce75b35a30..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-02-22-14-57.gh-issue-84436.hvMgwF.rst +++ /dev/null @@ -1,3 +0,0 @@ -The implementation of PEP-683 which adds Immortal Objects by using a fixed -reference count that skips reference counting to make objects truly -immutable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-07-12-18-41.gh-issue-103323.9802br.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-07-12-18-41.gh-issue-103323.9802br.rst deleted file mode 100644 index 347c91d973e5ce..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-07-12-18-41.gh-issue-103323.9802br.rst +++ /dev/null @@ -1,3 +0,0 @@ -We've replaced our use of ``_PyRuntime.tstate_current`` with a thread-local -variable. This is a fairly low-level implementation detail, and there -should be no change in behavior. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst deleted file mode 100644 index 38b107f3be174f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Migrate :meth:`~ssl.SSLContext.set_ecdh_curve` method not to use deprecated -OpenSSL APIs. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst deleted file mode 100644 index 793f02c2afdcff..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-09-04-30-02.gh-issue-103333.gKOetS.rst +++ /dev/null @@ -1 +0,0 @@ -:exc:`AttributeError` now retains the ``name`` attribute when pickled and unpickled. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-09-22-21-57.gh-issue-77757._Ow-u2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-09-22-21-57.gh-issue-77757._Ow-u2.rst deleted file mode 100644 index 85c8ecf7de8d1b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-09-22-21-57.gh-issue-77757._Ow-u2.rst +++ /dev/null @@ -1,3 +0,0 @@ -Exceptions raised in a typeobject's ``__set_name__`` method are no longer -wrapped by a :exc:`RuntimeError`. Context information is added to the -exception as a :pep:`678` note. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst deleted file mode 100644 index 819a2359bf6fae..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst +++ /dev/null @@ -1,5 +0,0 @@ -The bitwise inversion operator (``~``) on bool is deprecated. -It returns the bitwise inversion of the underlying ``int`` representation such that -``bool(~True) == True``, which can be confusing. Use ``not`` for logical negation -of bools. In the rare case that you really need the bitwise inversion of the underlying ``int``, -convert to int explicitly ``~int(x)``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-18-51.gh-issue-103488.vYvlHD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-18-51.gh-issue-103488.vYvlHD.rst deleted file mode 100644 index e7daa104e57105..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-18-51.gh-issue-103488.vYvlHD.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change the internal offset distinguishing yield and return target addresses, -so that the instruction pointer is correct for exception handling and other -stack unwinding. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-22-03.gh-issue-87729.99A7UO.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-22-03.gh-issue-87729.99A7UO.rst deleted file mode 100644 index 9d75de1565a170..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-20-22-03.gh-issue-87729.99A7UO.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :opcode:`LOAD_SUPER_ATTR` (and a specialization for ``super().method()``) to -speed up ``super().method()`` and ``super().attr``. This makes -``super().method()`` roughly 2.3x faster and brings it within 20% of the -performance of a simple method call. Patch by Vladimir Matveev and Carl Meyer. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-13-00-58-55.gh-issue-103492.P4k0Ay.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-13-00-58-55.gh-issue-103492.P4k0Ay.rst deleted file mode 100644 index 929650968173e7..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-13-00-58-55.gh-issue-103492.P4k0Ay.rst +++ /dev/null @@ -1 +0,0 @@ -Clarify :exc:`SyntaxWarning` with literal ``is`` comparison by specifying which literal is problematic, since comparisons using ``is`` with e.g. None and bool literals are idiomatic. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-14-22-35-23.gh-issue-101517.5EqM-S.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-14-22-35-23.gh-issue-101517.5EqM-S.rst deleted file mode 100644 index 730c6cd40d7235..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-14-22-35-23.gh-issue-101517.5EqM-S.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug in line numbers of instructions emitted for :keyword:`except* `. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-16-14-38-39.gh-issue-100530.OR6-sn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-16-14-38-39.gh-issue-100530.OR6-sn.rst deleted file mode 100644 index 5b1bcc4a680fc3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-16-14-38-39.gh-issue-100530.OR6-sn.rst +++ /dev/null @@ -1 +0,0 @@ -Clarify the error message raised when the called part of a class pattern isn't actually a class. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-17-16-00-32.gh-issue-102856.UunJ7y.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-17-16-00-32.gh-issue-102856.UunJ7y.rst deleted file mode 100644 index 35eceb83816bcb..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-17-16-00-32.gh-issue-102856.UunJ7y.rst +++ /dev/null @@ -1 +0,0 @@ -Implement the required C tokenizer changes for PEP 701. Patch by Pablo Galindo Salgado, Lysandros Nikolaou, Batuhan Taskaya, Marta Gómez Macías and sunmy2019. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst deleted file mode 100644 index 5434660e9d6ffb..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-20-16-17-51.gh-issue-103650.K1MFXR.rst +++ /dev/null @@ -1 +0,0 @@ -Change the perf map format to remove the '0x' prefix from the addresses diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-21-16-12-41.gh-issue-103590.7DHDOE.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-21-16-12-41.gh-issue-103590.7DHDOE.rst deleted file mode 100644 index af733a8207a2c1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-21-16-12-41.gh-issue-103590.7DHDOE.rst +++ /dev/null @@ -1 +0,0 @@ -Do not wrap a single exception raised from a ``try-except*`` construct in an :exc:`ExceptionGroup`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-21-17-03-14.gh-issue-102310.anLjDx.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-21-17-03-14.gh-issue-102310.anLjDx.rst deleted file mode 100644 index 15cb6c64adbab1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-21-17-03-14.gh-issue-102310.anLjDx.rst +++ /dev/null @@ -1 +0,0 @@ -Change the error range for invalid bytes literals. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst deleted file mode 100644 index c48348798e7142..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst +++ /dev/null @@ -1,3 +0,0 @@ -Optimized asyncio Task creation by deferring expensive string formatting -(task name generation) from Task creation to the first time ``get_name`` is -called. This makes asyncio benchmarks up to 5% faster. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-24-21-47-38.gh-issue-103801.WaBanq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-24-21-47-38.gh-issue-103801.WaBanq.rst deleted file mode 100644 index 6f07d72fafdfc3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-24-21-47-38.gh-issue-103801.WaBanq.rst +++ /dev/null @@ -1 +0,0 @@ -Adds three minor linting fixes to the wasm module caught that were caught by ruff. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst deleted file mode 100644 index cefe6429ab2041..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-08-43-11.gh-issue-103763.ZLBZk1.rst +++ /dev/null @@ -1,23 +0,0 @@ -Implement :pep:`695`, adding syntactic support for generic classes, generic -functions, and type aliases. - -A new ``type X = ...`` syntax is added for type aliases, which resolves at -runtime to an instance of the new class ``typing.TypeAliasType``. -The value is lazily evaluated and is accessible through the ``.__value__`` -attribute. This is implemented as a new AST node ``ast.TypeAlias``. - -New syntax (``class X[T]: ...``, ``def func[T](): ...``) is added for defining -generic functions and classes. This is implemented as a new -``typeparams`` attribute on the AST nodes for classes and functions. -This node holds instances of the new AST classes ``ast.TypeVar``, -``ast.ParamSpec``, and ``ast.TypeVarTuple``. - -``typing.TypeVar``, ``typing.ParamSpec``, ``typing.ParamSpecArgs``, -``typing.ParamSpecKwargs``, ``typing.TypeVarTuple``, and -``typing.Generic`` are now implemented in C rather than Python. - -There are new bytecode instructions ``LOAD_LOCALS``, -``LOAD_CLASSDICT_OR_GLOBAL``, and ``LOAD_CLASSDICT_OR_DEREF`` -to support correct resolution of names in class namespaces. - -Patch by Eric Traut, Larry Hastings, and Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst deleted file mode 100644 index e8434854cde632..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst +++ /dev/null @@ -1 +0,0 @@ -Remove both line and instruction instrumentation before adding new ones for monitoring, to avoid newly added instrumentation being removed immediately. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst deleted file mode 100644 index c12a6b9cb841f2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst +++ /dev/null @@ -1,3 +0,0 @@ -Provide a helpful hint in the :exc:`TypeError` message when accidentally -calling a :term:`module` object that has a callable attribute of the same -name (such as :func:`dis.dis` or :class:`datetime.datetime`). diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-16-26-35.gh-issue-103907.kiONZQ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-16-26-35.gh-issue-103907.kiONZQ.rst deleted file mode 100644 index cb2aaffed9ff14..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-16-26-35.gh-issue-103907.kiONZQ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Don't modify the refcounts of known immortal objects (:const:`True`, -:const:`False`, and :const:`None`) in the main interpreter loop. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst deleted file mode 100644 index 6fed304c9132b3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve handling of edge cases in showing ``Exception.__notes__``. Ensures -that the messages always end with a newline and that string/bytes are not -exploded over multiple lines. Patch by Carey Metcalfe. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst deleted file mode 100644 index 997bef226e713f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst +++ /dev/null @@ -1 +0,0 @@ -Fix performance loss when accessing an object's attributes with ``__getattr__`` defined. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst deleted file mode 100644 index f3cadaee0e32d9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst +++ /dev/null @@ -1 +0,0 @@ -Disallow the "z" format specifier in %-format of bytes objects. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst deleted file mode 100644 index 9c35ea88499dce..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst +++ /dev/null @@ -1,2 +0,0 @@ -Reduce object creation while calling callback function from gc. -Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst deleted file mode 100644 index 97e0c01689cb6f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve the performance of :func:`hasattr` for module objects with a missing -attribute. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst deleted file mode 100644 index 6f24529bac3e0c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst +++ /dev/null @@ -1 +0,0 @@ -Improve the performance of :c:func:`PyObject_HasAttrString` diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst deleted file mode 100644 index 6a19ae84057f4c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue where :class:`list` or :class:`tuple` repetition could fail to -respect :pep:`683`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst deleted file mode 100644 index dad843636493ae..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst +++ /dev/null @@ -1,6 +0,0 @@ -Multi-phase init extension modules may now indicate whether or not they -actually support multiple interpreters. By default such modules are -expected to support use in multiple interpreters. In the uncommon case that -one does not, it may use the new ``Py_mod_multiple_interpreters`` module def -slot. A value of ``0`` means the module does not support them. ``1`` means -it does. The default is ``1``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst deleted file mode 100644 index 42e26cb27b6e01..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-12-14-47.gh-issue-99113.-RAdnv.rst +++ /dev/null @@ -1,6 +0,0 @@ -The GIL is now (optionally) per-interpreter. This is the fundamental change -for PEP 684. This is all made possible by virtue of the isolated state of -each interpreter in the process. The behavior of the main interpreter -remains unchanged. Likewise, interpreters created using -``Py_NewInterpreter()`` are not affected. To get an interpreter with its -own GIL, call ``Py_NewInterpreterFromConfig()``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst deleted file mode 100644 index afd26750846167..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-05-13-18-56.gh-issue-99113.hT1ajK.rst +++ /dev/null @@ -1,11 +0,0 @@ -Multi-phase init extension modules may now indicate that they support -running in subinterpreters that have their own GIL. This is done by using -``Py_MOD_PER_INTERPRETER_GIL_SUPPORTED`` as the value for the -``Py_mod_multiple_interpreters`` module def slot. Otherwise the module, by -default, cannot be imported in such subinterpreters. (This does not affect -the main interpreter or subinterpreters that do not have their own GIL.) In -addition to the isolation that multi-phase init already normally requires, -support for per-interpreter GIL involves one additional constraint: -thread-safety. If the module has external (linked) dependencies and those -libraries have any state that isn't thread-safe then the module must do the -additional work to add thread-safety. This should be an uncommon case. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst deleted file mode 100644 index 342467cfcd4e75..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-08-10-34-55.gh-issue-104263.ctHWI8.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix ``float("nan")`` to produce a quiet NaN on platforms (like MIPS) where -the meaning of the signalling / quiet bit is inverted from its usual -meaning. Also introduce a new macro ``Py_INFINITY`` matching C99's -``INFINITY``, and refactor internals to rely on C99's ``NAN`` and -``INFINITY`` macros instead of hard-coding bit patterns for infinities and -NaNs. Thanks Sebastian Berg. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst deleted file mode 100644 index 40eee64dfa717d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst +++ /dev/null @@ -1,5 +0,0 @@ -Change behavior of ``sys.monitoring.events.LINE`` events in -``sys.monitoring``: Line events now occur when a new line is reached -dynamically, instead of using a static approximation, as before. This makes -the behavior very similar to that of "line" events in ``sys.settrace``. This -should ease porting of tools from 3.11 to 3.12. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst deleted file mode 100644 index 06ec5d7b0f0cc9..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-11-15-56-07.gh-issue-104405.tXV5fn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue where some :term:`bytecode` instructions could ignore -:pep:`523` when "inlining" calls. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-12-13-30-04.gh-issue-102818.rnv1mH.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-12-13-30-04.gh-issue-102818.rnv1mH.rst deleted file mode 100644 index 5cb77bff792c90..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-12-13-30-04.gh-issue-102818.rnv1mH.rst +++ /dev/null @@ -1,5 +0,0 @@ -Do not add a frame to the traceback in the ``sys.setprofile`` and -``sys.settrace`` trampoline functions. This ensures that frames are not -duplicated if an exception is raised in the callback function, and ensures -that frames are not omitted if a C callback is used and that does not add the -frame. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-14-18-56-54.gh-issue-104482.yaQsv8.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-14-18-56-54.gh-issue-104482.yaQsv8.rst deleted file mode 100644 index 07c09a3a84a62c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-14-18-56-54.gh-issue-104482.yaQsv8.rst +++ /dev/null @@ -1 +0,0 @@ -Fix three error handling bugs in ast.c's validation of pattern matching statements. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-16-19-17-48.gh-issue-104572.eBZQYS.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-16-19-17-48.gh-issue-104572.eBZQYS.rst deleted file mode 100644 index 25bad8aa6e3e6f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-16-19-17-48.gh-issue-104572.eBZQYS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve syntax error message for invalid constructs in :pep:`695` contexts -and in annotations when ``from __future__ import annotations`` is active. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-18-13-00-21.gh-issue-104615.h_rtw2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-18-13-00-21.gh-issue-104615.h_rtw2.rst deleted file mode 100644 index 447291a619dd67..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-18-13-00-21.gh-issue-104615.h_rtw2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix wrong ordering of assignments in code like ``a, a = x, y``. Contributed by -Carl Meyer. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-20-23-08-48.gh-issue-102856.Knv9WT.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-20-23-08-48.gh-issue-102856.Knv9WT.rst deleted file mode 100644 index ff831c9f935db3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-20-23-08-48.gh-issue-102856.Knv9WT.rst +++ /dev/null @@ -1 +0,0 @@ -Implement PEP 701 changes in the :mod:`tokenize` module. Patch by Marta Gómez Macías and Pablo Galindo Salgado diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-23-00-36-02.gh-issue-104770.poSkyY.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-23-00-36-02.gh-issue-104770.poSkyY.rst new file mode 100644 index 00000000000000..2103fb7d61c21a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-23-00-36-02.gh-issue-104770.poSkyY.rst @@ -0,0 +1,2 @@ +If a generator returns a value upon being closed, the value is now returned +by :meth:`generator.close`. diff --git a/Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst b/Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst deleted file mode 100644 index 1569e815ee50fa..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst +++ /dev/null @@ -1 +0,0 @@ -Fix extension type from documentation for compiling in C++20 mode diff --git a/Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst b/Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst deleted file mode 100644 index 39461f3f84c9ac..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add support for Unicode Path Extra Field in ZipFile. Patch by Yeojin Kim -and Andrea Giudiceandrea diff --git a/Misc/NEWS.d/next/Documentation/2023-04-25-22-58-08.gh-issue-48241.l1Gxxh.rst b/Misc/NEWS.d/next/Documentation/2023-04-25-22-58-08.gh-issue-48241.l1Gxxh.rst deleted file mode 100644 index 619505cf6ee5b8..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-04-25-22-58-08.gh-issue-48241.l1Gxxh.rst +++ /dev/null @@ -1 +0,0 @@ -Clarifying documentation about the url parameter to urllib.request.urlopen and urllib.request.Requst needing to be encoded properly. diff --git a/Misc/NEWS.d/next/Documentation/2023-04-26-23-55-31.gh-issue-103629.-0reqn.rst b/Misc/NEWS.d/next/Documentation/2023-04-26-23-55-31.gh-issue-103629.-0reqn.rst deleted file mode 100644 index 6dc0a1cb5a3e4f..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-04-26-23-55-31.gh-issue-103629.-0reqn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Mention the new way of typing ``**kwargs`` with ``Unpack`` and ``TypedDict`` -introduced in :pep:`692`. diff --git a/Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst b/Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst deleted file mode 100644 index 2c6ef17810723d..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-05-14-12-11-28.gh-issue-67056.nVC2Rf.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document that the effect of registering or unregistering an :mod:`atexit` -cleanup function from within a registered cleanup function is undefined. diff --git a/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst b/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst deleted file mode 100644 index 4f390d189d23b5..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst +++ /dev/null @@ -1 +0,0 @@ -Fix IDLE test hang on macOS. diff --git a/Misc/NEWS.d/next/IDLE/2023-05-17-15-11-11.gh-issue-104496.wjav-y.rst b/Misc/NEWS.d/next/IDLE/2023-05-17-15-11-11.gh-issue-104496.wjav-y.rst deleted file mode 100644 index 18d83150e94f01..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-05-17-15-11-11.gh-issue-104496.wjav-y.rst +++ /dev/null @@ -1 +0,0 @@ -About prints both tcl and tk versions if different (expected someday). diff --git a/Misc/NEWS.d/next/IDLE/2023-05-17-17-32-21.gh-issue-104499.hNeqV4.rst b/Misc/NEWS.d/next/IDLE/2023-05-17-17-32-21.gh-issue-104499.hNeqV4.rst deleted file mode 100644 index 20cbd9ab9e571f..00000000000000 --- a/Misc/NEWS.d/next/IDLE/2023-05-17-17-32-21.gh-issue-104499.hNeqV4.rst +++ /dev/null @@ -1 +0,0 @@ -Fix completions for Tk Aqua 8.7 (currently blank). diff --git a/Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst b/Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst deleted file mode 100644 index 00bcf38bbcdf51..00000000000000 --- a/Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst +++ /dev/null @@ -1,3 +0,0 @@ -http.client CONNECT method tunnel improvements: Use HTTP 1.1 protocol; send -a matching Host: header with CONNECT, if one is not provided; convert IDN -domain names to Punycode. Patch by Michael Handler. diff --git a/Misc/NEWS.d/next/Library/2020-02-25-00-43-22.bpo-39744.hgK689.rst b/Misc/NEWS.d/next/Library/2020-02-25-00-43-22.bpo-39744.hgK689.rst deleted file mode 100644 index 6e690f996569a4..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-02-25-00-43-22.bpo-39744.hgK689.rst +++ /dev/null @@ -1 +0,0 @@ -Make :func:`asyncio.subprocess.Process.communicate` close the subprocess's stdin even when called with ``input=None``. diff --git a/Misc/NEWS.d/next/Library/2020-05-25-12-42-36.bpo-17258.lf2554.rst b/Misc/NEWS.d/next/Library/2020-05-25-12-42-36.bpo-17258.lf2554.rst deleted file mode 100644 index 18ebd6e140cff7..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-05-25-12-42-36.bpo-17258.lf2554.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`multiprocessing` now supports stronger HMAC algorithms for inter-process -connection authentication rather than only HMAC-MD5. diff --git a/Misc/NEWS.d/next/Library/2020-09-16-16-53-06.bpo-41768.8_fWkC.rst b/Misc/NEWS.d/next/Library/2020-09-16-16-53-06.bpo-41768.8_fWkC.rst new file mode 100644 index 00000000000000..bfd3a294d44efa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-09-16-16-53-06.bpo-41768.8_fWkC.rst @@ -0,0 +1,2 @@ +:mod:`unittest.mock` speccing no longer calls class properties. +Patch by Melanie Witt. diff --git a/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst b/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst deleted file mode 100644 index 0904b162e63523..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added :meth:`http.client.HTTPConnection.get_proxy_response_headers` that -provides access to the HTTP headers on a proxy server response to the -``CONNECT`` request. diff --git a/Misc/NEWS.d/next/Library/2021-11-07-15-31-25.bpo-23041.564i32.rst b/Misc/NEWS.d/next/Library/2021-11-07-15-31-25.bpo-23041.564i32.rst deleted file mode 100644 index 53c32d397b206b..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-11-07-15-31-25.bpo-23041.564i32.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :data:`~csv.QUOTE_STRINGS` and :data:`~csv.QUOTE_NOTNULL` to the suite -of :mod:`csv` module quoting styles. diff --git a/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst b/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst deleted file mode 100644 index 531f4729220036..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fixed the bug in :meth:`pathlib.Path.glob` -- previously a dangling symlink -would not be found by this method when the pattern is an exact match, but -would be found when the pattern contains a wildcard or the recursive -wildcard (``**``). With this change, a dangling symlink will be found in -both cases. diff --git a/Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst b/Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst deleted file mode 100644 index f0c91236dfdf21..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst +++ /dev/null @@ -1 +0,0 @@ -Enables :mod:`webbrowser` to detect and launch Microsoft Edge browser. diff --git a/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst b/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst deleted file mode 100644 index 6539efbc9d0eb0..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-02-19-14-19-34.bpo-46797.6BXZX4.rst +++ /dev/null @@ -1,4 +0,0 @@ -Deprecation warnings are now emitted for :class:`!ast.Num`, -:class:`!ast.Bytes`, :class:`!ast.Str`, :class:`!ast.NameConstant` and -:class:`!ast.Ellipsis`. These have been documented as deprecated since Python -3.8, and will be removed in Python 3.14. diff --git a/Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst b/Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst deleted file mode 100644 index 65dbdc9f60371e..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-05-02-16-21-05.gh-issue-92184.hneGVW.rst +++ /dev/null @@ -1,3 +0,0 @@ -When creating zip files using :mod:`zipfile`, ``os.altsep``, if not ``None``, -will always be treated as a path separator even when it is not ``/``. -Patch by Carey Metcalfe. diff --git a/Misc/NEWS.d/next/Library/2022-05-17-10-46-44.gh-issue-92871.GVogrT.rst b/Misc/NEWS.d/next/Library/2022-05-17-10-46-44.gh-issue-92871.GVogrT.rst new file mode 100644 index 00000000000000..992f8afadbe912 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-17-10-46-44.gh-issue-92871.GVogrT.rst @@ -0,0 +1,2 @@ +Remove the ``typing.io`` and ``typing.re`` namespaces, deprecated since Python +3.8. All items are still available from the main :mod:`typing` module. diff --git a/Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst b/Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst deleted file mode 100644 index 7719b74b8e5ef1..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst +++ /dev/null @@ -1 +0,0 @@ -Convert private :meth:`_posixsubprocess.fork_exec` to use Argument Clinic. diff --git a/Misc/NEWS.d/next/Library/2022-07-06-11-10-37.gh-issue-51574.sveUeD.rst b/Misc/NEWS.d/next/Library/2022-07-06-11-10-37.gh-issue-51574.sveUeD.rst deleted file mode 100644 index 50a3d6a4629182..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-06-11-10-37.gh-issue-51574.sveUeD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make :func:`tempfile.mkdtemp` return absolute paths when its *dir* -parameter is relative. diff --git a/Misc/NEWS.d/next/Library/2022-07-16-17-15-29.gh-issue-94906.C4G8DG.rst b/Misc/NEWS.d/next/Library/2022-07-16-17-15-29.gh-issue-94906.C4G8DG.rst deleted file mode 100644 index 663343371d1b99..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-16-17-15-29.gh-issue-94906.C4G8DG.rst +++ /dev/null @@ -1 +0,0 @@ -Support multiple steps in :func:`math.nextafter`. Patch by Shantanu Jain and Matthias Gorgens. diff --git a/Misc/NEWS.d/next/Library/2022-08-27-21-41-41.gh-issue-87474.9X-kxt.rst b/Misc/NEWS.d/next/Library/2022-08-27-21-41-41.gh-issue-87474.9X-kxt.rst deleted file mode 100644 index eeb5308680e8af..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-08-27-21-41-41.gh-issue-87474.9X-kxt.rst +++ /dev/null @@ -1 +0,0 @@ -Fix potential file descriptor leaks in :class:`subprocess.Popen`. diff --git a/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst b/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst deleted file mode 100644 index 0497d9eb69163e..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst +++ /dev/null @@ -1 +0,0 @@ -Support divert(4) added in FreeBSD 14. diff --git a/Misc/NEWS.d/next/Library/2022-09-07-09-32-07.gh-issue-96522.t73oqp.rst b/Misc/NEWS.d/next/Library/2022-09-07-09-32-07.gh-issue-96522.t73oqp.rst deleted file mode 100644 index 12ed52f97129eb..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-07-09-32-07.gh-issue-96522.t73oqp.rst +++ /dev/null @@ -1 +0,0 @@ -Fix potential deadlock in pty.spawn() diff --git a/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst b/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst deleted file mode 100644 index ac185406844153..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove more deprecated importlib APIs: ``find_loader()``, ``find_module()``, -``importlib.abc.Finder``, ``pkgutil.ImpImporter``, ``pkgutil.ImpLoader``. diff --git a/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst b/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst deleted file mode 100644 index e3297d164fff6d..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deprecate :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` -in favor of :func:`importlib.util.find_spec`. diff --git a/Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst b/Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst deleted file mode 100644 index f67bffcb0ddc6c..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst +++ /dev/null @@ -1 +0,0 @@ -Remove the long-deprecated ``imp`` module. diff --git a/Misc/NEWS.d/next/Library/2022-11-10-16-26-47.gh-issue-99353.DQFjnt.rst b/Misc/NEWS.d/next/Library/2022-11-10-16-26-47.gh-issue-99353.DQFjnt.rst deleted file mode 100644 index 1ad42d5c9aa53d..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-10-16-26-47.gh-issue-99353.DQFjnt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Respect the :class:`http.client.HTTPConnection` ``.debuglevel`` flag -in :class:`urllib.request.AbstractHTTPHandler` when its constructor -parameter ``debuglevel`` is not set. And do the same for ``*HTTPS*``. diff --git a/Misc/NEWS.d/next/Library/2023-01-14-17-54-56.gh-issue-95299.vUhpKz.rst b/Misc/NEWS.d/next/Library/2023-01-14-17-54-56.gh-issue-95299.vUhpKz.rst deleted file mode 100644 index 29c30848e09a83..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-14-17-54-56.gh-issue-95299.vUhpKz.rst +++ /dev/null @@ -1 +0,0 @@ -Remove the bundled setuptools wheel from ``ensurepip``, and stop installing setuptools in environments created by ``venv``. diff --git a/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst b/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst deleted file mode 100644 index 556db0eae00c0b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-22-14-53-12.gh-issue-89550.c1U23f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Decrease execution time of some :mod:`gzip` file writes by 15% by -adding more appropriate buffering. diff --git a/Misc/NEWS.d/next/Library/2023-02-06-16-45-18.gh-issue-83861.mMbIU3.rst b/Misc/NEWS.d/next/Library/2023-02-06-16-45-18.gh-issue-83861.mMbIU3.rst deleted file mode 100644 index e85e7a4ff2e73a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-06-16-45-18.gh-issue-83861.mMbIU3.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix datetime.astimezone method return value when invoked on a naive datetime -instance that represents local time falling in a timezone transition gap. -PEP 495 requires that instances with fold=1 produce earlier times than those -with fold=0 in this case. diff --git a/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst b/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst deleted file mode 100644 index 917cf0f97b9e06..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-09-22-24-34.gh-issue-101640.oFuEpB.rst +++ /dev/null @@ -1 +0,0 @@ -:class:`argparse.ArgumentParser` now catches errors when writing messages, such as when :data:`sys.stderr` is ``None``. Patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Library/2023-02-11-15-01-32.gh-issue-101688.kwXmfM.rst b/Misc/NEWS.d/next/Library/2023-02-11-15-01-32.gh-issue-101688.kwXmfM.rst deleted file mode 100644 index 6df69463931494..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-11-15-01-32.gh-issue-101688.kwXmfM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement :func:`types.get_original_bases` to provide further introspection -for types. diff --git a/Misc/NEWS.d/next/Library/2023-02-11-21-18-10.gh-issue-85984.nvzOD0.rst b/Misc/NEWS.d/next/Library/2023-02-11-21-18-10.gh-issue-85984.nvzOD0.rst deleted file mode 100644 index 246530472165f7..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-11-21-18-10.gh-issue-85984.nvzOD0.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :func:`tty.cfmakeraw` and :func:`tty.cfmakecbreak` to :mod:`tty` and -modernize, the behavior of :func:`tty.setraw` and :func:`tty.setcbreak` to use -POSIX.1-2017 Chapter 11 "General Terminal Interface" flag masks by default. diff --git a/Misc/NEWS.d/next/Library/2023-02-17-21-14-40.gh-issue-78079.z3Szr6.rst b/Misc/NEWS.d/next/Library/2023-02-17-21-14-40.gh-issue-78079.z3Szr6.rst deleted file mode 100644 index bbb9ac3e3f8faa..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-21-14-40.gh-issue-78079.z3Szr6.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix incorrect normalization of UNC device path roots, and partial UNC share -path roots, in :class:`pathlib.PurePath`. Pathlib no longer appends a -trailing slash to such paths. diff --git a/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst b/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst deleted file mode 100644 index a8d66ea48c3278..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-19-12-37-08.gh-issue-62432.GnBFIB.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :mod:`unittest` runner will now exit with status code 5 if no tests -were run. It is common for test runner misconfiguration to fail to find -any tests, this should be an error. diff --git a/Misc/NEWS.d/next/Library/2023-02-21-14-57-34.gh-issue-102114.uUDQzb.rst b/Misc/NEWS.d/next/Library/2023-02-21-14-57-34.gh-issue-102114.uUDQzb.rst deleted file mode 100644 index 4140c9a96cd272..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-21-14-57-34.gh-issue-102114.uUDQzb.rst +++ /dev/null @@ -1 +0,0 @@ -Functions in the :mod:`dis` module that accept a source code string as argument now print a more concise traceback when the string contains a syntax or indentation error. diff --git a/Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst b/Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst deleted file mode 100644 index 87617a503c0dba..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :class:`pathlib.Path` construction by omitting the path anchor from -the internal list of path parts. diff --git a/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst b/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst deleted file mode 100644 index b5282d3d612916..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst +++ /dev/null @@ -1 +0,0 @@ -Deprecate :class:`collections.abc.ByteString` diff --git a/Misc/NEWS.d/next/Library/2023-03-08-19-30-53.gh-issue-102120.xkQ5Wr.rst b/Misc/NEWS.d/next/Library/2023-03-08-19-30-53.gh-issue-102120.xkQ5Wr.rst new file mode 100644 index 00000000000000..ca50242fdbe293 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-08-19-30-53.gh-issue-102120.xkQ5Wr.rst @@ -0,0 +1,2 @@ +Added a stream mode to ``tarfile`` that allows for reading +archives without caching info about the inner files. diff --git a/Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst b/Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst deleted file mode 100644 index 4a73bbf32b370e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-14-11-20-19.gh-issue-101819.0-h0it.rst +++ /dev/null @@ -1,2 +0,0 @@ -Isolate the :mod:`io` extension module by applying :pep:`687`. Patch by -Kumar Aditya, Victor Stinner, and Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst b/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst deleted file mode 100644 index ef5690533985d5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add *case_sensitive* keyword-only argument to :meth:`pathlib.Path.glob` and -:meth:`~pathlib.Path.rglob`. diff --git a/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst b/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst deleted file mode 100644 index 0b3854d74eb991..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst +++ /dev/null @@ -1,6 +0,0 @@ -Implemented an eager task factory in asyncio. -When used as a task factory on an event loop, it performs eager execution of -coroutines. Coroutines that are able to complete synchronously (e.g. return or -raise without blocking) are returned immediately as a finished task, and the -task is never scheduled to the event loop. If the coroutine blocks, the -(pending) task is scheduled and returned. diff --git a/Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst b/Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst deleted file mode 100644 index 48a105a4a17b29..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst +++ /dev/null @@ -1,4 +0,0 @@ -The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, -have a new a *filter* argument that allows limiting tar features than may be -surprising or dangerous, such as creating files outside the destination -directory. See :ref:`tarfile-extraction-filter` for details. diff --git a/Misc/NEWS.d/next/Library/2023-03-24-20-49-48.gh-issue-103000.6eVNZI.rst b/Misc/NEWS.d/next/Library/2023-03-24-20-49-48.gh-issue-103000.6eVNZI.rst deleted file mode 100644 index 15f16d9eb4c1bf..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-24-20-49-48.gh-issue-103000.6eVNZI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :func:`dataclasses.astuple` and -:func:`dataclasses.asdict` in cases where the contents are common Python types. diff --git a/Misc/NEWS.d/next/Library/2023-03-28-09-13-31.gh-issue-103015.ETTfNf.rst b/Misc/NEWS.d/next/Library/2023-03-28-09-13-31.gh-issue-103015.ETTfNf.rst deleted file mode 100644 index dcac1a28ca5847..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-28-09-13-31.gh-issue-103015.ETTfNf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add *entrypoint* keyword-only parameter to -:meth:`sqlite3.Connection.load_extension`, for overriding the SQLite -extension entry point. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-03-31-01-13-00.gh-issue-103143.6eMluy.rst b/Misc/NEWS.d/next/Library/2023-03-31-01-13-00.gh-issue-103143.6eMluy.rst deleted file mode 100644 index 32bd62d27c7c6d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-31-01-13-00.gh-issue-103143.6eMluy.rst +++ /dev/null @@ -1 +0,0 @@ -Polish the help messages and docstrings of :mod:`pdb`. diff --git a/Misc/NEWS.d/next/Library/2023-04-01-23-01-31.gh-issue-103176.FBsdxa.rst b/Misc/NEWS.d/next/Library/2023-04-01-23-01-31.gh-issue-103176.FBsdxa.rst deleted file mode 100644 index b89f9bae595457..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-01-23-01-31.gh-issue-103176.FBsdxa.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`sys._current_exceptions` now returns a mapping from thread-id to an -exception instance, rather than to a ``(typ, exc, tb)`` tuple. diff --git a/Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst b/Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst deleted file mode 100644 index f0b76a605a5610..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :func:`inspect.getattr_static`. Patch by Alex -Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst b/Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst deleted file mode 100644 index 8ec568ec4e4775..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst +++ /dev/null @@ -1 +0,0 @@ -Fix various Windows-specific issues with ``shutil.which``. diff --git a/Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst b/Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst deleted file mode 100644 index f8b3aa5151b622..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-02-23-05-22.gh-issue-103204.bbDmu0.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixes :mod:`http.server` accepting HTTP requests with HTTP version numbers -preceded by '+', or '-', or with digit-separating '_' characters. The length -of the version numbers is also constrained. diff --git a/Misc/NEWS.d/next/Library/2023-04-03-21-08-53.gh-issue-103220.OW_Bj5.rst b/Misc/NEWS.d/next/Library/2023-04-03-21-08-53.gh-issue-103220.OW_Bj5.rst deleted file mode 100644 index 9cf26c26873b2a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-03-21-08-53.gh-issue-103220.OW_Bj5.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where :func:`os.path.join` added a slash when joining onto an -incomplete UNC drive with a trailing slash on Windows. diff --git a/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst b/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst deleted file mode 100644 index 58db90480d2ff0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :meth:`pathlib.PurePath.with_segments`, which creates a path object from -arguments. This method is called whenever a derivative path is created, such -as from :attr:`pathlib.PurePath.parent`. Subclasses may override this method -to share information between path objects. diff --git a/Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst b/Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst deleted file mode 100644 index e7586a223c1415..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst +++ /dev/null @@ -1 +0,0 @@ -Isolate :mod:`!_socket` (apply :pep:`687`). Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-04-03-23-44-34.gh-issue-102978.gy9eVk.rst b/Misc/NEWS.d/next/Library/2023-04-03-23-44-34.gh-issue-102978.gy9eVk.rst deleted file mode 100644 index df63af10a385eb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-03-23-44-34.gh-issue-102978.gy9eVk.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixes :func:`unittest.mock.patch` not enforcing function signatures for methods -decorated with ``@classmethod`` or ``@staticmethod`` when patch is called with -``autospec=True``. diff --git a/Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst b/Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst deleted file mode 100644 index 783aefae0770a9..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst +++ /dev/null @@ -1 +0,0 @@ -Remove deprecation of enum ``memmber.member`` access. diff --git a/Misc/NEWS.d/next/Library/2023-04-04-21-27-51.gh-issue-103092.7s7Bzf.rst b/Misc/NEWS.d/next/Library/2023-04-04-21-27-51.gh-issue-103092.7s7Bzf.rst deleted file mode 100644 index 39c62ffbe8c659..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-04-21-27-51.gh-issue-103092.7s7Bzf.rst +++ /dev/null @@ -1 +0,0 @@ -Adapt the :mod:`winsound` extension module to :pep:`687`. diff --git a/Misc/NEWS.d/next/Library/2023-04-04-21-44-25.gh-issue-103092.Dz0_Xn.rst b/Misc/NEWS.d/next/Library/2023-04-04-21-44-25.gh-issue-103092.Dz0_Xn.rst deleted file mode 100644 index 7bd191e3c22b2b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-04-21-44-25.gh-issue-103092.Dz0_Xn.rst +++ /dev/null @@ -1 +0,0 @@ -Adapt the :mod:`msvcrt` extension module to :pep:`687`. diff --git a/Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst b/Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst deleted file mode 100644 index 5d1a063acdeb8c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in :mod:`pdb` when displaying line numbers of module-level source code. diff --git a/Misc/NEWS.d/next/Library/2023-04-06-04-35-59.gh-issue-103285.rCZ9-G.rst b/Misc/NEWS.d/next/Library/2023-04-06-04-35-59.gh-issue-103285.rCZ9-G.rst deleted file mode 100644 index 62b4364c2b1665..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-06-04-35-59.gh-issue-103285.rCZ9-G.rst +++ /dev/null @@ -1 +0,0 @@ -Improve performance of :func:`ast.get_source_segment`. diff --git a/Misc/NEWS.d/next/Library/2023-04-06-16-55-51.gh-issue-102778.BWeAmE.rst b/Misc/NEWS.d/next/Library/2023-04-06-16-55-51.gh-issue-102778.BWeAmE.rst deleted file mode 100644 index 64ae5b5b6d564b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-06-16-55-51.gh-issue-102778.BWeAmE.rst +++ /dev/null @@ -1 +0,0 @@ -Support ``sys.last_exc`` in :mod:`idlelib`. diff --git a/Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst b/Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst deleted file mode 100644 index 894c046dcdf0fd..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fixed a bug that caused :mod:`hmac` to raise an exception when the requested -hash algorithm was not available in OpenSSL despite being available -separately as part of ``hashlib`` itself. It now falls back properly to the -built-in. This could happen when, for example, your OpenSSL does not include -SHA3 support and you want to compute ``hmac.digest(b'K', b'M', -'sha3_256')``. diff --git a/Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst b/Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst deleted file mode 100644 index 0a103ae11970d4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst +++ /dev/null @@ -1,3 +0,0 @@ -The members of a runtime-checkable protocol are now considered "frozen" at -runtime as soon as the class has been created. See -:ref:`"What's new in Python 3.12" ` for more details. diff --git a/Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst b/Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst deleted file mode 100644 index 48f11aac692ddb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst +++ /dev/null @@ -1,8 +0,0 @@ -The performance of :func:`isinstance` checks against -:func:`runtime-checkable protocols ` has been -considerably improved for protocols that only have a few members. To achieve -this improvement, several internal implementation details of the -:mod:`typing` module have been refactored, including -``typing._ProtocolMeta.__instancecheck__``, -``typing._is_callable_members_only``, and ``typing._get_protocol_attrs``. -Patches by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-04-08-00-48-40.gh-issue-103092.5EFts0.rst b/Misc/NEWS.d/next/Library/2023-04-08-00-48-40.gh-issue-103092.5EFts0.rst deleted file mode 100644 index 0f2108fee763d0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-08-00-48-40.gh-issue-103092.5EFts0.rst +++ /dev/null @@ -1 +0,0 @@ -Adapt the :mod:`winreg` extension module to :pep:`687`. diff --git a/Misc/NEWS.d/next/Library/2023-04-08-01-33-12.gh-issue-103357.vjin28.rst b/Misc/NEWS.d/next/Library/2023-04-08-01-33-12.gh-issue-103357.vjin28.rst deleted file mode 100644 index 83dce56ed0b7c5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-08-01-33-12.gh-issue-103357.vjin28.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added support for :class:`logging.Formatter` ``defaults`` parameter to -:func:`logging.config.dictConfig` and :func:`logging.config.fileConfig`. -Patch by Bar Harel. diff --git a/Misc/NEWS.d/next/Library/2023-04-09-06-59-36.gh-issue-103092.vskbro.rst b/Misc/NEWS.d/next/Library/2023-04-09-06-59-36.gh-issue-103092.vskbro.rst deleted file mode 100644 index 6977c1489a29cb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-09-06-59-36.gh-issue-103092.vskbro.rst +++ /dev/null @@ -1 +0,0 @@ -Isolate :mod:`!_collections` (apply :pep:`687`). Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-04-11-21-38-39.gh-issue-103449.-nxmhb.rst b/Misc/NEWS.d/next/Library/2023-04-11-21-38-39.gh-issue-103449.-nxmhb.rst deleted file mode 100644 index 0b2b47af1cbaab..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-11-21-38-39.gh-issue-103449.-nxmhb.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in doc string generation in :func:`dataclasses.dataclass`. diff --git a/Misc/NEWS.d/next/Library/2023-04-12-06-00-02.gh-issue-103462.w6yBlM.rst b/Misc/NEWS.d/next/Library/2023-04-12-06-00-02.gh-issue-103462.w6yBlM.rst deleted file mode 100644 index 50758c89cc2856..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-12-06-00-02.gh-issue-103462.w6yBlM.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixed an issue with using :meth:`~asyncio.WriteTransport.writelines` in :mod:`asyncio` to send very -large payloads that exceed the amount of data that can be written in one -call to :meth:`socket.socket.send` or :meth:`socket.socket.sendmsg`, -resulting in the remaining buffer being left unwritten. diff --git a/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst b/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst deleted file mode 100644 index 01d84f024bd4a6..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid a potential :exc:`ResourceWarning` in :class:`http.client.HTTPConnection` -by closing the proxy / tunnel's CONNECT response explicitly. diff --git a/Misc/NEWS.d/next/Library/2023-04-12-17-59-55.gh-issue-103365.UBEE0U.rst b/Misc/NEWS.d/next/Library/2023-04-12-17-59-55.gh-issue-103365.UBEE0U.rst deleted file mode 100644 index 4d69f6f6fff713..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-12-17-59-55.gh-issue-103365.UBEE0U.rst +++ /dev/null @@ -1 +0,0 @@ -Set default Flag boundary to ``STRICT`` and fix bitwise operations. diff --git a/Misc/NEWS.d/next/Library/2023-04-13-13-17-47.gh-issue-103489.ZSZgmu.rst b/Misc/NEWS.d/next/Library/2023-04-13-13-17-47.gh-issue-103489.ZSZgmu.rst deleted file mode 100644 index 264564d018ceb4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-13-13-17-47.gh-issue-103489.ZSZgmu.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :meth:`~sqlite3.Connection.getconfig` and -:meth:`~sqlite3.Connection.setconfig` to :class:`~sqlite3.Connection` to -make configuration changes to a database connection. Patch by Erlend E. -Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst b/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst deleted file mode 100644 index 1414cb07dd9155..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix misleading exception message when mixed ``str`` and ``bytes`` arguments -are supplied to :class:`pathlib.PurePath` and :class:`~pathlib.Path`. diff --git a/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst b/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst deleted file mode 100644 index 1008ea076c71a0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst +++ /dev/null @@ -1 +0,0 @@ -Update :mod:`cProfile` to use PEP 669 API diff --git a/Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst b/Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst deleted file mode 100644 index 32788307d6f33b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-14-21-12-32.gh-issue-103538.M4FK_v.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove ``_tkinter`` module code guarded by definition of the ``TK_AQUA`` macro -which was only needed for Tk 8.4.7 or earlier and was never actually defined by -any build system or documented for manual use. diff --git a/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst b/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst deleted file mode 100644 index 238f2868867472..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-14-21-16-05.gh-issue-103548.lagdpp.rst +++ /dev/null @@ -1,4 +0,0 @@ -Improve performance of :meth:`pathlib.Path.absolute` and -:meth:`~pathlib.Path.cwd` by joining paths only when necessary. Also improve -performance of :meth:`pathlib.PurePath.is_absolute` on Posix by skipping path -parsing and normalization. diff --git a/Misc/NEWS.d/next/Library/2023-04-15-11-21-38.gh-issue-103559.a9rYHG.rst b/Misc/NEWS.d/next/Library/2023-04-15-11-21-38.gh-issue-103559.a9rYHG.rst deleted file mode 100644 index 2c9d67e2c4bf71..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-15-11-21-38.gh-issue-103559.a9rYHG.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copy of pip to version 23.1.1. diff --git a/Misc/NEWS.d/next/Library/2023-04-15-12-19-14.gh-issue-103556.TEf-2m.rst b/Misc/NEWS.d/next/Library/2023-04-15-12-19-14.gh-issue-103556.TEf-2m.rst deleted file mode 100644 index fe2267b7b79019..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-15-12-19-14.gh-issue-103556.TEf-2m.rst +++ /dev/null @@ -1,3 +0,0 @@ -Now creating :class:`inspect.Signature` objects with positional-only -parameter with a default followed by a positional-or-keyword parameter -without one is impossible. diff --git a/Misc/NEWS.d/next/Library/2023-04-16-18-29-04.gh-issue-103578.fly1wc.rst b/Misc/NEWS.d/next/Library/2023-04-16-18-29-04.gh-issue-103578.fly1wc.rst deleted file mode 100644 index 69986c2a15b39e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-16-18-29-04.gh-issue-103578.fly1wc.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug where :mod:`pdb` crashes when reading source file with different encoding by replacing :func:`io.open` with :func:`io.open_code`. The new method would also call into the hook set by :func:`PyFile_SetOpenCodeHook`. diff --git a/Misc/NEWS.d/next/Library/2023-04-16-19-48-21.gh-issue-103584.3mBTuM.rst b/Misc/NEWS.d/next/Library/2023-04-16-19-48-21.gh-issue-103584.3mBTuM.rst deleted file mode 100644 index 6d7c93ade9cd94..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-16-19-48-21.gh-issue-103584.3mBTuM.rst +++ /dev/null @@ -1,12 +0,0 @@ -Updated ``importlib.metadata`` with changes from ``importlib_metadata`` 5.2 -through 6.5.0, including: Support ``installed-files.txt`` for -``Distribution.files`` when present. ``PackageMetadata`` now stipulates an -additional ``get`` method allowing for easy querying of metadata keys that -may not be present. ``packages_distributions`` now honors packages and -modules with Python modules that not ``.py`` sources (e.g. ``.pyc``, -``.so``). Expand protocol for ``PackageMetadata.get_all`` to match the -upstream implementation of ``email.message.Message.get_all`` in -python/typeshed#9620. Deprecated use of ``Distribution`` without defining -abstract methods. Deprecated expectation that -``PackageMetadata.__getitem__`` will return ``None`` for missing keys. In -the future, it will raise a ``KeyError``. diff --git a/Misc/NEWS.d/next/Library/2023-04-17-14-47-28.gh-issue-103596.ME1y3_.rst b/Misc/NEWS.d/next/Library/2023-04-17-14-47-28.gh-issue-103596.ME1y3_.rst deleted file mode 100644 index 2fa27e60b58efe..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-17-14-47-28.gh-issue-103596.ME1y3_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Attributes/methods are no longer shadowed by same-named enum members, -although they may be shadowed by enum.property's. diff --git a/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst b/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst deleted file mode 100644 index 8658627aeba434..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst +++ /dev/null @@ -1,5 +0,0 @@ -Create a new ``Lib/_pydatetime.py`` file that defines the Python version of -the ``datetime`` module, and make ``datetime`` import the contents of the -new library only if the C implementation is missing. Currently, the full -Python implementation is defined and then deleted if the C implementation is -not available, slowing down ``import datetime`` unnecessarily. diff --git a/Misc/NEWS.d/next/Library/2023-04-21-10-25-39.gh-issue-103636.YK6NEa.rst b/Misc/NEWS.d/next/Library/2023-04-21-10-25-39.gh-issue-103636.YK6NEa.rst deleted file mode 100644 index b3b5085250f078..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-21-10-25-39.gh-issue-103636.YK6NEa.rst +++ /dev/null @@ -1 +0,0 @@ -Added Enum for months and days in the calendar module. diff --git a/Misc/NEWS.d/next/Library/2023-04-22-02-41-06.gh-issue-103673.oE7S_k.rst b/Misc/NEWS.d/next/Library/2023-04-22-02-41-06.gh-issue-103673.oE7S_k.rst deleted file mode 100644 index bd5317744ff140..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-22-02-41-06.gh-issue-103673.oE7S_k.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`socketserver` gains ``ForkingUnixStreamServer`` and -``ForkingUnixDatagramServer`` classes. Patch by Jay Berry. diff --git a/Misc/NEWS.d/next/Library/2023-04-22-11-20-27.gh-issue-89415.YHk760.rst b/Misc/NEWS.d/next/Library/2023-04-22-11-20-27.gh-issue-89415.YHk760.rst deleted file mode 100644 index a5b99a2f1360f0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-22-11-20-27.gh-issue-89415.YHk760.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :mod:`socket` constants for source-specific multicast. -Patch by Reese Hyde. diff --git a/Misc/NEWS.d/next/Library/2023-04-22-12-30-10.gh-issue-92248.NcVTKR.rst b/Misc/NEWS.d/next/Library/2023-04-22-12-30-10.gh-issue-92248.NcVTKR.rst deleted file mode 100644 index d4a02d82941927..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-22-12-30-10.gh-issue-92248.NcVTKR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deprecate ``type``, ``choices``, and ``metavar`` parameters of -``argparse.BooleanOptionalAction``. diff --git a/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst b/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst deleted file mode 100644 index 52c68bfc9ceea4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst +++ /dev/null @@ -1 +0,0 @@ -Add convenience variable feature to :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2023-04-22-22-37-39.gh-issue-103699.NizCjc.rst b/Misc/NEWS.d/next/Library/2023-04-22-22-37-39.gh-issue-103699.NizCjc.rst deleted file mode 100644 index 60547a25a109bc..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-22-22-37-39.gh-issue-103699.NizCjc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``__orig_bases__`` to non-generic TypedDicts, call-based TypedDicts, and -call-based NamedTuples. Other TypedDicts and NamedTuples already had the attribute. diff --git a/Misc/NEWS.d/next/Library/2023-04-23-15-39-17.gh-issue-81403.zVz9Td.rst b/Misc/NEWS.d/next/Library/2023-04-23-15-39-17.gh-issue-81403.zVz9Td.rst deleted file mode 100644 index 6adb71f7677229..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-23-15-39-17.gh-issue-81403.zVz9Td.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`urllib.request.CacheFTPHandler` no longer raises :class:`URLError` -if a cached FTP instance is reused. ftplib's endtransfer method calls -voidresp to drain the connection to handle FTP instance reuse properly. diff --git a/Misc/NEWS.d/next/Library/2023-04-24-00-34-23.gh-issue-103685.U14jBM.rst b/Misc/NEWS.d/next/Library/2023-04-24-00-34-23.gh-issue-103685.U14jBM.rst deleted file mode 100644 index 31df04790721a8..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-24-00-34-23.gh-issue-103685.U14jBM.rst +++ /dev/null @@ -1 +0,0 @@ -Prepare :meth:`tkinter.Menu.index` for Tk 8.7 so that it does not raise ``TclError: expected integer but got ""`` when it should return ``None``. diff --git a/Misc/NEWS.d/next/Library/2023-04-24-16-00-28.gh-issue-90750.da0Xi8.rst b/Misc/NEWS.d/next/Library/2023-04-24-16-00-28.gh-issue-90750.da0Xi8.rst deleted file mode 100644 index 99e10f140f5049..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-24-16-00-28.gh-issue-90750.da0Xi8.rst +++ /dev/null @@ -1,3 +0,0 @@ -Use :meth:`datetime.datetime.fromisocalendar` in the implementation of -:meth:`datetime.datetime.strptime`, which should now accept only valid ISO -dates. (Patch by Paul Ganssle) diff --git a/Misc/NEWS.d/next/Library/2023-04-24-23-07-56.gh-issue-103791.bBPWdS.rst b/Misc/NEWS.d/next/Library/2023-04-24-23-07-56.gh-issue-103791.bBPWdS.rst deleted file mode 100644 index f00384cde9706e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-24-23-07-56.gh-issue-103791.bBPWdS.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`contextlib.suppress` now supports suppressing exceptions raised as -part of an :exc:`ExceptionGroup`. If other exceptions exist on the group, they -are re-raised in a group that does not contain the suppressed exceptions. diff --git a/Misc/NEWS.d/next/Library/2023-04-25-16-31-00.gh-issue-103839.tpyLhI.rst b/Misc/NEWS.d/next/Library/2023-04-25-16-31-00.gh-issue-103839.tpyLhI.rst deleted file mode 100644 index 1d40a8fbcd3f59..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-25-16-31-00.gh-issue-103839.tpyLhI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid compilation error due to tommath.h not being found when building -Tkinter against Tcl 8.7 built with bundled libtommath. diff --git a/Misc/NEWS.d/next/Library/2023-04-25-17-03-18.gh-issue-103857.Mr2Cak.rst b/Misc/NEWS.d/next/Library/2023-04-25-17-03-18.gh-issue-103857.Mr2Cak.rst deleted file mode 100644 index 3bd370dabf4ed5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-25-17-03-18.gh-issue-103857.Mr2Cak.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deprecated :meth:`datetime.datetime.utcnow` and -:meth:`datetime.datetime.utcfromtimestamp`. (Patch by Paul Ganssle) diff --git a/Misc/NEWS.d/next/Library/2023-04-25-19-58-13.gh-issue-103861.JeozgD.rst b/Misc/NEWS.d/next/Library/2023-04-25-19-58-13.gh-issue-103861.JeozgD.rst deleted file mode 100644 index cc1c444449fe93..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-25-19-58-13.gh-issue-103861.JeozgD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``zipfile.Zipfile`` creating invalid zip files when ``force_zip64`` was -used to add files to them. Patch by Carey Metcalfe. diff --git a/Misc/NEWS.d/next/Library/2023-04-25-22-06-00.gh-issue-74940.TOacQ9.rst b/Misc/NEWS.d/next/Library/2023-04-25-22-06-00.gh-issue-74940.TOacQ9.rst deleted file mode 100644 index c37d795f3eb33d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-25-22-06-00.gh-issue-74940.TOacQ9.rst +++ /dev/null @@ -1,2 +0,0 @@ -The C.UTF-8 locale is no longer converted to en_US.UTF-8, enabling the use -of UTF-8 encoding on systems which have no locales installed. diff --git a/Misc/NEWS.d/next/Library/2023-04-25-22-59-06.gh-issue-99944.pst8iT.rst b/Misc/NEWS.d/next/Library/2023-04-25-22-59-06.gh-issue-99944.pst8iT.rst deleted file mode 100644 index 80238a65e32a41..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-25-22-59-06.gh-issue-99944.pst8iT.rst +++ /dev/null @@ -1 +0,0 @@ -Make :mod:`dis` display the value of oparg of :opcode:`KW_NAMES`. diff --git a/Misc/NEWS.d/next/Library/2023-04-26-09-38-47.gh-issue-103872.8LBsDz.rst b/Misc/NEWS.d/next/Library/2023-04-26-09-38-47.gh-issue-103872.8LBsDz.rst deleted file mode 100644 index b840f9f5769f08..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-26-09-38-47.gh-issue-103872.8LBsDz.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copy of pip to version 23.1.2. diff --git a/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst deleted file mode 100644 index 81e5904aa6cca2..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add checks to ensure that ``[`` bracketed ``]`` hosts found by -:func:`urllib.parse.urlsplit` are of IPv6 or IPvFuture format. diff --git a/Misc/NEWS.d/next/Library/2023-04-26-15-14-36.gh-issue-103583.iCMDFt.rst b/Misc/NEWS.d/next/Library/2023-04-26-15-14-36.gh-issue-103583.iCMDFt.rst deleted file mode 100644 index 8c92ee40831619..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-26-15-14-36.gh-issue-103583.iCMDFt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Isolate :mod:`!_multibytecodec` and codecs extension modules. Patches by -Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst b/Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst deleted file mode 100644 index a05a6f5cbcdb99..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst +++ /dev/null @@ -1 +0,0 @@ -Module-level attributes ``January`` and ``February`` are deprecated from :mod:`calendar`. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-00-05-32.gh-issue-102628.X230E-.rst b/Misc/NEWS.d/next/Library/2023-04-27-00-05-32.gh-issue-102628.X230E-.rst deleted file mode 100644 index eaaca5b41ba5e2..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-27-00-05-32.gh-issue-102628.X230E-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Substitute CTRL-D with CTRL-Z in :mod:`sqlite3` CLI banner when running on -Windows. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst b/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst deleted file mode 100644 index 9022d55c48cb11..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-27-00-45-41.gh-issue-100370.MgZ3KY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix potential :exc:`OverflowError` in :meth:`sqlite3.Connection.blobopen` -for 32-bit builds. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst b/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst deleted file mode 100644 index bf29b64793b933..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed garbled output of :meth:`~unittest.TestCase.assertEqual` when an input lacks final newline. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-20-03-08.gh-issue-103935.Uaf2M0.rst b/Misc/NEWS.d/next/Library/2023-04-27-20-03-08.gh-issue-103935.Uaf2M0.rst deleted file mode 100644 index 71b2d87249c47b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-27-20-03-08.gh-issue-103935.Uaf2M0.rst +++ /dev/null @@ -1 +0,0 @@ -Use :func:`io.open_code` for files to be executed instead of raw :func:`open` diff --git a/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst b/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst deleted file mode 100644 index f14c9533f3af87..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst +++ /dev/null @@ -1 +0,0 @@ -Added :func:`turtle.teleport` to the :mod:`turtle` module to move a turtle to a new point without tracing a line, visible or invisible. Patch by Liam Gersten. diff --git a/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst b/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst deleted file mode 100644 index ff4005774a95d2..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst +++ /dev/null @@ -1 +0,0 @@ -Improve import time of :mod:`platform` module. diff --git a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst b/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst deleted file mode 100644 index 48c6d149a8ed42..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-04-29-18-23-16.gh-issue-103987.sRgALL.rst +++ /dev/null @@ -1,2 +0,0 @@ -In :mod:`mmap`, fix several bugs that could lead to access to memory-mapped files after -they have been invalidated. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst b/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst deleted file mode 100644 index 8c8e3d6ba5fbc1..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Do not ignore user-defined ``__getstate__`` and ``__setstate__`` methods for -slotted frozen dataclasses. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst b/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst deleted file mode 100644 index cb06ad5d22e8a9..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst +++ /dev/null @@ -1 +0,0 @@ -Make :mod:`dis` display the names of the args for :opcode:`CALL_INTRINSIC_*`. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst b/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst deleted file mode 100644 index 7971ab66359c3d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst +++ /dev/null @@ -1 +0,0 @@ -Update the ``repr`` of :class:`typing.Unpack` according to :pep:`692`. diff --git a/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst b/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst deleted file mode 100644 index 3daf9cc093807b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst +++ /dev/null @@ -1 +0,0 @@ -Update the return type of ``weekday`` to the newly added Day attribute diff --git a/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst b/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst deleted file mode 100644 index 7101de908a5004..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :meth:`pathlib.Path.glob` when evaluating patterns -that contain ``'../'`` segments. diff --git a/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst b/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst deleted file mode 100644 index 935a0e2a2bff18..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :meth:`pathlib.Path.glob` by using -:data:`re.IGNORECASE` to implement case-insensitive matching. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst b/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst deleted file mode 100644 index e705fea8326e7a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix issue where :meth:`pathlib.Path.glob` returns paths using the case of -non-wildcard segments for corresponding path segments, rather than the real -filesystem case. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst deleted file mode 100644 index b975d48ed3385c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-03-16-50-24.gh-issue-104144.yNkjL8.rst +++ /dev/null @@ -1,3 +0,0 @@ -Optimize :func:`asyncio.gather` when using :func:`asyncio.eager_task_factory` -to complete eagerly if all fututres completed eagerly. -Avoid scheduling done callbacks for futures that complete eagerly. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst b/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst deleted file mode 100644 index ced3b7cea04954..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-03-16-51-53.gh-issue-104144.653Q0P.rst +++ /dev/null @@ -1,2 +0,0 @@ -Optimize :class:`asyncio.TaskGroup` when using :func:`asyncio.eager_task_factory`. -Skip scheduling a done callback if a TaskGroup task completes eagerly. diff --git a/Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst b/Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst deleted file mode 100644 index 1fd9588bebd5d8..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-03-19-22-24.gh-issue-90208.tI00da.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixed issue where :meth:`pathlib.Path.glob` returned incomplete results when -it encountered a :exc:`PermissionError`. This method now suppresses all -:exc:`OSError` exceptions, except those raised from calling -:meth:`~pathlib.Path.is_dir` on the top-level path. diff --git a/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst b/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst deleted file mode 100644 index 54b0190192863c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-05-18-52-22.gh-issue-65772.w5P5Wv.rst +++ /dev/null @@ -1 +0,0 @@ -Remove unneeded comments and code in turtle.py. diff --git a/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst b/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst deleted file mode 100644 index 01f8b948d2cb65..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-06-20-37-46.gh-issue-102613.QZG9iX.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve performance of :meth:`pathlib.Path.glob` when expanding recursive -wildcards ("``**``") by merging adjacent wildcards and de-duplicating -results only when necessary. diff --git a/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst b/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst deleted file mode 100644 index 9c582844bf909b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-07-19-56-45.gh-issue-104265.fVblry.rst +++ /dev/null @@ -1,4 +0,0 @@ -Prevent possible crash by disallowing instantiation of the -:class:`!_csv.Reader` and :class:`!_csv.Writer` types. -The regression was introduced in 3.10.0a4 with PR 23224 (:issue:`14935`). -Patch by Radislav Chugunov. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst b/Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst deleted file mode 100644 index 7b677656bea308..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-08-15-39-00.gh-issue-87695.f6iO7v.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where :meth:`pathlib.Path.glob` raised :exc:`OSError` when it -encountered a symlink to an overly long path. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst b/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst deleted file mode 100644 index 3743d569995f2e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-08-15-50-59.gh-issue-104310.fXVSPY.rst +++ /dev/null @@ -1,3 +0,0 @@ -Users may now use ``importlib.util.allowing_all_extensions()`` (a context -manager) to temporarily disable the strict compatibility checks for -importing extension modules in subinterpreters. diff --git a/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst b/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst deleted file mode 100644 index 03775845450caa..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-08-20-57-17.gh-issue-104307.DSB93G.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`socket.getnameinfo` now releases the GIL while contacting the DNS server diff --git a/Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst b/Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst deleted file mode 100644 index 145e75f6dea6f4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-08-23-01-59.gh-issue-104139.83Tnt-.rst +++ /dev/null @@ -1,3 +0,0 @@ -Teach :func:`urllib.parse.unsplit` to retain the ``"//"`` when assembling -``itms-services://?action=generate-bugs`` style `Apple Platform Deployment -`_ URLs. diff --git a/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst b/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst deleted file mode 100644 index a44ad765e62458..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst +++ /dev/null @@ -1 +0,0 @@ -Allow leading whitespace in disambiguated statements in :mod:`pdb`. diff --git a/Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst b/Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst deleted file mode 100644 index f84ec5c8b3caf6..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-10-19-33-36.gh-issue-103000.j0KSfD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve performance of :func:`dataclasses.asdict` for the common case where -*dict_factory* is ``dict``. Patch by David C Ellis. diff --git a/Misc/NEWS.d/next/Library/2023-05-11-01-07-42.gh-issue-102613.uMsokt.rst b/Misc/NEWS.d/next/Library/2023-05-11-01-07-42.gh-issue-102613.uMsokt.rst deleted file mode 100644 index 3b06964dc8d233..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-11-01-07-42.gh-issue-102613.uMsokt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where :meth:`pathlib.Path.glob` raised :exc:`RecursionError` when -walking deep directory trees. diff --git a/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst b/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst deleted file mode 100644 index b441b2acd19f1b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove undocumented and unused ``_paramspec_tvars`` attribute from some -classes in :mod:`typing`. diff --git a/Misc/NEWS.d/next/Library/2023-05-11-21-32-18.gh-issue-101520.l9MjRE.rst b/Misc/NEWS.d/next/Library/2023-05-11-21-32-18.gh-issue-101520.l9MjRE.rst deleted file mode 100644 index 5e8bf967bfdcca..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-11-21-32-18.gh-issue-101520.l9MjRE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Move the core functionality of the ``tracemalloc`` module in the ``Python/`` -folder, leaving just the module wrapper in ``Modules/``. diff --git a/Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst b/Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst deleted file mode 100644 index 6e8162d9ecc286..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-12-19-29-28.gh-issue-103857.0IzSxr.rst +++ /dev/null @@ -1 +0,0 @@ -Update datetime deprecations' stracktrace to point to the calling line diff --git a/Misc/NEWS.d/next/Library/2023-05-16-11-02-44.gh-issue-75367.qLWR35.rst b/Misc/NEWS.d/next/Library/2023-05-16-11-02-44.gh-issue-75367.qLWR35.rst deleted file mode 100644 index 554c425e6a7864..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-16-11-02-44.gh-issue-75367.qLWR35.rst +++ /dev/null @@ -1 +0,0 @@ -Fix data descriptor detection in :func:`inspect.getattr_static`. diff --git a/Misc/NEWS.d/next/Library/2023-05-17-03-14-07.gh-issue-104484.y6KxL6.rst b/Misc/NEWS.d/next/Library/2023-05-17-03-14-07.gh-issue-104484.y6KxL6.rst deleted file mode 100644 index 6d42078c35dd26..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-03-14-07.gh-issue-104484.y6KxL6.rst +++ /dev/null @@ -1 +0,0 @@ -Added *case_sensitive* argument to :meth:`pathlib.PurePath.match` diff --git a/Misc/NEWS.d/next/Library/2023-05-17-08-01-36.gh-issue-104372.jpoWs6.rst b/Misc/NEWS.d/next/Library/2023-05-17-08-01-36.gh-issue-104372.jpoWs6.rst deleted file mode 100644 index c228f503aab4bd..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-08-01-36.gh-issue-104372.jpoWs6.rst +++ /dev/null @@ -1 +0,0 @@ -Refactored the ``_posixsubprocess`` internals to avoid Python C API usage between fork and exec when marking ``pass_fds=`` file descriptors inheritable. diff --git a/Misc/NEWS.d/next/Library/2023-05-17-16-58-23.gh-issue-104555.5rb5oM.rst b/Misc/NEWS.d/next/Library/2023-05-17-16-58-23.gh-issue-104555.5rb5oM.rst deleted file mode 100644 index 2992346484c5fa..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-16-58-23.gh-issue-104555.5rb5oM.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fix issue where an :func:`issubclass` check comparing a class ``X`` against a -:func:`runtime-checkable protocol ` ``Y`` with -non-callable members would not cause :exc:`TypeError` to be raised if an -:func:`isinstance` call had previously been made comparing an instance of ``X`` -to ``Y``. This issue was present in edge cases on Python 3.11, but became more -prominent in 3.12 due to some unrelated changes that were made to -runtime-checkable protocols. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-05-17-20-03-01.gh-issue-104340.kp_XmX.rst b/Misc/NEWS.d/next/Library/2023-05-17-20-03-01.gh-issue-104340.kp_XmX.rst deleted file mode 100644 index 5b03622df6a2ab..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-20-03-01.gh-issue-104340.kp_XmX.rst +++ /dev/null @@ -1 +0,0 @@ -When an ``asyncio`` pipe protocol loses its connection due to an error, and the caller doesn't await ``wait_closed()`` on the corresponding ``StreamWriter``, don't log a warning about an exception that was never retrieved. After all, according to the ``StreamWriter.close()`` docs, the ``wait_closed()`` call is optional ("not mandatory"). diff --git a/Misc/NEWS.d/next/Library/2023-05-17-21-01-48.gh-issue-104600.E6CK35.rst b/Misc/NEWS.d/next/Library/2023-05-17-21-01-48.gh-issue-104600.E6CK35.rst deleted file mode 100644 index 64f81e140d9639..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-05-17-21-01-48.gh-issue-104600.E6CK35.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`functools.update_wrapper` now sets the ``__type_params__`` attribute -(added by :pep:`695`). diff --git a/Misc/NEWS.d/next/Library/2023-05-19-19-46-22.gh-issue-99108.wqCg0t.rst b/Misc/NEWS.d/next/Library/2023-05-19-19-46-22.gh-issue-99108.wqCg0t.rst new file mode 100644 index 00000000000000..b595f1893609cc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-19-19-46-22.gh-issue-99108.wqCg0t.rst @@ -0,0 +1,3 @@ +We now release the GIL around built-in :mod:`hashlib` computations of +reasonable size for the SHA families and MD5 hash functions, matching +what our OpenSSL backed hash computations already does. diff --git a/Misc/NEWS.d/next/Library/2023-05-23-02-20-13.gh-issue-104773.7K59zr.rst b/Misc/NEWS.d/next/Library/2023-05-23-02-20-13.gh-issue-104773.7K59zr.rst new file mode 100644 index 00000000000000..0f3162ef509a82 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-23-02-20-13.gh-issue-104773.7K59zr.rst @@ -0,0 +1,2 @@ +:pep:`594`: Remove the :mod:`!telnetlib` module, deprecated in Python 3.11. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2023-05-23-03-36-47.gh-issue-104780.P4e3Yf.rst b/Misc/NEWS.d/next/Library/2023-05-23-03-36-47.gh-issue-104780.P4e3Yf.rst new file mode 100644 index 00000000000000..acdca53b4bb7d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-23-03-36-47.gh-issue-104780.P4e3Yf.rst @@ -0,0 +1,2 @@ +Remove the ``2to3`` program and the :mod:`!lib2to3` module, deprecated in +Python 3.11. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2023-05-23-19-53-18.gh-issue-83863.eRI5JG.rst b/Misc/NEWS.d/next/Library/2023-05-23-19-53-18.gh-issue-83863.eRI5JG.rst new file mode 100644 index 00000000000000..7a073aa37dd6ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-23-19-53-18.gh-issue-83863.eRI5JG.rst @@ -0,0 +1,4 @@ +Support for using :class:`pathlib.Path` objects as context managers has been +removed. Before Python 3.9, exiting the context manager marked a path as +"closed", which caused some (but not all!) methods to raise when called. +Since Python 3.9, using a path as a context manager does nothing. diff --git a/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst deleted file mode 100644 index e57ac4ed3ac5d7..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`urllib.parse.urlsplit` now strips leading C0 control and space -characters following the specification for URLs defined by WHATWG in -response to CVE-2023-24329. Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst b/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst deleted file mode 100644 index f259acf753831c..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-04-17-14-38-12.gh-issue-99108.720lG8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Upgrade built-in :mod:`hashlib` SHA3 implementation to a verified implementation -from the ``HACL*`` project. Used when OpenSSL is not present or lacks SHA3. diff --git a/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst b/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst deleted file mode 100644 index 969deb26bfeb95..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst +++ /dev/null @@ -1,2 +0,0 @@ -Do not expose the local on-disk location in directory indexes -produced by :class:`http.client.SimpleHTTPRequestHandler`. diff --git a/Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst b/Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst deleted file mode 100644 index b7002e81b6b677..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-05-02-17-56-32.gh-issue-99889.l664SU.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a security in flaw in :func:`uu.decode` that could allow for -directory traversal based on the input if no ``out_file`` was specified. diff --git a/Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst b/Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst deleted file mode 100644 index 8baecdfc31881f..00000000000000 --- a/Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the :func:`os.spawn* ` tests failing on Windows -when the working directory or interpreter path contains spaces. diff --git a/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst b/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst deleted file mode 100644 index fe2afff91ece7a..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst +++ /dev/null @@ -1 +0,0 @@ -fix use of poll in test_epoll's test_control_and_wait diff --git a/Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst b/Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst deleted file mode 100644 index 79448ed728040d..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst +++ /dev/null @@ -1 +0,0 @@ -Regression tests for the behaviour of ``unittest.mock.PropertyMock`` were added. diff --git a/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst b/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst deleted file mode 100644 index 37ab74ffc441ed..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added property-based tests to the :mod:`zoneinfo` tests, along with stubs -for the ``hypothesis`` interface. (Patch by Paul Ganssle) diff --git a/Misc/NEWS.d/next/Tests/2023-05-14-03-00-00.gh-issue-104461.Rmex11.rst b/Misc/NEWS.d/next/Tests/2023-05-14-03-00-00.gh-issue-104461.Rmex11.rst deleted file mode 100644 index ae69f623e94558..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-05-14-03-00-00.gh-issue-104461.Rmex11.rst +++ /dev/null @@ -1,3 +0,0 @@ -Run test_configure_screen on X11 only, since the ``DISPLAY`` -environment variable and ``-screen`` option for toplevels -are not useful on Tk for Win32 or Aqua. diff --git a/Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst b/Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst deleted file mode 100644 index a320c48428b58b..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-05-15-02-22-44.gh-issue-104494.Bkrbfn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update ``test_pack_configure_in`` and ``test_place_configure_in`` -for changes to error message formatting in Tk 8.7. diff --git a/Misc/NEWS.d/next/Tests/2023-05-19-08-06-06.gh-issue-81005.-q7m9W.rst b/Misc/NEWS.d/next/Tests/2023-05-19-08-06-06.gh-issue-81005.-q7m9W.rst new file mode 100644 index 00000000000000..dfb653241e2607 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-05-19-08-06-06.gh-issue-81005.-q7m9W.rst @@ -0,0 +1,2 @@ +String tests are modified to reflect that ``str`` and ``unicode`` are merged +in Python 3. Patch by Daniel Fortunov. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst b/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst deleted file mode 100644 index 854e1cca967c42..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Argument Clinic C converters now accept the ``unused`` keyword, for wrapping -a parameter with :c:macro:`Py_UNUSED`. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst b/Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst deleted file mode 100644 index 4ca3185ea1f65e..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a bug where :exc:`TypeError` was raised when calling -:func:`ntpath.realpath` with a bytes parameter in some cases. diff --git a/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst b/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst deleted file mode 100644 index c8f7259aecba6f..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.41.2. diff --git a/Misc/NEWS.d/next/Windows/2023-04-11-09-22-22.gh-issue-103088.6AJEuR.rst b/Misc/NEWS.d/next/Windows/2023-04-11-09-22-22.gh-issue-103088.6AJEuR.rst deleted file mode 100644 index f9f5343f4210dc..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-04-11-09-22-22.gh-issue-103088.6AJEuR.rst +++ /dev/null @@ -1 +0,0 @@ -Fixes venvs not working in bash on Windows across different disks diff --git a/Misc/NEWS.d/next/Windows/2023-04-12-10-49-21.gh-issue-103088.Yjj-qJ.rst b/Misc/NEWS.d/next/Windows/2023-04-12-10-49-21.gh-issue-103088.Yjj-qJ.rst deleted file mode 100644 index 1fee99da240378..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-04-12-10-49-21.gh-issue-103088.Yjj-qJ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix virtual environment :file:`activate` script having incorrect line endings for Cygwin. diff --git a/Misc/NEWS.d/next/Windows/2023-04-24-15-51-11.gh-issue-82814.GI3UkZ.rst b/Misc/NEWS.d/next/Windows/2023-04-24-15-51-11.gh-issue-82814.GI3UkZ.rst deleted file mode 100644 index 5bd005ffacb800..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-04-24-15-51-11.gh-issue-82814.GI3UkZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a potential ``[Errno 13] Permission denied`` when using :func:`shutil.copystat` -within Windows Subsystem for Linux (WSL) on a mounted filesystem by adding -``errno.EACCES`` to the list of ignored errors within the internal implementation. diff --git a/Misc/NEWS.d/next/Windows/2023-05-18-22-46-03.gh-issue-104623.HJZhm1.rst b/Misc/NEWS.d/next/Windows/2023-05-18-22-46-03.gh-issue-104623.HJZhm1.rst deleted file mode 100644 index 3ffe8261e5a1b0..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-05-18-22-46-03.gh-issue-104623.HJZhm1.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.42.0. diff --git a/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst b/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst deleted file mode 100644 index d0b390a896b743..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.41.2. diff --git a/Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst b/Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst deleted file mode 100644 index 3c176e3a6b5310..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add instructions to the macOS installer welcome display on how to workaround -the macOS 13 Ventura “The installer encountered an error” failure. diff --git a/Misc/NEWS.d/next/macOS/2023-04-24-18-37-48.gh-issue-60436.in-IyF.rst b/Misc/NEWS.d/next/macOS/2023-04-24-18-37-48.gh-issue-60436.in-IyF.rst deleted file mode 100644 index f274d3b898f15d..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-04-24-18-37-48.gh-issue-60436.in-IyF.rst +++ /dev/null @@ -1 +0,0 @@ -update curses textbox to additionally handle backspace using the ``curses.ascii.DEL`` key press. diff --git a/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst b/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst deleted file mode 100644 index b6b18dcfd81394..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-05-04-21-47-59.gh-issue-104180.lEJCwd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Support reading SOCKS proxy configuration from macOS System Configuration. -Patch by Sam Schott. diff --git a/Misc/NEWS.d/next/macOS/2023-05-18-08-52-04.gh-issue-103545.pi5k2N.rst b/Misc/NEWS.d/next/macOS/2023-05-18-08-52-04.gh-issue-103545.pi5k2N.rst deleted file mode 100644 index c40f9460508dd7..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-05-18-08-52-04.gh-issue-103545.pi5k2N.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``os.PRIO_DARWIN_THREAD``, ``os.PRIO_DARWIN_PROCESS``, ``os.PRIO_DARWIN_BG`` and ``os.PRIO_DARWIN_NONUI``. These can be used with ``os.setpriority`` to run the process at a lower priority and make use of the efficiency cores on Apple Silicon systems. diff --git a/Misc/NEWS.d/next/macOS/2023-05-18-22-31-49.gh-issue-104623.6h7Xfx.rst b/Misc/NEWS.d/next/macOS/2023-05-18-22-31-49.gh-issue-104623.6h7Xfx.rst deleted file mode 100644 index ffb124631c218b..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-05-18-22-31-49.gh-issue-104623.6h7Xfx.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.42.0. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 8b417bdd0fb5b6..19e11780ec6e19 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5190,7 +5190,7 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.utcfromtimestamp() is deprecated and scheduled for removal " "in a future version. Use timezone-aware objects to represent " - "datetimes in UTC: datetime.now(datetime.UTC).", 1)) + "datetimes in UTC: datetime.fromtimestamp(timestamp, datetime.UTC).", 1)) { return NULL; } diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 99d0b72819137e..4b425f4147513e 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -227,12 +227,16 @@ get_hashlib_state(PyObject *module) typedef struct { PyObject_HEAD EVP_MD_CTX *ctx; /* OpenSSL message digest context */ + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. PyThread_type_lock lock; /* OpenSSL context lock */ } EVPobject; typedef struct { PyObject_HEAD HMAC_CTX *ctx; /* OpenSSL hmac context */ + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. PyThread_type_lock lock; /* HMAC context lock */ } HMACobject; @@ -896,6 +900,8 @@ py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj, if (view.buf && view.len) { if (view.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS result = EVP_hash(self, view.buf, view.len); Py_END_ALLOW_THREADS diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 016a5a5cbca548..59fd401cac270b 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1330,10 +1330,8 @@ _get_peer_alt_names (_sslmodulestate *state, X509 *certificate) { p[0], p[1], p[2], p[3] ); } else if (name->d.ip->length == 16) { - /* PyUnicode_FromFormat() does not support %X */ unsigned char *p = name->d.ip->data; - len = sprintf( - buf, + v = PyUnicode_FromFormat( "%X:%X:%X:%X:%X:%X:%X:%X", p[0] << 8 | p[1], p[2] << 8 | p[3], @@ -1344,7 +1342,6 @@ _get_peer_alt_names (_sslmodulestate *state, X509 *certificate) { p[12] << 8 | p[13], p[14] << 8 | p[15] ); - v = PyUnicode_FromStringAndSize(buf, len); } else { v = PyUnicode_FromString(""); } @@ -6153,6 +6150,18 @@ sslmodule_init_strings(PyObject *module) return 0; } +static int +sslmodule_init_lock(PyObject *module) +{ + _sslmodulestate *state = get_ssl_state(module); + state->keylog_lock = PyThread_allocate_lock(); + if (state->keylog_lock == NULL) { + PyErr_NoMemory(); + return -1; + } + return 0; +} + static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_types}, {Py_mod_exec, sslmodule_init_exceptions}, @@ -6161,9 +6170,8 @@ static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_constants}, {Py_mod_exec, sslmodule_init_versioninfo}, {Py_mod_exec, sslmodule_init_strings}, - // XXX gh-103092: fix isolation. - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, - //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_exec, sslmodule_init_lock}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; @@ -6222,6 +6230,8 @@ static void sslmodule_free(void *m) { sslmodule_clear((PyObject *)m); + _sslmodulestate *state = get_ssl_state(m); + PyThread_free_lock(state->keylog_lock); } static struct PyModuleDef _sslmodule_def = { diff --git a/Modules/_ssl.h b/Modules/_ssl.h index c1da8b46b536d3..22d93ddcc6d6eb 100644 --- a/Modules/_ssl.h +++ b/Modules/_ssl.h @@ -33,6 +33,8 @@ typedef struct { PyObject *str_reason; PyObject *str_verify_code; PyObject *str_verify_message; + /* keylog lock */ + PyThread_type_lock keylog_lock; } _sslmodulestate; static struct PyModuleDef _sslmodule_def; diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 217f224942556e..a81f0aad05a802 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -118,30 +118,22 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) PyGILState_STATE threadstate; PySSLSocket *ssl_obj = NULL; /* ssl._SSLSocket, borrowed ref */ int res, e; - static PyThread_type_lock *lock = NULL; threadstate = PyGILState_Ensure(); ssl_obj = (PySSLSocket *)SSL_get_app_data(ssl); assert(Py_IS_TYPE(ssl_obj, get_state_sock(ssl_obj)->PySSLSocket_Type)); + PyThread_type_lock lock = get_state_sock(ssl_obj)->keylog_lock; + assert(lock != NULL); if (ssl_obj->ctx->keylog_bio == NULL) { return; } - - /* Allocate a static lock to synchronize writes to keylog file. + /* * The lock is neither released on exit nor on fork(). The lock is * also shared between all SSLContexts although contexts may write to * their own files. IMHO that's good enough for a non-performance * critical debug helper. */ - if (lock == NULL) { - lock = PyThread_allocate_lock(); - if (lock == NULL) { - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - ssl_obj->exc = PyErr_GetRaisedException(); - return; - } - } PySSL_BEGIN_ALLOW_THREADS PyThread_acquire_lock(lock, 1); diff --git a/Modules/_testcapi/clinic/long.c.h b/Modules/_testcapi/clinic/long.c.h new file mode 100644 index 00000000000000..95885e0ae17a6c --- /dev/null +++ b/Modules/_testcapi/clinic/long.c.h @@ -0,0 +1,166 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(_testcapi_test_long_api__doc__, +"test_long_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_TEST_LONG_API_METHODDEF \ + {"test_long_api", (PyCFunction)_testcapi_test_long_api, METH_NOARGS, _testcapi_test_long_api__doc__}, + +static PyObject * +_testcapi_test_long_api_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_api_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_longlong_api__doc__, +"test_longlong_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_TEST_LONGLONG_API_METHODDEF \ + {"test_longlong_api", (PyCFunction)_testcapi_test_longlong_api, METH_NOARGS, _testcapi_test_longlong_api__doc__}, + +static PyObject * +_testcapi_test_longlong_api_impl(PyObject *module); + +static PyObject * +_testcapi_test_longlong_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_longlong_api_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_and_overflow__doc__, +"test_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongAndOverflow API.\n" +"\n" +"General conversion to PY_LONG is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_and_overflow", (PyCFunction)_testcapi_test_long_and_overflow, METH_NOARGS, _testcapi_test_long_and_overflow__doc__}, + +static PyObject * +_testcapi_test_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_long_and_overflow__doc__, +"test_long_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongLongAndOverflow API.\n" +"\n" +"General conversion to long long is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_long_and_overflow", (PyCFunction)_testcapi_test_long_long_and_overflow, METH_NOARGS, _testcapi_test_long_long_and_overflow__doc__}, + +static PyObject * +_testcapi_test_long_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_as_size_t__doc__, +"test_long_as_size_t($module, /)\n" +"--\n" +"\n" +"Test the PyLong_As{Size,Ssize}_t API.\n" +"\n" +"At present this just tests that non-integer arguments are handled correctly.\n" +"It should be extended to test overflow handling."); + +#define _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ + {"test_long_as_size_t", (PyCFunction)_testcapi_test_long_as_size_t, METH_NOARGS, _testcapi_test_long_as_size_t__doc__}, + +static PyObject * +_testcapi_test_long_as_size_t_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_as_size_t(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_as_size_t_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_as_unsigned_long_long_mask__doc__, +"test_long_as_unsigned_long_long_mask($module, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF \ + {"test_long_as_unsigned_long_long_mask", (PyCFunction)_testcapi_test_long_as_unsigned_long_long_mask, METH_NOARGS, _testcapi_test_long_as_unsigned_long_long_mask__doc__}, + +static PyObject * +_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_as_unsigned_long_long_mask(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_as_unsigned_long_long_mask_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_as_double__doc__, +"test_long_as_double($module, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF \ + {"test_long_as_double", (PyCFunction)_testcapi_test_long_as_double, METH_NOARGS, _testcapi_test_long_as_double__doc__}, + +static PyObject * +_testcapi_test_long_as_double_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_as_double_impl(module); +} + +PyDoc_STRVAR(_testcapi_test_long_numbits__doc__, +"test_long_numbits($module, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_TEST_LONG_NUMBITS_METHODDEF \ + {"test_long_numbits", (PyCFunction)_testcapi_test_long_numbits, METH_NOARGS, _testcapi_test_long_numbits__doc__}, + +static PyObject * +_testcapi_test_long_numbits_impl(PyObject *module); + +static PyObject * +_testcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testcapi_test_long_numbits_impl(module); +} + +PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, +"call_long_compact_api($module, arg, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF \ + {"call_long_compact_api", (PyCFunction)_testcapi_call_long_compact_api, METH_O, _testcapi_call_long_compact_api__doc__}, +/*[clinic end generated code: output=d000a1b58fa81eab input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 61dd96596dad1c..ede43f60d06f95 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -1,4 +1,10 @@ #include "parts.h" +#include "clinic/long.c.h" + +/*[clinic input] +module _testcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ static PyObject * @@ -40,8 +46,13 @@ raise_test_long_error(const char* msg) #include "testcapi_long.h" +/*[clinic input] +_testcapi.test_long_api +[clinic start generated code]*/ + static PyObject * -test_long_api(PyObject* self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_api_impl(PyObject *module) +/*[clinic end generated code: output=4405798ca1e9f444 input=e9b8880d7331c688]*/ { return TESTNAME(raise_test_long_error); } @@ -68,8 +79,13 @@ raise_test_longlong_error(const char* msg) #include "testcapi_long.h" +/*[clinic input] +_testcapi.test_longlong_api +[clinic start generated code]*/ + static PyObject * -test_longlong_api(PyObject* self, PyObject *args) +_testcapi_test_longlong_api_impl(PyObject *module) +/*[clinic end generated code: output=2b3414ba8c31dfe6 input=ccbb2a48c2b3c4a5]*/ { return TESTNAME(raise_test_longlong_error); } @@ -81,13 +97,19 @@ test_longlong_api(PyObject* self, PyObject *args) #undef F_U_TO_PY #undef F_PY_TO_U -/* Test the PyLong_AsLongAndOverflow API. General conversion to PY_LONG - is tested by test_long_api_inner. This test will concentrate on proper - handling of overflow. -*/ + +/*[clinic input] +_testcapi.test_long_and_overflow + +Test the PyLong_AsLongAndOverflow API. + +General conversion to PY_LONG is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ static PyObject * -test_long_and_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=f8460ca115e31d8e input=762f6b62da0a3cdc]*/ { PyObject *num, *one, *temp; long value; @@ -243,13 +265,18 @@ test_long_and_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -/* Test the PyLong_AsLongLongAndOverflow API. General conversion to - long long is tested by test_long_api_inner. This test will - concentrate on proper handling of overflow. -*/ +/*[clinic input] +_testcapi.test_long_long_and_overflow + +Test the PyLong_AsLongLongAndOverflow API. + +General conversion to long long is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ static PyObject * -test_long_long_and_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=0b92330786f45483 input=544bb0aefe5e8a9e]*/ { PyObject *num, *one, *temp; long long value; @@ -405,13 +432,18 @@ test_long_long_and_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -/* Test the PyLong_As{Size,Ssize}_t API. At present this just tests that - non-integer arguments are handled correctly. It should be extended to - test overflow handling. - */ +/*[clinic input] +_testcapi.test_long_as_size_t + +Test the PyLong_As{Size,Ssize}_t API. + +At present this just tests that non-integer arguments are handled correctly. +It should be extended to test overflow handling. +[clinic start generated code]*/ static PyObject * -test_long_as_size_t(PyObject *self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_as_size_t_impl(PyObject *module) +/*[clinic end generated code: output=f6490ea2b41e6173 input=922990c4a3edfb0d]*/ { size_t out_u; Py_ssize_t out_s; @@ -442,9 +474,13 @@ test_long_as_size_t(PyObject *self, PyObject *Py_UNUSED(ignored)) return Py_None; } +/*[clinic input] +_testcapi.test_long_as_unsigned_long_long_mask +[clinic start generated code]*/ + static PyObject * -test_long_as_unsigned_long_long_mask(PyObject *self, - PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module) +/*[clinic end generated code: output=e3e16cd0189440cc input=eb2438493ae7b9af]*/ { unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL); @@ -462,12 +498,13 @@ test_long_as_unsigned_long_long_mask(PyObject *self, Py_RETURN_NONE; } -/* Test the PyLong_AsDouble API. At present this just tests that - non-integer arguments are handled correctly. - */ +/*[clinic input] +_testcapi.test_long_as_double +[clinic start generated code]*/ static PyObject * -test_long_as_double(PyObject *self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_as_double_impl(PyObject *module) +/*[clinic end generated code: output=deca0898e15adde5 input=c77bc88ef5a1de76]*/ { double out; @@ -487,9 +524,13 @@ test_long_as_double(PyObject *self, PyObject *Py_UNUSED(ignored)) return Py_None; } -/* Simple test of _PyLong_NumBits and _PyLong_Sign. */ +/*[clinic input] +_testcapi.test_long_numbits +[clinic start generated code]*/ + static PyObject * -test_long_numbits(PyObject *self, PyObject *Py_UNUSED(ignored)) +_testcapi_test_long_numbits_impl(PyObject *module) +/*[clinic end generated code: output=9eaf8458cb15d7f7 input=265c02d48a13059e]*/ { struct triple { long input; @@ -534,8 +575,16 @@ test_long_numbits(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } +/*[clinic input] +_testcapi.call_long_compact_api + arg: object + / +[clinic start generated code]*/ + static PyObject * -check_long_compact_api(PyObject *self, PyObject *arg) +_testcapi_call_long_compact_api(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=7e3894f611b1b2b7 input=87b87396967af14c]*/ + { assert(PyLong_Check(arg)); int is_compact = PyUnstable_Long_IsCompact((PyLongObject*)arg); @@ -547,15 +596,15 @@ check_long_compact_api(PyObject *self, PyObject *arg) } static PyMethodDef test_methods[] = { - {"test_long_and_overflow", test_long_and_overflow, METH_NOARGS}, - {"test_long_api", test_long_api, METH_NOARGS}, - {"test_long_as_double", test_long_as_double, METH_NOARGS}, - {"test_long_as_size_t", test_long_as_size_t, METH_NOARGS}, - {"test_long_as_unsigned_long_long_mask", test_long_as_unsigned_long_long_mask, METH_NOARGS}, - {"test_long_long_and_overflow",test_long_long_and_overflow, METH_NOARGS}, - {"test_long_numbits", test_long_numbits, METH_NOARGS}, - {"test_longlong_api", test_longlong_api, METH_NOARGS}, - {"call_long_compact_api", check_long_compact_api, METH_O}, + _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF + _TESTCAPI_TEST_LONG_API_METHODDEF + _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF + _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF + _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF + _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF + _TESTCAPI_TEST_LONG_NUMBITS_METHODDEF + _TESTCAPI_TEST_LONGLONG_API_METHODDEF + _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF {NULL}, }; diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 7dd3b9c0c03e50..73929eaffc676d 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -1,3 +1,5 @@ +#include // ptrdiff_t + #define PY_SSIZE_T_CLEAN #include "parts.h" @@ -1130,25 +1132,48 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1( "%c", "c", 'c'); CHECK_FORMAT_1( "%0c", "c", 'c'); CHECK_FORMAT_1("%00c", "c", 'c'); - CHECK_FORMAT_1( "%2c", "c", 'c'); - CHECK_FORMAT_1("%02c", "c", 'c'); - CHECK_FORMAT_1("%.0c", "c", 'c'); - CHECK_FORMAT_1("%.2c", "c", 'c'); + CHECK_FORMAT_1( "%2c", NULL, 'c'); + CHECK_FORMAT_1("%02c", NULL, 'c'); + CHECK_FORMAT_1("%.0c", NULL, 'c'); + CHECK_FORMAT_1("%.2c", NULL, 'c'); // Integers CHECK_FORMAT_1("%d", "123", (int)123); CHECK_FORMAT_1("%i", "123", (int)123); CHECK_FORMAT_1("%u", "123", (unsigned int)123); + CHECK_FORMAT_1("%x", "7b", (unsigned int)123); + CHECK_FORMAT_1("%X", "7B", (unsigned int)123); + CHECK_FORMAT_1("%o", "173", (unsigned int)123); CHECK_FORMAT_1("%ld", "123", (long)123); CHECK_FORMAT_1("%li", "123", (long)123); CHECK_FORMAT_1("%lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%lx", "7b", (unsigned long)123); + CHECK_FORMAT_1("%lX", "7B", (unsigned long)123); + CHECK_FORMAT_1("%lo", "173", (unsigned long)123); CHECK_FORMAT_1("%lld", "123", (long long)123); CHECK_FORMAT_1("%lli", "123", (long long)123); CHECK_FORMAT_1("%llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%llx", "7b", (unsigned long long)123); + CHECK_FORMAT_1("%llX", "7B", (unsigned long long)123); + CHECK_FORMAT_1("%llo", "173", (unsigned long long)123); CHECK_FORMAT_1("%zd", "123", (Py_ssize_t)123); CHECK_FORMAT_1("%zi", "123", (Py_ssize_t)123); CHECK_FORMAT_1("%zu", "123", (size_t)123); - CHECK_FORMAT_1("%x", "7b", (int)123); + CHECK_FORMAT_1("%zx", "7b", (size_t)123); + CHECK_FORMAT_1("%zX", "7B", (size_t)123); + CHECK_FORMAT_1("%zo", "173", (size_t)123); + CHECK_FORMAT_1("%td", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%ti", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tu", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tx", "7b", (ptrdiff_t)123); + CHECK_FORMAT_1("%tX", "7B", (ptrdiff_t)123); + CHECK_FORMAT_1("%to", "173", (ptrdiff_t)123); + CHECK_FORMAT_1("%jd", "123", (intmax_t)123); + CHECK_FORMAT_1("%ji", "123", (intmax_t)123); + CHECK_FORMAT_1("%ju", "123", (uintmax_t)123); + CHECK_FORMAT_1("%jx", "7b", (uintmax_t)123); + CHECK_FORMAT_1("%jX", "7B", (uintmax_t)123); + CHECK_FORMAT_1("%jo", "173", (uintmax_t)123); CHECK_FORMAT_1("%d", "-123", (int)-123); CHECK_FORMAT_1("%i", "-123", (int)-123); @@ -1158,7 +1183,10 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%lli", "-123", (long long)-123); CHECK_FORMAT_1("%zd", "-123", (Py_ssize_t)-123); CHECK_FORMAT_1("%zi", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%x", "ffffff85", (int)-123); + CHECK_FORMAT_1("%td", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%ti", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%jd", "-123", (intmax_t)-123); + CHECK_FORMAT_1("%ji", "-123", (intmax_t)-123); // Integers: width < length CHECK_FORMAT_1("%1d", "123", (int)123); @@ -1183,7 +1211,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%1lli", "-123", (long long)-123); CHECK_FORMAT_1("%1zd", "-123", (Py_ssize_t)-123); CHECK_FORMAT_1("%1zi", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%1x", "ffffff85", (int)-123); // Integers: width > length CHECK_FORMAT_1("%5d", " 123", (int)123); @@ -1208,7 +1235,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%5lli", " -123", (long long)-123); CHECK_FORMAT_1("%5zd", " -123", (Py_ssize_t)-123); CHECK_FORMAT_1("%5zi", " -123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%9x", " ffffff85", (int)-123); // Integers: width > length, 0-flag CHECK_FORMAT_1("%05d", "00123", (int)123); @@ -1233,7 +1259,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%05lli", "-0123", (long long)-123); CHECK_FORMAT_1("%05zd", "-0123", (Py_ssize_t)-123); CHECK_FORMAT_1("%05zi", "-0123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%09x", "0ffffff85", (int)-123); // Integers: precision < length CHECK_FORMAT_1("%.1d", "123", (int)123); @@ -1258,7 +1283,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%.1lli", "-123", (long long)-123); CHECK_FORMAT_1("%.1zd", "-123", (Py_ssize_t)-123); CHECK_FORMAT_1("%.1zi", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.1x", "ffffff85", (int)-123); // Integers: precision > length CHECK_FORMAT_1("%.5d", "00123", (int)123); @@ -1283,7 +1307,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%.5lli", "-00123", (long long)-123); CHECK_FORMAT_1("%.5zd", "-00123", (Py_ssize_t)-123); CHECK_FORMAT_1("%.5zi", "-00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.9x", "0ffffff85", (int)-123); // Integers: width > precision > length CHECK_FORMAT_1("%7.5d", " 00123", (int)123); @@ -1308,7 +1331,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%7.5lli", " -00123", (long long)-123); CHECK_FORMAT_1("%7.5zd", " -00123", (Py_ssize_t)-123); CHECK_FORMAT_1("%7.5zi", " -00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%10.9x", " 0ffffff85", (int)-123); // Integers: width > precision > length, 0-flag CHECK_FORMAT_1("%07.5d", "0000123", (int)123); @@ -1333,7 +1355,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%07.5lli", "-000123", (long long)-123); CHECK_FORMAT_1("%07.5zd", "-000123", (Py_ssize_t)-123); CHECK_FORMAT_1("%07.5zi", "-000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%010.9x", "00ffffff85", (int)-123); // Integers: precision > width > length CHECK_FORMAT_1("%5.7d", "0000123", (int)123); @@ -1358,7 +1379,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%5.7lli", "-0000123", (long long)-123); CHECK_FORMAT_1("%5.7zd", "-0000123", (Py_ssize_t)-123); CHECK_FORMAT_1("%5.7zi", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%9.10x", "00ffffff85", (int)-123); // Integers: precision > width > length, 0-flag CHECK_FORMAT_1("%05.7d", "0000123", (int)123); @@ -1383,7 +1403,6 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) CHECK_FORMAT_1("%05.7lli", "-0000123", (long long)-123); CHECK_FORMAT_1("%05.7zd", "-0000123", (Py_ssize_t)-123); CHECK_FORMAT_1("%05.7zi", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%09.10x", "00ffffff85", (int)-123); // Integers: precision = 0, arg = 0 (empty string in C) CHECK_FORMAT_1("%.0d", "0", (int)0); @@ -1402,66 +1421,80 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) // Strings CHECK_FORMAT_1("%s", "None", "None"); + CHECK_FORMAT_1("%ls", "None", L"None"); CHECK_FORMAT_1("%U", "None", unicode); CHECK_FORMAT_1("%A", "None", Py_None); CHECK_FORMAT_1("%S", "None", Py_None); CHECK_FORMAT_1("%R", "None", Py_None); CHECK_FORMAT_2("%V", "None", unicode, "ignored"); CHECK_FORMAT_2("%V", "None", NULL, "None"); + CHECK_FORMAT_2("%lV", "None", NULL, L"None"); // Strings: width < length CHECK_FORMAT_1("%1s", "None", "None"); + CHECK_FORMAT_1("%1ls", "None", L"None"); CHECK_FORMAT_1("%1U", "None", unicode); CHECK_FORMAT_1("%1A", "None", Py_None); CHECK_FORMAT_1("%1S", "None", Py_None); CHECK_FORMAT_1("%1R", "None", Py_None); CHECK_FORMAT_2("%1V", "None", unicode, "ignored"); CHECK_FORMAT_2("%1V", "None", NULL, "None"); + CHECK_FORMAT_2("%1lV", "None", NULL, L"None"); // Strings: width > length CHECK_FORMAT_1("%5s", " None", "None"); + CHECK_FORMAT_1("%5ls", " None", L"None"); CHECK_FORMAT_1("%5U", " None", unicode); CHECK_FORMAT_1("%5A", " None", Py_None); CHECK_FORMAT_1("%5S", " None", Py_None); CHECK_FORMAT_1("%5R", " None", Py_None); CHECK_FORMAT_2("%5V", " None", unicode, "ignored"); CHECK_FORMAT_2("%5V", " None", NULL, "None"); + CHECK_FORMAT_2("%5lV", " None", NULL, L"None"); // Strings: precision < length CHECK_FORMAT_1("%.1s", "N", "None"); + CHECK_FORMAT_1("%.1ls", "N", L"None"); CHECK_FORMAT_1("%.1U", "N", unicode); CHECK_FORMAT_1("%.1A", "N", Py_None); CHECK_FORMAT_1("%.1S", "N", Py_None); CHECK_FORMAT_1("%.1R", "N", Py_None); CHECK_FORMAT_2("%.1V", "N", unicode, "ignored"); CHECK_FORMAT_2("%.1V", "N", NULL, "None"); + CHECK_FORMAT_2("%.1lV", "N", NULL, L"None"); // Strings: precision > length CHECK_FORMAT_1("%.5s", "None", "None"); + CHECK_FORMAT_1("%.5ls", "None", L"None"); CHECK_FORMAT_1("%.5U", "None", unicode); CHECK_FORMAT_1("%.5A", "None", Py_None); CHECK_FORMAT_1("%.5S", "None", Py_None); CHECK_FORMAT_1("%.5R", "None", Py_None); CHECK_FORMAT_2("%.5V", "None", unicode, "ignored"); CHECK_FORMAT_2("%.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%.5lV", "None", NULL, L"None"); // Strings: precision < length, width > length CHECK_FORMAT_1("%5.1s", " N", "None"); + CHECK_FORMAT_1("%5.1ls"," N", L"None"); CHECK_FORMAT_1("%5.1U", " N", unicode); CHECK_FORMAT_1("%5.1A", " N", Py_None); CHECK_FORMAT_1("%5.1S", " N", Py_None); CHECK_FORMAT_1("%5.1R", " N", Py_None); CHECK_FORMAT_2("%5.1V", " N", unicode, "ignored"); CHECK_FORMAT_2("%5.1V", " N", NULL, "None"); + CHECK_FORMAT_2("%5.1lV"," N", NULL, L"None"); // Strings: width < length, precision > length CHECK_FORMAT_1("%1.5s", "None", "None"); + CHECK_FORMAT_1("%1.5ls", "None", L"None"); CHECK_FORMAT_1("%1.5U", "None", unicode); CHECK_FORMAT_1("%1.5A", "None", Py_None); CHECK_FORMAT_1("%1.5S", "None", Py_None); CHECK_FORMAT_1("%1.5R", "None", Py_None); CHECK_FORMAT_2("%1.5V", "None", unicode, "ignored"); CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); Py_XDECREF(unicode); Py_RETURN_NONE; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index b91f7b620fdb4e..8267dbf6779017 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -762,19 +762,24 @@ clear_extension(PyObject *self, PyObject *args) static PyObject * write_perf_map_entry(PyObject *self, PyObject *args) { + PyObject *code_addr_v; const void *code_addr; unsigned int code_size; const char *entry_name; - if (!PyArg_ParseTuple(args, "KIs", &code_addr, &code_size, &entry_name)) + if (!PyArg_ParseTuple(args, "OIs", &code_addr_v, &code_size, &entry_name)) return NULL; + code_addr = PyLong_AsVoidPtr(code_addr_v); + if (code_addr == NULL) { + return NULL; + } int ret = PyUnstable_WritePerfMapEntry(code_addr, code_size, entry_name); - if (ret == -1) { - PyErr_SetString(PyExc_OSError, "Failed to write performance map entry"); + if (ret < 0) { + PyErr_SetFromErrno(PyExc_OSError); return NULL; } - return Py_BuildValue("i", ret); + return PyLong_FromLong(ret); } static PyObject * diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index c8c791b6d7c0d8..0dcdb4da47d5db 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -2381,7 +2381,12 @@ get_local_timestamp(PyObject *dt, int64_t *local_ts) ///// // Functions for cache handling -/* Constructor for StrongCacheNode */ +/* Constructor for StrongCacheNode + * + * This function doesn't set MemoryError if PyMem_Malloc fails, + * as the cache intentionally doesn't propagate exceptions + * and fails silently if error occurs. + */ static StrongCacheNode * strong_cache_node_new(PyObject *key, PyObject *zone) { @@ -2572,6 +2577,9 @@ update_strong_cache(zoneinfo_state *state, const PyTypeObject *const type, } StrongCacheNode *new_node = strong_cache_node_new(key, zone); + if (new_node == NULL) { + return; + } StrongCacheNode **root = &(state->ZONEINFO_STRONG_CACHE); move_strong_cache_node_to_front(state, root, new_node); diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index fddde960a5fe9a..3d0c2d7ae945bc 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -84,6 +84,7 @@ errno_exec(PyObject *module) PyObject *module_dict = PyModule_GetDict(module); PyObject *error_dict = PyDict_New(); if (!module_dict || !error_dict) { + Py_XDECREF(error_dict); return -1; } if (PyDict_SetItemString(module_dict, "errorcode", error_dict) < 0) { diff --git a/Modules/hashlib.h b/Modules/hashlib.h index 56ae7a5e50bf58..a8bad9dd87a939 100644 --- a/Modules/hashlib.h +++ b/Modules/hashlib.h @@ -37,6 +37,13 @@ * LEAVE_HASHLIB block or explicitly acquire and release the lock inside * a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for * an operation. + * + * These only drop the GIL if the lock acquisition itself is likely to + * block. Thus the non-blocking acquire gating the GIL release for a + * blocking lock acquisition. The intent of these macros is to surround + * the assumed always "fast" operations that you aren't releasing the + * GIL around. Otherwise use code similar to what you see in hash + * function update() methods. */ #include "pythread.h" @@ -53,7 +60,7 @@ PyThread_release_lock((obj)->lock); \ } -/* TODO(gps): We should probably make this a module or EVPobject attribute +/* TODO(gpshead): We should make this a module or class attribute * to allow the user to optimize based on the platform they're using. */ #define HASHLIB_GIL_MINSIZE 2048 diff --git a/Modules/md5module.c b/Modules/md5module.c index 86605771d9643f..2122f8b18baf6e 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -49,7 +49,9 @@ typedef long long MD5_INT64; /* 64-bit integer */ typedef struct { PyObject_HEAD - + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. + PyThread_type_lock lock; Hacl_Streaming_MD5_state *hash_state; } MD5object; @@ -72,6 +74,7 @@ static MD5object * newMD5object(MD5State * st) { MD5object *md5 = (MD5object *)PyObject_GC_New(MD5object, st->md5_type); + md5->lock = NULL; PyObject_GC_Track(md5); return md5; } @@ -88,6 +91,9 @@ static void MD5_dealloc(MD5object *ptr) { Hacl_Streaming_MD5_legacy_free(ptr->hash_state); + if (ptr->lock != NULL) { + PyThread_free_lock(ptr->lock); + } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -115,7 +121,9 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls) if ((newobj = newMD5object(st))==NULL) return NULL; + ENTER_HASHLIB(self); newobj->hash_state = Hacl_Streaming_MD5_legacy_copy(self->hash_state); + LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -130,7 +138,9 @@ MD5Type_digest_impl(MD5object *self) /*[clinic end generated code: output=eb691dc4190a07ec input=bc0c4397c2994be6]*/ { unsigned char digest[MD5_DIGESTSIZE]; + ENTER_HASHLIB(self); Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, MD5_DIGESTSIZE); } @@ -145,7 +155,9 @@ MD5Type_hexdigest_impl(MD5object *self) /*[clinic end generated code: output=17badced1f3ac932 input=b60b19de644798dd]*/ { unsigned char digest[MD5_DIGESTSIZE]; + ENTER_HASHLIB(self); Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return _Py_strhex((const char*)digest, MD5_DIGESTSIZE); } @@ -177,7 +189,18 @@ MD5Type_update(MD5object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - update(self->hash_state, buf.buf, buf.len); + if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + } + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + update(self->hash_state, buf.buf, buf.len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + update(self->hash_state, buf.buf, buf.len); + } PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -279,7 +302,15 @@ _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update(new->hash_state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update(new->hash_state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update(new->hash_state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 531f26ba8bc86f..2fe9973ef5bf7a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9023,6 +9023,10 @@ os_setgroups(PyObject *module, PyObject *groups) } gid_t *grouplist = PyMem_New(gid_t, len); + if (grouplist == NULL) { + PyErr_NoMemory(); + return NULL; + } for (Py_ssize_t i = 0; i < len; i++) { PyObject *elem; elem = PySequence_GetItem(groups, i); diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 79bd5b59ab68f9..9a4943c9eb2f75 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -1849,14 +1849,11 @@ static PyObject * kqueue_event_repr(kqueue_event_Object *s) { - char buf[1024]; - PyOS_snprintf( - buf, sizeof(buf), + return PyUnicode_FromFormat( "", (size_t)(s->e.ident), (int)s->e.filter, (unsigned int)s->e.flags, (unsigned int)s->e.fflags, (long long)(s->e.data), (void *)s->e.udata); - return PyUnicode_FromString(buf); } static int diff --git a/Modules/sha1module.c b/Modules/sha1module.c index bdb76c56f1a6e8..ef8e067dd337b3 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -48,7 +48,9 @@ typedef long long SHA1_INT64; /* 64-bit integer */ typedef struct { PyObject_HEAD - + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. + PyThread_type_lock lock; Hacl_Streaming_SHA1_state *hash_state; } SHA1object; @@ -71,6 +73,10 @@ static SHA1object * newSHA1object(SHA1State *st) { SHA1object *sha = (SHA1object *)PyObject_GC_New(SHA1object, st->sha1_type); + if (sha == NULL) { + return NULL; + } + sha->lock = NULL; PyObject_GC_Track(sha); return sha; } @@ -88,6 +94,9 @@ static void SHA1_dealloc(SHA1object *ptr) { Hacl_Streaming_SHA1_legacy_free(ptr->hash_state); + if (ptr->lock != NULL) { + PyThread_free_lock(ptr->lock); + } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -115,7 +124,9 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls) if ((newobj = newSHA1object(st)) == NULL) return NULL; + ENTER_HASHLIB(self); newobj->hash_state = Hacl_Streaming_SHA1_legacy_copy(self->hash_state); + LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -130,7 +141,9 @@ SHA1Type_digest_impl(SHA1object *self) /*[clinic end generated code: output=2f05302a7aa2b5cb input=13824b35407444bd]*/ { unsigned char digest[SHA1_DIGESTSIZE]; + ENTER_HASHLIB(self); Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, SHA1_DIGESTSIZE); } @@ -145,7 +158,9 @@ SHA1Type_hexdigest_impl(SHA1object *self) /*[clinic end generated code: output=4161fd71e68c6659 input=97691055c0c74ab0]*/ { unsigned char digest[SHA1_DIGESTSIZE]; + ENTER_HASHLIB(self); Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, SHA1_DIGESTSIZE); } @@ -177,7 +192,18 @@ SHA1Type_update(SHA1object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - update(self->hash_state, buf.buf, buf.len); + if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + } + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + update(self->hash_state, buf.buf, buf.len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + update(self->hash_state, buf.buf, buf.len); + } PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -279,7 +305,15 @@ _sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update(new->hash_state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update(new->hash_state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update(new->hash_state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 37d9b5c538fd0b..6c7c3917198d18 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -52,12 +52,18 @@ class SHA512Type "SHA512object *" "&PyType_Type" typedef struct { PyObject_HEAD int digestsize; + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. + PyThread_type_lock lock; Hacl_Streaming_SHA2_state_sha2_256 *state; } SHA256object; typedef struct { PyObject_HEAD int digestsize; + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. + PyThread_type_lock lock; Hacl_Streaming_SHA2_state_sha2_512 *state; } SHA512object; @@ -100,6 +106,7 @@ newSHA224object(sha2_state *state) if (!sha) { return NULL; } + sha->lock = NULL; PyObject_GC_Track(sha); return sha; } @@ -112,6 +119,7 @@ newSHA256object(sha2_state *state) if (!sha) { return NULL; } + sha->lock = NULL; PyObject_GC_Track(sha); return sha; } @@ -124,6 +132,7 @@ newSHA384object(sha2_state *state) if (!sha) { return NULL; } + sha->lock = NULL; PyObject_GC_Track(sha); return sha; } @@ -136,6 +145,7 @@ newSHA512object(sha2_state *state) if (!sha) { return NULL; } + sha->lock = NULL; PyObject_GC_Track(sha); return sha; } @@ -153,6 +163,9 @@ static void SHA256_dealloc(SHA256object *ptr) { Hacl_Streaming_SHA2_free_256(ptr->state); + if (ptr->lock != NULL) { + PyThread_free_lock(ptr->lock); + } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -163,6 +176,9 @@ static void SHA512_dealloc(SHA512object *ptr) { Hacl_Streaming_SHA2_free_512(ptr->state); + if (ptr->lock != NULL) { + PyThread_free_lock(ptr->lock); + } PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -229,7 +245,9 @@ SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls) } } + ENTER_HASHLIB(self); SHA256copy(self, newobj); + LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -259,7 +277,9 @@ SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls) } } + ENTER_HASHLIB(self); SHA512copy(self, newobj); + LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -275,9 +295,11 @@ SHA256Type_digest_impl(SHA256object *self) { uint8_t digest[SHA256_DIGESTSIZE]; assert(self->digestsize <= SHA256_DIGESTSIZE); + ENTER_HASHLIB(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. Hacl_Streaming_SHA2_finish_256(self->state, digest); + LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -293,9 +315,11 @@ SHA512Type_digest_impl(SHA512object *self) { uint8_t digest[SHA512_DIGESTSIZE]; assert(self->digestsize <= SHA512_DIGESTSIZE); + ENTER_HASHLIB(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. Hacl_Streaming_SHA2_finish_512(self->state, digest); + LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -311,7 +335,9 @@ SHA256Type_hexdigest_impl(SHA256object *self) { uint8_t digest[SHA256_DIGESTSIZE]; assert(self->digestsize <= SHA256_DIGESTSIZE); + ENTER_HASHLIB(self); Hacl_Streaming_SHA2_finish_256(self->state, digest); + LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -327,7 +353,9 @@ SHA512Type_hexdigest_impl(SHA512object *self) { uint8_t digest[SHA512_DIGESTSIZE]; assert(self->digestsize <= SHA512_DIGESTSIZE); + ENTER_HASHLIB(self); Hacl_Streaming_SHA2_finish_512(self->state, digest); + LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -348,7 +376,18 @@ SHA256Type_update(SHA256object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - update_256(self->state, buf.buf, buf.len); + if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + } + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + update_256(self->state, buf.buf, buf.len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + update_256(self->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -371,7 +410,18 @@ SHA512Type_update(SHA512object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - update_512(self->state, buf.buf, buf.len); + if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + } + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + update_512(self->state, buf.buf, buf.len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + update_512(self->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -560,7 +610,15 @@ _sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update_256(new->state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update_256(new->state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update_256(new->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } @@ -606,7 +664,15 @@ _sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update_256(new->state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update_256(new->state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update_256(new->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } @@ -651,7 +717,15 @@ _sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update_512(new->state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update_512(new->state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update_512(new->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } @@ -696,7 +770,15 @@ _sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - update_512(new->state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + update_512(new->state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + update_512(new->state, buf.buf, buf.len); + } PyBuffer_Release(&buf); } diff --git a/Modules/sha3module.c b/Modules/sha3module.c index f05187498a19b3..558d2005cff617 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -60,6 +60,9 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type" typedef struct { PyObject_HEAD + // Prevents undefined behavior via multiple threads entering the C API. + // The lock will be NULL before threaded access has been enabled. + PyThread_type_lock lock; Hacl_Streaming_Keccak_state *hash_state; } SHA3object; @@ -73,6 +76,7 @@ newSHA3object(PyTypeObject *type) if (newobj == NULL) { return NULL; } + newobj->lock = NULL; return newobj; } @@ -133,7 +137,15 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) if (data) { GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - sha3_update(self->hash_state, buf.buf, buf.len); + if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* We do not initialize self->lock here as this is the constructor + * where it is not yet possible to have concurrent access. */ + Py_BEGIN_ALLOW_THREADS + sha3_update(self->hash_state, buf.buf, buf.len); + Py_END_ALLOW_THREADS + } else { + sha3_update(self->hash_state, buf.buf, buf.len); + } } PyBuffer_Release(&buf); @@ -157,6 +169,9 @@ static void SHA3_dealloc(SHA3object *self) { Hacl_Streaming_Keccak_free(self->hash_state); + if (self->lock != NULL) { + PyThread_free_lock(self->lock); + } PyTypeObject *tp = Py_TYPE(self); PyObject_Free(self); Py_DECREF(tp); @@ -181,7 +196,9 @@ _sha3_sha3_224_copy_impl(SHA3object *self) if ((newobj = newSHA3object(Py_TYPE(self))) == NULL) { return NULL; } + ENTER_HASHLIB(self); newobj->hash_state = Hacl_Streaming_Keccak_copy(self->hash_state); + LEAVE_HASHLIB(self); return (PyObject *)newobj; } @@ -199,7 +216,9 @@ _sha3_sha3_224_digest_impl(SHA3object *self) unsigned char digest[SHA3_MAX_DIGESTSIZE]; // This function errors out if the algorithm is Shake. Here, we know this // not to be the case, and therefore do not perform error checking. + ENTER_HASHLIB(self); Hacl_Streaming_Keccak_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return PyBytes_FromStringAndSize((const char *)digest, Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -216,7 +235,9 @@ _sha3_sha3_224_hexdigest_impl(SHA3object *self) /*[clinic end generated code: output=75ad03257906918d input=2d91bb6e0d114ee3]*/ { unsigned char digest[SHA3_MAX_DIGESTSIZE]; + ENTER_HASHLIB(self); Hacl_Streaming_Keccak_finish(self->hash_state, digest); + LEAVE_HASHLIB(self); return _Py_strhex((const char *)digest, Hacl_Streaming_Keccak_hash_len(self->hash_state)); } @@ -237,7 +258,18 @@ _sha3_sha3_224_update(SHA3object *self, PyObject *data) { Py_buffer buf; GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - sha3_update(self->hash_state, buf.buf, buf.len); + if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { + self->lock = PyThread_allocate_lock(); + } + if (self->lock != NULL) { + Py_BEGIN_ALLOW_THREADS + PyThread_acquire_lock(self->lock, 1); + sha3_update(self->hash_state, buf.buf, buf.len); + PyThread_release_lock(self->lock); + Py_END_ALLOW_THREADS + } else { + sha3_update(self->hash_state, buf.buf, buf.len); + } PyBuffer_Release(&buf); Py_RETURN_NONE; } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index c11fb4400eab2f..a86aaed501fa33 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1339,8 +1339,6 @@ setbdaddr(const char *name, bdaddr_t *bdaddr) static PyObject * makebdaddr(bdaddr_t *bdaddr) { - char buf[(6 * 2) + 5 + 1]; - #ifdef MS_WINDOWS int i; unsigned int octets[6]; @@ -1349,16 +1347,14 @@ makebdaddr(bdaddr_t *bdaddr) octets[i] = ((*bdaddr) >> (8 * i)) & 0xFF; } - sprintf(buf, "%02X:%02X:%02X:%02X:%02X:%02X", + return PyUnicode_FromFormat("%02X:%02X:%02X:%02X:%02X:%02X", octets[5], octets[4], octets[3], octets[2], octets[1], octets[0]); #else - sprintf(buf, "%02X:%02X:%02X:%02X:%02X:%02X", + return PyUnicode_FromFormat("%02X:%02X:%02X:%02X:%02X:%02X", bdaddr->b[5], bdaddr->b[4], bdaddr->b[3], bdaddr->b[2], bdaddr->b[1], bdaddr->b[0]); #endif - - return PyUnicode_FromString(buf); } #endif diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index b67844a67c315c..534d065765f0f9 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1722,6 +1722,9 @@ ZlibDecompressor__new__(PyTypeObject *cls, return NULL; } ZlibDecompressor *self = PyObject_New(ZlibDecompressor, cls); + if (self == NULL) { + return NULL; + } self->eof = 0; self->needs_input = 1; self->avail_in_real = 0; diff --git a/Objects/genobject.c b/Objects/genobject.c index 9252c654934565..1abfc83ab678ef 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -408,11 +408,16 @@ gen_close(PyGenObject *gen, PyObject *args) PyErr_SetString(PyExc_RuntimeError, msg); return NULL; } - if (PyErr_ExceptionMatches(PyExc_StopIteration) - || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { - PyErr_Clear(); /* ignore these errors */ + assert(PyErr_Occurred()); + if (PyErr_ExceptionMatches(PyExc_GeneratorExit)) { + PyErr_Clear(); /* ignore this error */ Py_RETURN_NONE; } + /* if the generator returned a value while closing, StopIteration was + * raised in gen_send_ex() above; retrieve and return the value here */ + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + return retval; + } return NULL; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7726f2fb17afde..ec5684b1d09502 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -56,6 +56,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_unicodeobject_generated.h" // _PyUnicode_InitStaticStrings() #include "stringlib/eq.h" // unicode_eq() +#include // ptrdiff_t #ifdef MS_WINDOWS #include @@ -2285,14 +2286,15 @@ PyUnicode_AsUCS4Copy(PyObject *string) return as_ucs4(string, NULL, 0, 1); } -/* maximum number of characters required for output of %lld or %p. - We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits, - plus 1 for the sign. 53/22 is an upper bound for log10(256). */ -#define MAX_LONG_LONG_CHARS (2 + (SIZEOF_LONG_LONG*53-1) / 22) +/* maximum number of characters required for output of %jo or %jd or %p. + We need at most ceil(log8(256)*sizeof(intmax_t)) digits, + plus 1 for the sign, plus 2 for the 0x prefix (for %p), + plus 1 for the terminal NUL. */ +#define MAX_INTMAX_CHARS (5 + (sizeof(intmax_t)*8-1) / 3) static int unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, - Py_ssize_t width, Py_ssize_t precision) + Py_ssize_t width, Py_ssize_t precision, int flags) { Py_ssize_t length, fill, arglen; Py_UCS4 maxchar; @@ -2314,8 +2316,8 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, if (_PyUnicodeWriter_Prepare(writer, arglen, maxchar) == -1) return -1; - if (width > length) { - fill = width - length; + fill = Py_MAX(width - length, 0); + if (fill && !(flags & F_LJUST)) { if (PyUnicode_Fill(writer->buffer, writer->pos, fill, ' ') == -1) return -1; writer->pos += fill; @@ -2324,12 +2326,19 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, str, 0, length); writer->pos += length; + + if (fill && (flags & F_LJUST)) { + if (PyUnicode_Fill(writer->buffer, writer->pos, fill, ' ') == -1) + return -1; + writer->pos += fill; + } + return 0; } static int unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, - Py_ssize_t width, Py_ssize_t precision) + Py_ssize_t width, Py_ssize_t precision, int flags) { /* UTF-8 */ Py_ssize_t length; @@ -2349,24 +2358,58 @@ unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, if (unicode == NULL) return -1; - res = unicode_fromformat_write_str(writer, unicode, width, -1); + res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); + Py_DECREF(unicode); + return res; +} + +static int +unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, + Py_ssize_t width, Py_ssize_t precision, int flags) +{ + /* UTF-8 */ + Py_ssize_t length; + PyObject *unicode; + int res; + + if (precision == -1) { + length = wcslen(str); + } + else { + length = 0; + while (length < precision && str[length]) { + length++; + } + } + unicode = PyUnicode_FromWideChar(str, length); + if (unicode == NULL) + return -1; + + res = unicode_fromformat_write_str(writer, unicode, width, -1, flags); Py_DECREF(unicode); return res; } +#define F_LONG 1 +#define F_LONGLONG 2 +#define F_SIZE 3 +#define F_PTRDIFF 4 +#define F_INTMAX 5 +static const char * const formats[] = {"%d", "%ld", "%lld", "%zd", "%td", "%jd"}; +static const char * const formats_o[] = {"%o", "%lo", "%llo", "%zo", "%to", "%jo"}; +static const char * const formats_u[] = {"%u", "%lu", "%llu", "%zu", "%tu", "%ju"}; +static const char * const formats_x[] = {"%x", "%lx", "%llx", "%zx", "%tx", "%jx"}; +static const char * const formats_X[] = {"%X", "%lX", "%llX", "%zX", "%tX", "%jX"}; + static const char* unicode_fromformat_arg(_PyUnicodeWriter *writer, const char *f, va_list *vargs) { const char *p; Py_ssize_t len; - int zeropad; + int flags = 0; Py_ssize_t width; Py_ssize_t precision; - int longflag; - int longlongflag; - int size_tflag; - Py_ssize_t fill; p = f; f++; @@ -2377,15 +2420,31 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, return f; } - zeropad = 0; - if (*f == '0') { - zeropad = 1; - f++; + /* Parse flags. Example: "%-i" => flags=F_LJUST. */ + /* Flags '+', ' ' and '#' are not particularly useful. + * They are not worth the implementation and maintenance costs. + * In addition, '#' should add "0" for "o" conversions for compatibility + * with printf, but it would confuse Python users. */ + while (1) { + switch (*f++) { + case '-': flags |= F_LJUST; continue; + case '0': flags |= F_ZERO; continue; + } + f--; + break; } /* parse the width.precision part, e.g. "%2.5s" => width=2, precision=5 */ width = -1; - if (Py_ISDIGIT((unsigned)*f)) { + if (*f == '*') { + width = va_arg(*vargs, int); + if (width < 0) { + flags |= F_LJUST; + width = -width; + } + f++; + } + else if (Py_ISDIGIT((unsigned)*f)) { width = *f - '0'; f++; while (Py_ISDIGIT((unsigned)*f)) { @@ -2401,7 +2460,14 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, precision = -1; if (*f == '.') { f++; - if (Py_ISDIGIT((unsigned)*f)) { + if (*f == '*') { + precision = va_arg(*vargs, int); + if (precision < 0) { + precision = -2; + } + f++; + } + else if (Py_ISDIGIT((unsigned)*f)) { precision = (*f - '0'); f++; while (Py_ISDIGIT((unsigned)*f)) { @@ -2416,30 +2482,47 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, } } - /* Handle %ld, %lu, %lld and %llu. */ - longflag = 0; - longlongflag = 0; - size_tflag = 0; + int sizemod = 0; if (*f == 'l') { - if (f[1] == 'd' || f[1] == 'u' || f[1] == 'i') { - longflag = 1; - ++f; - } - else if (f[1] == 'l' && - (f[2] == 'd' || f[2] == 'u' || f[2] == 'i')) { - longlongflag = 1; + if (f[1] == 'l') { + sizemod = F_LONGLONG; f += 2; } + else { + sizemod = F_LONG; + ++f; + } } - /* handle the size_t flag. */ - else if (*f == 'z' && (f[1] == 'd' || f[1] == 'u' || f[1] == 'i')) { - size_tflag = 1; + else if (*f == 'z') { + sizemod = F_SIZE; + ++f; + } + else if (*f == 't') { + sizemod = F_PTRDIFF; + ++f; + } + else if (*f == 'j') { + sizemod = F_INTMAX; ++f; } - if (f[0] != '\0' && f[1] == '\0') writer->overallocate = 0; + switch (*f) { + case 'd': case 'i': case 'o': case 'u': case 'x': case 'X': + break; + case 'c': case 'p': + if (sizemod || width >= 0 || precision >= 0) goto invalid_format; + break; + case 's': + case 'V': + if (sizemod && sizemod != F_LONG) goto invalid_format; + break; + default: + if (sizemod) goto invalid_format; + break; + } + switch (*f) { case 'c': { @@ -2454,91 +2537,98 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, break; } - case 'i': - case 'd': - case 'u': - case 'x': + case 'd': case 'i': + case 'o': case 'u': case 'x': case 'X': { /* used by sprintf */ - char buffer[MAX_LONG_LONG_CHARS]; - Py_ssize_t arglen; - - if (*f == 'u') { - if (longflag) { - len = sprintf(buffer, "%lu", va_arg(*vargs, unsigned long)); - } - else if (longlongflag) { - len = sprintf(buffer, "%llu", va_arg(*vargs, unsigned long long)); - } - else if (size_tflag) { - len = sprintf(buffer, "%zu", va_arg(*vargs, size_t)); - } - else { - len = sprintf(buffer, "%u", va_arg(*vargs, unsigned int)); - } - } - else if (*f == 'x') { - len = sprintf(buffer, "%x", va_arg(*vargs, int)); - } - else { - if (longflag) { - len = sprintf(buffer, "%li", va_arg(*vargs, long)); - } - else if (longlongflag) { - len = sprintf(buffer, "%lli", va_arg(*vargs, long long)); - } - else if (size_tflag) { - len = sprintf(buffer, "%zi", va_arg(*vargs, Py_ssize_t)); - } - else { - len = sprintf(buffer, "%i", va_arg(*vargs, int)); - } + char buffer[MAX_INTMAX_CHARS]; + const char *fmt = NULL; + switch (*f) { + case 'o': fmt = formats_o[sizemod]; break; + case 'u': fmt = formats_u[sizemod]; break; + case 'x': fmt = formats_x[sizemod]; break; + case 'X': fmt = formats_X[sizemod]; break; + default: fmt = formats[sizemod]; break; + } + int issigned = (*f == 'd' || *f == 'i'); + switch (sizemod) { + case F_LONG: + len = issigned ? + sprintf(buffer, fmt, va_arg(*vargs, long)) : + sprintf(buffer, fmt, va_arg(*vargs, unsigned long)); + break; + case F_LONGLONG: + len = issigned ? + sprintf(buffer, fmt, va_arg(*vargs, long long)) : + sprintf(buffer, fmt, va_arg(*vargs, unsigned long long)); + break; + case F_SIZE: + len = issigned ? + sprintf(buffer, fmt, va_arg(*vargs, Py_ssize_t)) : + sprintf(buffer, fmt, va_arg(*vargs, size_t)); + break; + case F_PTRDIFF: + len = sprintf(buffer, fmt, va_arg(*vargs, ptrdiff_t)); + break; + case F_INTMAX: + len = issigned ? + sprintf(buffer, fmt, va_arg(*vargs, intmax_t)) : + sprintf(buffer, fmt, va_arg(*vargs, uintmax_t)); + break; + default: + len = issigned ? + sprintf(buffer, fmt, va_arg(*vargs, int)) : + sprintf(buffer, fmt, va_arg(*vargs, unsigned int)); + break; } assert(len >= 0); - int negative = (buffer[0] == '-'); - len -= negative; + int sign = (buffer[0] == '-'); + len -= sign; precision = Py_MAX(precision, len); - width = Py_MAX(width, precision + negative); + width = Py_MAX(width, precision + sign); + if ((flags & F_ZERO) && !(flags & F_LJUST)) { + precision = width - sign; + } - arglen = Py_MAX(precision, width); - if (_PyUnicodeWriter_Prepare(writer, arglen, 127) == -1) - return NULL; + Py_ssize_t spacepad = Py_MAX(width - precision - sign, 0); + Py_ssize_t zeropad = Py_MAX(precision - len, 0); - if (width > precision) { - if (negative && zeropad) { - if (_PyUnicodeWriter_WriteChar(writer, '-') == -1) - return NULL; - } + if (_PyUnicodeWriter_Prepare(writer, width, 127) == -1) + return NULL; - Py_UCS4 fillchar = zeropad?'0':' '; - fill = width - precision - negative; - if (PyUnicode_Fill(writer->buffer, writer->pos, fill, fillchar) == -1) + if (spacepad && !(flags & F_LJUST)) { + if (PyUnicode_Fill(writer->buffer, writer->pos, spacepad, ' ') == -1) return NULL; - writer->pos += fill; + writer->pos += spacepad; + } - if (negative && !zeropad) { - if (_PyUnicodeWriter_WriteChar(writer, '-') == -1) - return NULL; - } + if (sign) { + if (_PyUnicodeWriter_WriteChar(writer, '-') == -1) + return NULL; } - if (precision > len) { - fill = precision - len; - if (PyUnicode_Fill(writer->buffer, writer->pos, fill, '0') == -1) + if (zeropad) { + if (PyUnicode_Fill(writer->buffer, writer->pos, zeropad, '0') == -1) return NULL; - writer->pos += fill; + writer->pos += zeropad; } - if (_PyUnicodeWriter_WriteASCIIString(writer, &buffer[negative], len) < 0) + if (_PyUnicodeWriter_WriteASCIIString(writer, &buffer[sign], len) < 0) return NULL; + + if (spacepad && (flags & F_LJUST)) { + if (PyUnicode_Fill(writer->buffer, writer->pos, spacepad, ' ') == -1) + return NULL; + writer->pos += spacepad; + } break; } case 'p': { - char number[MAX_LONG_LONG_CHARS]; + char number[MAX_INTMAX_CHARS]; len = sprintf(number, "%p", va_arg(*vargs, void*)); assert(len >= 0); @@ -2561,10 +2651,17 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, case 's': { - /* UTF-8 */ - const char *s = va_arg(*vargs, const char*); - if (unicode_fromformat_write_cstr(writer, s, width, precision) < 0) - return NULL; + if (sizemod) { + const wchar_t *s = va_arg(*vargs, const wchar_t*); + if (unicode_fromformat_write_wcstr(writer, s, width, precision, flags) < 0) + return NULL; + } + else { + /* UTF-8 */ + const char *s = va_arg(*vargs, const char*); + if (unicode_fromformat_write_cstr(writer, s, width, precision, flags) < 0) + return NULL; + } break; } @@ -2573,7 +2670,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, PyObject *obj = va_arg(*vargs, PyObject *); assert(obj && _PyUnicode_CHECK(obj)); - if (unicode_fromformat_write_str(writer, obj, width, precision) == -1) + if (unicode_fromformat_write_str(writer, obj, width, precision, flags) == -1) return NULL; break; } @@ -2581,15 +2678,27 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, case 'V': { PyObject *obj = va_arg(*vargs, PyObject *); - const char *str = va_arg(*vargs, const char *); + const char *str; + const wchar_t *wstr; + if (sizemod) { + wstr = va_arg(*vargs, const wchar_t*); + } + else { + str = va_arg(*vargs, const char *); + } if (obj) { assert(_PyUnicode_CHECK(obj)); - if (unicode_fromformat_write_str(writer, obj, width, precision) == -1) + if (unicode_fromformat_write_str(writer, obj, width, precision, flags) == -1) + return NULL; + } + else if (sizemod) { + assert(wstr != NULL); + if (unicode_fromformat_write_wcstr(writer, wstr, width, precision, flags) < 0) return NULL; } else { assert(str != NULL); - if (unicode_fromformat_write_cstr(writer, str, width, precision) < 0) + if (unicode_fromformat_write_cstr(writer, str, width, precision, flags) < 0) return NULL; } break; @@ -2603,7 +2712,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, str = PyObject_Str(obj); if (!str) return NULL; - if (unicode_fromformat_write_str(writer, str, width, precision) == -1) { + if (unicode_fromformat_write_str(writer, str, width, precision, flags) == -1) { Py_DECREF(str); return NULL; } @@ -2619,7 +2728,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, repr = PyObject_Repr(obj); if (!repr) return NULL; - if (unicode_fromformat_write_str(writer, repr, width, precision) == -1) { + if (unicode_fromformat_write_str(writer, repr, width, precision, flags) == -1) { Py_DECREF(repr); return NULL; } @@ -2635,7 +2744,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, ascii = PyObject_ASCII(obj); if (!ascii) return NULL; - if (unicode_fromformat_write_str(writer, ascii, width, precision) == -1) { + if (unicode_fromformat_write_str(writer, ascii, width, precision, flags) == -1) { Py_DECREF(ascii); return NULL; } @@ -2644,6 +2753,7 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, } default: + invalid_format: PyErr_Format(PyExc_SystemError, "invalid format string: %s", p); return NULL; } diff --git a/PC/layout/main.py b/PC/layout/main.py index c9246007d47d18..cb2e4878da26b1 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -49,8 +49,6 @@ REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*", "libffi*") -LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt") - PY_FILES = FileSuffixSet(".py") PYC_FILES = FileSuffixSet(".pyc") CAT_FILES = FileSuffixSet(".cat") @@ -298,27 +296,6 @@ def _write_to_zip(zf, dest, src, ns, checked=True): log_exception("Failed to delete {}", pyc) return - if src in LIB2TO3_GRAMMAR_FILES: - from lib2to3.pgen2.driver import load_grammar - - tmp = ns.temp / src.name - try: - shutil.copy(src, tmp) - load_grammar(str(tmp)) - for f in ns.temp.glob(src.stem + "*.pickle"): - zf.write(str(f), str(dest.parent / f.name)) - try: - f.unlink() - except: - log_exception("Failed to delete {}", f) - except: - log_exception("Failed to compile {}", src) - finally: - try: - tmp.unlink() - except: - log_exception("Failed to delete {}", tmp) - zf.write(str(src), str(dest)) diff --git a/PC/layout/support/props.py b/PC/layout/support/props.py index c7a7a0ce777a6e..f751a4b3f1e437 100644 --- a/PC/layout/support/props.py +++ b/PC/layout/support/props.py @@ -36,7 +36,6 @@ {PYTHON_VERSION} true - false false {PYTHON_TARGET};$(GetPythonRuntimeFilesDependsOn) @@ -67,7 +66,6 @@ DLLs\%(Filename)%(Extension) <_PythonRuntimeLib Include="$(PythonHome)\Lib\**\*" Exclude="$(PythonHome)\Lib\**\*.pyc;$(PythonHome)\Lib\site-packages\**\*" /> - <_PythonRuntimeLib Remove="$(PythonHome)\Lib\lib2to3\**\*" Condition="$(IncludeLib2To3) != 'true'" /> <_PythonRuntimeLib Remove="$(PythonHome)\Lib\ensurepip\**\*" Condition="$(IncludeVEnv) != 'true'" /> <_PythonRuntimeLib Remove="$(PythonHome)\Lib\venv\**\*" Condition="$(IncludeVEnv) != 'true'" /> <_PythonRuntimeLib> diff --git a/PC/layout/support/python.props b/PC/layout/support/python.props index e46891aafcb9fc..c48ecb2f9c1b57 100644 --- a/PC/layout/support/python.props +++ b/PC/layout/support/python.props @@ -8,7 +8,6 @@ $$PYTHON_VERSION$$ true - false false $$PYTHON_TARGET$$;$(GetPythonRuntimeFilesDependsOn) @@ -40,7 +39,6 @@ DLLs\%(Filename)%(Extension) <_PythonRuntimeLib Include="$(PythonHome)\Lib\**\*" Exclude="$(PythonHome)\Lib\**\*.pyc;$(PythonHome)\Lib\site-packages\**\*" /> - <_PythonRuntimeLib Remove="$(PythonHome)\Lib\lib2to3\**\*" Condition="$(IncludeLib2To3) != 'true'" /> <_PythonRuntimeLib Remove="$(PythonHome)\Lib\ensurepip\**\*" Condition="$(IncludeVEnv) != 'true'" /> <_PythonRuntimeLib Remove="$(PythonHome)\Lib\venv\**\*" Condition="$(IncludeVEnv) != 'true'" /> <_PythonRuntimeLib> diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 3415efe2dea117..f9250d6faccbea 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -308,11 +308,11 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ file in their Makefile (other compilers are generally taken care of by distutils.) */ # if defined(_DEBUG) -# pragma comment(lib,"python312_d.lib") +# pragma comment(lib,"python313_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3.lib") # else -# pragma comment(lib,"python312.lib") +# pragma comment(lib,"python313.lib") # endif /* _DEBUG */ # endif /* _MSC_VER */ # endif /* Py_BUILD_CORE */ diff --git a/Parser/Python.asdl b/Parser/Python.asdl index cfc41ef45b568b..dc2eb802b0436c 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -8,15 +8,15 @@ module Python | Expression(expr body) | FunctionType(expr* argtypes, expr returns) - stmt = FunctionDef(identifier name, typeparam* typeparams, arguments args, + stmt = FunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) - | AsyncFunctionDef(identifier name, typeparam* typeparams, arguments args, + | AsyncFunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) | ClassDef(identifier name, - typeparam* typeparams, + type_param* type_params, expr* bases, keyword* keywords, stmt* body, @@ -25,7 +25,7 @@ module Python | Delete(expr* targets) | Assign(expr* targets, expr value, string? type_comment) - | TypeAlias(expr name, typeparam* typeparams, expr value) + | TypeAlias(expr name, type_param* type_params, expr value) | AugAssign(expr target, operator op, expr value) -- 'simple' indicates that we annotate simple name without parens | AnnAssign(expr target, expr annotation, expr? value, int simple) @@ -145,8 +145,8 @@ module Python type_ignore = TypeIgnore(int lineno, string tag) - typeparam = TypeVar(identifier name, expr? bound) - | ParamSpec(identifier name) - | TypeVarTuple(identifier name) - attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + type_param = TypeVar(identifier name, expr? bound) + | ParamSpec(identifier name) + | TypeVarTuple(identifier name) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) } diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 0134e6f16ba8f0..06d77b64cacbcc 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -752,7 +752,7 @@ _PyPegen_function_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty f assert(function_def != NULL); if (function_def->kind == AsyncFunctionDef_kind) { return _PyAST_AsyncFunctionDef( - function_def->v.FunctionDef.name, function_def->v.FunctionDef.typeparams, + function_def->v.FunctionDef.name, function_def->v.FunctionDef.type_params, function_def->v.FunctionDef.args, function_def->v.FunctionDef.body, decorators, function_def->v.FunctionDef.returns, function_def->v.FunctionDef.type_comment, function_def->lineno, @@ -761,7 +761,7 @@ _PyPegen_function_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty f } return _PyAST_FunctionDef( - function_def->v.FunctionDef.name, function_def->v.FunctionDef.typeparams, + function_def->v.FunctionDef.name, function_def->v.FunctionDef.type_params, function_def->v.FunctionDef.args, function_def->v.FunctionDef.body, decorators, function_def->v.FunctionDef.returns, @@ -776,7 +776,7 @@ _PyPegen_class_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty clas { assert(class_def != NULL); return _PyAST_ClassDef( - class_def->v.ClassDef.name, class_def->v.ClassDef.typeparams, + class_def->v.ClassDef.name, class_def->v.ClassDef.type_params, class_def->v.ClassDef.bases, class_def->v.ClassDef.keywords, class_def->v.ClassDef.body, decorators, class_def->lineno, class_def->col_offset, class_def->end_lineno, diff --git a/Parser/myreadline.c b/Parser/myreadline.c index 3f0e29f051a438..7074aba74b728c 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -45,7 +45,10 @@ my_fgets(PyThreadState* tstate, char *buf, int len, FILE *fp) #endif while (1) { - if (PyOS_InputHook != NULL) { + if (PyOS_InputHook != NULL && + // GH-104668: See PyOS_ReadlineFunctionPointer's comment below... + _Py_IsMainInterpreter(tstate->interp)) + { (void)(PyOS_InputHook)(); } @@ -131,7 +134,10 @@ _PyOS_WindowsConsoleReadline(PyThreadState *tstate, HANDLE hStdIn) wbuf = wbuf_local; wbuflen = sizeof(wbuf_local) / sizeof(wbuf_local[0]) - 1; while (1) { - if (PyOS_InputHook != NULL) { + if (PyOS_InputHook != NULL && + // GH-104668: See PyOS_ReadlineFunctionPointer's comment below... + _Py_IsMainInterpreter(tstate->interp)) + { (void)(PyOS_InputHook)(); } if (!ReadConsoleW(hStdIn, &wbuf[total_read], wbuflen - total_read, &n_read, NULL)) { @@ -389,11 +395,23 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) * a tty. This can happen, for example if python is run like * this: python -i < test1.py */ - if (!isatty (fileno (sys_stdin)) || !isatty (fileno (sys_stdout))) - rv = PyOS_StdioReadline (sys_stdin, sys_stdout, prompt); - else - rv = (*PyOS_ReadlineFunctionPointer)(sys_stdin, sys_stdout, - prompt); + if (!isatty(fileno(sys_stdin)) || !isatty(fileno(sys_stdout)) || + // GH-104668: Don't call global callbacks like PyOS_InputHook or + // PyOS_ReadlineFunctionPointer from subinterpreters, since it seems + // like there's no good way for users (like readline and tkinter) to + // avoid using global state to manage them. Plus, we generally don't + // want to cause trouble for libraries that don't know/care about + // subinterpreter support. If libraries really need better APIs that + // work per-interpreter and have ways to access module state, we can + // certainly add them later (but for now we'll cross our fingers and + // hope that nobody actually cares): + !_Py_IsMainInterpreter(tstate->interp)) + { + rv = PyOS_StdioReadline(sys_stdin, sys_stdout, prompt); + } + else { + rv = (*PyOS_ReadlineFunctionPointer)(sys_stdin, sys_stdout, prompt); + } Py_END_ALLOW_THREADS PyThread_release_lock(_PyOS_ReadlineLock); diff --git a/Parser/parser.c b/Parser/parser.c index 894846714eff94..fc5466fea2b3fc 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -696,9 +696,9 @@ static asdl_pattern_seq* positional_patterns_rule(Parser *p); static asdl_seq* keyword_patterns_rule(Parser *p); static KeyPatternPair* keyword_pattern_rule(Parser *p); static stmt_ty type_alias_rule(Parser *p); -static asdl_typeparam_seq* type_params_rule(Parser *p); -static asdl_typeparam_seq* type_param_seq_rule(Parser *p); -static typeparam_ty type_param_rule(Parser *p); +static asdl_type_param_seq* type_params_rule(Parser *p); +static asdl_type_param_seq* type_param_seq_rule(Parser *p); +static type_param_ty type_param_rule(Parser *p); static expr_ty type_param_bound_rule(Parser *p); static expr_ty expressions_rule(Parser *p); static expr_ty expression_rule(Parser *p); @@ -10653,7 +10653,7 @@ type_alias_rule(Parser *p) } // type_params: '[' type_param_seq ']' -static asdl_typeparam_seq* +static asdl_type_param_seq* type_params_rule(Parser *p) { if (p->level++ == MAXSTACK) { @@ -10664,7 +10664,7 @@ type_params_rule(Parser *p) p->level--; return NULL; } - asdl_typeparam_seq* _res = NULL; + asdl_type_param_seq* _res = NULL; int _mark = p->mark; { // '[' type_param_seq ']' if (p->error_indicator) { @@ -10674,7 +10674,7 @@ type_params_rule(Parser *p) D(fprintf(stderr, "%*c> type_params[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'[' type_param_seq ']'")); Token * _literal; Token * _literal_1; - asdl_typeparam_seq* t; + asdl_type_param_seq* t; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && @@ -10684,7 +10684,7 @@ type_params_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ type_params[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'[' type_param_seq ']'")); - _res = CHECK_VERSION ( asdl_typeparam_seq* , 12 , "Type parameter lists are" , t ); + _res = CHECK_VERSION ( asdl_type_param_seq* , 12 , "Type parameter lists are" , t ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10703,7 +10703,7 @@ type_params_rule(Parser *p) } // type_param_seq: ','.type_param+ ','? -static asdl_typeparam_seq* +static asdl_type_param_seq* type_param_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { @@ -10714,7 +10714,7 @@ type_param_seq_rule(Parser *p) p->level--; return NULL; } - asdl_typeparam_seq* _res = NULL; + asdl_type_param_seq* _res = NULL; int _mark = p->mark; { // ','.type_param+ ','? if (p->error_indicator) { @@ -10724,9 +10724,9 @@ type_param_seq_rule(Parser *p) D(fprintf(stderr, "%*c> type_param_seq[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.type_param+ ','?")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - asdl_typeparam_seq* a; + asdl_type_param_seq* a; if ( - (a = (asdl_typeparam_seq*)_gather_81_rule(p)) // ','.type_param+ + (a = (asdl_type_param_seq*)_gather_81_rule(p)) // ','.type_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -10756,7 +10756,7 @@ type_param_seq_rule(Parser *p) // | '*' NAME // | '**' NAME ":" expression // | '**' NAME -static typeparam_ty +static type_param_ty type_param_rule(Parser *p) { if (p->level++ == MAXSTACK) { @@ -10767,7 +10767,7 @@ type_param_rule(Parser *p) p->level--; return NULL; } - typeparam_ty _res = NULL; + type_param_ty _res = NULL; if (_PyPegen_is_memoized(p, type_param_type, &_res)) { p->level--; return _res; @@ -30199,7 +30199,7 @@ _loop0_82_rule(Parser *p) } D(fprintf(stderr, "%*c> _loop0_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' type_param")); Token * _literal; - typeparam_ty elem; + type_param_ty elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -30266,7 +30266,7 @@ _gather_81_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _gather_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "type_param _loop0_82")); - typeparam_ty elem; + type_param_ty elem; asdl_seq * seq; if ( (elem = type_param_rule(p)) // type_param diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index fb94fbeac42bae..472d4174726354 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1556,14 +1556,11 @@ verify_identifier(struct tok_state *tok) tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s); } Py_DECREF(s); - // PyUnicode_FromFormatV() does not support %X - char hex[9]; - (void)PyOS_snprintf(hex, sizeof(hex), "%04X", ch); if (Py_UNICODE_ISPRINTABLE(ch)) { - syntaxerror(tok, "invalid character '%c' (U+%s)", ch, hex); + syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch); } else { - syntaxerror(tok, "invalid non-printable character U+%s", hex); + syntaxerror(tok, "invalid non-printable character U+%04X", ch); } return 0; } @@ -1821,10 +1818,6 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t /* Skip comment, unless it's a type comment */ if (c == '#') { - if (INSIDE_FSTRING(tok)) { - return MAKE_TOKEN(syntaxerror(tok, "f-string expression part cannot include '#'")); - } - const char* p = NULL; const char *prefix, *type_start; int current_starting_col_offset; @@ -2541,9 +2534,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } if (!Py_UNICODE_ISPRINTABLE(c)) { - char hex[9]; - (void)PyOS_snprintf(hex, sizeof(hex), "%04X", c); - return MAKE_TOKEN(syntaxerror(tok, "invalid non-printable character U+%s", hex)); + return MAKE_TOKEN(syntaxerror(tok, "invalid non-printable character U+%04X", c)); } if( c == '=' && INSIDE_FSTRING_EXPR(current_tok)) { diff --git a/Python/Python-ast.c b/Python/Python-ast.c index d62cccbb6e2415..87906d975d7414 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -261,8 +261,8 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->type_comment); Py_CLEAR(state->type_ignore_type); Py_CLEAR(state->type_ignores); - Py_CLEAR(state->typeparam_type); - Py_CLEAR(state->typeparams); + Py_CLEAR(state->type_param_type); + Py_CLEAR(state->type_params); Py_CLEAR(state->unaryop_type); Py_CLEAR(state->upper); Py_CLEAR(state->value); @@ -362,7 +362,7 @@ static int init_identifiers(struct ast_state *state) if ((state->type = PyUnicode_InternFromString("type")) == NULL) return 0; if ((state->type_comment = PyUnicode_InternFromString("type_comment")) == NULL) return 0; if ((state->type_ignores = PyUnicode_InternFromString("type_ignores")) == NULL) return 0; - if ((state->typeparams = PyUnicode_InternFromString("typeparams")) == NULL) return 0; + if ((state->type_params = PyUnicode_InternFromString("type_params")) == NULL) return 0; if ((state->upper = PyUnicode_InternFromString("upper")) == NULL) return 0; if ((state->value = PyUnicode_InternFromString("value")) == NULL) return 0; if ((state->values = PyUnicode_InternFromString("values")) == NULL) return 0; @@ -383,7 +383,7 @@ GENERATE_ASDL_SEQ_CONSTRUCTOR(withitem, withitem_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(match_case, match_case_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(pattern, pattern_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty) -GENERATE_ASDL_SEQ_CONSTRUCTOR(typeparam, typeparam_ty) +GENERATE_ASDL_SEQ_CONSTRUCTOR(type_param, type_param_ty) static PyObject* ast2obj_mod(struct ast_state *state, void*); static const char * const Module_fields[]={ @@ -409,7 +409,7 @@ static const char * const stmt_attributes[] = { static PyObject* ast2obj_stmt(struct ast_state *state, void*); static const char * const FunctionDef_fields[]={ "name", - "typeparams", + "type_params", "args", "body", "decorator_list", @@ -418,7 +418,7 @@ static const char * const FunctionDef_fields[]={ }; static const char * const AsyncFunctionDef_fields[]={ "name", - "typeparams", + "type_params", "args", "body", "decorator_list", @@ -427,7 +427,7 @@ static const char * const AsyncFunctionDef_fields[]={ }; static const char * const ClassDef_fields[]={ "name", - "typeparams", + "type_params", "bases", "keywords", "body", @@ -446,7 +446,7 @@ static const char * const Assign_fields[]={ }; static const char * const TypeAlias_fields[]={ "name", - "typeparams", + "type_params", "value", }; static const char * const AugAssign_fields[]={ @@ -775,13 +775,13 @@ static const char * const TypeIgnore_fields[]={ "lineno", "tag", }; -static const char * const typeparam_attributes[] = { +static const char * const type_param_attributes[] = { "lineno", "col_offset", "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_typeparam(struct ast_state *state, void*); +static PyObject* ast2obj_type_param(struct ast_state *state, void*); static const char * const TypeVar_fields[]={ "name", "bound", @@ -1169,13 +1169,13 @@ init_types(struct ast_state *state) "FunctionType(expr* argtypes, expr returns)"); if (!state->FunctionType_type) return 0; state->stmt_type = make_type(state, "stmt", state->AST_type, NULL, 0, - "stmt = FunctionDef(identifier name, typeparam* typeparams, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)\n" - " | AsyncFunctionDef(identifier name, typeparam* typeparams, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)\n" - " | ClassDef(identifier name, typeparam* typeparams, expr* bases, keyword* keywords, stmt* body, expr* decorator_list)\n" + "stmt = FunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)\n" + " | AsyncFunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)\n" + " | ClassDef(identifier name, type_param* type_params, expr* bases, keyword* keywords, stmt* body, expr* decorator_list)\n" " | Return(expr? value)\n" " | Delete(expr* targets)\n" " | Assign(expr* targets, expr value, string? type_comment)\n" - " | TypeAlias(expr name, typeparam* typeparams, expr value)\n" + " | TypeAlias(expr name, type_param* type_params, expr value)\n" " | AugAssign(expr target, operator op, expr value)\n" " | AnnAssign(expr target, expr annotation, expr? value, int simple)\n" " | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)\n" @@ -1206,7 +1206,7 @@ init_types(struct ast_state *state) return 0; state->FunctionDef_type = make_type(state, "FunctionDef", state->stmt_type, FunctionDef_fields, 7, - "FunctionDef(identifier name, typeparam* typeparams, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)"); + "FunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)"); if (!state->FunctionDef_type) return 0; if (PyObject_SetAttr(state->FunctionDef_type, state->returns, Py_None) == -1) @@ -1217,7 +1217,7 @@ init_types(struct ast_state *state) state->AsyncFunctionDef_type = make_type(state, "AsyncFunctionDef", state->stmt_type, AsyncFunctionDef_fields, 7, - "AsyncFunctionDef(identifier name, typeparam* typeparams, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)"); + "AsyncFunctionDef(identifier name, type_param* type_params, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)"); if (!state->AsyncFunctionDef_type) return 0; if (PyObject_SetAttr(state->AsyncFunctionDef_type, state->returns, Py_None) == -1) @@ -1227,7 +1227,7 @@ init_types(struct ast_state *state) return 0; state->ClassDef_type = make_type(state, "ClassDef", state->stmt_type, ClassDef_fields, 6, - "ClassDef(identifier name, typeparam* typeparams, expr* bases, keyword* keywords, stmt* body, expr* decorator_list)"); + "ClassDef(identifier name, type_param* type_params, expr* bases, keyword* keywords, stmt* body, expr* decorator_list)"); if (!state->ClassDef_type) return 0; state->Return_type = make_type(state, "Return", state->stmt_type, Return_fields, 1, @@ -1248,7 +1248,7 @@ init_types(struct ast_state *state) return 0; state->TypeAlias_type = make_type(state, "TypeAlias", state->stmt_type, TypeAlias_fields, 3, - "TypeAlias(expr name, typeparam* typeparams, expr value)"); + "TypeAlias(expr name, type_param* type_params, expr value)"); if (!state->TypeAlias_type) return 0; state->AugAssign_type = make_type(state, "AugAssign", state->stmt_type, AugAssign_fields, 3, @@ -1894,33 +1894,33 @@ init_types(struct ast_state *state) TypeIgnore_fields, 2, "TypeIgnore(int lineno, string tag)"); if (!state->TypeIgnore_type) return 0; - state->typeparam_type = make_type(state, "typeparam", state->AST_type, - NULL, 0, - "typeparam = TypeVar(identifier name, expr? bound)\n" - " | ParamSpec(identifier name)\n" - " | TypeVarTuple(identifier name)"); - if (!state->typeparam_type) return 0; - if (!add_attributes(state, state->typeparam_type, typeparam_attributes, 4)) - return 0; - if (PyObject_SetAttr(state->typeparam_type, state->end_lineno, Py_None) == + state->type_param_type = make_type(state, "type_param", state->AST_type, + NULL, 0, + "type_param = TypeVar(identifier name, expr? bound)\n" + " | ParamSpec(identifier name)\n" + " | TypeVarTuple(identifier name)"); + if (!state->type_param_type) return 0; + if (!add_attributes(state, state->type_param_type, type_param_attributes, + 4)) return 0; + if (PyObject_SetAttr(state->type_param_type, state->end_lineno, Py_None) == -1) return 0; - if (PyObject_SetAttr(state->typeparam_type, state->end_col_offset, Py_None) - == -1) + if (PyObject_SetAttr(state->type_param_type, state->end_col_offset, + Py_None) == -1) return 0; - state->TypeVar_type = make_type(state, "TypeVar", state->typeparam_type, + state->TypeVar_type = make_type(state, "TypeVar", state->type_param_type, TypeVar_fields, 2, "TypeVar(identifier name, expr? bound)"); if (!state->TypeVar_type) return 0; if (PyObject_SetAttr(state->TypeVar_type, state->bound, Py_None) == -1) return 0; state->ParamSpec_type = make_type(state, "ParamSpec", - state->typeparam_type, ParamSpec_fields, + state->type_param_type, ParamSpec_fields, 1, "ParamSpec(identifier name)"); if (!state->ParamSpec_type) return 0; state->TypeVarTuple_type = make_type(state, "TypeVarTuple", - state->typeparam_type, + state->type_param_type, TypeVarTuple_fields, 1, "TypeVarTuple(identifier name)"); if (!state->TypeVarTuple_type) return 0; @@ -1967,8 +1967,8 @@ static int obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, PyArena* arena); static int obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* out, PyArena* arena); -static int obj2ast_typeparam(struct ast_state *state, PyObject* obj, - typeparam_ty* out, PyArena* arena); +static int obj2ast_type_param(struct ast_state *state, PyObject* obj, + type_param_ty* out, PyArena* arena); mod_ty _PyAST_Module(asdl_stmt_seq * body, asdl_type_ignore_seq * type_ignores, @@ -2032,7 +2032,7 @@ _PyAST_FunctionType(asdl_expr_seq * argtypes, expr_ty returns, PyArena *arena) } stmt_ty -_PyAST_FunctionDef(identifier name, asdl_typeparam_seq * typeparams, +_PyAST_FunctionDef(identifier name, asdl_type_param_seq * type_params, arguments_ty args, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, expr_ty returns, string type_comment, int lineno, int col_offset, int end_lineno, int end_col_offset, @@ -2054,7 +2054,7 @@ _PyAST_FunctionDef(identifier name, asdl_typeparam_seq * typeparams, return NULL; p->kind = FunctionDef_kind; p->v.FunctionDef.name = name; - p->v.FunctionDef.typeparams = typeparams; + p->v.FunctionDef.type_params = type_params; p->v.FunctionDef.args = args; p->v.FunctionDef.body = body; p->v.FunctionDef.decorator_list = decorator_list; @@ -2068,7 +2068,7 @@ _PyAST_FunctionDef(identifier name, asdl_typeparam_seq * typeparams, } stmt_ty -_PyAST_AsyncFunctionDef(identifier name, asdl_typeparam_seq * typeparams, +_PyAST_AsyncFunctionDef(identifier name, asdl_type_param_seq * type_params, arguments_ty args, asdl_stmt_seq * body, asdl_expr_seq * decorator_list, expr_ty returns, string type_comment, int lineno, int col_offset, int end_lineno, int @@ -2090,7 +2090,7 @@ _PyAST_AsyncFunctionDef(identifier name, asdl_typeparam_seq * typeparams, return NULL; p->kind = AsyncFunctionDef_kind; p->v.AsyncFunctionDef.name = name; - p->v.AsyncFunctionDef.typeparams = typeparams; + p->v.AsyncFunctionDef.type_params = type_params; p->v.AsyncFunctionDef.args = args; p->v.AsyncFunctionDef.body = body; p->v.AsyncFunctionDef.decorator_list = decorator_list; @@ -2104,10 +2104,11 @@ _PyAST_AsyncFunctionDef(identifier name, asdl_typeparam_seq * typeparams, } stmt_ty -_PyAST_ClassDef(identifier name, asdl_typeparam_seq * typeparams, asdl_expr_seq - * bases, asdl_keyword_seq * keywords, asdl_stmt_seq * body, - asdl_expr_seq * decorator_list, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_ClassDef(identifier name, asdl_type_param_seq * type_params, + asdl_expr_seq * bases, asdl_keyword_seq * keywords, + asdl_stmt_seq * body, asdl_expr_seq * decorator_list, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { stmt_ty p; if (!name) { @@ -2120,7 +2121,7 @@ _PyAST_ClassDef(identifier name, asdl_typeparam_seq * typeparams, asdl_expr_seq return NULL; p->kind = ClassDef_kind; p->v.ClassDef.name = name; - p->v.ClassDef.typeparams = typeparams; + p->v.ClassDef.type_params = type_params; p->v.ClassDef.bases = bases; p->v.ClassDef.keywords = keywords; p->v.ClassDef.body = body; @@ -2192,8 +2193,8 @@ _PyAST_Assign(asdl_expr_seq * targets, expr_ty value, string type_comment, int } stmt_ty -_PyAST_TypeAlias(expr_ty name, asdl_typeparam_seq * typeparams, expr_ty value, - int lineno, int col_offset, int end_lineno, int +_PyAST_TypeAlias(expr_ty name, asdl_type_param_seq * type_params, expr_ty + value, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { stmt_ty p; @@ -2212,7 +2213,7 @@ _PyAST_TypeAlias(expr_ty name, asdl_typeparam_seq * typeparams, expr_ty value, return NULL; p->kind = TypeAlias_kind; p->v.TypeAlias.name = name; - p->v.TypeAlias.typeparams = typeparams; + p->v.TypeAlias.type_params = type_params; p->v.TypeAlias.value = value; p->lineno = lineno; p->col_offset = col_offset; @@ -3713,17 +3714,17 @@ _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena) return p; } -typeparam_ty +type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { - typeparam_ty p; + type_param_ty p; if (!name) { PyErr_SetString(PyExc_ValueError, "field 'name' is required for TypeVar"); return NULL; } - p = (typeparam_ty)_PyArena_Malloc(arena, sizeof(*p)); + p = (type_param_ty)_PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = TypeVar_kind; @@ -3736,17 +3737,17 @@ _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int return p; } -typeparam_ty +type_param_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { - typeparam_ty p; + type_param_ty p; if (!name) { PyErr_SetString(PyExc_ValueError, "field 'name' is required for ParamSpec"); return NULL; } - p = (typeparam_ty)_PyArena_Malloc(arena, sizeof(*p)); + p = (type_param_ty)_PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = ParamSpec_kind; @@ -3758,17 +3759,17 @@ _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, return p; } -typeparam_ty +type_param_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { - typeparam_ty p; + type_param_ty p; if (!name) { PyErr_SetString(PyExc_ValueError, "field 'name' is required for TypeVarTuple"); return NULL; } - p = (typeparam_ty)_PyArena_Malloc(arena, sizeof(*p)); + p = (type_param_ty)_PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = TypeVarTuple_kind; @@ -3882,10 +3883,10 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.typeparams, - ast2obj_typeparam); + value = ast2obj_list(state, (asdl_seq*)o->v.FunctionDef.type_params, + ast2obj_type_param); if (!value) goto failed; - if (PyObject_SetAttr(result, state->typeparams, value) == -1) + if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_arguments(state, o->v.FunctionDef.args); @@ -3926,10 +3927,10 @@ ast2obj_stmt(struct ast_state *state, void* _o) goto failed; Py_DECREF(value); value = ast2obj_list(state, - (asdl_seq*)o->v.AsyncFunctionDef.typeparams, - ast2obj_typeparam); + (asdl_seq*)o->v.AsyncFunctionDef.type_params, + ast2obj_type_param); if (!value) goto failed; - if (PyObject_SetAttr(result, state->typeparams, value) == -1) + if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_arguments(state, o->v.AsyncFunctionDef.args); @@ -3970,10 +3971,10 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.typeparams, - ast2obj_typeparam); + value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.type_params, + ast2obj_type_param); if (!value) goto failed; - if (PyObject_SetAttr(result, state->typeparams, value) == -1) + if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(state, (asdl_seq*)o->v.ClassDef.bases, @@ -4052,10 +4053,10 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.TypeAlias.typeparams, - ast2obj_typeparam); + value = ast2obj_list(state, (asdl_seq*)o->v.TypeAlias.type_params, + ast2obj_type_param); if (!value) goto failed; - if (PyObject_SetAttr(result, state->typeparams, value) == -1) + if (PyObject_SetAttr(result, state->type_params, value) == -1) goto failed; Py_DECREF(value); value = ast2obj_expr(state, o->v.TypeAlias.value); @@ -5656,9 +5657,9 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) } PyObject* -ast2obj_typeparam(struct ast_state *state, void* _o) +ast2obj_type_param(struct ast_state *state, void* _o) { - typeparam_ty o = (typeparam_ty)_o; + type_param_ty o = (type_param_ty)_o; PyObject *result = NULL, *value = NULL; PyTypeObject *tp; if (!o) { @@ -6074,7 +6075,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (isinstance) { identifier name; - asdl_typeparam_seq* typeparams; + asdl_type_param_seq* type_params; arguments_ty args; asdl_stmt_seq* body; asdl_expr_seq* decorator_list; @@ -6098,11 +6099,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->typeparams, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->type_params, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"typeparams\" missing from FunctionDef"); + PyErr_SetString(PyExc_TypeError, "required field \"type_params\" missing from FunctionDef"); return 1; } else { @@ -6110,27 +6111,27 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_ssize_t len; Py_ssize_t i; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "FunctionDef field \"typeparams\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + PyErr_Format(PyExc_TypeError, "FunctionDef field \"type_params\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); goto failed; } len = PyList_GET_SIZE(tmp); - typeparams = _Py_asdl_typeparam_seq_new(len, arena); - if (typeparams == NULL) goto failed; + type_params = _Py_asdl_type_param_seq_new(len, arena); + if (type_params == NULL) goto failed; for (i = 0; i < len; i++) { - typeparam_ty val; + type_param_ty val; PyObject *tmp2 = Py_NewRef(PyList_GET_ITEM(tmp, i)); if (_Py_EnterRecursiveCall(" while traversing 'FunctionDef' node")) { goto failed; } - res = obj2ast_typeparam(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "FunctionDef field \"typeparams\" changed size during iteration"); + PyErr_SetString(PyExc_RuntimeError, "FunctionDef field \"type_params\" changed size during iteration"); goto failed; } - asdl_seq_SET(typeparams, i, val); + asdl_seq_SET(type_params, i, val); } Py_CLEAR(tmp); } @@ -6257,9 +6258,10 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_FunctionDef(name, typeparams, args, body, decorator_list, - returns, type_comment, lineno, col_offset, - end_lineno, end_col_offset, arena); + *out = _PyAST_FunctionDef(name, type_params, args, body, + decorator_list, returns, type_comment, + lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6270,7 +6272,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (isinstance) { identifier name; - asdl_typeparam_seq* typeparams; + asdl_type_param_seq* type_params; arguments_ty args; asdl_stmt_seq* body; asdl_expr_seq* decorator_list; @@ -6294,11 +6296,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->typeparams, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->type_params, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"typeparams\" missing from AsyncFunctionDef"); + PyErr_SetString(PyExc_TypeError, "required field \"type_params\" missing from AsyncFunctionDef"); return 1; } else { @@ -6306,27 +6308,27 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_ssize_t len; Py_ssize_t i; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"typeparams\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"type_params\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); goto failed; } len = PyList_GET_SIZE(tmp); - typeparams = _Py_asdl_typeparam_seq_new(len, arena); - if (typeparams == NULL) goto failed; + type_params = _Py_asdl_type_param_seq_new(len, arena); + if (type_params == NULL) goto failed; for (i = 0; i < len; i++) { - typeparam_ty val; + type_param_ty val; PyObject *tmp2 = Py_NewRef(PyList_GET_ITEM(tmp, i)); if (_Py_EnterRecursiveCall(" while traversing 'AsyncFunctionDef' node")) { goto failed; } - res = obj2ast_typeparam(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "AsyncFunctionDef field \"typeparams\" changed size during iteration"); + PyErr_SetString(PyExc_RuntimeError, "AsyncFunctionDef field \"type_params\" changed size during iteration"); goto failed; } - asdl_seq_SET(typeparams, i, val); + asdl_seq_SET(type_params, i, val); } Py_CLEAR(tmp); } @@ -6453,7 +6455,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_AsyncFunctionDef(name, typeparams, args, body, + *out = _PyAST_AsyncFunctionDef(name, type_params, args, body, decorator_list, returns, type_comment, lineno, col_offset, end_lineno, end_col_offset, arena); @@ -6467,7 +6469,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (isinstance) { identifier name; - asdl_typeparam_seq* typeparams; + asdl_type_param_seq* type_params; asdl_expr_seq* bases; asdl_keyword_seq* keywords; asdl_stmt_seq* body; @@ -6490,11 +6492,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->typeparams, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->type_params, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"typeparams\" missing from ClassDef"); + PyErr_SetString(PyExc_TypeError, "required field \"type_params\" missing from ClassDef"); return 1; } else { @@ -6502,27 +6504,27 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_ssize_t len; Py_ssize_t i; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "ClassDef field \"typeparams\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + PyErr_Format(PyExc_TypeError, "ClassDef field \"type_params\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); goto failed; } len = PyList_GET_SIZE(tmp); - typeparams = _Py_asdl_typeparam_seq_new(len, arena); - if (typeparams == NULL) goto failed; + type_params = _Py_asdl_type_param_seq_new(len, arena); + if (type_params == NULL) goto failed; for (i = 0; i < len; i++) { - typeparam_ty val; + type_param_ty val; PyObject *tmp2 = Py_NewRef(PyList_GET_ITEM(tmp, i)); if (_Py_EnterRecursiveCall(" while traversing 'ClassDef' node")) { goto failed; } - res = obj2ast_typeparam(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"typeparams\" changed size during iteration"); + PyErr_SetString(PyExc_RuntimeError, "ClassDef field \"type_params\" changed size during iteration"); goto failed; } - asdl_seq_SET(typeparams, i, val); + asdl_seq_SET(type_params, i, val); } Py_CLEAR(tmp); } @@ -6670,7 +6672,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } Py_CLEAR(tmp); } - *out = _PyAST_ClassDef(name, typeparams, bases, keywords, body, + *out = _PyAST_ClassDef(name, type_params, bases, keywords, body, decorator_list, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; @@ -6847,7 +6849,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } if (isinstance) { expr_ty name; - asdl_typeparam_seq* typeparams; + asdl_type_param_seq* type_params; expr_ty value; if (_PyObject_LookupAttr(obj, state->name, &tmp) < 0) { @@ -6867,11 +6869,11 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->typeparams, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->type_params, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"typeparams\" missing from TypeAlias"); + PyErr_SetString(PyExc_TypeError, "required field \"type_params\" missing from TypeAlias"); return 1; } else { @@ -6879,27 +6881,27 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* Py_ssize_t len; Py_ssize_t i; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TypeAlias field \"typeparams\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + PyErr_Format(PyExc_TypeError, "TypeAlias field \"type_params\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); goto failed; } len = PyList_GET_SIZE(tmp); - typeparams = _Py_asdl_typeparam_seq_new(len, arena); - if (typeparams == NULL) goto failed; + type_params = _Py_asdl_type_param_seq_new(len, arena); + if (type_params == NULL) goto failed; for (i = 0; i < len; i++) { - typeparam_ty val; + type_param_ty val; PyObject *tmp2 = Py_NewRef(PyList_GET_ITEM(tmp, i)); if (_Py_EnterRecursiveCall(" while traversing 'TypeAlias' node")) { goto failed; } - res = obj2ast_typeparam(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "TypeAlias field \"typeparams\" changed size during iteration"); + PyErr_SetString(PyExc_RuntimeError, "TypeAlias field \"type_params\" changed size during iteration"); goto failed; } - asdl_seq_SET(typeparams, i, val); + asdl_seq_SET(type_params, i, val); } Py_CLEAR(tmp); } @@ -6920,7 +6922,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_TypeAlias(name, typeparams, value, lineno, col_offset, + *out = _PyAST_TypeAlias(name, type_params, value, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; @@ -12293,8 +12295,8 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* } int -obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, - PyArena* arena) +obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, + PyArena* arena) { int isinstance; @@ -12313,12 +12315,12 @@ obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from typeparam"); + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from type_param"); return 1; } else { int res; - if (_Py_EnterRecursiveCall(" while traversing 'typeparam' node")) { + if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } res = obj2ast_int(state, tmp, &lineno, arena); @@ -12330,12 +12332,12 @@ obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from typeparam"); + PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from type_param"); return 1; } else { int res; - if (_Py_EnterRecursiveCall(" while traversing 'typeparam' node")) { + if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } res = obj2ast_int(state, tmp, &col_offset, arena); @@ -12352,7 +12354,7 @@ obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, } else { int res; - if (_Py_EnterRecursiveCall(" while traversing 'typeparam' node")) { + if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } res = obj2ast_int(state, tmp, &end_lineno, arena); @@ -12369,7 +12371,7 @@ obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, } else { int res; - if (_Py_EnterRecursiveCall(" while traversing 'typeparam' node")) { + if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } res = obj2ast_int(state, tmp, &end_col_offset, arena); @@ -12486,7 +12488,7 @@ obj2ast_typeparam(struct ast_state *state, PyObject* obj, typeparam_ty* out, return 0; } - PyErr_Format(PyExc_TypeError, "expected some sort of typeparam, but got %R", obj); + PyErr_Format(PyExc_TypeError, "expected some sort of type_param, but got %R", obj); failed: Py_XDECREF(tmp); return 1; @@ -12880,7 +12882,7 @@ astmodule_exec(PyObject *m) if (PyModule_AddObjectRef(m, "TypeIgnore", state->TypeIgnore_type) < 0) { return -1; } - if (PyModule_AddObjectRef(m, "typeparam", state->typeparam_type) < 0) { + if (PyModule_AddObjectRef(m, "type_param", state->type_param_type) < 0) { return -1; } if (PyModule_AddObjectRef(m, "TypeVar", state->TypeVar_type) < 0) { diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index ece238672e34fd..f7e32d3af9a9f7 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -89,11 +89,9 @@ _tokenizer_error(struct tok_state *tok) } return -1; case E_DEDENT: - PyErr_Format(PyExc_IndentationError, - "unindent does not match any outer indentation level " - "(, line %d)", - tok->lineno); - return -1; + msg = "unindent does not match any outer indentation level"; + errtype = PyExc_IndentationError; + break; case E_INTR: if (!PyErr_Occurred()) { PyErr_SetNone(PyExc_KeyboardInterrupt); @@ -131,7 +129,12 @@ _tokenizer_error(struct tok_state *tok) goto exit; } - tmp = Py_BuildValue("(OnnOii)", tok->filename, tok->lineno, 0, error_line, 0, 0); + Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, tok->inp - tok->buf); + if (offset == -1) { + result = -1; + goto exit; + } + tmp = Py_BuildValue("(OnnOOO)", tok->filename, tok->lineno, offset, error_line, Py_None, Py_None); if (!tmp) { result = -1; goto exit; @@ -207,7 +210,22 @@ tokenizeriter_next(tokenizeriterobject *it) end_col_offset = _PyPegen_byte_offset_to_character_offset(line, token.end - it->tok->line_start); } - result = Py_BuildValue("(NinnnnN)", str, type, lineno, end_lineno, col_offset, end_col_offset, line); + if (it->tok->tok_extra_tokens) { + // Necessary adjustments to match the original Python tokenize + // implementation + if (type > DEDENT && type < OP) { + type = OP; + } + else if (type == ASYNC || type == AWAIT) { + type = NAME; + } + else if (type == NEWLINE) { + str = PyUnicode_FromString("\n"); + end_col_offset++; + } + } + + result = Py_BuildValue("(iN(nn)(nn)N)", type, str, lineno, col_offset, end_lineno, end_col_offset, line); exit: _PyToken_Free(&token); return result; diff --git a/Python/ast.c b/Python/ast.c index 0844f2afa06bfe..68600ce683b974 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -17,12 +17,12 @@ struct validator { static int validate_stmts(struct validator *, asdl_stmt_seq *); static int validate_exprs(struct validator *, asdl_expr_seq *, expr_context_ty, int); static int validate_patterns(struct validator *, asdl_pattern_seq *, int); -static int validate_typeparams(struct validator *, asdl_typeparam_seq *); +static int validate_type_params(struct validator *, asdl_type_param_seq *); static int _validate_nonempty_seq(asdl_seq *, const char *, const char *); static int validate_stmt(struct validator *, stmt_ty); static int validate_expr(struct validator *, expr_ty, expr_context_ty); static int validate_pattern(struct validator *, pattern_ty, int); -static int validate_typeparam(struct validator *, typeparam_ty); +static int validate_typeparam(struct validator *, type_param_ty); #define VALIDATE_POSITIONS(node) \ if (node->lineno > node->end_lineno) { \ @@ -728,7 +728,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) switch (stmt->kind) { case FunctionDef_kind: ret = validate_body(state, stmt->v.FunctionDef.body, "FunctionDef") && - validate_typeparams(state, stmt->v.FunctionDef.typeparams) && + validate_type_params(state, stmt->v.FunctionDef.type_params) && validate_arguments(state, stmt->v.FunctionDef.args) && validate_exprs(state, stmt->v.FunctionDef.decorator_list, Load, 0) && (!stmt->v.FunctionDef.returns || @@ -736,7 +736,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) break; case ClassDef_kind: ret = validate_body(state, stmt->v.ClassDef.body, "ClassDef") && - validate_typeparams(state, stmt->v.ClassDef.typeparams) && + validate_type_params(state, stmt->v.ClassDef.type_params) && validate_exprs(state, stmt->v.ClassDef.bases, Load, 0) && validate_keywords(state, stmt->v.ClassDef.keywords) && validate_exprs(state, stmt->v.ClassDef.decorator_list, Load, 0); @@ -769,7 +769,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) break; case TypeAlias_kind: ret = validate_expr(state, stmt->v.TypeAlias.name, Store) && - validate_typeparams(state, stmt->v.TypeAlias.typeparams) && + validate_type_params(state, stmt->v.TypeAlias.type_params) && validate_expr(state, stmt->v.TypeAlias.value, Load); break; case For_kind: @@ -919,7 +919,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) break; case AsyncFunctionDef_kind: ret = validate_body(state, stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef") && - validate_typeparams(state, stmt->v.AsyncFunctionDef.typeparams) && + validate_type_params(state, stmt->v.AsyncFunctionDef.type_params) && validate_arguments(state, stmt->v.AsyncFunctionDef.args) && validate_exprs(state, stmt->v.AsyncFunctionDef.decorator_list, Load, 0) && (!stmt->v.AsyncFunctionDef.returns || @@ -993,7 +993,7 @@ validate_patterns(struct validator *state, asdl_pattern_seq *patterns, int star_ } static int -validate_typeparam(struct validator *state, typeparam_ty tp) +validate_typeparam(struct validator *state, type_param_ty tp) { VALIDATE_POSITIONS(tp); int ret = -1; @@ -1014,11 +1014,11 @@ validate_typeparam(struct validator *state, typeparam_ty tp) } static int -validate_typeparams(struct validator *state, asdl_typeparam_seq *tps) +validate_type_params(struct validator *state, asdl_type_param_seq *tps) { Py_ssize_t i; for (i = 0; i < asdl_seq_LEN(tps); i++) { - typeparam_ty tp = asdl_seq_GET(tps, i); + type_param_ty tp = asdl_seq_GET(tps, i); if (tp) { if (!validate_typeparam(state, tp)) return 0; diff --git a/Python/ast_opt.c b/Python/ast_opt.c index c5b3e075467334..274bd134e1435b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -642,7 +642,7 @@ static int astfold_withitem(withitem_ty node_, PyArena *ctx_, _PyASTOptimizeStat static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); static int astfold_match_case(match_case_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); static int astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); -static int astfold_typeparam(typeparam_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); +static int astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); #define CALL(FUNC, TYPE, ARG) \ if (!FUNC((ARG), ctx_, state)) \ @@ -881,7 +881,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) } switch (node_->kind) { case FunctionDef_kind: - CALL_SEQ(astfold_typeparam, typeparam, node_->v.FunctionDef.typeparams); + CALL_SEQ(astfold_type_param, type_param, node_->v.FunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args); CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body); CALL_SEQ(astfold_expr, expr, node_->v.FunctionDef.decorator_list); @@ -890,7 +890,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) } break; case AsyncFunctionDef_kind: - CALL_SEQ(astfold_typeparam, typeparam, node_->v.AsyncFunctionDef.typeparams); + CALL_SEQ(astfold_type_param, type_param, node_->v.AsyncFunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args); CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body); CALL_SEQ(astfold_expr, expr, node_->v.AsyncFunctionDef.decorator_list); @@ -899,7 +899,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) } break; case ClassDef_kind: - CALL_SEQ(astfold_typeparam, typeparam, node_->v.ClassDef.typeparams); + CALL_SEQ(astfold_type_param, type_param, node_->v.ClassDef.type_params); CALL_SEQ(astfold_expr, expr, node_->v.ClassDef.bases); CALL_SEQ(astfold_keyword, keyword, node_->v.ClassDef.keywords); CALL(astfold_body, asdl_seq, node_->v.ClassDef.body); @@ -928,7 +928,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) break; case TypeAlias_kind: CALL(astfold_expr, expr_ty, node_->v.TypeAlias.name); - CALL_SEQ(astfold_typeparam, typeparam, node_->v.TypeAlias.typeparams); + CALL_SEQ(astfold_type_param, type_param, node_->v.TypeAlias.type_params); CALL(astfold_expr, expr_ty, node_->v.TypeAlias.value); break; case For_kind: @@ -1084,7 +1084,7 @@ astfold_match_case(match_case_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat } static int -astfold_typeparam(typeparam_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) +astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { switch (node_->kind) { case TypeVar_kind: diff --git a/Python/compile.c b/Python/compile.c index cfe8224a7e27c6..f2314ae11c417b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2113,15 +2113,15 @@ wrap_in_stopiteration_handler(struct compiler *c) } static int -compiler_type_params(struct compiler *c, asdl_typeparam_seq *typeparams) +compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) { - if (!typeparams) { + if (!type_params) { return SUCCESS; } - Py_ssize_t n = asdl_seq_LEN(typeparams); + Py_ssize_t n = asdl_seq_LEN(type_params); for (Py_ssize_t i = 0; i < n; i++) { - typeparam_ty typeparam = asdl_seq_GET(typeparams, i); + type_param_ty typeparam = asdl_seq_GET(type_params, i); location loc = LOC(typeparam); switch(typeparam->kind) { case TypeVar_kind: @@ -2170,7 +2170,7 @@ compiler_type_params(struct compiler *c, asdl_typeparam_seq *typeparams) break; } } - ADDOP_I(c, LOC(asdl_seq_GET(typeparams, 0)), BUILD_TUPLE, n); + ADDOP_I(c, LOC(asdl_seq_GET(type_params, 0)), BUILD_TUPLE, n); return SUCCESS; } @@ -2248,7 +2248,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) expr_ty returns; identifier name; asdl_expr_seq *decos; - asdl_typeparam_seq *typeparams; + asdl_type_param_seq *type_params; Py_ssize_t funcflags; int annotations; int firstlineno; @@ -2260,7 +2260,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) returns = s->v.AsyncFunctionDef.returns; decos = s->v.AsyncFunctionDef.decorator_list; name = s->v.AsyncFunctionDef.name; - typeparams = s->v.AsyncFunctionDef.typeparams; + type_params = s->v.AsyncFunctionDef.type_params; } else { assert(s->kind == FunctionDef_kind); @@ -2268,7 +2268,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) returns = s->v.FunctionDef.returns; decos = s->v.FunctionDef.decorator_list; name = s->v.FunctionDef.name; - typeparams = s->v.FunctionDef.typeparams; + type_params = s->v.FunctionDef.type_params; } RETURN_IF_ERROR(compiler_check_debug_args(c, args)); @@ -2281,7 +2281,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) location loc = LOC(s); - int is_generic = asdl_seq_LEN(typeparams) > 0; + int is_generic = asdl_seq_LEN(type_params) > 0; if (is_generic) { // Used by the CALL to the type parameters function. @@ -2305,17 +2305,17 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (num_typeparam_args == 2) { ADDOP_I(c, loc, SWAP, 2); } - PyObject *typeparams_name = PyUnicode_FromFormat("", name); - if (!typeparams_name) { + PyObject *type_params_name = PyUnicode_FromFormat("", name); + if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, typeparams_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)typeparams, firstlineno) == -1) { - Py_DECREF(typeparams_name); + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + (void *)type_params, firstlineno) == -1) { + Py_DECREF(type_params_name); return ERROR; } - Py_DECREF(typeparams_name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, typeparams)); + Py_DECREF(type_params_name); + RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); if ((funcflags & 0x01) || (funcflags & 0x02)) { RETURN_IF_ERROR_IN_SCOPE(c, codegen_addop_i(INSTR_SEQUENCE(c), LOAD_FAST, 0, loc)); } @@ -2416,8 +2416,8 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) compiler_exit_scope(c); return ERROR; } - asdl_typeparam_seq *typeparams = s->v.ClassDef.typeparams; - if (asdl_seq_LEN(typeparams) > 0) { + asdl_type_param_seq *type_params = s->v.ClassDef.type_params; + if (asdl_seq_LEN(type_params) > 0) { if (!compiler_set_type_params_in_class(c, loc)) { compiler_exit_scope(c); return ERROR; @@ -2486,6 +2486,10 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) } /* 2. load the 'build_class' function */ + + // these instructions should be attributed to the class line, + // not a decorator line + loc = LOC(s); ADDOP(c, loc, PUSH_NULL); ADDOP(c, loc, LOAD_BUILD_CLASS); @@ -2515,23 +2519,23 @@ compiler_class(struct compiler *c, stmt_ty s) } location loc = LOC(s); - asdl_typeparam_seq *typeparams = s->v.ClassDef.typeparams; - int is_generic = asdl_seq_LEN(typeparams) > 0; + asdl_type_param_seq *type_params = s->v.ClassDef.type_params; + int is_generic = asdl_seq_LEN(type_params) > 0; if (is_generic) { Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); ADDOP(c, loc, PUSH_NULL); - PyObject *typeparams_name = PyUnicode_FromFormat("", + PyObject *type_params_name = PyUnicode_FromFormat("", s->v.ClassDef.name); - if (!typeparams_name) { + if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, typeparams_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)typeparams, firstlineno) == -1) { - Py_DECREF(typeparams_name); + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + (void *)type_params, firstlineno) == -1) { + Py_DECREF(type_params_name); return ERROR; } - Py_DECREF(typeparams_name); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, typeparams)); + Py_DECREF(type_params_name); + RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); _Py_DECLARE_STR(type_params, ".type_params"); RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store)); } @@ -2633,26 +2637,26 @@ static int compiler_typealias(struct compiler *c, stmt_ty s) { location loc = LOC(s); - asdl_typeparam_seq *typeparams = s->v.TypeAlias.typeparams; - int is_generic = asdl_seq_LEN(typeparams) > 0; + asdl_type_param_seq *type_params = s->v.TypeAlias.type_params; + int is_generic = asdl_seq_LEN(type_params) > 0; PyObject *name = s->v.TypeAlias.name->v.Name.id; if (is_generic) { ADDOP(c, loc, PUSH_NULL); - PyObject *typeparams_name = PyUnicode_FromFormat("", + PyObject *type_params_name = PyUnicode_FromFormat("", name); - if (!typeparams_name) { + if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, typeparams_name, COMPILER_SCOPE_TYPEPARAMS, - (void *)typeparams, loc.lineno) == -1) { - Py_DECREF(typeparams_name); + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + (void *)type_params, loc.lineno) == -1) { + Py_DECREF(type_params_name); return ERROR; } - Py_DECREF(typeparams_name); + Py_DECREF(type_params_name); RETURN_IF_ERROR_IN_SCOPE( c, compiler_addop_load_const(c->c_const_cache, c->u, loc, name) ); - RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, typeparams)); + RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); } else { ADDOP_LOAD_CONST(c, loc, name); diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index ed4a0ac2dd32de..b926caf4a57907 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -172,7 +172,6 @@ static const char* _Py_stdlib_module_names[] = { "itertools", "json", "keyword", -"lib2to3", "linecache", "locale", "logging", @@ -261,7 +260,6 @@ static const char* _Py_stdlib_module_names[] = { "syslog", "tabnanny", "tarfile", -"telnetlib", "tempfile", "termios", "textwrap", diff --git a/Python/symtable.c b/Python/symtable.c index 73cbb2b8e995b2..e2c00d17480dd1 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -231,7 +231,7 @@ static int symtable_enter_block(struct symtable *st, identifier name, static int symtable_exit_block(struct symtable *st); static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); -static int symtable_visit_typeparam(struct symtable *st, typeparam_ty s); +static int symtable_visit_type_param(struct symtable *st, type_param_ty s); static int symtable_visit_genexp(struct symtable *st, expr_ty s); static int symtable_visit_listcomp(struct symtable *st, expr_ty s); static int symtable_visit_setcomp(struct symtable *st, expr_ty s); @@ -528,7 +528,7 @@ error_at_directive(PySTEntryObject *ste, PyObject *name) static int analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, PyObject *bound, PyObject *local, PyObject *free, - PyObject *global, PyObject *typeparams, PySTEntryObject *class_entry) + PyObject *global, PyObject *type_params, PySTEntryObject *class_entry) { if (flags & DEF_GLOBAL) { if (flags & DEF_NONLOCAL) { @@ -557,7 +557,7 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, return error_at_directive(ste, name); } - if (PySet_Contains(typeparams, name)) { + if (PySet_Contains(type_params, name)) { PyErr_Format(PyExc_SyntaxError, "nonlocal binding not allowed for type parameter '%U'", name); @@ -574,11 +574,11 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, if (PySet_Discard(global, name) < 0) return 0; if (flags & DEF_TYPE_PARAM) { - if (PySet_Add(typeparams, name) < 0) + if (PySet_Add(type_params, name) < 0) return 0; } else { - if (PySet_Discard(typeparams, name) < 0) + if (PySet_Discard(type_params, name) < 0) return 0; } return 1; @@ -871,12 +871,12 @@ update_symbols(PyObject *symbols, PyObject *scopes, static int analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, - PyObject *global, PyObject *typeparams, + PyObject *global, PyObject *type_params, PySTEntryObject *class_entry, PyObject **child_free); static int analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, - PyObject *global, PyObject *typeparams, + PyObject *global, PyObject *type_params, PySTEntryObject *class_entry) { PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL; @@ -939,7 +939,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { long flags = PyLong_AS_LONG(v); if (!analyze_name(ste, scopes, name, flags, - bound, local, free, global, typeparams, class_entry)) + bound, local, free, global, type_params, class_entry)) goto error; } @@ -1002,7 +1002,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, !entry->ste_generator; if (!analyze_child_block(entry, newbound, newfree, newglobal, - typeparams, new_class_entry, &child_free)) + type_params, new_class_entry, &child_free)) { goto error; } @@ -1066,11 +1066,11 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, static int analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, - PyObject *global, PyObject *typeparams, + PyObject *global, PyObject *type_params, PySTEntryObject *class_entry, PyObject** child_free) { PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL; - PyObject *temp_typeparams = NULL; + PyObject *temp_type_params = NULL; /* Copy the bound/global/free sets. @@ -1088,30 +1088,30 @@ analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free, temp_global = PySet_New(global); if (!temp_global) goto error; - temp_typeparams = PySet_New(typeparams); - if (!temp_typeparams) + temp_type_params = PySet_New(type_params); + if (!temp_type_params) goto error; if (!analyze_block(entry, temp_bound, temp_free, temp_global, - temp_typeparams, class_entry)) + temp_type_params, class_entry)) goto error; *child_free = temp_free; Py_DECREF(temp_bound); Py_DECREF(temp_global); - Py_DECREF(temp_typeparams); + Py_DECREF(temp_type_params); return 1; error: Py_XDECREF(temp_bound); Py_XDECREF(temp_free); Py_XDECREF(temp_global); - Py_XDECREF(temp_typeparams); + Py_XDECREF(temp_type_params); return 0; } static int symtable_analyze(struct symtable *st) { - PyObject *free, *global, *typeparams; + PyObject *free, *global, *type_params; int r; free = PySet_New(NULL); @@ -1122,16 +1122,16 @@ symtable_analyze(struct symtable *st) Py_DECREF(free); return 0; } - typeparams = PySet_New(NULL); - if (!typeparams) { + type_params = PySet_New(NULL); + if (!type_params) { Py_DECREF(free); Py_DECREF(global); return 0; } - r = analyze_block(st->st_top, NULL, free, global, typeparams, NULL); + r = analyze_block(st->st_top, NULL, free, global, type_params, NULL); Py_DECREF(free); Py_DECREF(global); - Py_DECREF(typeparams); + Py_DECREF(type_params); return r; } @@ -1313,7 +1313,7 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag, } static int -symtable_enter_typeparam_block(struct symtable *st, identifier name, +symtable_enter_type_param_block(struct symtable *st, identifier name, void *ast, int has_defaults, int has_kwdefaults, enum _stmt_kind kind, int lineno, int col_offset, @@ -1472,10 +1472,10 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults); if (s->v.FunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list); - if (asdl_seq_LEN(s->v.FunctionDef.typeparams) > 0) { - if (!symtable_enter_typeparam_block( + if (asdl_seq_LEN(s->v.FunctionDef.type_params) > 0) { + if (!symtable_enter_type_param_block( st, s->v.FunctionDef.name, - (void *)s->v.FunctionDef.typeparams, + (void *)s->v.FunctionDef.type_params, s->v.FunctionDef.args->defaults != NULL, has_kwonlydefaults(s->v.FunctionDef.args->kwonlyargs, s->v.FunctionDef.args->kw_defaults), @@ -1483,7 +1483,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) LOCATION(s))) { VISIT_QUIT(st, 0); } - VISIT_SEQ(st, typeparam, s->v.FunctionDef.typeparams); + VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params); } if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args, s->v.FunctionDef.returns)) @@ -1496,7 +1496,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); - if (asdl_seq_LEN(s->v.FunctionDef.typeparams) > 0) { + if (asdl_seq_LEN(s->v.FunctionDef.type_params) > 0) { if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); } @@ -1507,14 +1507,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); if (s->v.ClassDef.decorator_list) VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list); - if (asdl_seq_LEN(s->v.ClassDef.typeparams) > 0) { - if (!symtable_enter_typeparam_block(st, s->v.ClassDef.name, - (void *)s->v.ClassDef.typeparams, + if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { + if (!symtable_enter_type_param_block(st, s->v.ClassDef.name, + (void *)s->v.ClassDef.type_params, false, false, s->kind, LOCATION(s))) { VISIT_QUIT(st, 0); } - VISIT_SEQ(st, typeparam, s->v.ClassDef.typeparams); + VISIT_SEQ(st, type_param, s->v.ClassDef.type_params); } VISIT_SEQ(st, expr, s->v.ClassDef.bases); VISIT_SEQ(st, keyword, s->v.ClassDef.keywords); @@ -1524,7 +1524,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); tmp = st->st_private; st->st_private = s->v.ClassDef.name; - if (asdl_seq_LEN(s->v.ClassDef.typeparams) > 0) { + if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_add_def(st, &_Py_ID(__type_params__), DEF_LOCAL, LOCATION(s))) { VISIT_QUIT(st, 0); @@ -1539,7 +1539,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) st->st_private = tmp; if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); - if (asdl_seq_LEN(s->v.ClassDef.typeparams) > 0) { + if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); } @@ -1550,16 +1550,16 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) assert(s->v.TypeAlias.name->kind == Name_kind); PyObject *name = s->v.TypeAlias.name->v.Name.id; int is_in_class = st->st_cur->ste_type == ClassBlock; - int is_generic = asdl_seq_LEN(s->v.TypeAlias.typeparams) > 0; + int is_generic = asdl_seq_LEN(s->v.TypeAlias.type_params) > 0; if (is_generic) { - if (!symtable_enter_typeparam_block( + if (!symtable_enter_type_param_block( st, name, - (void *)s->v.TypeAlias.typeparams, + (void *)s->v.TypeAlias.type_params, false, false, s->kind, LOCATION(s))) { VISIT_QUIT(st, 0); } - VISIT_SEQ(st, typeparam, s->v.TypeAlias.typeparams); + VISIT_SEQ(st, type_param, s->v.TypeAlias.type_params); } if (!symtable_enter_block(st, name, TypeAliasBlock, (void *)s, LOCATION(s))) @@ -1785,10 +1785,10 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) s->v.AsyncFunctionDef.args->kw_defaults); if (s->v.AsyncFunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list); - if (asdl_seq_LEN(s->v.AsyncFunctionDef.typeparams) > 0) { - if (!symtable_enter_typeparam_block( + if (asdl_seq_LEN(s->v.AsyncFunctionDef.type_params) > 0) { + if (!symtable_enter_type_param_block( st, s->v.AsyncFunctionDef.name, - (void *)s->v.AsyncFunctionDef.typeparams, + (void *)s->v.AsyncFunctionDef.type_params, s->v.AsyncFunctionDef.args->defaults != NULL, has_kwonlydefaults(s->v.AsyncFunctionDef.args->kwonlyargs, s->v.AsyncFunctionDef.args->kw_defaults), @@ -1796,7 +1796,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) LOCATION(s))) { VISIT_QUIT(st, 0); } - VISIT_SEQ(st, typeparam, s->v.AsyncFunctionDef.typeparams); + VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params); } if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, s->v.AsyncFunctionDef.returns)) @@ -1811,7 +1811,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); - if (asdl_seq_LEN(s->v.AsyncFunctionDef.typeparams) > 0) { + if (asdl_seq_LEN(s->v.AsyncFunctionDef.type_params) > 0) { if (!symtable_exit_block(st)) VISIT_QUIT(st, 0); } @@ -2110,7 +2110,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) } static int -symtable_visit_typeparam(struct symtable *st, typeparam_ty tp) +symtable_visit_type_param(struct symtable *st, type_param_ty tp) { if (++st->recursion_depth > st->recursion_limit) { PyErr_SetString(PyExc_RecursionError, diff --git a/README.rst b/README.rst index 06c4f0a1fa3917..4a4c8ab65dceff 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.12.0 alpha 7 +This is Python version 3.13.0 alpha 0 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg @@ -133,7 +133,7 @@ What's New ---------- We have a comprehensive overview of the changes in the `What's New in Python -3.12 `_ document. For a more +3.13 `_ document. For a more detailed change log, read `Misc/NEWS `_, but a full accounting of changes can only be gleaned from the `commit history @@ -146,7 +146,7 @@ entitled "Installing multiple versions". Documentation ------------- -`Documentation for Python 3.12 `_ is online, +`Documentation for Python 3.13 `_ is online, updated daily. It can also be downloaded in many formats for faster access. The documentation @@ -206,8 +206,8 @@ intend to install multiple versions using the same prefix you must decide which version (if any) is your "primary" version. Install that version using ``make install``. Install all other versions using ``make altinstall``. -For example, if you want to install Python 2.7, 3.6, and 3.12 with 3.12 being the -primary version, you would execute ``make install`` in your 3.12 build directory +For example, if you want to install Python 2.7, 3.6, and 3.13 with 3.13 being the +primary version, you would execute ``make install`` in your 3.13 build directory and ``make altinstall`` in the others. @@ -238,7 +238,7 @@ All current PEPs, as well as guidelines for submitting a new PEP, are listed at Release Schedule ---------------- -See :pep:`693` for Python 3.12 release details. +See :pep:`XXX` for Python 3.13 release details. Copyright and License Information diff --git a/Tools/README b/Tools/README index e51624f453c5bb..9c4b6d86e990ba 100644 --- a/Tools/README +++ b/Tools/README @@ -39,10 +39,8 @@ patchcheck Tools for checking and applying patches to the Python source cod peg_generator PEG-based parser generator (pegen) used for new parser. -scripts A number of useful single-file programs, e.g. tabnanny.py - by Tim Peters, which checks for inconsistent mixing of - tabs and spaces, and 2to3, which converts Python 2 code - to Python 3 code. +scripts A number of useful single-file programs, e.g. run_tests.py + which runs the Python test suite. ssl Scripts to generate ssl_data.h from OpenSSL sources, and run tests against multiple installations of OpenSSL and LibreSSL. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 863bd66bef754d..d182e5e7764e46 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1966,15 +1966,6 @@ def dump(self): extensions['py'] = PythonLanguage -# maps strings to callables. -# these callables must be of the form: -# def foo(*, ...) -# The callable may have any number of keyword-only parameters. -# The callable must return a CConverter object. -# The callable should not call builtins.print. -return_converters = {} - - def file_changed(filename: str, new_contents: str) -> bool: """Return true if file contents changed (meaning we must update it)""" try: @@ -1998,6 +1989,11 @@ def write_file(filename: str, new_contents: str): raise +ClassDict = dict[str, "Class"] +DestinationDict = dict[str, Destination] +ModuleDict = dict[str, "Module"] +ParserDict = dict[str, "DSLParser"] + clinic = None class Clinic: @@ -2044,23 +2040,30 @@ class Clinic: """ - def __init__(self, language, printer=None, *, verify=True, filename=None): + def __init__( + self, + language: CLanguage, + printer: BlockPrinter | None = None, + *, + verify: bool = True, + filename: str | None = None + ) -> None: # maps strings to Parser objects. # (instantiated from the "parsers" global.) - self.parsers = {} - self.language = language + self.parsers: ParserDict = {} + self.language: CLanguage = language if printer: fail("Custom printers are broken right now") self.printer = printer or BlockPrinter(language) self.verify = verify self.filename = filename - self.modules = {} - self.classes = {} - self.functions = [] + self.modules: ModuleDict = {} + self.classes: ClassDict = {} + self.functions: list[Function] = [] self.line_prefix = self.line_suffix = '' - self.destinations = {} + self.destinations: DestinationDict = {} self.add_destination("block", "buffer") self.add_destination("suppress", "suppress") self.add_destination("buffer", "buffer") @@ -2081,10 +2084,13 @@ def __init__(self, language, printer=None, *, verify=True, filename=None): 'impl_definition': d('block'), } - self.destination_buffers_stack = [] - self.ifndef_symbols = set() + DestBufferType = dict[str, Callable[..., Any]] + DestBufferList = list[DestBufferType] - self.presets = {} + self.destination_buffers_stack: DestBufferList = [] + self.ifndef_symbols: set[str] = set() + + self.presets: dict[str, dict[Any, Any]] = {} preset = None for line in self.presets_text.strip().split('\n'): line = line.strip() @@ -2112,18 +2118,27 @@ def __init__(self, language, printer=None, *, verify=True, filename=None): global clinic clinic = self - def add_destination(self, name, type, *args): + def add_destination( + self, + name: str, + type: str, + *args + ) -> None: if name in self.destinations: fail("Destination already exists: " + repr(name)) self.destinations[name] = Destination(name, type, self, *args) - def get_destination(self, name): + def get_destination(self, name: str) -> Destination: d = self.destinations.get(name) if not d: fail("Destination does not exist: " + repr(name)) return d - def get_destination_buffer(self, name, item=0): + def get_destination_buffer( + self, + name: str, + item: int = 0 + ): d = self.get_destination(name) return d.buffers[item] @@ -2249,6 +2264,7 @@ def parse_file( if not find_start_re.search(raw): return + assert isinstance(language, CLanguage) clinic = Clinic(language, verify=verify, filename=filename) src_out, clinic_out = clinic.parse(raw) @@ -2284,8 +2300,6 @@ def parse(self, block: Block) -> None: block.output = s.getvalue() -ModuleDict = dict[str, "Module"] - class Module: def __init__( self, @@ -2303,8 +2317,6 @@ def __repr__(self) -> str: return "" -ClassDict = dict[str, "Class"] - class Class: def __init__( self, @@ -2404,6 +2416,9 @@ def __repr__(self) -> str: INVALID, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW """.replace(",", "").strip().split() +ParamDict = dict[str, "Parameter"] +ReturnConverterType = Callable[..., "CReturnConverter"] + class Function: """ Mutable duck type for inspect.Function. @@ -2416,12 +2431,22 @@ class Function: (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) """ - def __init__(self, parameters=None, *, name, - module, cls=None, c_basename=None, - full_name=None, - return_converter, return_annotation=inspect.Signature.empty, - docstring=None, kind=CALLABLE, coexist=False, - docstring_only=False): + def __init__( + self, + parameters: ParamDict | None = None, + *, + name: str, + module: Module, + cls: Class | None = None, + c_basename: str | None = None, + full_name: str | None = None, + return_converter: ReturnConverterType, + return_annotation = inspect.Signature.empty, + docstring: str | None = None, + kind: str = CALLABLE, + coexist: bool = False, + docstring_only: bool = False + ) -> None: self.parameters = parameters or {} self.return_annotation = return_annotation self.name = name @@ -2455,7 +2480,7 @@ def render_parameters(self): return self.__render_parameters__ @property - def methoddef_flags(self): + def methoddef_flags(self) -> str | None: if self.kind in (METHOD_INIT, METHOD_NEW): return None flags = [] @@ -2469,10 +2494,10 @@ def methoddef_flags(self): flags.append('METH_COEXIST') return '|'.join(flags) - def __repr__(self): + def __repr__(self) -> str: return '' - def copy(self, **overrides): + def copy(self, **overrides) -> "Function": kwargs = { 'name': self.name, 'module': self.module, 'parameters': self.parameters, 'cls': self.cls, 'c_basename': self.c_basename, @@ -2495,9 +2520,18 @@ class Parameter: Mutable duck type of inspect.Parameter. """ - def __init__(self, name, kind, *, default=inspect.Parameter.empty, - function, converter, annotation=inspect.Parameter.empty, - docstring=None, group=0): + def __init__( + self, + name: str, + kind: str, + *, + default = inspect.Parameter.empty, + function: Function, + converter: "CConverter", + annotation = inspect.Parameter.empty, + docstring: str | None = None, + group: int = 0 + ) -> None: self.name = name self.kind = kind self.default = default @@ -2507,22 +2541,22 @@ def __init__(self, name, kind, *, default=inspect.Parameter.empty, self.docstring = docstring or '' self.group = group - def __repr__(self): + def __repr__(self) -> str: return '' - def is_keyword_only(self): + def is_keyword_only(self) -> bool: return self.kind == inspect.Parameter.KEYWORD_ONLY - def is_positional_only(self): + def is_positional_only(self) -> bool: return self.kind == inspect.Parameter.POSITIONAL_ONLY - def is_vararg(self): + def is_vararg(self) -> bool: return self.kind == inspect.Parameter.VAR_POSITIONAL - def is_optional(self): + def is_optional(self) -> bool: return not self.is_vararg() and (self.default is not unspecified) - def copy(self, **overrides): + def copy(self, **overrides) -> "Parameter": kwargs = { 'name': self.name, 'kind': self.kind, 'default':self.default, 'function': self.function, 'converter': self.converter, 'annotation': self.annotation, @@ -2535,7 +2569,7 @@ def copy(self, **overrides): kwargs['converter'] = converter return Parameter(**kwargs) - def get_displayname(self, i): + def get_displayname(self, i: int) -> str: if i == 0: return '"argument"' if not self.is_positional_only(): @@ -2546,13 +2580,13 @@ def get_displayname(self, i): class LandMine: # try to access any - def __init__(self, message): + def __init__(self, message: str) -> None: self.__message__ = message - def __repr__(self): + def __repr__(self) -> str: return '" - def __getattribute__(self, name): + def __getattribute__(self, name: str): if name in ('__repr__', '__message__'): return super().__getattribute__(name) # raise RuntimeError(repr(name)) @@ -3005,6 +3039,15 @@ def parser_name(self): # note however that they will never be called with keyword-only parameters. legacy_converters: ConverterDict = {} +# maps strings to callables. +# these callables must be of the form: +# def foo(*, ...) +# The callable may have any number of keyword-only parameters. +# The callable must return a CReturnConverter object. +# The callable should not call builtins.print. +ReturnConverterDict = dict[str, ReturnConverterType] +return_converters: ReturnConverterDict = {} + TypeSet = set[bltns.type[Any]] @@ -3966,8 +4009,10 @@ def set_template_dict(self, template_dict): template_dict['base_type_ptr'] = type_ptr - -def add_c_return_converter(f, name=None): +def add_c_return_converter( + f: ReturnConverterType, + name: str | None = None +) -> ReturnConverterType: if not name: name = f.__name__ if not name.endswith('_return_converter'): @@ -3978,9 +4023,15 @@ def add_c_return_converter(f, name=None): class CReturnConverterAutoRegister(type): - def __init__(cls, name, bases, classdict): + def __init__( + cls: ReturnConverterType, + name: str, + bases: tuple[type, ...], + classdict: dict[str, Any] + ) -> None: add_c_return_converter(cls) + class CReturnConverter(metaclass=CReturnConverterAutoRegister): # The C type to use for this variable. @@ -3992,7 +4043,12 @@ class CReturnConverter(metaclass=CReturnConverterAutoRegister): # Or the magic value "unspecified" if there is no default. default: object = None - def __init__(self, *, py_default=None, **kwargs): + def __init__( + self, + *, + py_default: str | None = None, + **kwargs + ) -> None: self.py_default = py_default try: self.return_converter_init(**kwargs) @@ -4000,11 +4056,10 @@ def __init__(self, *, py_default=None, **kwargs): s = ', '.join(name + '=' + repr(value) for name, value in kwargs.items()) sys.exit(self.__class__.__name__ + '(' + s + ')\n' + str(e)) - def return_converter_init(self): - pass + def return_converter_init(self) -> None: ... - def declare(self, data): - line = [] + def declare(self, data: CRenderData) -> None: + line: list[str] = [] add = line.append add(self.type) if not self.type.endswith('*'): @@ -4013,50 +4068,70 @@ def declare(self, data): data.declarations.append(''.join(line)) data.return_value = data.converter_retval - def err_occurred_if(self, expr, data): + def err_occurred_if( + self, + expr: str, + data: CRenderData + ) -> None: line = f'if (({expr}) && PyErr_Occurred()) {{\n goto exit;\n}}\n' data.return_conversion.append(line) - def err_occurred_if_null_pointer(self, variable, data): + def err_occurred_if_null_pointer( + self, + variable: str, + data: CRenderData + ) -> None: line = f'if ({variable} == NULL) {{\n goto exit;\n}}\n' data.return_conversion.append(line) - def render(self, function, data): - """ - function is a clinic.Function instance. - data is a CRenderData instance. - """ - pass + def render( + self, + function: Function, + data: CRenderData + ) -> None: ... + add_c_return_converter(CReturnConverter, 'object') + class bool_return_converter(CReturnConverter): type = 'int' - def render(self, function, data): + def render( + self, + function: Function, + data: CRenderData + ) -> None: self.declare(data) self.err_occurred_if(f"{data.converter_retval} == -1", data) data.return_conversion.append( f'return_value = PyBool_FromLong((long){data.converter_retval});\n' ) + class long_return_converter(CReturnConverter): type = 'long' conversion_fn = 'PyLong_FromLong' cast = '' unsigned_cast = '' - def render(self, function, data): + def render( + self, + function: Function, + data: CRenderData + ) -> None: self.declare(data) self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) data.return_conversion.append( f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' ) + class int_return_converter(long_return_converter): type = 'int' cast = '(long)' + class init_return_converter(long_return_converter): """ Special return converter for __init__ functions. @@ -4064,23 +4139,30 @@ class init_return_converter(long_return_converter): type = 'int' cast = '(long)' - def render(self, function, data): - pass + def render( + self, + function: Function, + data: CRenderData + ) -> None: ... + class unsigned_long_return_converter(long_return_converter): type = 'unsigned long' conversion_fn = 'PyLong_FromUnsignedLong' unsigned_cast = '(unsigned long)' + class unsigned_int_return_converter(unsigned_long_return_converter): type = 'unsigned int' cast = '(unsigned long)' unsigned_cast = '(unsigned int)' + class Py_ssize_t_return_converter(long_return_converter): type = 'Py_ssize_t' conversion_fn = 'PyLong_FromSsize_t' + class size_t_return_converter(long_return_converter): type = 'size_t' conversion_fn = 'PyLong_FromSize_t' @@ -4091,13 +4173,18 @@ class double_return_converter(CReturnConverter): type = 'double' cast = '' - def render(self, function, data): + def render( + self, + function: Function, + data: CRenderData + ) -> None: self.declare(data) self.err_occurred_if(f"{data.converter_retval} == -1.0", data) data.return_conversion.append( f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' ) + class float_return_converter(double_return_converter): type = 'float' cast = '(double)' @@ -4929,22 +5016,26 @@ def bad_node(self, node): key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name self.function.parameters[key] = p - def parse_converter(self, annotation): - if (isinstance(annotation, ast.Constant) and - type(annotation.value) is str): - return annotation.value, True, {} - - if isinstance(annotation, ast.Name): - return annotation.id, False, {} - - if not isinstance(annotation, ast.Call): - fail("Annotations must be either a name, a function call, or a string.") + KwargDict = dict[str | None, Any] - name = annotation.func.id - symbols = globals() - - kwargs = {node.arg: eval_ast_expr(node.value, symbols) for node in annotation.keywords} - return name, False, kwargs + @staticmethod + def parse_converter(annotation: ast.expr | None) -> tuple[str, bool, KwargDict]: + match annotation: + case ast.Constant(value=str() as value): + return value, True, {} + case ast.Name(name): + return name, False, {} + case ast.Call(func=ast.Name(name)): + symbols = globals() + kwargs = { + node.arg: eval_ast_expr(node.value, symbols) + for node in annotation.keywords + } + return name, False, kwargs + case _: + fail( + "Annotations must be either a name, a function call, or a string." + ) def parse_special_symbol(self, symbol): if symbol == '*': diff --git a/Tools/msi/bundle/packagegroups/postinstall.wxs b/Tools/msi/bundle/packagegroups/postinstall.wxs index 64f42dd30e8ba0..24c72e6357ab08 100644 --- a/Tools/msi/bundle/packagegroups/postinstall.wxs +++ b/Tools/msi/bundle/packagegroups/postinstall.wxs @@ -42,7 +42,7 @@ - + - - - - diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index 06aa0b8bbca277..cfb3ca9e76e24c 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -156,9 +156,6 @@ <_Uuid Include="PythonwExeComponentGuid"> pythonw.exe - <_Uuid Include="RemoveLib2to3PickleComponentGuid"> - lib2to3/pickles - <_Uuid Include="CommonPythonRegComponentGuid"> registry @@ -187,4 +184,4 @@ $(DefineConstants);@(_UuidValue,';'); - \ No newline at end of file + diff --git a/Tools/peg_generator/Makefile b/Tools/peg_generator/Makefile index 084da154919e3e..eef2d807e5c5ce 100644 --- a/Tools/peg_generator/Makefile +++ b/Tools/peg_generator/Makefile @@ -87,8 +87,7 @@ time_stdlib: $(CPYTHON) venv $(VENVPYTHON) scripts/test_parse_directory.py \ -d $(CPYTHON) \ $(TESTFLAGS) \ - --exclude "*/bad*" \ - --exclude "*/test/test_lib2to3/data/*" + --exclude "*/bad*" mypy: regen-metaparser $(MYPY) # For list of files, see mypy.ini diff --git a/Tools/peg_generator/scripts/benchmark.py b/Tools/peg_generator/scripts/benchmark.py index 053f8ef06d42c5..6354e1a17167af 100644 --- a/Tools/peg_generator/scripts/benchmark.py +++ b/Tools/peg_generator/scripts/benchmark.py @@ -78,7 +78,6 @@ def run_benchmark_stdlib(subcommand): verbose=False, excluded_files=[ "*/bad*", - "*/test/test_lib2to3/data/*", ], short=True, mode=modes[subcommand], diff --git a/Tools/scripts/2to3 b/Tools/scripts/2to3 deleted file mode 100755 index f27d18ecf67086..00000000000000 --- a/Tools/scripts/2to3 +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -import sys -from lib2to3.main import main - -sys.exit(main("lib2to3.fixes")) diff --git a/Tools/scripts/README b/Tools/scripts/README index 9dbb89a8dae633..a078bfbf662a37 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -1,7 +1,6 @@ This directory contains a collection of executable Python scripts that are useful while building, extending or managing Python. -2to3 Main script for running the 2to3 conversion tool checkpip.py Checks the version of the projects bundled in ensurepip are the latest available combinerefs.py A helper for analyzing PYTHONDUMPREFS output diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/wasm_assets.py index 1fc97fd5e70a10..fcd99405ee0d8e 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/wasm_assets.py @@ -41,8 +41,6 @@ # package management "ensurepip/", "venv/", - # build system - "lib2to3/", # deprecated "uu.py", "xdrlib.py", @@ -78,7 +76,6 @@ "poplib.py", "smtplib.py", "socketserver.py", - "telnetlib.py", # keep urllib.parse for pydoc "urllib/error.py", "urllib/request.py", diff --git a/configure b/configure index 7aad4fe89e3cbf..bcf2a5c7c7f73c 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for python 3.12. +# Generated by GNU Autoconf 2.69 for python 3.13. # # Report bugs to . # @@ -580,8 +580,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='python' PACKAGE_TARNAME='python' -PACKAGE_VERSION='3.12' -PACKAGE_STRING='python 3.12' +PACKAGE_VERSION='3.13' +PACKAGE_STRING='python 3.13' PACKAGE_BUGREPORT='https://github.com/python/cpython/issues/' PACKAGE_URL='' @@ -883,10 +883,11 @@ CFLAGS_NODIST BASECFLAGS CFLAGS_ALIASING OPT +BOLT_APPLY_FLAGS +BOLT_INSTRUMENT_FLAGS +BOLT_BINARIES MERGE_FDATA LLVM_BOLT -ac_ct_READELF -READELF PREBOLT_RULE LLVM_PROF_FOUND LLVM_PROFDATA @@ -1105,6 +1106,8 @@ CPPFLAGS CPP HOSTRUNNER PROFILE_TASK +BOLT_INSTRUMENT_FLAGS +BOLT_APPLY_FLAGS LIBUUID_CFLAGS LIBUUID_LIBS LIBFFI_CFLAGS @@ -1687,7 +1690,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures python 3.12 to adapt to many kinds of systems. +\`configure' configures python 3.13 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1753,7 +1756,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of python 3.12:";; + short | recursive ) echo "Configuration of python 3.13:";; esac cat <<\_ACEOF @@ -1797,9 +1800,9 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-build-python=python3.12 + --with-build-python=python3.13 path to build python binary for cross compiling - (default: _bootstrap_python or python3.12) + (default: _bootstrap_python or python3.13) --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) @@ -1916,6 +1919,10 @@ Some influential environment variables: HOSTRUNNER Program to run CPython for the host platform PROFILE_TASK Python args for PGO generation task + BOLT_INSTRUMENT_FLAGS + Arguments to llvm-bolt when instrumenting binaries + BOLT_APPLY_FLAGS + Arguments to llvm-bolt when creating a BOLT optimized binary LIBUUID_CFLAGS C compiler flags for LIBUUID, overriding pkg-config LIBUUID_LIBS @@ -2034,7 +2041,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -python configure 3.12 +python configure 3.13 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2743,7 +2750,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by python $as_me 3.12, which was +It was created by python $as_me 3.13, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3359,7 +3366,7 @@ fi -for ac_prog in python$PACKAGE_VERSION python3.12 python3.11 python3.10 python3 python +for ac_prog in python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 @@ -3429,7 +3436,7 @@ rm confdefs.h mv confdefs.h.new confdefs.h -VERSION=3.12 +VERSION=3.13 # Version number of Python's own shared library file. @@ -8106,112 +8113,6 @@ if test "$Py_BOLT" = 'true' ; then DEF_MAKE_ALL_RULE="bolt-opt" DEF_MAKE_RULE="build_all" - - if test -n "$ac_tool_prefix"; then - for ac_prog in readelf - do - # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. -set dummy $ac_tool_prefix$ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_READELF+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$READELF"; then - ac_cv_prog_READELF="$READELF" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_READELF="$ac_tool_prefix$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -READELF=$ac_cv_prog_READELF -if test -n "$READELF"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $READELF" >&5 -$as_echo "$READELF" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$READELF" && break - done -fi -if test -z "$READELF"; then - ac_ct_READELF=$READELF - for ac_prog in readelf -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_READELF+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_READELF"; then - ac_cv_prog_ac_ct_READELF="$ac_ct_READELF" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_READELF="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_READELF=$ac_cv_prog_ac_ct_READELF -if test -n "$ac_ct_READELF"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_READELF" >&5 -$as_echo "$ac_ct_READELF" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$ac_ct_READELF" && break -done - - if test "x$ac_ct_READELF" = x; then - READELF=""notfound"" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - READELF=$ac_ct_READELF - fi -fi - - if test "$READELF" == "notfound" - then - as_fn_error $? "readelf is required for a --enable-bolt build but could not be found." "$LINENO" 5 - fi - # -fno-reorder-blocks-and-partition is required for bolt to work. # Possibly GCC only. { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fno-reorder-blocks-and-partition" >&5 @@ -8474,6 +8375,36 @@ $as_echo "\"Found merge-fdata\"" >&6; } fi fi + +BOLT_BINARIES='$(BUILDPYTHON)' +if test "x$enable_shared" = xyes; then : + + BOLT_BINARIES="${BOLT_BINARIES} \$(INSTSONAME)" + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking BOLT_INSTRUMENT_FLAGS" >&5 +$as_echo_n "checking BOLT_INSTRUMENT_FLAGS... " >&6; } +if test -z "${BOLT_INSTRUMENT_FLAGS}" +then + BOLT_INSTRUMENT_FLAGS= +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $BOLT_INSTRUMENT_FLAGS" >&5 +$as_echo "$BOLT_INSTRUMENT_FLAGS" >&6; } + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking BOLT_APPLY_FLAGS" >&5 +$as_echo_n "checking BOLT_APPLY_FLAGS... " >&6; } +if test -z "${BOLT_APPLY_FLAGS}" +then + BOLT_APPLY_FLAGS=-update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot + + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $BOLT_APPLY_FLAGS" >&5 +$as_echo "$BOLT_APPLY_FLAGS" >&6; } + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code? @@ -29040,7 +28971,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by python $as_me 3.12, which was +This file was extended by python $as_me 3.13, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -29102,7 +29033,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -python config.status 3.12 +python config.status 3.13 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 115998e0753b26..d5cb822ae8cb6b 100644 --- a/configure.ac +++ b/configure.ac @@ -6,7 +6,7 @@ dnl Python's configure.ac file requires autoconf 2.69 and autoconf-archive. dnl # Set VERSION so we only need to edit in one place (i.e., here) -m4_define(PYTHON_VERSION, 3.12) +m4_define(PYTHON_VERSION, 3.13) AC_PREREQ([2.69]) @@ -202,7 +202,7 @@ AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], - [python$PACKAGE_VERSION python3.12 python3.11 python3.10 python3 python], + [python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python], [python3]) AC_SUBST(PYTHON_FOR_REGEN) @@ -2028,13 +2028,6 @@ if test "$Py_BOLT" = 'true' ; then DEF_MAKE_ALL_RULE="bolt-opt" DEF_MAKE_RULE="build_all" - AC_SUBST(READELF) - AC_CHECK_TOOLS(READELF, [readelf], "notfound") - if test "$READELF" == "notfound" - then - AC_MSG_ERROR([readelf is required for a --enable-bolt build but could not be found.]) - fi - # -fno-reorder-blocks-and-partition is required for bolt to work. # Possibly GCC only. AX_CHECK_COMPILE_FLAG([-fno-reorder-blocks-and-partition],[ @@ -2067,6 +2060,54 @@ if test "$Py_BOLT" = 'true' ; then fi fi +dnl Enable BOLT of libpython if built. +AC_SUBST(BOLT_BINARIES) +BOLT_BINARIES='$(BUILDPYTHON)' +AS_VAR_IF([enable_shared], [yes], [ + BOLT_BINARIES="${BOLT_BINARIES} \$(INSTSONAME)" +]) + +AC_ARG_VAR( + [BOLT_INSTRUMENT_FLAGS], + [Arguments to llvm-bolt when instrumenting binaries] +) +AC_MSG_CHECKING([BOLT_INSTRUMENT_FLAGS]) +if test -z "${BOLT_INSTRUMENT_FLAGS}" +then + BOLT_INSTRUMENT_FLAGS= +fi +AC_MSG_RESULT([$BOLT_INSTRUMENT_FLAGS]) + +AC_ARG_VAR( + [BOLT_APPLY_FLAGS], + [Arguments to llvm-bolt when creating a BOLT optimized binary] +) +AC_MSG_CHECKING([BOLT_APPLY_FLAGS]) +if test -z "${BOLT_APPLY_FLAGS}" +then + AS_VAR_SET( + [BOLT_APPLY_FLAGS], + [m4_join([ ], + [-update-debug-sections], + [-reorder-blocks=ext-tsp], + [-reorder-functions=hfsort+], + [-split-functions], + [-icf=1], + [-inline-all], + [-split-eh], + [-reorder-functions-use-hot-size], + [-peepholes=none], + [-jump-tables=aggressive], + [-inline-ap], + [-indirect-call-promotion=all], + [-dyno-stats], + [-use-gnu-stack], + [-frame-opt=hot] + )] + ) +fi +AC_MSG_RESULT([$BOLT_APPLY_FLAGS]) + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code?