From 2fe178488acbfe8eed32727680c884bacbcec7c2 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:06 +0300 Subject: [PATCH 01/13] code/source: expose deindent kwarg in signature Probably was done to avoid the shadowing issue, but work around it instead. --- src/_pytest/_code/source.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index 2ccbaf657c2..1c69498ea70 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -31,9 +31,8 @@ class Source: _compilecounter = 0 - def __init__(self, *parts, **kwargs) -> None: + def __init__(self, *parts, deindent: bool = True) -> None: self.lines = lines = [] # type: List[str] - de = kwargs.get("deindent", True) for part in parts: if not part: partlines = [] # type: List[str] @@ -44,9 +43,9 @@ def __init__(self, *parts, **kwargs) -> None: elif isinstance(part, str): partlines = part.split("\n") else: - partlines = getsource(part, deindent=de).lines - if de: - partlines = deindent(partlines) + partlines = getsource(part, deindent=deindent).lines + if deindent: + partlines = _deindent_function(partlines) lines.extend(partlines) def __eq__(self, other): @@ -307,20 +306,24 @@ def getrawcode(obj, trycall: bool = True): return obj -def getsource(obj, **kwargs) -> Source: +def getsource(obj, *, deindent: bool = True) -> Source: obj = getrawcode(obj) try: strsrc = inspect.getsource(obj) except IndentationError: strsrc = '"Buggy python version consider upgrading, cannot get source"' assert isinstance(strsrc, str) - return Source(strsrc, **kwargs) + return Source(strsrc, deindent=deindent) def deindent(lines: Sequence[str]) -> List[str]: return textwrap.dedent("\n".join(lines)).splitlines() +# Internal alias to avoid shadowing with `deindent` parameter. +_deindent_function = deindent + + def get_statement_startend2(lineno: int, node: ast.AST) -> Tuple[int, Optional[int]]: # flatten all statements and except handlers into one lineno-list # AST's line numbers start indexing at 1 From 410817477763bd27c3fc72f1d3227e84df1e2a35 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:07 +0300 Subject: [PATCH 02/13] code/source: remove Source(deindent: bool) parameter Not used, except in tests. --- src/_pytest/_code/source.py | 20 ++++++++------------ testing/code/test_source.py | 11 ++++++----- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index 1c69498ea70..0bc2e243e2d 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -25,13 +25,14 @@ class Source: - """ an immutable object holding a source code fragment, - possibly deindenting it. + """An immutable object holding a source code fragment. + + When using Source(...), the source lines are deindented. """ _compilecounter = 0 - def __init__(self, *parts, deindent: bool = True) -> None: + def __init__(self, *parts) -> None: self.lines = lines = [] # type: List[str] for part in parts: if not part: @@ -43,9 +44,8 @@ def __init__(self, *parts, deindent: bool = True) -> None: elif isinstance(part, str): partlines = part.split("\n") else: - partlines = getsource(part, deindent=deindent).lines - if deindent: - partlines = _deindent_function(partlines) + partlines = getsource(part).lines + partlines = deindent(partlines) lines.extend(partlines) def __eq__(self, other): @@ -306,24 +306,20 @@ def getrawcode(obj, trycall: bool = True): return obj -def getsource(obj, *, deindent: bool = True) -> Source: +def getsource(obj) -> Source: obj = getrawcode(obj) try: strsrc = inspect.getsource(obj) except IndentationError: strsrc = '"Buggy python version consider upgrading, cannot get source"' assert isinstance(strsrc, str) - return Source(strsrc, deindent=deindent) + return Source(strsrc) def deindent(lines: Sequence[str]) -> List[str]: return textwrap.dedent("\n".join(lines)).splitlines() -# Internal alias to avoid shadowing with `deindent` parameter. -_deindent_function = deindent - - def get_statement_startend2(lineno: int, node: ast.AST) -> Tuple[int, Optional[int]]: # flatten all statements and except handlers into one lineno-list # AST's line numbers start indexing at 1 diff --git a/testing/code/test_source.py b/testing/code/test_source.py index 35728c33443..014034dec90 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -4,6 +4,7 @@ import ast import inspect import sys +import textwrap from types import CodeType from typing import Any from typing import Dict @@ -64,8 +65,6 @@ def test_source_from_inner_function() -> None: def f(): pass - source = _pytest._code.Source(f, deindent=False) - assert str(source).startswith(" def f():") source = _pytest._code.Source(f) assert str(source).startswith("def f():") @@ -557,7 +556,7 @@ def __call__(self) -> None: def getstatement(lineno: int, source) -> Source: from _pytest._code.source import getstatementrange_ast - src = _pytest._code.Source(source, deindent=False) + src = _pytest._code.Source(source) ast, start, end = getstatementrange_ast(lineno, src) return src[start:end] @@ -633,7 +632,7 @@ def deco_mark(): assert False src = inspect.getsource(deco_mark) - assert str(Source(deco_mark, deindent=False)) == src + assert textwrap.indent(str(Source(deco_mark)), " ") + "\n" == src assert src.startswith(" @pytest.mark.foo") @pytest.fixture @@ -646,7 +645,9 @@ def deco_fixture(): # existing behavior here for explicitness, but perhaps we should revisit/change this # in the future assert str(Source(deco_fixture)).startswith("@functools.wraps(function)") - assert str(Source(get_real_func(deco_fixture), deindent=False)) == src + assert ( + textwrap.indent(str(Source(get_real_func(deco_fixture))), " ") + "\n" == src + ) def test_single_line_else() -> None: From c6083ab970e436b14987cbc7074fc3a894943fcc Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:08 +0300 Subject: [PATCH 03/13] code/source: remove old IndentationError workaround in getsource() This has been there since as far as the git history goes (2007), is not covered by any test, and says "Buggy python version consider upgrading". Hopefully everyone have upgraded... --- src/_pytest/_code/source.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index 0bc2e243e2d..eb4c4df78e6 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -308,10 +308,7 @@ def getrawcode(obj, trycall: bool = True): def getsource(obj) -> Source: obj = getrawcode(obj) - try: - strsrc = inspect.getsource(obj) - except IndentationError: - strsrc = '"Buggy python version consider upgrading, cannot get source"' + strsrc = inspect.getsource(obj) assert isinstance(strsrc, str) return Source(strsrc) From c83e16ab2e3218c8bf12293fd6d8d7e68d959ecf Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:09 +0300 Subject: [PATCH 04/13] code/source: remove unneeded assert inspect.getsource() definitely returns str. --- src/_pytest/_code/source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index eb4c4df78e6..1089b6ef0f0 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -309,7 +309,6 @@ def getrawcode(obj, trycall: bool = True): def getsource(obj) -> Source: obj = getrawcode(obj) strsrc = inspect.getsource(obj) - assert isinstance(strsrc, str) return Source(strsrc) From 2b99bfbc60a0f344e6ef2c73e2dc5a8b0d9d764f Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:09 +0300 Subject: [PATCH 05/13] code/source: remove support for passing multiple parts to Source It isn't used, so keep it simple. --- src/_pytest/_code/source.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index 1089b6ef0f0..6cc12320251 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -8,10 +8,10 @@ from bisect import bisect_right from types import CodeType from types import FrameType +from typing import Iterable from typing import Iterator from typing import List from typing import Optional -from typing import Sequence from typing import Tuple from typing import Union @@ -32,21 +32,17 @@ class Source: _compilecounter = 0 - def __init__(self, *parts) -> None: - self.lines = lines = [] # type: List[str] - for part in parts: - if not part: - partlines = [] # type: List[str] - elif isinstance(part, Source): - partlines = part.lines - elif isinstance(part, (tuple, list)): - partlines = [x.rstrip("\n") for x in part] - elif isinstance(part, str): - partlines = part.split("\n") - else: - partlines = getsource(part).lines - partlines = deindent(partlines) - lines.extend(partlines) + def __init__(self, obj: object = None) -> None: + if not obj: + self.lines = [] # type: List[str] + elif isinstance(obj, Source): + self.lines = obj.lines + elif isinstance(obj, (tuple, list)): + self.lines = deindent(x.rstrip("\n") for x in obj) + elif isinstance(obj, str): + self.lines = deindent(obj.split("\n")) + else: + self.lines = deindent(getsource(obj).lines) def __eq__(self, other): try: @@ -312,7 +308,7 @@ def getsource(obj) -> Source: return Source(strsrc) -def deindent(lines: Sequence[str]) -> List[str]: +def deindent(lines: Iterable[str]) -> List[str]: return textwrap.dedent("\n".join(lines)).splitlines() From a127a22d13ce637b10244f2bf60e0df0b0313f57 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:10 +0300 Subject: [PATCH 06/13] code/source: remove support for comparing Source with str Cross-type comparisons like this are a bad idea. This isn't used. --- src/_pytest/_code/source.py | 11 ++++------- testing/code/test_code.py | 3 ++- testing/code/test_source.py | 8 ++++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index 6cc12320251..f7dcdeff956 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -44,13 +44,10 @@ def __init__(self, obj: object = None) -> None: else: self.lines = deindent(getsource(obj).lines) - def __eq__(self, other): - try: - return self.lines == other.lines - except AttributeError: - if isinstance(other, str): - return str(self) == other - return False + def __eq__(self, other: object) -> bool: + if not isinstance(other, Source): + return NotImplemented + return self.lines == other.lines # Ignore type because of https://github.com/python/mypy/issues/4266. __hash__ = None # type: ignore diff --git a/testing/code/test_code.py b/testing/code/test_code.py index 5cbd899905b..25a3e9aeb59 100644 --- a/testing/code/test_code.py +++ b/testing/code/test_code.py @@ -6,6 +6,7 @@ from _pytest._code import Code from _pytest._code import ExceptionInfo from _pytest._code import Frame +from _pytest._code import Source from _pytest._code.code import ExceptionChainRepr from _pytest._code.code import ReprFuncArgs @@ -67,7 +68,7 @@ def func() -> FrameType: f = Frame(func()) with mock.patch.object(f.code.__class__, "fullsource", None): - assert f.statement == "" + assert f.statement == Source("") def test_code_from_func() -> None: diff --git a/testing/code/test_source.py b/testing/code/test_source.py index 014034dec90..8616b2f25b5 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -227,9 +227,9 @@ def test_getstatementrange_triple_quoted(self) -> None: ''')""" ) s = source.getstatement(0) - assert s == str(source) + assert s == source s = source.getstatement(1) - assert s == str(source) + assert s == source def test_getstatementrange_within_constructs(self) -> None: source = Source( @@ -445,7 +445,7 @@ def test_getsource_fallback() -> None: expected = """def x(): pass""" src = getsource(x) - assert src == expected + assert str(src) == expected def test_idem_compile_and_getsource() -> None: @@ -454,7 +454,7 @@ def test_idem_compile_and_getsource() -> None: expected = "def x(): pass" co = _pytest._code.compile(expected) src = getsource(co) - assert src == expected + assert str(src) == expected def test_compile_ast() -> None: From a7303b52db2ae319324ec4f09c470ff1f932cf7b Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:11 +0300 Subject: [PATCH 07/13] code/source: remove unused method Source.isparseable() --- src/_pytest/_code/source.py | 15 --------------- testing/code/test_source.py | 10 ---------- 2 files changed, 25 deletions(-) diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index f7dcdeff956..6c9aaa7e647 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -132,21 +132,6 @@ def deindent(self) -> "Source": newsource.lines[:] = deindent(self.lines) return newsource - def isparseable(self, deindent: bool = True) -> bool: - """ return True if source is parseable, heuristically - deindenting it by default. - """ - if deindent: - source = str(self.deindent()) - else: - source = str(self) - try: - ast.parse(source) - except (SyntaxError, ValueError, TypeError): - return False - else: - return True - def __str__(self) -> str: return "\n".join(self.lines) diff --git a/testing/code/test_source.py b/testing/code/test_source.py index 8616b2f25b5..0bf8c0b17b0 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -125,15 +125,6 @@ def test_syntaxerror_rerepresentation() -> None: assert ex.value.text.rstrip("\n") == "xyz xyz" -def test_isparseable() -> None: - assert Source("hello").isparseable() - assert Source("if 1:\n pass").isparseable() - assert Source(" \nif 1:\n pass").isparseable() - assert not Source("if 1:\n").isparseable() - assert not Source(" \nif 1:\npass").isparseable() - assert not Source(chr(0)).isparseable() - - class TestAccesses: def setup_class(self) -> None: self.source = Source( @@ -147,7 +138,6 @@ def g(x): def test_getrange(self) -> None: x = self.source[0:2] - assert x.isparseable() assert len(x.lines) == 2 assert str(x) == "def f(x):\n pass" From 4a27d7d9738e4281a28c8449363a9b89705267fb Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:11 +0300 Subject: [PATCH 08/13] code/source: remove unused method Source.putaround() --- src/_pytest/_code/source.py | 13 ------------- testing/code/test_source.py | 33 --------------------------------- 2 files changed, 46 deletions(-) diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index 6c9aaa7e647..019da5765ff 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -89,19 +89,6 @@ def strip(self) -> "Source": source.lines[:] = self.lines[start:end] return source - def putaround( - self, before: str = "", after: str = "", indent: str = " " * 4 - ) -> "Source": - """ return a copy of the source object with - 'before' and 'after' wrapped around it. - """ - beforesource = Source(before) - aftersource = Source(after) - newsource = Source() - lines = [(indent + line) for line in self.lines] - newsource.lines = beforesource.lines + lines + aftersource.lines - return newsource - def indent(self, indent: str = " " * 4) -> "Source": """ return a copy of the source object with all lines indented by the given indent-string. diff --git a/testing/code/test_source.py b/testing/code/test_source.py index 0bf8c0b17b0..97a00964b09 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -69,39 +69,6 @@ def f(): assert str(source).startswith("def f():") -def test_source_putaround_simple() -> None: - source = Source("raise ValueError") - source = source.putaround( - "try:", - """\ - except ValueError: - x = 42 - else: - x = 23""", - ) - assert ( - str(source) - == """\ -try: - raise ValueError -except ValueError: - x = 42 -else: - x = 23""" - ) - - -def test_source_putaround() -> None: - source = Source() - source = source.putaround( - """ - if 1: - x=1 - """ - ) - assert str(source).strip() == "if 1:\n x=1" - - def test_source_strips() -> None: source = Source("") assert source == Source() From 9640c9c9eb2f1ccdfea67dbfe541bdf3b0b8a128 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:12 +0300 Subject: [PATCH 09/13] skipping: use plain compile() instead of _pytest._code.compile() eval() is used for evaluating string conditions in skipif/xfail e.g. @pytest.mark.skipif("1 == 0") This is the only code that uses `_pytest._code.compile()`, so removing its last use enables us to remove it entirely. In this case it doesn't add much. Plain compile() gives a good enough error message. For regular exceptions, the message is the same. For SyntaxError exceptions, e.g. "1 ==", the previous code adds a little bit of useful context: ``` invalid syntax (skipping.py:108>, line 1) The above exception was the direct cause of the following exception: 1 == ^ (code was compiled probably from here: <0-codegen /pytest/src/_pytest/skipping.py:108>) (line 1) During handling of the above exception, another exception occurred: Error evaluating 'skipif' condition 1 == ^ SyntaxError: invalid syntax ``` The new code loses it: ``` unexpected EOF while parsing (, line 1) During handling of the above exception, another exception occurred: Error evaluating 'skipif' condition 1 == ^ SyntaxError: invalid syntax ``` Since the old message is a minor improvement to an unlikely error condition in a deprecated feature, I think it is not worth all the code that it requires. --- src/_pytest/skipping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_pytest/skipping.py b/src/_pytest/skipping.py index 7bd975e5a09..a72bdaabf27 100644 --- a/src/_pytest/skipping.py +++ b/src/_pytest/skipping.py @@ -9,7 +9,6 @@ import attr -import _pytest._code from _pytest.compat import TYPE_CHECKING from _pytest.config import Config from _pytest.config import hookimpl @@ -105,7 +104,8 @@ def evaluate_condition(item: Item, mark: Mark, condition: object) -> Tuple[bool, if hasattr(item, "obj"): globals_.update(item.obj.__globals__) # type: ignore[attr-defined] try: - condition_code = _pytest._code.compile(condition, mode="eval") + filename = "<{} condition>".format(mark.name) + condition_code = compile(condition, filename, "eval") result = eval(condition_code, globals_) except SyntaxError as exc: msglines = [ From ef39115001b70990ab7dee6fda61d9f1fec6a293 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:12 +0300 Subject: [PATCH 10/13] code/source: remove compiling functions A lot of complex code that isn't used anymore outside of tests after the previous commit. --- src/_pytest/_code/__init__.py | 2 - src/_pytest/_code/source.py | 130 ------------------------------- testing/code/test_excinfo.py | 74 +++++++++--------- testing/code/test_source.py | 140 +++++----------------------------- 4 files changed, 56 insertions(+), 290 deletions(-) diff --git a/src/_pytest/_code/__init__.py b/src/_pytest/_code/__init__.py index 76963c0eb59..136da31959e 100644 --- a/src/_pytest/_code/__init__.py +++ b/src/_pytest/_code/__init__.py @@ -7,7 +7,6 @@ from .code import getrawcode from .code import Traceback from .code import TracebackEntry -from .source import compile_ as compile from .source import Source __all__ = [ @@ -19,6 +18,5 @@ "getrawcode", "Traceback", "TracebackEntry", - "compile", "Source", ] diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index 019da5765ff..6cb602a9328 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -1,13 +1,9 @@ import ast import inspect -import linecache -import sys import textwrap import tokenize import warnings from bisect import bisect_right -from types import CodeType -from types import FrameType from typing import Iterable from typing import Iterator from typing import List @@ -15,13 +11,7 @@ from typing import Tuple from typing import Union -import py - from _pytest.compat import overload -from _pytest.compat import TYPE_CHECKING - -if TYPE_CHECKING: - from typing_extensions import Literal class Source: @@ -30,8 +20,6 @@ class Source: When using Source(...), the source lines are deindented. """ - _compilecounter = 0 - def __init__(self, obj: object = None) -> None: if not obj: self.lines = [] # type: List[str] @@ -122,124 +110,6 @@ def deindent(self) -> "Source": def __str__(self) -> str: return "\n".join(self.lines) - @overload - def compile( - self, - filename: Optional[str] = ..., - mode: str = ..., - flag: "Literal[0]" = ..., - dont_inherit: int = ..., - _genframe: Optional[FrameType] = ..., - ) -> CodeType: - raise NotImplementedError() - - @overload # noqa: F811 - def compile( # noqa: F811 - self, - filename: Optional[str] = ..., - mode: str = ..., - flag: int = ..., - dont_inherit: int = ..., - _genframe: Optional[FrameType] = ..., - ) -> Union[CodeType, ast.AST]: - raise NotImplementedError() - - def compile( # noqa: F811 - self, - filename: Optional[str] = None, - mode: str = "exec", - flag: int = 0, - dont_inherit: int = 0, - _genframe: Optional[FrameType] = None, - ) -> Union[CodeType, ast.AST]: - """ return compiled code object. if filename is None - invent an artificial filename which displays - the source/line position of the caller frame. - """ - if not filename or py.path.local(filename).check(file=0): - if _genframe is None: - _genframe = sys._getframe(1) # the caller - fn, lineno = _genframe.f_code.co_filename, _genframe.f_lineno - base = "<%d-codegen " % self._compilecounter - self.__class__._compilecounter += 1 - if not filename: - filename = base + "%s:%d>" % (fn, lineno) - else: - filename = base + "%r %s:%d>" % (filename, fn, lineno) - source = "\n".join(self.lines) + "\n" - try: - co = compile(source, filename, mode, flag) - except SyntaxError as ex: - # re-represent syntax errors from parsing python strings - msglines = self.lines[: ex.lineno] - if ex.offset: - msglines.append(" " * ex.offset + "^") - msglines.append("(code was compiled probably from here: %s)" % filename) - newex = SyntaxError("\n".join(msglines)) - newex.offset = ex.offset - newex.lineno = ex.lineno - newex.text = ex.text - raise newex from ex - else: - if flag & ast.PyCF_ONLY_AST: - assert isinstance(co, ast.AST) - return co - assert isinstance(co, CodeType) - lines = [(x + "\n") for x in self.lines] - # Type ignored because linecache.cache is private. - linecache.cache[filename] = (1, None, lines, filename) # type: ignore - return co - - -# -# public API shortcut functions -# - - -@overload -def compile_( - source: Union[str, bytes, ast.mod, ast.AST], - filename: Optional[str] = ..., - mode: str = ..., - flags: "Literal[0]" = ..., - dont_inherit: int = ..., -) -> CodeType: - raise NotImplementedError() - - -@overload # noqa: F811 -def compile_( # noqa: F811 - source: Union[str, bytes, ast.mod, ast.AST], - filename: Optional[str] = ..., - mode: str = ..., - flags: int = ..., - dont_inherit: int = ..., -) -> Union[CodeType, ast.AST]: - raise NotImplementedError() - - -def compile_( # noqa: F811 - source: Union[str, bytes, ast.mod, ast.AST], - filename: Optional[str] = None, - mode: str = "exec", - flags: int = 0, - dont_inherit: int = 0, -) -> Union[CodeType, ast.AST]: - """ compile the given source to a raw code object, - and maintain an internal cache which allows later - retrieval of the source code for the code object - and any recursively created code objects. - """ - if isinstance(source, ast.AST): - # XXX should Source support having AST? - assert filename is not None - co = compile(source, filename, mode, flags, dont_inherit) - assert isinstance(co, (CodeType, ast.AST)) - return co - _genframe = sys._getframe(1) # the caller - s = Source(source) - return s.compile(filename, mode, flags, _genframe=_genframe) - # # helper functions diff --git a/testing/code/test_excinfo.py b/testing/code/test_excinfo.py index 75c937612e9..52d5286b8ad 100644 --- a/testing/code/test_excinfo.py +++ b/testing/code/test_excinfo.py @@ -127,24 +127,28 @@ def test_traceback_entry_getsource(self): assert s.endswith("raise ValueError") def test_traceback_entry_getsource_in_construct(self): - source = _pytest._code.Source( - """\ - def xyz(): - try: - raise ValueError - except somenoname: - pass - xyz() - """ - ) + def xyz(): + try: + raise ValueError + except somenoname: # type: ignore[name-defined] # noqa: F821 + pass + try: - exec(source.compile()) + xyz() except NameError: - tb = _pytest._code.ExceptionInfo.from_current().traceback - print(tb[-1].getsource()) - s = str(tb[-1].getsource()) - assert s.startswith("def xyz():\n try:") - assert s.strip().endswith("except somenoname:") + excinfo = _pytest._code.ExceptionInfo.from_current() + else: + assert False, "did not raise NameError" + + tb = excinfo.traceback + source = tb[-1].getsource() + assert source is not None + assert source.deindent().lines == [ + "def xyz():", + " try:", + " raise ValueError", + " except somenoname: # type: ignore[name-defined] # noqa: F821", + ] def test_traceback_cut(self): co = _pytest._code.Code(f) @@ -445,16 +449,6 @@ def importasmod(source): return importasmod - def excinfo_from_exec(self, source): - source = _pytest._code.Source(source).strip() - try: - exec(source.compile()) - except KeyboardInterrupt: - raise - except BaseException: - return _pytest._code.ExceptionInfo.from_current() - assert 0, "did not raise" - def test_repr_source(self): pr = FormattedExcinfo() source = _pytest._code.Source( @@ -471,19 +465,29 @@ def f(x): def test_repr_source_excinfo(self) -> None: """ check if indentation is right """ - pr = FormattedExcinfo() - excinfo = self.excinfo_from_exec( - """ - def f(): - assert 0 - f() - """ - ) + try: + + def f(): + 1 / 0 + + f() + + except BaseException: + excinfo = _pytest._code.ExceptionInfo.from_current() + else: + assert 0, "did not raise" + pr = FormattedExcinfo() source = pr._getentrysource(excinfo.traceback[-1]) assert source is not None lines = pr.get_source(source, 1, excinfo) - assert lines == [" def f():", "> assert 0", "E AssertionError"] + for line in lines: + print(line) + assert lines == [ + " def f():", + "> 1 / 0", + "E ZeroDivisionError: division by zero", + ] def test_repr_source_not_existing(self): pr = FormattedExcinfo() diff --git a/testing/code/test_source.py b/testing/code/test_source.py index 97a00964b09..11f2f53cfd6 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -3,6 +3,7 @@ # or redundant on purpose and can't be disable on a line-by-line basis import ast import inspect +import linecache import sys import textwrap from types import CodeType @@ -33,14 +34,6 @@ def test_source_str_function() -> None: assert str(x) == "\n3" -def test_unicode() -> None: - x = Source("4") - assert str(x) == "4" - co = _pytest._code.compile('"å"', mode="eval") - val = eval(co) - assert isinstance(val, str) - - def test_source_from_function() -> None: source = _pytest._code.Source(test_source_str_function) assert str(source).startswith("def test_source_str_function() -> None:") @@ -83,15 +76,6 @@ def test_source_strip_multiline() -> None: assert source2.lines == [" hello"] -def test_syntaxerror_rerepresentation() -> None: - ex = pytest.raises(SyntaxError, _pytest._code.compile, "xyz xyz") - assert ex is not None - assert ex.value.lineno == 1 - assert ex.value.offset in {5, 7} # cpython: 7, pypy3.6 7.1.1: 5 - assert ex.value.text - assert ex.value.text.rstrip("\n") == "xyz xyz" - - class TestAccesses: def setup_class(self) -> None: self.source = Source( @@ -124,7 +108,7 @@ def test_iter(self) -> None: assert len(values) == 4 -class TestSourceParsingAndCompiling: +class TestSourceParsing: def setup_class(self) -> None: self.source = Source( """\ @@ -135,39 +119,6 @@ def f(x): """ ).strip() - def test_compile(self) -> None: - co = _pytest._code.compile("x=3") - d = {} # type: Dict[str, Any] - exec(co, d) - assert d["x"] == 3 - - def test_compile_and_getsource_simple(self) -> None: - co = _pytest._code.compile("x=3") - exec(co) - source = _pytest._code.Source(co) - assert str(source) == "x=3" - - def test_compile_and_getsource_through_same_function(self) -> None: - def gensource(source): - return _pytest._code.compile(source) - - co1 = gensource( - """ - def f(): - raise KeyError() - """ - ) - co2 = gensource( - """ - def f(): - raise ValueError() - """ - ) - source1 = inspect.getsource(co1) - assert "KeyError" in source1 - source2 = inspect.getsource(co2) - assert "ValueError" in source2 - def test_getstatement(self) -> None: # print str(self.source) ass = str(self.source[1:]) @@ -264,44 +215,6 @@ def test_getstatementrange_with_syntaxerror_issue7(self) -> None: source = Source(":") pytest.raises(SyntaxError, lambda: source.getstatementrange(0)) - def test_compile_to_ast(self) -> None: - source = Source("x = 4") - mod = source.compile(flag=ast.PyCF_ONLY_AST) - assert isinstance(mod, ast.Module) - compile(mod, "", "exec") - - def test_compile_and_getsource(self) -> None: - co = self.source.compile() - exec(co, globals()) - f(7) # type: ignore - excinfo = pytest.raises(AssertionError, f, 6) # type: ignore - assert excinfo is not None - frame = excinfo.traceback[-1].frame - assert isinstance(frame.code.fullsource, Source) - stmt = frame.code.fullsource.getstatement(frame.lineno) - assert str(stmt).strip().startswith("assert") - - @pytest.mark.parametrize("name", ["", None, "my"]) - def test_compilefuncs_and_path_sanity(self, name: Optional[str]) -> None: - def check(comp, name) -> None: - co = comp(self.source, name) - if not name: - expected = "codegen %s:%d>" % (mypath, mylineno + 2 + 2) # type: ignore - else: - expected = "codegen %r %s:%d>" % (name, mypath, mylineno + 2 + 2) # type: ignore - fn = co.co_filename - assert fn.endswith(expected) - - mycode = _pytest._code.Code(self.test_compilefuncs_and_path_sanity) - mylineno = mycode.firstlineno - mypath = mycode.path - - for comp in _pytest._code.compile, _pytest._code.Source.compile: - check(comp, name) - - def test_offsetless_synerr(self): - pytest.raises(SyntaxError, _pytest._code.compile, "lambda a,a: 0", mode="eval") - def test_getstartingblock_singleline() -> None: class A: @@ -331,18 +244,16 @@ def c() -> None: def test_getfuncsource_dynamic() -> None: - source = """ - def f(): - raise ValueError + def f(): + raise ValueError - def g(): pass - """ - co = _pytest._code.compile(source) - exec(co, globals()) - f_source = _pytest._code.Source(f) # type: ignore - g_source = _pytest._code.Source(g) # type: ignore + def g(): + pass + + f_source = _pytest._code.Source(f) + g_source = _pytest._code.Source(g) assert str(f_source).strip() == "def f():\n raise ValueError" - assert str(g_source).strip() == "def g(): pass" + assert str(g_source).strip() == "def g():\n pass" def test_getfuncsource_with_multine_string() -> None: @@ -405,23 +316,6 @@ def test_getsource_fallback() -> None: assert str(src) == expected -def test_idem_compile_and_getsource() -> None: - from _pytest._code.source import getsource - - expected = "def x(): pass" - co = _pytest._code.compile(expected) - src = getsource(co) - assert str(src) == expected - - -def test_compile_ast() -> None: - # We don't necessarily want to support this. - # This test was added just for coverage. - stmt = ast.parse("def x(): pass") - co = _pytest._code.compile(stmt, filename="foo.py") - assert isinstance(co, CodeType) - - def test_findsource_fallback() -> None: from _pytest._code.source import findsource @@ -431,15 +325,15 @@ def test_findsource_fallback() -> None: assert src[lineno] == " def x():" -def test_findsource() -> None: +def test_findsource(monkeypatch) -> None: from _pytest._code.source import findsource - co = _pytest._code.compile( - """if 1: - def x(): - pass -""" - ) + filename = "" + lines = ["if 1:\n", " def x():\n", " pass\n"] + co = compile("".join(lines), filename, "exec") + + # Type ignored because linecache.cache is private. + monkeypatch.setitem(linecache.cache, filename, (1, None, lines, filename)) # type: ignore[attr-defined] src, lineno = findsource(co) assert src is not None From f5c69f3eb2ea43d363e30c73b83209e13e953139 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 1 Jul 2020 20:20:13 +0300 Subject: [PATCH 11/13] code/source: inline getsource() The recursive way in which Source and getsource interact is a bit confusing, just inline it. --- src/_pytest/_code/source.py | 10 +++------- testing/code/test_source.py | 6 ++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index 6cb602a9328..65560be2a5e 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -30,7 +30,9 @@ def __init__(self, obj: object = None) -> None: elif isinstance(obj, str): self.lines = deindent(obj.split("\n")) else: - self.lines = deindent(getsource(obj).lines) + rawcode = getrawcode(obj) + src = inspect.getsource(rawcode) + self.lines = deindent(src.split("\n")) def __eq__(self, other: object) -> bool: if not isinstance(other, Source): @@ -141,12 +143,6 @@ def getrawcode(obj, trycall: bool = True): return obj -def getsource(obj) -> Source: - obj = getrawcode(obj) - strsrc = inspect.getsource(obj) - return Source(strsrc) - - def deindent(lines: Iterable[str]) -> List[str]: return textwrap.dedent("\n".join(lines)).splitlines() diff --git a/testing/code/test_source.py b/testing/code/test_source.py index 11f2f53cfd6..ea5b7a4a577 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -307,12 +307,10 @@ def x(): pass -def test_getsource_fallback() -> None: - from _pytest._code.source import getsource - +def test_source_fallback() -> None: + src = Source(x) expected = """def x(): pass""" - src = getsource(x) assert str(src) == expected From 40301effb8ffa3859efe9fca7f460e1af522d275 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sat, 4 Jul 2020 11:45:28 +0300 Subject: [PATCH 12/13] Add changelog entry for code/source changes --- changelog/7438.breaking.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/7438.breaking.rst diff --git a/changelog/7438.breaking.rst b/changelog/7438.breaking.rst new file mode 100644 index 00000000000..5d5d239fc99 --- /dev/null +++ b/changelog/7438.breaking.rst @@ -0,0 +1,11 @@ +Some changes were made to the internal ``_pytest._code.source``, listed here +for the benefit of plugin authors who may be using it: + +- The ``deindent`` argument to ``Source()`` has been removed, now it is always true. +- Support for zero or multiple arguments to ``Source()`` has been removed. +- Support for comparing ``Source`` with an ``str`` has been removed. +- The methods ``Source.isparseable()`` and ``Source.putaround()`` have been removed. +- The method ``Source.compile()`` and function ``_pytest._code.compile()`` have + been removed; use plain ``compile()`` instead. +- The function ``_pytest._code.source.getsource()`` has been removed; use + ``Source()`` directly instead. From 11efe057ea0a46341c14ab230145a7c8accbc30c Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sat, 4 Jul 2020 12:12:52 +0300 Subject: [PATCH 13/13] testing: skip some unreachable code in coverage --- testing/code/test_excinfo.py | 4 ++-- testing/code/test_source.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/testing/code/test_excinfo.py b/testing/code/test_excinfo.py index 52d5286b8ad..6ee848e54ae 100644 --- a/testing/code/test_excinfo.py +++ b/testing/code/test_excinfo.py @@ -131,7 +131,7 @@ def xyz(): try: raise ValueError except somenoname: # type: ignore[name-defined] # noqa: F821 - pass + pass # pragma: no cover try: xyz() @@ -475,7 +475,7 @@ def f(): except BaseException: excinfo = _pytest._code.ExceptionInfo.from_current() else: - assert 0, "did not raise" + assert False, "did not raise" pr = FormattedExcinfo() source = pr._getentrysource(excinfo.traceback[-1]) diff --git a/testing/code/test_source.py b/testing/code/test_source.py index ea5b7a4a577..4222eb172f2 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -56,7 +56,7 @@ def test_source_from_lines() -> None: def test_source_from_inner_function() -> None: def f(): - pass + raise NotImplementedError() source = _pytest._code.Source(f) assert str(source).startswith("def f():") @@ -245,15 +245,15 @@ def c() -> None: def test_getfuncsource_dynamic() -> None: def f(): - raise ValueError + raise NotImplementedError() def g(): - pass + pass # pragma: no cover f_source = _pytest._code.Source(f) g_source = _pytest._code.Source(g) - assert str(f_source).strip() == "def f():\n raise ValueError" - assert str(g_source).strip() == "def g():\n pass" + assert str(f_source).strip() == "def f():\n raise NotImplementedError()" + assert str(g_source).strip() == "def g():\n pass # pragma: no cover" def test_getfuncsource_with_multine_string() -> None: