From 4687cec37a2a28e477e0fcf7eb95d2701bea55eb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 27 Jul 2022 22:40:42 +0100 Subject: [PATCH] Add daemon command to get type of an expression (#13209) Implementation is straightforward (I also tried to make it future-proof, so it is easy to add new inspections). Few things to point out: * I added a more flexible traverser class, that supports shared visit logic and allows for `O(log n)` search by early return. * I noticed a bunch of places where line/column/full name were not set. Did a quick pass and fixed as many as I can. * I also implement basic support for expression attributes and go to definition (there are few tricky corner cases left, but they can be addressed later). The current _default_ logic is optimized for speed, while it may give stale results (e.g if file was edited, but has full tree loaded in memory). My thinking here is that users typically want to get inspections really fast (like on mouse hover over an expression), so the editor integrations that would use this can call with default flag values, and only use `--force-reload` or a full `dmypy recheck` if they know some file(s) were edited. Co-authored-by: Ivan Levkivskyi --- docs/source/mypy_daemon.rst | 131 ++++- mypy/checker.py | 10 +- mypy/checkmember.py | 4 + mypy/dmypy/client.py | 124 ++++- mypy/dmypy_server.py | 67 ++- mypy/errors.py | 23 +- mypy/fastparse.py | 37 +- mypy/inspections.py | 622 +++++++++++++++++++++++ mypy/messages.py | 49 +- mypy/nodes.py | 17 +- mypy/options.py | 5 + mypy/semanal.py | 1 + mypy/semanal_enum.py | 2 +- mypy/semanal_namedtuple.py | 2 +- mypy/semanal_typeddict.py | 2 +- mypy/test/testdaemon.py | 4 + mypy/test/testfinegrained.py | 69 ++- mypy/traverser.py | 462 ++++++++++++++++- mypy/treetransform.py | 18 +- mypyc/irbuild/classdef.py | 2 +- test-data/unit/check-ignore.test | 17 + test-data/unit/daemon.test | 217 +++++++- test-data/unit/fine-grained-inspect.test | 269 ++++++++++ 23 files changed, 2060 insertions(+), 94 deletions(-) create mode 100644 mypy/inspections.py create mode 100644 test-data/unit/fine-grained-inspect.test diff --git a/docs/source/mypy_daemon.rst b/docs/source/mypy_daemon.rst index 29b554db82a9..ce4f1582558c 100644 --- a/docs/source/mypy_daemon.rst +++ b/docs/source/mypy_daemon.rst @@ -152,6 +152,12 @@ Additional daemon flags Write performance profiling information to ``FILE``. This is only available for the ``check``, ``recheck``, and ``run`` commands. +.. option:: --export-types + + Store all expression types in memory for future use. This is useful to speed + up future calls to ``dmypy inspect`` (but uses more memory). Only valid for + ``check``, ``recheck``, and ``run`` command. + Static inference of annotations ******************************* @@ -243,8 +249,129 @@ command. Set the maximum number of types to try for a function (default: ``64``). -.. TODO: Add similar sections about go to definition, find usages, and - reveal type when added, and then move this to a separate file. +Statically inspect expressions +****************************** + +The daemon allows to get declared or inferred type of an expression (or other +information about an expression, such as known attributes or definition location) +using ``dmypy inspect LOCATION`` command. The location of the expression should be +specified in the format ``path/to/file.py:line:column[:end_line:end_column]``. +Both line and column are 1-based. Both start and end position are inclusive. +These rules match how mypy prints the error location in error messages. + +If a span is given (i.e. all 4 numbers), then only an exactly matching expression +is inspected. If only a position is given (i.e. 2 numbers, line and column), mypy +will inspect all *expressions*, that include this position, starting from the +innermost one. + +Consider this Python code snippet: + +.. code-block:: python + + def foo(x: int, longer_name: str) -> None: + x + longer_name + +Here to find the type of ``x`` one needs to call ``dmypy inspect src.py:2:5:2:5`` +or ``dmypy inspect src.py:2:5``. While for ``longer_name`` one needs to call +``dmypy inspect src.py:3:5:3:15`` or, for example, ``dmypy inspect src.py:3:10``. +Please note that this command is only valid after daemon had a successful type +check (without parse errors), so that types are populated, e.g. using +``dmypy check``. In case where multiple expressions match the provided location, +their types are returned separated by a newline. + +Important note: it is recommended to check files with :option:`--export-types` +since otherwise most inspections will not work without :option:`--force-reload`. + +.. option:: --show INSPECTION + + What kind of inspection to run for expression(s) found. Currently the supported + inspections are: + + * ``type`` (default): Show the best known type of a given expression. + * ``attrs``: Show which attributes are valid for an expression (e.g. for + auto-completion). Format is ``{"Base1": ["name_1", "name_2", ...]; "Base2": ...}``. + Names are sorted by method resolution order. If expression refers to a module, + then module attributes will be under key like ``""``. + * ``definition`` (experimental): Show the definition location for a name + expression or member expression. Format is ``path/to/file.py:line:column:Symbol``. + If multiple definitions are found (e.g. for a Union attribute), they are + separated by comma. + +.. option:: --verbose + + Increase verbosity of types string representation (can be repeated). + For example, this will print fully qualified names of instance types (like + ``"builtins.str"``), instead of just a short name (like ``"str"``). + +.. option:: --limit NUM + + If the location is given as ``line:column``, this will cause daemon to + return only at most ``NUM`` inspections of innermost expressions. + Value of 0 means no limit (this is the default). For example, if one calls + ``dmypy inspect src.py:4:10 --limit=1`` with this code + + .. code-block:: python + + def foo(x: int) -> str: .. + def bar(x: str) -> None: ... + baz: int + bar(foo(baz)) + + This will output just one type ``"int"`` (for ``baz`` name expression). + While without the limit option, it would output all three types: ``"int"``, + ``"str"``, and ``"None"``. + +.. option:: --include-span + + With this option on, the daemon will prepend each inspection result with + the full span of corresponding expression, formatted as ``1:2:1:4 -> "int"``. + This may be useful in case multiple expressions match a location. + +.. option:: --include-kind + + With this option on, the daemon will prepend each inspection result with + the kind of corresponding expression, formatted as ``NameExpr -> "int"``. + If both this option and :option:`--include-span` are on, the kind will + appear first, for example ``NameExpr:1:2:1:4 -> "int"``. + +.. option:: --include-object-attrs + + This will make the daemon include attributes of ``object`` (excluded by + default) in case of an ``atts`` inspection. + +.. option:: --union-attrs + + Include attributes valid for some of possible expression types (by default + an intersection is returned). This is useful for union types of type variables + with values. For example, with this code: + + .. code-block:: python + + from typing import Union + + class A: + x: int + z: int + class B: + y: int + z: int + var: Union[A, B] + var + + The command ``dmypy inspect --show attrs src.py:10:1`` will return + ``{"A": ["z"], "B": ["z"]}``, while with ``--union-attrs`` it will return + ``{"A": ["x", "z"], "B": ["y", "z"]}``. + +.. option:: --force-reload + + Force re-parsing and re-type-checking file before inspection. By default + this is done only when needed (for example file was not loaded from cache + or daemon was initially run without ``--export-types`` mypy option), + since reloading may be slow (up to few seconds for very large files). + +.. TODO: Add similar section about find usages when added, and then move + this to a separate file. .. _watchman: https://facebook.github.io/watchman/ diff --git a/mypy/checker.py b/mypy/checker.py index 384e6c47d331..e110a5aa4188 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1727,7 +1727,7 @@ def expand_typevars( # Make a copy of the function to check for each combination of # value restricted type variables. (Except when running mypyc, # where we need one canonical version of the function.) - if subst and not self.options.mypyc: + if subst and not (self.options.mypyc or self.options.inspections): result: List[Tuple[FuncItem, CallableType]] = [] for substitutions in itertools.product(*subst): mapping = dict(substitutions) @@ -3205,7 +3205,7 @@ def check_assignment_to_multiple_lvalues( lr_pairs = list(zip(left_lvs, left_rvs)) if star_lv: rv_list = ListExpr(star_rvs) - rv_list.set_line(rvalue.get_line()) + rv_list.set_line(rvalue) lr_pairs.append((star_lv.expr, rv_list)) lr_pairs.extend(zip(right_lvs, right_rvs)) @@ -3406,7 +3406,7 @@ def check_multi_assignment_from_tuple( list_expr = ListExpr( [self.temp_node(rv_type, context) for rv_type in star_rv_types] ) - list_expr.set_line(context.get_line()) + list_expr.set_line(context) self.check_assignment(star_lv.expr, list_expr, infer_lvalue_type) for lv, rv_type in zip(right_lvs, right_rv_types): self.check_assignment(lv, self.temp_node(rv_type, context), infer_lvalue_type) @@ -4065,7 +4065,7 @@ def visit_if_stmt(self, s: IfStmt) -> None: def visit_while_stmt(self, s: WhileStmt) -> None: """Type check a while statement.""" if_stmt = IfStmt([s.expr], [s.body], None) - if_stmt.set_line(s.get_line(), s.get_column()) + if_stmt.set_line(s) self.accept_loop(if_stmt, s.else_body, exit_condition=s.expr) def visit_operator_assignment_stmt(self, s: OperatorAssignmentStmt) -> None: @@ -6540,7 +6540,7 @@ def flatten_types(t: Type) -> List[Type]: def expand_func(defn: FuncItem, map: Dict[TypeVarId, Type]) -> FuncItem: visitor = TypeTransformVisitor(map) - ret = defn.accept(visitor) + ret = visitor.node(defn) assert isinstance(ret, FuncItem) return ret diff --git a/mypy/checkmember.py b/mypy/checkmember.py index e86cda4a7e62..3cd977ac8b0d 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -224,6 +224,10 @@ def _analyze_member_access( elif isinstance(typ, NoneType): return analyze_none_member_access(name, typ, mx) elif isinstance(typ, TypeVarLikeType): + if isinstance(typ, TypeVarType) and typ.values: + return _analyze_member_access( + name, make_simplified_union(typ.values), mx, override_info + ) return _analyze_member_access(name, typ.upper_bound, mx, override_info) elif isinstance(typ, DeletedType): mx.msg.deleted_as_rvalue(typ, mx.context) diff --git a/mypy/dmypy/client.py b/mypy/dmypy/client.py index 5b6a6a0a072f..d67f25d0d9b4 100644 --- a/mypy/dmypy/client.py +++ b/mypy/dmypy/client.py @@ -83,6 +83,11 @@ def __init__(self, prog: str) -> None: p.add_argument("--junit-xml", help="Write junit.xml to the given file") p.add_argument("--perf-stats-file", help="write performance information to the given file") p.add_argument("files", metavar="FILE", nargs="+", help="File (or directory) to check") +p.add_argument( + "--export-types", + action="store_true", + help="Store types of all expressions in a shared location (useful for inspections)", +) run_parser = p = subparsers.add_parser( "run", @@ -96,6 +101,11 @@ def __init__(self, prog: str) -> None: "--timeout", metavar="TIMEOUT", type=int, help="Server shutdown timeout (in seconds)" ) p.add_argument("--log-file", metavar="FILE", type=str, help="Direct daemon stdout/stderr to FILE") +p.add_argument( + "--export-types", + action="store_true", + help="Store types of all expressions in a shared location (useful for inspections)", +) p.add_argument( "flags", metavar="ARG", @@ -113,6 +123,11 @@ def __init__(self, prog: str) -> None: p.add_argument("-q", "--quiet", action="store_true", help=argparse.SUPPRESS) # Deprecated p.add_argument("--junit-xml", help="Write junit.xml to the given file") p.add_argument("--perf-stats-file", help="write performance information to the given file") +p.add_argument( + "--export-types", + action="store_true", + help="Store types of all expressions in a shared location (useful for inspections)", +) p.add_argument( "--update", metavar="FILE", @@ -164,6 +179,68 @@ def __init__(self, prog: str) -> None: help="Set the maximum number of types to try for a function (default 64)", ) +inspect_parser = p = subparsers.add_parser( + "inspect", help="Locate and statically inspect expression(s)" +) +p.add_argument( + "location", + metavar="LOCATION", + type=str, + help="Location specified as path/to/file.py:line:column[:end_line:end_column]." + " If position is given (i.e. only line and column), this will return all" + " enclosing expressions", +) +p.add_argument( + "--show", + metavar="INSPECTION", + type=str, + default="type", + choices=["type", "attrs", "definition"], + help="What kind of inspection to run", +) +p.add_argument( + "--verbose", + "-v", + action="count", + default=0, + help="Increase verbosity of the type string representation (can be repeated)", +) +p.add_argument( + "--limit", + metavar="NUM", + type=int, + default=0, + help="Return at most NUM innermost expressions (if position is given); 0 means no limit", +) +p.add_argument( + "--include-span", + action="store_true", + help="Prepend each inspection result with the span of corresponding expression" + ' (e.g. 1:2:3:4:"int")', +) +p.add_argument( + "--include-kind", + action="store_true", + help="Prepend each inspection result with the kind of corresponding expression" + ' (e.g. NameExpr:"int")', +) +p.add_argument( + "--include-object-attrs", + action="store_true", + help='Include attributes of "object" in "attrs" inspection', +) +p.add_argument( + "--union-attrs", + action="store_true", + help="Include attributes valid for some of possible expression types" + " (by default an intersection is returned)", +) +p.add_argument( + "--force-reload", + action="store_true", + help="Re-parse and re-type-check file before inspection (may be slow)", +) + hang_parser = p = subparsers.add_parser("hang", help="Hang for 100 seconds") daemon_parser = p = subparsers.add_parser("daemon", help="Run daemon in foreground") @@ -321,12 +398,24 @@ def do_run(args: argparse.Namespace) -> None: # Bad or missing status file or dead process; good to start. start_server(args, allow_sources=True) t0 = time.time() - response = request(args.status_file, "run", version=__version__, args=args.flags) + response = request( + args.status_file, + "run", + version=__version__, + args=args.flags, + export_types=args.export_types, + ) # If the daemon signals that a restart is necessary, do it if "restart" in response: print(f"Restarting: {response['restart']}") restart_server(args, allow_sources=True) - response = request(args.status_file, "run", version=__version__, args=args.flags) + response = request( + args.status_file, + "run", + version=__version__, + args=args.flags, + export_types=args.export_types, + ) t1 = time.time() response["roundtrip_time"] = t1 - t0 @@ -383,7 +472,7 @@ def do_kill(args: argparse.Namespace) -> None: def do_check(args: argparse.Namespace) -> None: """Ask the daemon to check a list of files.""" t0 = time.time() - response = request(args.status_file, "check", files=args.files) + response = request(args.status_file, "check", files=args.files, export_types=args.export_types) t1 = time.time() response["roundtrip_time"] = t1 - t0 check_output(response, args.verbose, args.junit_xml, args.perf_stats_file) @@ -406,9 +495,15 @@ def do_recheck(args: argparse.Namespace) -> None: """ t0 = time.time() if args.remove is not None or args.update is not None: - response = request(args.status_file, "recheck", remove=args.remove, update=args.update) + response = request( + args.status_file, + "recheck", + export_types=args.export_types, + remove=args.remove, + update=args.update, + ) else: - response = request(args.status_file, "recheck") + response = request(args.status_file, "recheck", export_types=args.export_types) t1 = time.time() response["roundtrip_time"] = t1 - t0 check_output(response, args.verbose, args.junit_xml, args.perf_stats_file) @@ -437,6 +532,25 @@ def do_suggest(args: argparse.Namespace) -> None: check_output(response, verbose=False, junit_xml=None, perf_stats_file=None) +@action(inspect_parser) +def do_inspect(args: argparse.Namespace) -> None: + """Ask daemon to print the type of an expression.""" + response = request( + args.status_file, + "inspect", + show=args.show, + location=args.location, + verbosity=args.verbose, + limit=args.limit, + include_span=args.include_span, + include_kind=args.include_kind, + include_object_attrs=args.include_object_attrs, + union_attrs=args.union_attrs, + force_reload=args.force_reload, + ) + check_output(response, verbose=False, junit_xml=None, perf_stats_file=None) + + def check_output( response: Dict[str, Any], verbose: bool, diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index a271b20792be..fa804ca32d44 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -26,6 +26,7 @@ from mypy.find_sources import InvalidSourceList, create_source_list from mypy.fscache import FileSystemCache from mypy.fswatcher import FileData, FileSystemWatcher +from mypy.inspections import InspectionEngine from mypy.ipc import IPCServer from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, compute_search_paths from mypy.options import Options @@ -297,7 +298,12 @@ def cmd_stop(self) -> Dict[str, object]: return {} def cmd_run( - self, version: str, args: Sequence[str], is_tty: bool, terminal_width: int + self, + version: str, + args: Sequence[str], + export_types: bool, + is_tty: bool, + terminal_width: int, ) -> Dict[str, object]: """Check a list of files, triggering a restart if needed.""" stderr = io.StringIO() @@ -332,22 +338,23 @@ def cmd_run( return {"out": "", "err": str(err), "status": 2} except SystemExit as e: return {"out": stdout.getvalue(), "err": stderr.getvalue(), "status": e.code} - return self.check(sources, is_tty, terminal_width) + return self.check(sources, export_types, is_tty, terminal_width) def cmd_check( - self, files: Sequence[str], is_tty: bool, terminal_width: int + self, files: Sequence[str], export_types: bool, is_tty: bool, terminal_width: int ) -> Dict[str, object]: """Check a list of files.""" try: sources = create_source_list(files, self.options, self.fscache) except InvalidSourceList as err: return {"out": "", "err": str(err), "status": 2} - return self.check(sources, is_tty, terminal_width) + return self.check(sources, export_types, is_tty, terminal_width) def cmd_recheck( self, is_tty: bool, terminal_width: int, + export_types: bool, remove: Optional[List[str]] = None, update: Optional[List[str]] = None, ) -> Dict[str, object]: @@ -374,6 +381,7 @@ def cmd_recheck( t1 = time.time() manager = self.fine_grained_manager.manager manager.log(f"fine-grained increment: cmd_recheck: {t1 - t0:.3f}s") + self.options.export_types = export_types if not self.following_imports(): messages = self.fine_grained_increment(sources, remove, update) else: @@ -385,13 +393,14 @@ def cmd_recheck( return res def check( - self, sources: List[BuildSource], is_tty: bool, terminal_width: int + self, sources: List[BuildSource], export_types: bool, is_tty: bool, terminal_width: int ) -> Dict[str, Any]: """Check using fine-grained incremental mode. If is_tty is True format the output nicely with colors and summary line (unless disabled in self.options). Also pass the terminal_width to formatter. """ + self.options.export_types = export_types if not self.fine_grained_manager: res = self.initialize_fine_grained(sources, is_tty, terminal_width) else: @@ -851,6 +860,54 @@ def _find_changed( return changed, removed + def cmd_inspect( + self, + show: str, + location: str, + verbosity: int = 0, + limit: int = 0, + include_span: bool = False, + include_kind: bool = False, + include_object_attrs: bool = False, + union_attrs: bool = False, + force_reload: bool = False, + ) -> Dict[str, object]: + """Locate and inspect expression(s).""" + if sys.version_info < (3, 8): + return {"error": 'Python 3.8 required for "inspect" command'} + if not self.fine_grained_manager: + return { + "error": 'Command "inspect" is only valid after a "check" command' + " (that produces no parse errors)" + } + engine = InspectionEngine( + self.fine_grained_manager, + verbosity=verbosity, + limit=limit, + include_span=include_span, + include_kind=include_kind, + include_object_attrs=include_object_attrs, + union_attrs=union_attrs, + force_reload=force_reload, + ) + old_inspections = self.options.inspections + self.options.inspections = True + try: + if show == "type": + result = engine.get_type(location) + elif show == "attrs": + result = engine.get_attrs(location) + elif show == "definition": + result = engine.get_definition(location) + else: + assert False, "Unknown inspection kind" + finally: + self.options.inspections = old_inspections + if "out" in result: + assert isinstance(result["out"], str) + result["out"] += "\n" + return result + def cmd_suggest(self, function: str, callsites: bool, **kwargs: Any) -> Dict[str, object]: """Suggest a signature for a function.""" if not self.fine_grained_manager: diff --git a/mypy/errors.py b/mypy/errors.py index 2656a6edf2c5..00421040a9c9 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -360,7 +360,7 @@ def report( file: Optional[str] = None, only_once: bool = False, allow_dups: bool = False, - origin_line: Optional[int] = None, + origin_span: Optional[Tuple[int, int]] = None, offset: int = 0, end_line: Optional[int] = None, end_column: Optional[int] = None, @@ -377,7 +377,8 @@ def report( file: if non-None, override current file as context only_once: if True, only report this exact message once per build allow_dups: if True, allow duplicate copies of this message (ignored if only_once) - origin_line: if non-None, override current context as origin + origin_span: if non-None, override current context as origin + (type: ignores have effect here) end_line: if non-None, override current context as end """ if self.scope: @@ -402,11 +403,11 @@ def report( if offset: message = " " * offset + message - if origin_line is None: - origin_line = line + if origin_span is None: + origin_span = (line, line) if end_line is None: - end_line = origin_line + end_line = line code = code or (codes.MISC if not blocker else None) @@ -426,7 +427,7 @@ def report( blocker, only_once, allow_dups, - origin=(self.file, origin_line, end_line), + origin=(self.file, *origin_span), target=self.current_target(), ) self.add_error_info(info) @@ -468,12 +469,6 @@ def add_error_info(self, info: ErrorInfo) -> None: return if not info.blocker: # Blockers cannot be ignored if file in self.ignored_lines: - # It's okay if end_line is *before* line. - # Function definitions do this, for example, because the correct - # error reporting line is at the *end* of the ignorable range - # (for compatibility reasons). If so, just flip 'em! - if end_line < line: - line, end_line = end_line, line # Check each line in this context for "type: ignore" comments. # line == end_line for most nodes, so we only loop once. for scope_line in range(line, end_line + 1): @@ -576,14 +571,14 @@ def is_ignored_error(self, line: int, info: ErrorInfo, ignores: Dict[int, List[s if info.blocker: # Blocking errors can never be ignored return False - if info.code and self.is_error_code_enabled(info.code) is False: + if info.code and not self.is_error_code_enabled(info.code): return True if line not in ignores: return False if not ignores[line]: # Empty list means that we ignore all errors return True - if info.code and self.is_error_code_enabled(info.code) is True: + if info.code and self.is_error_code_enabled(info.code): return info.code.code in ignores[line] return False diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 0bfb0b4265c2..f7e9c13a7274 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -480,8 +480,8 @@ def visit(self, node: Optional[AST]) -> Any: def set_line(self, node: N, n: AstNode) -> N: node.line = n.lineno node.column = n.col_offset - node.end_line = getattr(n, "end_lineno", None) if isinstance(n, ast3.expr) else None - node.end_column = getattr(n, "end_col_offset", None) if isinstance(n, ast3.expr) else None + node.end_line = getattr(n, "end_lineno", None) + node.end_column = getattr(n, "end_col_offset", None) return node @@ -593,6 +593,8 @@ def as_block(self, stmts: List[ast3.stmt], lineno: int) -> Optional[Block]: def as_required_block(self, stmts: List[ast3.stmt], lineno: int) -> Block: assert stmts # must be non-empty b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) + # TODO: in most call sites line is wrong (includes first line of enclosing statement) + # TODO: also we need to set the column, and the end position here. b.set_line(lineno) return b @@ -983,6 +985,10 @@ def do_func_def( _dummy_fallback, ) + # End position is always the same. + end_line = getattr(n, "end_lineno", None) + end_column = getattr(n, "end_col_offset", None) + func_def = FuncDef(n.name, args, self.as_required_block(n.body, lineno), func_type) if isinstance(func_def.type, CallableType): # semanal.py does some in-place modifications we want to avoid @@ -997,28 +1003,31 @@ def do_func_def( if sys.version_info < (3, 8): # Before 3.8, [typed_]ast the line number points to the first decorator. # In 3.8, it points to the 'def' line, where we want it. - lineno += len(n.decorator_list) - end_lineno: Optional[int] = None + deco_line = lineno + lineno += len(n.decorator_list) # this is only approximately true else: - # Set end_lineno to the old pre-3.8 lineno, in order to keep + # Set deco_line to the old pre-3.8 lineno, in order to keep # existing "# type: ignore" comments working: - end_lineno = n.decorator_list[0].lineno + len(n.decorator_list) + deco_line = n.decorator_list[0].lineno var = Var(func_def.name) var.is_ready = False var.set_line(lineno) func_def.is_decorated = True - func_def.set_line(lineno, n.col_offset, end_lineno) - func_def.body.set_line(lineno) # TODO: Why? + func_def.deco_line = deco_line + func_def.set_line(lineno, n.col_offset, end_line, end_column) + # Set the line again after we updated it (to make value same in Python 3.7/3.8) + # Note that TODOs in as_required_block() apply here as well. + func_def.body.set_line(lineno) deco = Decorator(func_def, self.translate_expr_list(n.decorator_list), var) first = n.decorator_list[0] - deco.set_line(first.lineno, first.col_offset) + deco.set_line(first.lineno, first.col_offset, end_line, end_column) retval: Union[FuncDef, Decorator] = deco else: # FuncDef overrides set_line -- can't use self.set_line - func_def.set_line(lineno, n.col_offset) + func_def.set_line(lineno, n.col_offset, end_line, end_column) retval = func_def self.class_and_function_stack.pop() return retval @@ -1121,15 +1130,17 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: keywords=keywords, ) cdef.decorators = self.translate_expr_list(n.decorator_list) - # Set end_lineno to the old mypy 0.700 lineno, in order to keep + # Set lines to match the old mypy 0.700 lines, in order to keep # existing "# type: ignore" comments working: if sys.version_info < (3, 8): cdef.line = n.lineno + len(n.decorator_list) - cdef.end_line = n.lineno + cdef.deco_line = n.lineno else: cdef.line = n.lineno - cdef.end_line = n.decorator_list[0].lineno if n.decorator_list else None + cdef.deco_line = n.decorator_list[0].lineno if n.decorator_list else None cdef.column = n.col_offset + cdef.end_line = getattr(n, "end_lineno", None) + cdef.end_column = getattr(n, "end_col_offset", None) self.class_and_function_stack.pop() return cdef diff --git a/mypy/inspections.py b/mypy/inspections.py new file mode 100644 index 000000000000..d2c090351b5b --- /dev/null +++ b/mypy/inspections.py @@ -0,0 +1,622 @@ +import os +from collections import defaultdict +from functools import cmp_to_key +from typing import Callable, Dict, List, Optional, Set, Tuple, Union + +from mypy.build import State +from mypy.find_sources import InvalidSourceList, SourceFinder +from mypy.messages import format_type +from mypy.modulefinder import PYTHON_EXTENSIONS +from mypy.nodes import ( + LDEF, + Decorator, + Expression, + FuncBase, + MemberExpr, + MypyFile, + Node, + OverloadedFuncDef, + RefExpr, + SymbolNode, + TypeInfo, + Var, +) +from mypy.server.update import FineGrainedBuildManager +from mypy.traverser import ExtendedTraverserVisitor +from mypy.typeops import tuple_fallback +from mypy.types import ( + FunctionLike, + Instance, + LiteralType, + ProperType, + TupleType, + TypedDictType, + TypeVarType, + UnionType, + get_proper_type, +) +from mypy.typevars import fill_typevars_with_any + + +def node_starts_after(o: Node, line: int, column: int) -> bool: + return o.line > line or o.line == line and o.column > column + + +def node_ends_before(o: Node, line: int, column: int) -> bool: + # Unfortunately, end positions for some statements are a mess, + # e.g. overloaded functions, so we return False when we don't know. + if o.end_line is not None and o.end_column is not None: + if o.end_line < line or o.end_line == line and o.end_column < column: + return True + return False + + +def expr_span(expr: Expression) -> str: + """Format expression span as in mypy error messages.""" + return f"{expr.line}:{expr.column + 1}:{expr.end_line}:{expr.end_column}" + + +def get_instance_fallback(typ: ProperType) -> List[Instance]: + """Returns the Instance fallback for this type if one exists or None.""" + if isinstance(typ, Instance): + return [typ] + elif isinstance(typ, TupleType): + return [tuple_fallback(typ)] + elif isinstance(typ, TypedDictType): + return [typ.fallback] + elif isinstance(typ, FunctionLike): + return [typ.fallback] + elif isinstance(typ, LiteralType): + return [typ.fallback] + elif isinstance(typ, TypeVarType): + if typ.values: + res = [] + for t in typ.values: + res.extend(get_instance_fallback(get_proper_type(t))) + return res + return get_instance_fallback(get_proper_type(typ.upper_bound)) + elif isinstance(typ, UnionType): + res = [] + for t in typ.items: + res.extend(get_instance_fallback(get_proper_type(t))) + return res + return [] + + +def find_node(name: str, info: TypeInfo) -> Optional[Union[Var, FuncBase]]: + """Find the node defining member 'name' in given TypeInfo.""" + # TODO: this code shares some logic with checkmember.py + method = info.get_method(name) + if method: + if isinstance(method, Decorator): + return method.var + if method.is_property: + assert isinstance(method, OverloadedFuncDef) + dec = method.items[0] + assert isinstance(dec, Decorator) + return dec.var + return method + else: + # don't have such method, maybe variable? + node = info.get(name) + v = node.node if node else None + if isinstance(v, Var): + return v + return None + + +def find_module_by_fullname(fullname: str, modules: Dict[str, State]) -> Optional[State]: + """Find module by a node fullname. + + This logic mimics the one we use in fixup, so should be good enough. + """ + head = fullname + # Special case: a module symbol is considered to be defined in itself, not in enclosing + # package, since this is what users want when clicking go to definition on a module. + if head in modules: + return modules[head] + while True: + if "." not in head: + return None + head, tail = head.rsplit(".", maxsplit=1) + mod = modules.get(head) + if mod is not None: + return mod + return None + + +class SearchVisitor(ExtendedTraverserVisitor): + """Visitor looking for an expression whose span matches given one exactly.""" + + def __init__(self, line: int, column: int, end_line: int, end_column: int) -> None: + self.line = line + self.column = column + self.end_line = end_line + self.end_column = end_column + self.result: Optional[Expression] = None + + def visit(self, o: Node) -> bool: + if node_starts_after(o, self.line, self.column): + return False + if node_ends_before(o, self.end_line, self.end_column): + return False + if ( + o.line == self.line + and o.end_line == self.end_line + and o.column == self.column + and o.end_column == self.end_column + ): + if isinstance(o, Expression): + self.result = o + return self.result is None + + +def find_by_location( + tree: MypyFile, line: int, column: int, end_line: int, end_column: int +) -> Optional[Expression]: + """Find an expression matching given span, or None if not found.""" + if end_line < line: + raise ValueError('"end_line" must not be before "line"') + if end_line == line and end_column <= column: + raise ValueError('"end_column" must be after "column"') + visitor = SearchVisitor(line, column, end_line, end_column) + tree.accept(visitor) + return visitor.result + + +class SearchAllVisitor(ExtendedTraverserVisitor): + """Visitor looking for all expressions whose spans enclose given position.""" + + def __init__(self, line: int, column: int) -> None: + self.line = line + self.column = column + self.result: List[Expression] = [] + + def visit(self, o: Node) -> bool: + if node_starts_after(o, self.line, self.column): + return False + if node_ends_before(o, self.line, self.column): + return False + if isinstance(o, Expression): + self.result.append(o) + return True + + +def find_all_by_location(tree: MypyFile, line: int, column: int) -> List[Expression]: + """Find all expressions enclosing given position starting from innermost.""" + visitor = SearchAllVisitor(line, column) + tree.accept(visitor) + return list(reversed(visitor.result)) + + +class InspectionEngine: + """Engine for locating and statically inspecting expressions.""" + + def __init__( + self, + fg_manager: FineGrainedBuildManager, + *, + verbosity: int = 0, + limit: int = 0, + include_span: bool = False, + include_kind: bool = False, + include_object_attrs: bool = False, + union_attrs: bool = False, + force_reload: bool = False, + ) -> None: + self.fg_manager = fg_manager + self.finder = SourceFinder( + self.fg_manager.manager.fscache, self.fg_manager.manager.options + ) + self.verbosity = verbosity + self.limit = limit + self.include_span = include_span + self.include_kind = include_kind + self.include_object_attrs = include_object_attrs + self.union_attrs = union_attrs + self.force_reload = force_reload + # Module for which inspection was requested. + self.module: Optional[State] = None + + def parse_location(self, location: str) -> Tuple[str, List[int]]: + if location.count(":") not in [2, 4]: + raise ValueError("Format should be file:line:column[:end_line:end_column]") + parts = location.split(":") + module, *rest = parts + return module, [int(p) for p in rest] + + def reload_module(self, state: State) -> None: + """Reload given module while temporary exporting types.""" + old = self.fg_manager.manager.options.export_types + self.fg_manager.manager.options.export_types = True + try: + self.fg_manager.flush_cache() + assert state.path is not None + self.fg_manager.update([(state.id, state.path)], []) + finally: + self.fg_manager.manager.options.export_types = old + + def expr_type(self, expression: Expression) -> Tuple[str, bool]: + """Format type for an expression using current options. + + If type is known, second item returned is True. If type is not known, an error + message is returned instead, and second item returned is False. + """ + expr_type = self.fg_manager.manager.all_types.get(expression) + if expr_type is None: + return self.missing_type(expression), False + + type_str = format_type(expr_type, verbosity=self.verbosity) + return self.add_prefixes(type_str, expression), True + + def object_type(self) -> Instance: + builtins = self.fg_manager.graph["builtins"].tree + assert builtins is not None + object_node = builtins.names["object"].node + assert isinstance(object_node, TypeInfo) + return Instance(object_node, []) + + def collect_attrs(self, instances: List[Instance]) -> Dict[TypeInfo, List[str]]: + """Collect attributes from all union/typevar variants.""" + + def item_attrs(attr_dict: Dict[TypeInfo, List[str]]) -> Set[str]: + attrs = set() + for base in attr_dict: + attrs |= set(attr_dict[base]) + return attrs + + def cmp_types(x: TypeInfo, y: TypeInfo) -> int: + if x in y.mro: + return 1 + if y in x.mro: + return -1 + return 0 + + # First gather all attributes for every union variant. + assert instances + all_attrs = [] + for instance in instances: + attrs = {} + mro = instance.type.mro + if not self.include_object_attrs: + mro = mro[:-1] + for base in mro: + attrs[base] = sorted(base.names) + all_attrs.append(attrs) + + # Find attributes valid for all variants in a union or type variable. + intersection = item_attrs(all_attrs[0]) + for item in all_attrs[1:]: + intersection &= item_attrs(item) + + # Combine attributes from all variants into a single dict while + # also removing invalid attributes (unless using --union-attrs). + combined_attrs = defaultdict(list) + for item in all_attrs: + for base in item: + if base in combined_attrs: + continue + for name in item[base]: + if self.union_attrs or name in intersection: + combined_attrs[base].append(name) + + # Sort bases by MRO, unrelated will appear in the order they appeared as union variants. + sorted_bases = sorted(combined_attrs.keys(), key=cmp_to_key(cmp_types)) + result = {} + for base in sorted_bases: + if not combined_attrs[base]: + # Skip bases where everytihng was filtered out. + continue + result[base] = combined_attrs[base] + return result + + def _fill_from_dict( + self, attrs_strs: List[str], attrs_dict: Dict[TypeInfo, List[str]] + ) -> None: + for base in attrs_dict: + cls_name = base.name if self.verbosity < 1 else base.fullname + attrs = [f'"{attr}"' for attr in attrs_dict[base]] + attrs_strs.append(f'"{cls_name}": [{", ".join(attrs)}]') + + def expr_attrs(self, expression: Expression) -> Tuple[str, bool]: + """Format attributes that are valid for a given expression. + + If expression type is not an Instance, try using fallback. Attributes are + returned as a JSON (ordered by MRO) that maps base class name to list of + attributes. Attributes may appear in multiple bases if overridden (we simply + follow usual mypy logic for creating new Vars etc). + """ + expr_type = self.fg_manager.manager.all_types.get(expression) + if expr_type is None: + return self.missing_type(expression), False + + expr_type = get_proper_type(expr_type) + instances = get_instance_fallback(expr_type) + if not instances: + # Everything is an object in Python. + instances = [self.object_type()] + + attrs_dict = self.collect_attrs(instances) + + # Special case: modules have names apart from those from ModuleType. + if isinstance(expression, RefExpr) and isinstance(expression.node, MypyFile): + node = expression.node + names = sorted(node.names) + if "__builtins__" in names: + # This is just to make tests stable. No one will really need ths name. + names.remove("__builtins__") + mod_dict = {f'"<{node.fullname}>"': [f'"{name}"' for name in names]} + else: + mod_dict = {} + + # Special case: for class callables, prepend with the class attributes. + # TODO: also handle cases when such callable appears in a union. + if isinstance(expr_type, FunctionLike) and expr_type.is_type_obj(): + template = fill_typevars_with_any(expr_type.type_object()) + class_dict = self.collect_attrs(get_instance_fallback(template)) + else: + class_dict = {} + + # We don't use JSON dump to be sure keys order is always preserved. + base_attrs = [] + if mod_dict: + for mod in mod_dict: + base_attrs.append(f'{mod}: [{", ".join(mod_dict[mod])}]') + self._fill_from_dict(base_attrs, class_dict) + self._fill_from_dict(base_attrs, attrs_dict) + return self.add_prefixes(f'{{{", ".join(base_attrs)}}}', expression), True + + def format_node(self, module: State, node: Union[FuncBase, SymbolNode]) -> str: + return f"{module.path}:{node.line}:{node.column + 1}:{node.name}" + + def collect_nodes(self, expression: RefExpr) -> List[Union[FuncBase, SymbolNode]]: + """Collect nodes that can be referred to by an expression. + + Note: it can be more than one for example in case of a union attribute. + """ + node: Optional[Union[FuncBase, SymbolNode]] = expression.node + nodes: List[Union[FuncBase, SymbolNode]] + if node is None: + # Tricky case: instance attribute + if isinstance(expression, MemberExpr) and expression.kind is None: + base_type = self.fg_manager.manager.all_types.get(expression.expr) + if base_type is None: + return [] + + # Now we use the base type to figure out where the attribute is defined. + base_type = get_proper_type(base_type) + instances = get_instance_fallback(base_type) + nodes = [] + for instance in instances: + node = find_node(expression.name, instance.type) + if node: + nodes.append(node) + if not nodes: + # Try checking class namespace if attribute is on a class object. + if isinstance(base_type, FunctionLike) and base_type.is_type_obj(): + instances = get_instance_fallback( + fill_typevars_with_any(base_type.type_object()) + ) + for instance in instances: + node = find_node(expression.name, instance.type) + if node: + nodes.append(node) + else: + # Still no luck, give up. + return [] + else: + return [] + else: + # Easy case: a module-level definition + nodes = [node] + return nodes + + def modules_for_nodes( + self, nodes: List[Union[FuncBase, SymbolNode]], expression: RefExpr + ) -> Tuple[Dict[Union[FuncBase, SymbolNode], State], bool]: + """Gather modules where given nodes where defined. + + Also check if they need to be refreshed (cached nodes may have + lines/columns missing). + """ + modules = {} + reload_needed = False + for node in nodes: + module = find_module_by_fullname(node.fullname, self.fg_manager.graph) + if not module: + if expression.kind == LDEF and self.module: + module = self.module + else: + continue + modules[node] = module + if not module.tree or module.tree.is_cache_skeleton or self.force_reload: + reload_needed |= not module.tree or module.tree.is_cache_skeleton + self.reload_module(module) + return modules, reload_needed + + def expression_def(self, expression: Expression) -> Tuple[str, bool]: + """Find and format definition location for an expression. + + If it is not a RefExpr, it is effectively skipped by returning an + empty result. + """ + if not isinstance(expression, RefExpr): + # If there are no suitable matches at all, we return error later. + return "", True + + nodes = self.collect_nodes(expression) + + if not nodes: + return self.missing_node(expression), False + + modules, reload_needed = self.modules_for_nodes(nodes, expression) + if reload_needed: + # TODO: line/column are not stored in cache for vast majority of symbol nodes. + # Adding them will make thing faster, but will have visible memory impact. + nodes = self.collect_nodes(expression) + modules, reload_needed = self.modules_for_nodes(nodes, expression) + assert not reload_needed + + result = [] + for node in modules: + result.append(self.format_node(modules[node], node)) + + if not result: + return self.missing_node(expression), False + + return self.add_prefixes(", ".join(result), expression), True + + def missing_type(self, expression: Expression) -> str: + alt_suggestion = "" + if not self.force_reload: + alt_suggestion = " or try --force-reload" + return ( + f'No known type available for "{type(expression).__name__}"' + f" (maybe unreachable{alt_suggestion})" + ) + + def missing_node(self, expression: Expression) -> str: + return ( + f'Cannot find definition for "{type(expression).__name__}"' + f" at {expr_span(expression)}" + ) + + def add_prefixes(self, result: str, expression: Expression) -> str: + prefixes = [] + if self.include_kind: + prefixes.append(f"{type(expression).__name__}") + if self.include_span: + prefixes.append(expr_span(expression)) + if prefixes: + prefix = ":".join(prefixes) + " -> " + else: + prefix = "" + return prefix + result + + def run_inspection_by_exact_location( + self, + tree: MypyFile, + line: int, + column: int, + end_line: int, + end_column: int, + method: Callable[[Expression], Tuple[str, bool]], + ) -> Dict[str, object]: + """Get type of an expression matching a span. + + Type or error is returned as a standard daemon response dict. + """ + try: + expression = find_by_location(tree, line, column - 1, end_line, end_column) + except ValueError as err: + return {"error": str(err)} + + if expression is None: + span = f"{line}:{column}:{end_line}:{end_column}" + return {"out": f"Can't find expression at span {span}", "err": "", "status": 1} + + inspection_str, success = method(expression) + return {"out": inspection_str, "err": "", "status": 0 if success else 1} + + def run_inspection_by_position( + self, + tree: MypyFile, + line: int, + column: int, + method: Callable[[Expression], Tuple[str, bool]], + ) -> Dict[str, object]: + """Get types of all expressions enclosing a position. + + Types and/or errors are returned as a standard daemon response dict. + """ + expressions = find_all_by_location(tree, line, column - 1) + if not expressions: + position = f"{line}:{column}" + return { + "out": f"Can't find any expressions at position {position}", + "err": "", + "status": 1, + } + + inspection_strs = [] + status = 0 + for expression in expressions: + inspection_str, success = method(expression) + if not success: + status = 1 + if inspection_str: + inspection_strs.append(inspection_str) + if self.limit: + inspection_strs = inspection_strs[: self.limit] + return {"out": "\n".join(inspection_strs), "err": "", "status": status} + + def find_module(self, file: str) -> Tuple[Optional[State], Dict[str, object]]: + """Find module by path, or return a suitable error message. + + Note we don't use exceptions to simplify handling 1 vs 2 statuses. + """ + if not any(file.endswith(ext) for ext in PYTHON_EXTENSIONS): + return None, {"error": "Source file is not a Python file"} + + try: + module, _ = self.finder.crawl_up(os.path.normpath(file)) + except InvalidSourceList: + return None, {"error": "Invalid source file name: " + file} + + state = self.fg_manager.graph.get(module) + self.module = state + return ( + state, + {"out": f"Unknown module: {module}", "err": "", "status": 1} if state is None else {}, + ) + + def run_inspection( + self, location: str, method: Callable[[Expression], Tuple[str, bool]] + ) -> Dict[str, object]: + """Top-level logic to inspect expression(s) at a location. + + This can be re-used by various simple inspections. + """ + try: + file, pos = self.parse_location(location) + except ValueError as err: + return {"error": str(err)} + + state, err_dict = self.find_module(file) + if state is None: + assert err_dict + return err_dict + + # Force reloading to load from cache, account for any edits, etc. + if not state.tree or state.tree.is_cache_skeleton or self.force_reload: + self.reload_module(state) + assert state.tree is not None + + if len(pos) == 4: + # Full span, return an exact match only. + line, column, end_line, end_column = pos + return self.run_inspection_by_exact_location( + state.tree, line, column, end_line, end_column, method + ) + assert len(pos) == 2 + # Inexact location, return all expressions. + line, column = pos + return self.run_inspection_by_position(state.tree, line, column, method) + + def get_type(self, location: str) -> Dict[str, object]: + """Get types of expression(s) at a location.""" + return self.run_inspection(location, self.expr_type) + + def get_attrs(self, location: str) -> Dict[str, object]: + """Get attributes of expression(s) at a location.""" + return self.run_inspection(location, self.expr_attrs) + + def get_definition(self, location: str) -> Dict[str, object]: + """Get symbol definitions of expression(s) at a location.""" + result = self.run_inspection(location, self.expression_def) + if "out" in result and not result["out"]: + # None of the expressions found turns out to be a RefExpr. + _, location = location.split(":", maxsplit=1) + result["out"] = f"No name or member expressions at {location}" + result["status"] = 1 + return result diff --git a/mypy/messages.py b/mypy/messages.py index 3068390ad30c..1957a5238eab 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -46,7 +46,9 @@ SYMBOL_FUNCBASE_TYPES, ArgKind, CallExpr, + ClassDef, Context, + Expression, FuncDef, IndexExpr, MypyFile, @@ -205,13 +207,33 @@ def report( offset: int = 0, allow_dups: bool = False, ) -> None: - """Report an error or note (unless disabled).""" + """Report an error or note (unless disabled). + + Note that context controls where error is reported, while origin controls + where # type: ignore comments have effect. + """ + + def span_from_context(ctx: Context) -> Tuple[int, int]: + """This determines where a type: ignore for a given context has effect. + + Current logic is a bit tricky, to keep as much backwards compatibility as + possible. We may reconsider this to always be a single line (or otherwise + simplify it) when we drop Python 3.7. + """ + if isinstance(ctx, (ClassDef, FuncDef)): + return ctx.deco_line or ctx.line, ctx.line + elif not isinstance(ctx, Expression): + return ctx.line, ctx.line + else: + return ctx.line, ctx.end_line or ctx.line + + origin_span: Optional[Tuple[int, int]] if origin is not None: - end_line = origin.end_line + origin_span = span_from_context(origin) elif context is not None: - end_line = context.end_line + origin_span = span_from_context(context) else: - end_line = None + origin_span = None self.errors.report( context.get_line() if context else -1, context.get_column() if context else -1, @@ -219,8 +241,8 @@ def report( severity=severity, file=file, offset=offset, - origin_line=origin.get_line() if origin else None, - end_line=end_line, + origin_span=origin_span, + end_line=context.end_line if context else -1, end_column=context.end_column if context else -1, code=code, allow_dups=allow_dups, @@ -233,13 +255,10 @@ def fail( *, code: Optional[ErrorCode] = None, file: Optional[str] = None, - origin: Optional[Context] = None, allow_dups: bool = False, ) -> None: """Report an error message (unless disabled).""" - self.report( - msg, context, "error", code=code, file=file, origin=origin, allow_dups=allow_dups - ) + self.report(msg, context, "error", code=code, file=file, allow_dups=allow_dups) def note( self, @@ -269,7 +288,6 @@ def note_multiline( messages: str, context: Context, file: Optional[str] = None, - origin: Optional[Context] = None, offset: int = 0, allow_dups: bool = False, code: Optional[ErrorCode] = None, @@ -277,14 +295,7 @@ def note_multiline( """Report as many notes as lines in the message (unless disabled).""" for msg in messages.splitlines(): self.report( - msg, - context, - "note", - file=file, - origin=origin, - offset=offset, - allow_dups=allow_dups, - code=code, + msg, context, "note", file=file, offset=offset, allow_dups=allow_dups, code=code ) # diff --git a/mypy/nodes.py b/mypy/nodes.py index 4ac5c766853f..4ab5eb655b63 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -338,6 +338,7 @@ def __init__( super().__init__() self.defs = defs self.line = 1 # Dummy line number + self.column = 0 # Dummy column self.imports = imports self.is_bom = is_bom self.alias_deps = defaultdict(set) @@ -599,6 +600,7 @@ def __init__(self, items: List["OverloadPart"]) -> None: self.unanalyzed_items = items.copy() self.impl = None if len(items) > 0: + # TODO: figure out how to reliably set end position (we don't know the impl here). self.set_line(items[0].line, items[0].column) self.is_final = False @@ -749,6 +751,7 @@ def set_line( ) -> None: super().set_line(target, column, end_line, end_column) for arg in self.arguments: + # TODO: set arguments line/column to their precise locations. arg.set_line(self.line, self.column, self.end_line, end_column) def is_dynamic(self) -> bool: @@ -764,7 +767,14 @@ class FuncDef(FuncItem, SymbolNode, Statement): This is a non-lambda function defined using 'def'. """ - __slots__ = ("_name", "is_decorated", "is_conditional", "is_abstract", "original_def") + __slots__ = ( + "_name", + "is_decorated", + "is_conditional", + "is_abstract", + "original_def", + "deco_line", + ) # Note that all __init__ args must have default values def __init__( @@ -782,6 +792,8 @@ def __init__( self.is_final = False # Original conditional definition self.original_def: Union[None, FuncDef, Var, Decorator] = None + # Used for error reporting (to keep backwad compatibility with pre-3.8) + self.deco_line: Optional[int] = None @property def name(self) -> str: @@ -1057,6 +1069,7 @@ class ClassDef(Statement): "keywords", "analyzed", "has_incompatible_baseclass", + "deco_line", ) name: str # Name of the class without module prefix @@ -1096,6 +1109,8 @@ def __init__( self.keywords = OrderedDict(keywords or []) self.analyzed = None self.has_incompatible_baseclass = False + # Used for error reporting (to keep backwad compatibility with pre-3.8) + self.deco_line: Optional[int] = None def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_class_def(self) diff --git a/mypy/options.py b/mypy/options.py index 15b474466e31..860c296cfbb0 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -240,6 +240,11 @@ def __init__(self) -> None: # in modules being compiled. Not in the config file or command line. self.mypyc = False + # An internal flag to modify some type-checking logic while + # running inspections (e.g. don't expand function definitions). + # Not in the config file or command line. + self.inspections = False + # Disable the memory optimization of freeing ASTs when # possible. This isn't exposed as a command line option # because it is intended for software integrating with diff --git a/mypy/semanal.py b/mypy/semanal.py index a5fd094a4057..0595f710c289 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1222,6 +1222,7 @@ def visit_decorator(self, dec: Decorator) -> None: if not dec.is_overload: self.add_symbol(dec.name, dec, dec) dec.func._fullname = self.qualified_name(dec.name) + dec.var._fullname = self.qualified_name(dec.name) for d in dec.decorators: d.accept(self) removed: List[int] = [] diff --git a/mypy/semanal_enum.py b/mypy/semanal_enum.py index 2b1481a90ba5..159d5ac73ca6 100644 --- a/mypy/semanal_enum.py +++ b/mypy/semanal_enum.py @@ -117,7 +117,7 @@ class A(enum.Enum): if name != var_name or is_func_scope: self.api.add_symbol_skip_local(name, info) call.analyzed = EnumCallExpr(info, items, values) - call.analyzed.set_line(call.line, call.column) + call.analyzed.set_line(call) info.line = node.line return info diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index fb7e2e532398..425b81df8016 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -294,7 +294,7 @@ def store_namedtuple_info( ) -> None: self.api.add_symbol(name, info, call) call.analyzed = NamedTupleExpr(info, is_typed=is_typed) - call.analyzed.set_line(call.line, call.column) + call.analyzed.set_line(call) def parse_namedtuple_args( self, call: CallExpr, fullname: str diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index dd6659bf1065..6eeba7cea38d 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -270,7 +270,7 @@ def check_typeddict( if var_name: self.api.add_symbol(var_name, info, node) call.analyzed = TypedDictExpr(info) - call.analyzed.set_line(call.line, call.column) + call.analyzed.set_line(call) return True, info def parse_typeddict_args( diff --git a/mypy/test/testdaemon.py b/mypy/test/testdaemon.py index 87a9877267e2..38b0c5397c51 100644 --- a/mypy/test/testdaemon.py +++ b/mypy/test/testdaemon.py @@ -12,6 +12,8 @@ import unittest from typing import List, Tuple +import pytest + from mypy.dmypy_server import filter_out_missing_top_level_packages from mypy.fscache import FileSystemCache from mypy.modulefinder import SearchPaths @@ -27,6 +29,8 @@ class DaemonSuite(DataSuite): files = daemon_files def run_case(self, testcase: DataDrivenTestCase) -> None: + if testcase.name.endswith("_python38") and sys.version_info < (3, 8): + pytest.skip("Not supported on this version of Python") try: test_daemon(testcase) finally: diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index c88ab595443f..61a5b90d7421 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -14,6 +14,7 @@ import os import re +import sys from typing import Any, Dict, List, Tuple, Union, cast import pytest @@ -66,6 +67,8 @@ def should_skip(self, testcase: DataDrivenTestCase) -> bool: if testcase.only_when == "-only_when_cache": return True + if "Inspect" in testcase.name and sys.version_info < (3, 8): + return True return False def run_case(self, testcase: DataDrivenTestCase) -> None: @@ -96,6 +99,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: assert testcase.tmpdir a.extend(self.maybe_suggest(step, server, main_src, testcase.tmpdir.name)) + a.extend(self.maybe_inspect(step, server, main_src)) if server.fine_grained_manager: if CHECK_CONSISTENCY: @@ -157,7 +161,7 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo return options def run_check(self, server: Server, sources: List[BuildSource]) -> List[str]: - response = server.check(sources, is_tty=False, terminal_width=-1) + response = server.check(sources, export_types=True, is_tty=False, terminal_width=-1) out = cast(str, response["out"] or response["err"]) return out.splitlines() @@ -238,6 +242,7 @@ def perform_step( a = new_messages assert testcase.tmpdir a.extend(self.maybe_suggest(step, server, main_src, testcase.tmpdir.name)) + a.extend(self.maybe_inspect(step, server, main_src)) return a, triggered @@ -294,19 +299,16 @@ def maybe_suggest(self, step: int, server: Server, src: str, tmp_dir: str) -> Li use_fixme = m.group(1) if m else None m = re.match("--max-guesses=([0-9]+)", flags) max_guesses = int(m.group(1)) if m else None - res = cast( - Dict[str, Any], - server.cmd_suggest( - target.strip(), - json=json, - no_any=no_any, - no_errors=no_errors, - try_text=try_text, - flex_any=flex_any, - use_fixme=use_fixme, - callsites=callsites, - max_guesses=max_guesses, - ), + res: Dict[str, Any] = server.cmd_suggest( + target.strip(), + json=json, + no_any=no_any, + no_errors=no_errors, + try_text=try_text, + flex_any=flex_any, + use_fixme=use_fixme, + callsites=callsites, + max_guesses=max_guesses, ) val = res["error"] if "error" in res else res["out"] + res["err"] if json: @@ -316,12 +318,51 @@ def maybe_suggest(self, step: int, server: Server, src: str, tmp_dir: str) -> Li output.extend(val.strip().split("\n")) return normalize_messages(output) + def maybe_inspect(self, step: int, server: Server, src: str) -> List[str]: + output: List[str] = [] + targets = self.get_inspect(src, step) + for flags, location in targets: + m = re.match(r"--show=(\w+)", flags) + show = m.group(1) if m else "type" + verbosity = 0 + if "-v" in flags: + verbosity = 1 + if "-vv" in flags: + verbosity = 2 + m = re.match(r"--limit=([0-9]+)", flags) + limit = int(m.group(1)) if m else 0 + include_span = "--include-span" in flags + include_kind = "--include-kind" in flags + include_object_attrs = "--include-object-attrs" in flags + union_attrs = "--union-attrs" in flags + force_reload = "--force-reload" in flags + res: Dict[str, Any] = server.cmd_inspect( + show, + location, + verbosity=verbosity, + limit=limit, + include_span=include_span, + include_kind=include_kind, + include_object_attrs=include_object_attrs, + union_attrs=union_attrs, + force_reload=force_reload, + ) + val = res["error"] if "error" in res else res["out"] + res["err"] + output.extend(val.strip().split("\n")) + return normalize_messages(output) + def get_suggest(self, program_text: str, incremental_step: int) -> List[Tuple[str, str]]: step_bit = "1?" if incremental_step == 1 else str(incremental_step) regex = f"# suggest{step_bit}: (--[a-zA-Z0-9_\\-./=?^ ]+ )*([a-zA-Z0-9_.:/?^ ]+)$" m = re.findall(regex, program_text, flags=re.MULTILINE) return m + def get_inspect(self, program_text: str, incremental_step: int) -> List[Tuple[str, str]]: + step_bit = "1?" if incremental_step == 1 else str(incremental_step) + regex = f"# inspect{step_bit}: (--[a-zA-Z0-9_\\-=?^ ]+ )*([a-zA-Z0-9_.:/?^ ]+)$" + m = re.findall(regex, program_text, flags=re.MULTILINE) + return m + def normalize_messages(messages: List[str]) -> List[str]: return [re.sub("^tmp" + re.escape(os.sep), "", message) for message in messages] diff --git a/mypy/traverser.py b/mypy/traverser.py index 1c2fa8c04dcb..e245fb181069 100644 --- a/mypy/traverser.py +++ b/mypy/traverser.py @@ -13,38 +13,53 @@ AwaitExpr, BackquoteExpr, Block, + BreakStmt, + BytesExpr, CallExpr, CastExpr, ClassDef, ComparisonExpr, + ComplexExpr, ConditionalExpr, + ContinueStmt, Decorator, DelStmt, DictExpr, DictionaryComprehension, + EllipsisExpr, + EnumCallExpr, ExecStmt, Expression, ExpressionStmt, + FloatExpr, ForStmt, FuncBase, FuncDef, FuncItem, GeneratorExpr, + GlobalDecl, IfStmt, Import, + ImportAll, ImportFrom, IndexExpr, + IntExpr, LambdaExpr, ListComprehension, ListExpr, MatchStmt, MemberExpr, MypyFile, + NamedTupleExpr, NameExpr, + NewTypeExpr, Node, + NonlocalDecl, OperatorAssignmentStmt, OpExpr, OverloadedFuncDef, + ParamSpecExpr, + PassStmt, PrintStmt, RaiseStmt, ReturnStmt, @@ -53,11 +68,18 @@ SetExpr, SliceExpr, StarExpr, + StrExpr, SuperExpr, TryStmt, TupleExpr, + TypeAlias, + TypeAliasExpr, TypeApplication, + TypedDictExpr, + TypeVarExpr, + TypeVarTupleExpr, UnaryExpr, + UnicodeExpr, WhileStmt, WithStmt, YieldExpr, @@ -69,6 +91,7 @@ MappingPattern, OrPattern, SequencePattern, + SingletonPattern, StarredPattern, ValuePattern, ) @@ -364,7 +387,7 @@ def visit_sequence_pattern(self, o: SequencePattern) -> None: for p in o.patterns: p.accept(self) - def visit_starred_patten(self, o: StarredPattern) -> None: + def visit_starred_pattern(self, o: StarredPattern) -> None: if o.capture is not None: o.capture.accept(self) @@ -403,6 +426,443 @@ def visit_exec_stmt(self, o: ExecStmt) -> None: o.locals.accept(self) +class ExtendedTraverserVisitor(TraverserVisitor): + """This is a more flexible traverser. + + In addition to the base traverser it: + * has visit_ methods for leaf nodes + * has common method that is called for all nodes + * allows to skip recursing into a node + + Note that this traverser still doesn't visit some internal + mypy constructs like _promote expression and Var. + """ + + def visit(self, o: Node) -> bool: + # If returns True, will continue to nested nodes. + return True + + def visit_mypy_file(self, o: MypyFile) -> None: + if not self.visit(o): + return + super().visit_mypy_file(o) + + # Module structure + + def visit_import(self, o: Import) -> None: + if not self.visit(o): + return + super().visit_import(o) + + def visit_import_from(self, o: ImportFrom) -> None: + if not self.visit(o): + return + super().visit_import_from(o) + + def visit_import_all(self, o: ImportAll) -> None: + if not self.visit(o): + return + super().visit_import_all(o) + + # Definitions + + def visit_func_def(self, o: FuncDef) -> None: + if not self.visit(o): + return + super().visit_func_def(o) + + def visit_overloaded_func_def(self, o: OverloadedFuncDef) -> None: + if not self.visit(o): + return + super().visit_overloaded_func_def(o) + + def visit_class_def(self, o: ClassDef) -> None: + if not self.visit(o): + return + super().visit_class_def(o) + + def visit_global_decl(self, o: GlobalDecl) -> None: + if not self.visit(o): + return + super().visit_global_decl(o) + + def visit_nonlocal_decl(self, o: NonlocalDecl) -> None: + if not self.visit(o): + return + super().visit_nonlocal_decl(o) + + def visit_decorator(self, o: Decorator) -> None: + if not self.visit(o): + return + super().visit_decorator(o) + + def visit_type_alias(self, o: TypeAlias) -> None: + if not self.visit(o): + return + super().visit_type_alias(o) + + # Statements + + def visit_block(self, block: Block) -> None: + if not self.visit(block): + return + super().visit_block(block) + + def visit_expression_stmt(self, o: ExpressionStmt) -> None: + if not self.visit(o): + return + super().visit_expression_stmt(o) + + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: + if not self.visit(o): + return + super().visit_assignment_stmt(o) + + def visit_operator_assignment_stmt(self, o: OperatorAssignmentStmt) -> None: + if not self.visit(o): + return + super().visit_operator_assignment_stmt(o) + + def visit_while_stmt(self, o: WhileStmt) -> None: + if not self.visit(o): + return + super().visit_while_stmt(o) + + def visit_for_stmt(self, o: ForStmt) -> None: + if not self.visit(o): + return + super().visit_for_stmt(o) + + def visit_return_stmt(self, o: ReturnStmt) -> None: + if not self.visit(o): + return + super().visit_return_stmt(o) + + def visit_assert_stmt(self, o: AssertStmt) -> None: + if not self.visit(o): + return + super().visit_assert_stmt(o) + + def visit_del_stmt(self, o: DelStmt) -> None: + if not self.visit(o): + return + super().visit_del_stmt(o) + + def visit_if_stmt(self, o: IfStmt) -> None: + if not self.visit(o): + return + super().visit_if_stmt(o) + + def visit_break_stmt(self, o: BreakStmt) -> None: + if not self.visit(o): + return + super().visit_break_stmt(o) + + def visit_continue_stmt(self, o: ContinueStmt) -> None: + if not self.visit(o): + return + super().visit_continue_stmt(o) + + def visit_pass_stmt(self, o: PassStmt) -> None: + if not self.visit(o): + return + super().visit_pass_stmt(o) + + def visit_raise_stmt(self, o: RaiseStmt) -> None: + if not self.visit(o): + return + super().visit_raise_stmt(o) + + def visit_try_stmt(self, o: TryStmt) -> None: + if not self.visit(o): + return + super().visit_try_stmt(o) + + def visit_with_stmt(self, o: WithStmt) -> None: + if not self.visit(o): + return + super().visit_with_stmt(o) + + def visit_print_stmt(self, o: PrintStmt) -> None: + if not self.visit(o): + return + super().visit_print_stmt(o) + + def visit_exec_stmt(self, o: ExecStmt) -> None: + if not self.visit(o): + return + super().visit_exec_stmt(o) + + def visit_match_stmt(self, o: MatchStmt) -> None: + if not self.visit(o): + return + super().visit_match_stmt(o) + + # Expressions (default no-op implementation) + + def visit_int_expr(self, o: IntExpr) -> None: + if not self.visit(o): + return + super().visit_int_expr(o) + + def visit_str_expr(self, o: StrExpr) -> None: + if not self.visit(o): + return + super().visit_str_expr(o) + + def visit_bytes_expr(self, o: BytesExpr) -> None: + if not self.visit(o): + return + super().visit_bytes_expr(o) + + def visit_unicode_expr(self, o: UnicodeExpr) -> None: + if not self.visit(o): + return + super().visit_unicode_expr(o) + + def visit_float_expr(self, o: FloatExpr) -> None: + if not self.visit(o): + return + super().visit_float_expr(o) + + def visit_complex_expr(self, o: ComplexExpr) -> None: + if not self.visit(o): + return + super().visit_complex_expr(o) + + def visit_ellipsis(self, o: EllipsisExpr) -> None: + if not self.visit(o): + return + super().visit_ellipsis(o) + + def visit_star_expr(self, o: StarExpr) -> None: + if not self.visit(o): + return + super().visit_star_expr(o) + + def visit_name_expr(self, o: NameExpr) -> None: + if not self.visit(o): + return + super().visit_name_expr(o) + + def visit_member_expr(self, o: MemberExpr) -> None: + if not self.visit(o): + return + super().visit_member_expr(o) + + def visit_yield_from_expr(self, o: YieldFromExpr) -> None: + if not self.visit(o): + return + super().visit_yield_from_expr(o) + + def visit_yield_expr(self, o: YieldExpr) -> None: + if not self.visit(o): + return + super().visit_yield_expr(o) + + def visit_call_expr(self, o: CallExpr) -> None: + if not self.visit(o): + return + super().visit_call_expr(o) + + def visit_op_expr(self, o: OpExpr) -> None: + if not self.visit(o): + return + super().visit_op_expr(o) + + def visit_comparison_expr(self, o: ComparisonExpr) -> None: + if not self.visit(o): + return + super().visit_comparison_expr(o) + + def visit_cast_expr(self, o: CastExpr) -> None: + if not self.visit(o): + return + super().visit_cast_expr(o) + + def visit_assert_type_expr(self, o: AssertTypeExpr) -> None: + if not self.visit(o): + return + super().visit_assert_type_expr(o) + + def visit_reveal_expr(self, o: RevealExpr) -> None: + if not self.visit(o): + return + super().visit_reveal_expr(o) + + def visit_super_expr(self, o: SuperExpr) -> None: + if not self.visit(o): + return + super().visit_super_expr(o) + + def visit_assignment_expr(self, o: AssignmentExpr) -> None: + if not self.visit(o): + return + super().visit_assignment_expr(o) + + def visit_unary_expr(self, o: UnaryExpr) -> None: + if not self.visit(o): + return + super().visit_unary_expr(o) + + def visit_list_expr(self, o: ListExpr) -> None: + if not self.visit(o): + return + super().visit_list_expr(o) + + def visit_dict_expr(self, o: DictExpr) -> None: + if not self.visit(o): + return + super().visit_dict_expr(o) + + def visit_tuple_expr(self, o: TupleExpr) -> None: + if not self.visit(o): + return + super().visit_tuple_expr(o) + + def visit_set_expr(self, o: SetExpr) -> None: + if not self.visit(o): + return + super().visit_set_expr(o) + + def visit_index_expr(self, o: IndexExpr) -> None: + if not self.visit(o): + return + super().visit_index_expr(o) + + def visit_type_application(self, o: TypeApplication) -> None: + if not self.visit(o): + return + super().visit_type_application(o) + + def visit_lambda_expr(self, o: LambdaExpr) -> None: + if not self.visit(o): + return + super().visit_lambda_expr(o) + + def visit_list_comprehension(self, o: ListComprehension) -> None: + if not self.visit(o): + return + super().visit_list_comprehension(o) + + def visit_set_comprehension(self, o: SetComprehension) -> None: + if not self.visit(o): + return + super().visit_set_comprehension(o) + + def visit_dictionary_comprehension(self, o: DictionaryComprehension) -> None: + if not self.visit(o): + return + super().visit_dictionary_comprehension(o) + + def visit_generator_expr(self, o: GeneratorExpr) -> None: + if not self.visit(o): + return + super().visit_generator_expr(o) + + def visit_slice_expr(self, o: SliceExpr) -> None: + if not self.visit(o): + return + super().visit_slice_expr(o) + + def visit_conditional_expr(self, o: ConditionalExpr) -> None: + if not self.visit(o): + return + super().visit_conditional_expr(o) + + def visit_backquote_expr(self, o: BackquoteExpr) -> None: + if not self.visit(o): + return + super().visit_backquote_expr(o) + + def visit_type_var_expr(self, o: TypeVarExpr) -> None: + if not self.visit(o): + return + super().visit_type_var_expr(o) + + def visit_paramspec_expr(self, o: ParamSpecExpr) -> None: + if not self.visit(o): + return + super().visit_paramspec_expr(o) + + def visit_type_var_tuple_expr(self, o: TypeVarTupleExpr) -> None: + if not self.visit(o): + return + super().visit_type_var_tuple_expr(o) + + def visit_type_alias_expr(self, o: TypeAliasExpr) -> None: + if not self.visit(o): + return + super().visit_type_alias_expr(o) + + def visit_namedtuple_expr(self, o: NamedTupleExpr) -> None: + if not self.visit(o): + return + super().visit_namedtuple_expr(o) + + def visit_enum_call_expr(self, o: EnumCallExpr) -> None: + if not self.visit(o): + return + super().visit_enum_call_expr(o) + + def visit_typeddict_expr(self, o: TypedDictExpr) -> None: + if not self.visit(o): + return + super().visit_typeddict_expr(o) + + def visit_newtype_expr(self, o: NewTypeExpr) -> None: + if not self.visit(o): + return + super().visit_newtype_expr(o) + + def visit_await_expr(self, o: AwaitExpr) -> None: + if not self.visit(o): + return + super().visit_await_expr(o) + + # Patterns + + def visit_as_pattern(self, o: AsPattern) -> None: + if not self.visit(o): + return + super().visit_as_pattern(o) + + def visit_or_pattern(self, o: OrPattern) -> None: + if not self.visit(o): + return + super().visit_or_pattern(o) + + def visit_value_pattern(self, o: ValuePattern) -> None: + if not self.visit(o): + return + super().visit_value_pattern(o) + + def visit_singleton_pattern(self, o: SingletonPattern) -> None: + if not self.visit(o): + return + super().visit_singleton_pattern(o) + + def visit_sequence_pattern(self, o: SequencePattern) -> None: + if not self.visit(o): + return + super().visit_sequence_pattern(o) + + def visit_starred_pattern(self, o: StarredPattern) -> None: + if not self.visit(o): + return + super().visit_starred_pattern(o) + + def visit_mapping_pattern(self, o: MappingPattern) -> None: + if not self.visit(o): + return + super().visit_mapping_pattern(o) + + def visit_class_pattern(self, o: ClassPattern) -> None: + if not self.visit(o): + return + super().visit_class_pattern(o) + + class ReturnSeeker(TraverserVisitor): def __init__(self) -> None: self.found = False diff --git a/mypy/treetransform.py b/mypy/treetransform.py index 7ac73d36ca15..cda8a5747c7a 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -160,7 +160,7 @@ def copy_argument(self, argument: Argument) -> Argument: ) # Refresh lines of the inner things - arg.set_line(argument.line) + arg.set_line(argument) return arg @@ -291,7 +291,7 @@ def visit_var(self, node: Var) -> Var: new.final_value = node.final_value new.final_unset_in_class = node.final_unset_in_class new.final_set_in_init = node.final_set_in_init - new.set_line(node.line) + new.set_line(node) self.var_map[node] = new return new @@ -535,7 +535,7 @@ def visit_index_expr(self, node: IndexExpr) -> IndexExpr: new.analyzed = self.visit_type_application(node.analyzed) else: new.analyzed = self.visit_type_alias_expr(node.analyzed) - new.analyzed.set_line(node.analyzed.line) + new.analyzed.set_line(node.analyzed) return new def visit_type_application(self, node: TypeApplication) -> TypeApplication: @@ -543,12 +543,12 @@ def visit_type_application(self, node: TypeApplication) -> TypeApplication: def visit_list_comprehension(self, node: ListComprehension) -> ListComprehension: generator = self.duplicate_generator(node.generator) - generator.set_line(node.generator.line, node.generator.column) + generator.set_line(node.generator) return ListComprehension(generator) def visit_set_comprehension(self, node: SetComprehension) -> SetComprehension: generator = self.duplicate_generator(node.generator) - generator.set_line(node.generator.line, node.generator.column) + generator.set_line(node.generator) return SetComprehension(generator) def visit_dictionary_comprehension( @@ -634,25 +634,25 @@ def visit_temp_node(self, node: TempNode) -> TempNode: def node(self, node: Node) -> Node: new = node.accept(self) - new.set_line(node.line) + new.set_line(node) return new def mypyfile(self, node: MypyFile) -> MypyFile: new = node.accept(self) assert isinstance(new, MypyFile) - new.set_line(node.line) + new.set_line(node) return new def expr(self, expr: Expression) -> Expression: new = expr.accept(self) assert isinstance(new, Expression) - new.set_line(expr.line, expr.column) + new.set_line(expr) return new def stmt(self, stmt: Statement) -> Statement: new = stmt.accept(self) assert isinstance(new, Statement) - new.set_line(stmt.line, stmt.column) + new.set_line(stmt) return new # Helpers diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 113741a021d3..f59475750de5 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -524,7 +524,7 @@ def setup_non_ext_dict( non_ext_dict = Register(dict_rprimitive) - true_block, false_block, exit_block = (BasicBlock(), BasicBlock(), BasicBlock()) + true_block, false_block, exit_block = BasicBlock(), BasicBlock(), BasicBlock() builder.add_bool_branch(has_prepare, true_block, false_block) builder.activate_block(true_block) diff --git a/test-data/unit/check-ignore.test b/test-data/unit/check-ignore.test index d304da96ceea..982fb67f4e7f 100644 --- a/test-data/unit/check-ignore.test +++ b/test-data/unit/check-ignore.test @@ -262,3 +262,20 @@ ERROR # E: Name "ERROR" is not defined def f(): ... ERROR # E: Name "ERROR" is not defined + +[case testIgnoreInsideFunctionDoesntAffectWhole] +# flags: --disallow-untyped-defs + +def f(): # E: Function is missing a return type annotation + 42 + 'no way' # type: ignore + return 0 + +[case testIgnoreInsideClassDoesntAffectWhole] +import six +class M(type): pass + +@six.add_metaclass(M) +class CD(six.with_metaclass(M)): # E: Multiple metaclass definitions + 42 + 'no way' # type: ignore + +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index c73be05e1be3..370413ee774b 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -72,7 +72,7 @@ Restarting: configuration changed Daemon stopped Daemon started foo.py:1: error: Function is missing a return type annotation - def f(): pass + def f(): ^ foo.py:1: note: Use "-> None" if function does not return a value Found 1 error in 1 file (checked 1 source file) @@ -83,7 +83,8 @@ Success: no issues found in 1 source file $ dmypy stop Daemon stopped [file foo.py] -def f(): pass +def f(): + pass [case testDaemonRunRestartPluginVersion] $ dmypy run -- foo.py --no-error-summary @@ -295,3 +296,215 @@ from foo import foo def bar() -> None: x = foo('abc') # type: str foo(arg='xyz') + +[case testDaemonGetType_python38] +$ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy inspect foo:1:2:3:4 +Command "inspect" is only valid after a "check" command (that produces no parse errors) +== Return code: 2 +$ dmypy check foo.py --export-types +foo.py:3: error: Incompatible types in assignment (expression has type "str", variable has type "int") +== Return code: 1 +$ dmypy inspect foo:1 +Format should be file:line:column[:end_line:end_column] +== Return code: 2 +$ dmypy inspect foo.py:1:2:a:b +invalid literal for int() with base 10: 'a' +== Return code: 2 +$ dmypy inspect foo.pyc:1:1:2:2 +Source file is not a Python file +== Return code: 2 +$ dmypy inspect bar/baz.py:1:1:2:2 +Unknown module: baz +== Return code: 1 +$ dmypy inspect foo.py:3:1:1:1 +"end_line" must not be before "line" +== Return code: 2 +$ dmypy inspect foo.py:3:3:3:1 +"end_column" must be after "column" +== Return code: 2 +$ dmypy inspect foo.py:3:10:3:17 +"str" +$ dmypy inspect foo.py:3:10:3:17 -vv +"builtins.str" +$ dmypy inspect foo.py:9:9:9:11 +"int" +$ dmypy inspect foo.py:11:1:11:3 +"Callable[[Optional[int]], None]" +$ dmypy inspect foo.py:11:1:13:1 +"None" +$ dmypy inspect foo.py:1:2:3:4 +Can't find expression at span 1:2:3:4 +== Return code: 1 +$ dmypy inspect foo.py:17:5:17:5 +No known type available for "NameExpr" (maybe unreachable or try --force-reload) +== Return code: 1 + +[file foo.py] +from typing import Optional + +x: int = "no way" # line 3 + +def foo(arg: Optional[int] = None) -> None: + if arg is None: + arg + else: + arg # line 9 + +foo( + # multiline +) + +def unreachable(x: int) -> None: + return + x # line 17 + +[case testDaemonGetTypeInexact_python38] +$ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy check foo.py --export-types +$ dmypy inspect foo.py:1:a +invalid literal for int() with base 10: 'a' +== Return code: 2 +$ dmypy inspect foo.pyc:1:2 +Source file is not a Python file +== Return code: 2 +$ dmypy inspect bar/baz.py:1:2 +Unknown module: baz +== Return code: 1 +$ dmypy inspect foo.py:7:5 --include-span +7:5:7:5 -> "int" +7:5:7:11 -> "int" +7:1:7:12 -> "None" +$ dmypy inspect foo.py:7:5 --include-kind +NameExpr -> "int" +OpExpr -> "int" +CallExpr -> "None" +$ dmypy inspect foo.py:7:5 --include-span --include-kind +NameExpr:7:5:7:5 -> "int" +OpExpr:7:5:7:11 -> "int" +CallExpr:7:1:7:12 -> "None" +$ dmypy inspect foo.py:7:5 -vv +"builtins.int" +"builtins.int" +"None" +$ dmypy inspect foo.py:7:5 -vv --limit=1 +"builtins.int" +$ dmypy inspect foo.py:7:3 +"Callable[[int], None]" +"None" +$ dmypy inspect foo.py:1:2 +Can't find any expressions at position 1:2 +== Return code: 1 +$ dmypy inspect foo.py:11:5 --force-reload +No known type available for "NameExpr" (maybe unreachable) +No known type available for "OpExpr" (maybe unreachable) +== Return code: 1 + +[file foo.py] +from typing import Optional + +def foo(x: int) -> None: ... + +a: int +b: int +foo(a and b) # line 7 + +def unreachable(x: int, y: int) -> None: + return + x and y # line 11 + +[case testDaemonGetAttrs_python38] +$ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy check foo.py bar.py --export-types +$ dmypy inspect foo.py:9:1 --show attrs --include-span --include-kind -vv +NameExpr:9:1:9:1 -> {"foo.C": ["a", "x", "y"], "foo.B": ["a", "b"]} +$ dmypy inspect foo.py:11:10 --show attrs +No known type available for "StrExpr" (maybe unreachable or try --force-reload) +== Return code: 1 +$ dmypy inspect foo.py:1:1 --show attrs +Can't find any expressions at position 1:1 +== Return code: 1 +$ dmypy inspect --show attrs bar.py:10:1 +{"A": ["z"], "B": ["z"]} +$ dmypy inspect --show attrs bar.py:10:1 --union-attrs +{"A": ["x", "z"], "B": ["y", "z"]} + +[file foo.py] +class B: + def b(self) -> int: ... + a: int +class C(B): + a: int + y: int + def x(self) -> int: ... + +v: C # line 9 +if False: + "unreachable" + +[file bar.py] +from typing import Union + +class A: + x: int + z: int +class B: + y: int + z: int +var: Union[A, B] +var # line 10 + +[case testDaemonGetDefinition_python38] +$ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary +Daemon started +$ dmypy check foo.py bar/baz.py bar/__init__.py --export-types +$ dmypy inspect foo.py:5:1 --show definition +foo.py:4:1:y +$ dmypy inspect foo.py:2:3 --show definition --include-span --include-kind -vv +MemberExpr:2:1:2:7 -> bar/baz.py:3:5:Alias +$ dmypy inspect foo.py:3:1 --show definition +Cannot find definition for "NameExpr" at 3:1:3:1 +== Return code: 1 +$ dmypy inspect foo.py:4:6 --show definition +No name or member expressions at 4:6 +== Return code: 1 +$ dmypy inspect foo.py:7:1:7:6 --show definition +bar/baz.py:4:5:attr +$ dmypy inspect foo.py:10:10 --show definition --include-span +10:1:10:12 -> bar/baz.py:6:1:test +$ dmypy inspect foo.py:14:6 --show definition --include-span --include-kind +NameExpr:14:5:14:7 -> foo.py:13:1:arg +MemberExpr:14:5:14:9 -> bar/baz.py:9:5:x, bar/baz.py:11:5:x + +[file foo.py] +from bar.baz import A, B, C +C.Alias +x # type: ignore +y = 42 +y # line 5 +z = C() +z.attr + +import bar +bar.baz.test() # line 10 + +from typing import Union +def foo(arg: Union[A, B]) -> None: + arg.x + +[file bar/__init__.py] +[file bar/baz.py] +from typing import Union +class C: + Alias = Union[int, str] + attr = 42 + +def test() -> None: ... # line 6 + +class A: + x: int +class B: + x: int diff --git a/test-data/unit/fine-grained-inspect.test b/test-data/unit/fine-grained-inspect.test new file mode 100644 index 000000000000..5661c14bc093 --- /dev/null +++ b/test-data/unit/fine-grained-inspect.test @@ -0,0 +1,269 @@ +[case testInspectTypeBasic] +# inspect2: --include-kind foo.py:10:13 +# inspect2: --show=type --include-kind foo.py:10:13 +# inspect2: --include-span -vv foo.py:12:5 +# inspect2: --include-span --include-kind foo.py:12:5:12:9 +import foo +[file foo.py] +from typing import TypeVar, Generic + +T = TypeVar('T') + +class C(Generic[T]): + def __init__(self, x: T) -> None: ... + x: T + +def foo(arg: C[T]) -> T: + return arg.x + +foo(C(42)) +[out] +== +NameExpr -> "C[T]" +MemberExpr -> "T" +NameExpr -> "C[T]" +MemberExpr -> "T" +12:5:12:5 -> "Type[foo.C[Any]]" +12:5:12:9 -> "foo.C[builtins.int]" +12:1:12:10 -> "builtins.int" +CallExpr:12:5:12:9 -> "C[int]" + +[case testInspectAttrsBasic] +# inspect2: --show=attrs foo.py:6:1 +# inspect2: --show=attrs foo.py:7:1 +# inspect2: --show=attrs foo.py:10:1 +# inspect2: --show=attrs --include-object-attrs foo.py:10:1 +import foo +[file foo.py] +from bar import Meta +class C(metaclass=Meta): + x: int + def meth(self) -> None: ... + +c: C +C + +def foo() -> int: ... +foo +[file bar.py] +class Meta(type): + y: int +[out] +== +{"C": ["meth", "x"]} +{"C": ["meth", "x"], "Meta": ["y"], "type": ["__init__"]} +{} +{"object": ["__init__"]} + +[case testInspectDefBasic] +# inspect2: --show=definition foo.py:5:5 +# inspect2: --show=definition --include-kind foo.py:6:3 +# inspect2: --show=definition --include-span foo.py:7:5 +# inspect2: --show=definition foo.py:8:1:8:4 +# inspect2: --show=definition foo.py:8:6:8:8 +# inspect2: --show=definition foo.py:9:3 +import foo +[file foo.py] +from bar import var, test, A +from baz import foo + +a: A +a.meth() +a.x +A.B.y +test(var) +foo +[file bar.py] +class A: + x: int + @classmethod + def meth(cls) -> None: ... + class B: + y: int + +var = 42 +def test(x: int) -> None: ... +[file baz.py] +from typing import overload, Union + +@overload +def foo(x: int) -> None: ... +@overload +def foo(x: str) -> None: ... +def foo(x: Union[int, str]) -> None: + pass +[builtins fixtures/classmethod.pyi] +[out] +== +bar.py:4:0:meth +MemberExpr -> tmp/bar.py:2:5:x +7:1:7:5 -> tmp/bar.py:6:9:y +bar.py:9:1:test +bar.py:8:1:var +baz.py:3:2:foo + +[case testInspectFallbackAttributes] +# inspect2: --show=attrs --include-object-attrs foo.py:5:1 +# inspect2: --show=attrs foo.py:8:1 +# inspect2: --show=attrs --include-kind foo.py:10:1 +# inspect2: --show=attrs --include-kind --include-object-attrs foo.py:10:1 +import foo +[file foo.py] +class B: ... +class C(B): + x: int +c: C +c # line 5 + +t = 42, "foo" +t # line 8 + +None +[builtins fixtures/args.pyi] +[out] +== +{"C": ["x"], "object": ["__eq__", "__init__", "__ne__"]} +{"Iterable": ["__iter__"]} +NameExpr -> {} +NameExpr -> {"object": ["__eq__", "__init__", "__ne__"]} + +[case testInspectTypeVarBoundAttrs] +# inspect2: --show=attrs foo.py:8:13 +import foo +[file foo.py] +from typing import TypeVar + +class C: + x: int + +T = TypeVar('T', bound=C) +def foo(arg: T) -> T: + return arg +[out] +== +{"C": ["x"]} + +[case testInspectTypeVarValuesAttrs] +# inspect2: --show=attrs --force-reload foo.py:13:13 +# inspect2: --show=attrs --force-reload --union-attrs foo.py:13:13 +# inspect2: --show=attrs foo.py:16:5 +# inspect2: --show=attrs --union-attrs foo.py:16:5 +import foo +[file foo.py] +from typing import TypeVar, Generic + +class A: + x: int + z: int + +class B: + y: int + z: int + +T = TypeVar('T', A, B) +def foo(arg: T) -> T: + return arg + +class C(Generic[T]): + x: T +[out] +== +{"A": ["z"], "B": ["z"]} +{"A": ["x", "z"], "B": ["y", "z"]} +{"A": ["z"], "B": ["z"]} +{"A": ["x", "z"], "B": ["y", "z"]} + +[case testInspectTypeVarBoundDef] +# inspect2: --show=definition foo.py:9:13 +# inspect2: --show=definition foo.py:8:9 +import foo +[file foo.py] +from typing import TypeVar + +class C: + x: int + +T = TypeVar('T', bound=C) +def foo(arg: T) -> T: + arg.x + return arg +[out] +== +foo.py:7:1:arg +foo.py:4:5:x + +[case testInspectTypeVarValuesDef] +# inspect2: --show=definition --force-reload foo.py:13:9 +# inspect2: --show=definition --force-reload foo.py:14:13 +# inspect2: --show=definition foo.py:18:7 +import foo +[file foo.py] +from typing import TypeVar, Generic + +class A: + x: int + z: int + +class B: + y: int + z: int + +T = TypeVar('T', A, B) +def foo(arg: T) -> T: + arg.z + return arg + +class C(Generic[T]): + x: T + x.z +[out] +== +foo.py:5:5:z, tmp/foo.py:9:5:z +foo.py:12:1:arg +foo.py:5:5:z, tmp/foo.py:9:5:z + +[case testInspectModuleAttrs] +# inspect2: --show=attrs foo.py:2:1 +import foo +[file foo.py] +from pack import bar +bar +[file pack/__init__.py] +[file pack/bar.py] +x: int +def bar() -> None: ... +class C: ... +[builtins fixtures/module.pyi] +[out] +== +{"": ["C", "__annotations__", "__doc__", "__file__", "__name__", "__package__", "bar", "x"], "ModuleType": ["__file__"]} + +[case testInspectModuleDef] +# inspect2: --show=definition --include-kind foo.py:2:1 +import foo +[file foo.py] +from pack import bar +bar.x +[file pack/__init__.py] +[file pack/bar.py] +pass +if True: + x: int +[out] +== +NameExpr -> tmp/pack/bar.py:1:1:bar +MemberExpr -> tmp/pack/bar.py:3:5:x + +[case testInspectFunctionArgDef] +# inspect2: --show=definition --include-span foo.py:4:13 +# TODO: for now all arguments have line/column set to function definition. +import foo +[file foo.py] +def foo(arg: int) -> int: + pass + pass + return arg + +[out] +== +4:12:4:14 -> tmp/foo.py:1:1:arg