Skip to content

Commit

Permalink
Make dict expression inference more consistent (#15174)
Browse files Browse the repository at this point in the history
Fixes #12977

IMO current dict expression inference logic is quite arbitrary: we only
take the non-star items to infer resulting type, then enforce it on the
remaining (star) items. In this PR I simplify the logic to simply put
all expressions as arguments into the same call. This has following
benefits:
* Makes everything more consistent/predictable.
* Fixes one of top upvoted bugs
* Makes dict item indexes correct (previously we reshuffled them causing
wrong indexes for non-star items after star items)
* No more weird wordings like `List item <n>` or `Argument <n> to
"update" of "dict"`
* I also fix the end position of generated expressions to show correct
spans in errors

The only downside is that we will see `Cannot infer type argument` error
instead of `Incompatible type` more often. This is because
`SupportsKeysAndGetItem` (used for star items) is invariant in key type.
I think this is fine however, since:
* This only affects key types, that are mixed much less often than value
types (they are usually just strings), and for latter we use joins.
* I added a dedicated note for this case
  • Loading branch information
ilevkivskyi authored May 5, 2023
1 parent fea5c93 commit 541639e
Show file tree
Hide file tree
Showing 7 changed files with 89 additions and 65 deletions.
74 changes: 26 additions & 48 deletions mypy/checkexpr.py
Original file line number Diff line number Diff line change
Expand Up @@ -4319,12 +4319,19 @@ def visit_dict_expr(self, e: DictExpr) -> Type:
if dt:
return dt

# Define type variables (used in constructors below).
kt = TypeVarType("KT", "KT", -1, [], self.object_type())
vt = TypeVarType("VT", "VT", -2, [], self.object_type())

# Collect function arguments, watching out for **expr.
args: list[Expression] = [] # Regular "key: value"
stargs: list[Expression] = [] # For "**expr"
args: list[Expression] = []
expected_types: list[Type] = []
for key, value in e.items:
if key is None:
stargs.append(value)
args.append(value)
expected_types.append(
self.chk.named_generic_type("_typeshed.SupportsKeysAndGetItem", [kt, vt])
)
else:
tup = TupleExpr([key, value])
if key.line >= 0:
Expand All @@ -4333,52 +4340,23 @@ def visit_dict_expr(self, e: DictExpr) -> Type:
else:
tup.line = value.line
tup.column = value.column
tup.end_line = value.end_line
tup.end_column = value.end_column
args.append(tup)
# Define type variables (used in constructors below).
kt = TypeVarType("KT", "KT", -1, [], self.object_type())
vt = TypeVarType("VT", "VT", -2, [], self.object_type())
rv = None
# Call dict(*args), unless it's empty and stargs is not.
if args or not stargs:
# The callable type represents a function like this:
#
# def <unnamed>(*v: Tuple[kt, vt]) -> Dict[kt, vt]: ...
constructor = CallableType(
[TupleType([kt, vt], self.named_type("builtins.tuple"))],
[nodes.ARG_STAR],
[None],
self.chk.named_generic_type("builtins.dict", [kt, vt]),
self.named_type("builtins.function"),
name="<dict>",
variables=[kt, vt],
)
rv = self.check_call(constructor, args, [nodes.ARG_POS] * len(args), e)[0]
else:
# dict(...) will be called below.
pass
# Call rv.update(arg) for each arg in **stargs,
# except if rv isn't set yet, then set rv = dict(arg).
if stargs:
for arg in stargs:
if rv is None:
constructor = CallableType(
[
self.chk.named_generic_type(
"_typeshed.SupportsKeysAndGetItem", [kt, vt]
)
],
[nodes.ARG_POS],
[None],
self.chk.named_generic_type("builtins.dict", [kt, vt]),
self.named_type("builtins.function"),
name="<list>",
variables=[kt, vt],
)
rv = self.check_call(constructor, [arg], [nodes.ARG_POS], arg)[0]
else:
self.check_method_call_by_name("update", rv, [arg], [nodes.ARG_POS], arg)
assert rv is not None
return rv
expected_types.append(TupleType([kt, vt], self.named_type("builtins.tuple")))

# The callable type represents a function like this (except we adjust for **expr):
# def <dict>(*v: Tuple[kt, vt]) -> Dict[kt, vt]: ...
constructor = CallableType(
expected_types,
[nodes.ARG_POS] * len(expected_types),
[None] * len(expected_types),
self.chk.named_generic_type("builtins.dict", [kt, vt]),
self.named_type("builtins.function"),
name="<dict>",
variables=[kt, vt],
)
return self.check_call(constructor, args, [nodes.ARG_POS] * len(args), e)[0]

def find_typeddict_context(
self, context: Type | None, dict_expr: DictExpr
Expand Down
20 changes: 18 additions & 2 deletions mypy/messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -679,11 +679,13 @@ def incompatible_argument(
name.title(), n, actual_type_str, expected_type_str
)
code = codes.LIST_ITEM
elif callee_name == "<dict>":
elif callee_name == "<dict>" and isinstance(
get_proper_type(callee.arg_types[n - 1]), TupleType
):
name = callee_name[1:-1]
n -= 1
key_type, value_type = cast(TupleType, arg_type).items
expected_key_type, expected_value_type = cast(TupleType, callee.arg_types[0]).items
expected_key_type, expected_value_type = cast(TupleType, callee.arg_types[n]).items

# don't increase verbosity unless there is need to do so
if is_subtype(key_type, expected_key_type):
Expand All @@ -710,6 +712,14 @@ def incompatible_argument(
expected_value_type_str,
)
code = codes.DICT_ITEM
elif callee_name == "<dict>":
value_type_str, expected_value_type_str = format_type_distinctly(
arg_type, callee.arg_types[n - 1], options=self.options
)
msg = "Unpacked dict entry {} has incompatible type {}; expected {}".format(
n - 1, value_type_str, expected_value_type_str
)
code = codes.DICT_ITEM
elif callee_name == "<list-comprehension>":
actual_type_str, expected_type_str = map(
strip_quotes,
Expand Down Expand Up @@ -1301,6 +1311,12 @@ def could_not_infer_type_arguments(
callee_name = callable_name(callee_type)
if callee_name is not None and n > 0:
self.fail(f"Cannot infer type argument {n} of {callee_name}", context)
if callee_name == "<dict>":
# Invariance in key type causes more of these errors than we would want.
self.note(
"Try assigning the literal to a variable annotated as dict[<key>, <val>]",
context,
)
else:
self.fail("Cannot infer function type argument", context)

Expand Down
19 changes: 6 additions & 13 deletions test-data/unit/check-expressions.test
Original file line number Diff line number Diff line change
Expand Up @@ -1800,11 +1800,12 @@ a = {'a': 1}
b = {'z': 26, **a}
c = {**b}
d = {**a, **b, 'c': 3}
e = {1: 'a', **a} # E: Argument 1 to "update" of "dict" has incompatible type "Dict[str, int]"; expected "SupportsKeysAndGetItem[int, str]"
f = {**b} # type: Dict[int, int] # E: List item 0 has incompatible type "Dict[str, int]"; expected "SupportsKeysAndGetItem[int, int]"
e = {1: 'a', **a} # E: Cannot infer type argument 1 of <dict> \
# N: Try assigning the literal to a variable annotated as dict[<key>, <val>]
f = {**b} # type: Dict[int, int] # E: Unpacked dict entry 0 has incompatible type "Dict[str, int]"; expected "SupportsKeysAndGetItem[int, int]"
g = {**Thing()}
h = {**a, **Thing()}
i = {**Thing()} # type: Dict[int, int] # E: List item 0 has incompatible type "Thing"; expected "SupportsKeysAndGetItem[int, int]" \
i = {**Thing()} # type: Dict[int, int] # E: Unpacked dict entry 0 has incompatible type "Thing"; expected "SupportsKeysAndGetItem[int, int]" \
# N: Following member(s) of "Thing" have conflicts: \
# N: Expected: \
# N: def __getitem__(self, int, /) -> int \
Expand All @@ -1814,16 +1815,8 @@ i = {**Thing()} # type: Dict[int, int] # E: List item 0 has incompatible type
# N: def keys(self) -> Iterable[int] \
# N: Got: \
# N: def keys(self) -> Iterable[str]
j = {1: 'a', **Thing()} # E: Argument 1 to "update" of "dict" has incompatible type "Thing"; expected "SupportsKeysAndGetItem[int, str]" \
# N: Following member(s) of "Thing" have conflicts: \
# N: Expected: \
# N: def __getitem__(self, int, /) -> str \
# N: Got: \
# N: def __getitem__(self, str, /) -> int \
# N: Expected: \
# N: def keys(self) -> Iterable[int] \
# N: Got: \
# N: def keys(self) -> Iterable[str]
j = {1: 'a', **Thing()} # E: Cannot infer type argument 1 of <dict> \
# N: Try assigning the literal to a variable annotated as dict[<key>, <val>]
[builtins fixtures/dict.pyi]
[typing fixtures/typing-medium.pyi]

Expand Down
22 changes: 22 additions & 0 deletions test-data/unit/check-generics.test
Original file line number Diff line number Diff line change
Expand Up @@ -2711,3 +2711,25 @@ class G(Generic[T]):
def g(self, x: S) -> Union[S, T]: ...

f(lambda x: x.g(0)) # E: Cannot infer type argument 1 of "f"

[case testDictStarInference]
class B: ...
class C1(B): ...
class C2(B): ...

dict1 = {"a": C1()}
dict2 = {"a": C2(), **dict1}
reveal_type(dict2) # N: Revealed type is "builtins.dict[builtins.str, __main__.B]"
[builtins fixtures/dict.pyi]

[case testDictStarAnyKeyJoinValue]
from typing import Any

class B: ...
class C1(B): ...
class C2(B): ...

dict1: Any
dict2 = {"a": C1(), **{x: C2() for x in dict1}}
reveal_type(dict2) # N: Revealed type is "builtins.dict[Any, __main__.B]"
[builtins fixtures/dict.pyi]
15 changes: 15 additions & 0 deletions test-data/unit/check-python38.test
Original file line number Diff line number Diff line change
Expand Up @@ -812,3 +812,18 @@ if sys.version_info < (3, 6):
else:
42 # type: ignore # E: Unused "type: ignore" comment
[builtins fixtures/ops.pyi]

[case testDictExpressionErrorLocations]
# flags: --pretty
from typing import Dict

other: Dict[str, str]
dct: Dict[str, int] = {"a": "b", **other}
[builtins fixtures/dict.pyi]
[out]
main:5: error: Dict entry 0 has incompatible type "str": "str"; expected "str": "int"
dct: Dict[str, int] = {"a": "b", **other}
^~~~~~~~
main:5: error: Unpacked dict entry 1 has incompatible type "Dict[str, str]"; expected "SupportsKeysAndGetItem[str, int]"
dct: Dict[str, int] = {"a": "b", **other}
^~~~~
2 changes: 1 addition & 1 deletion test-data/unit/fine-grained.test
Original file line number Diff line number Diff line change
Expand Up @@ -7546,7 +7546,7 @@ def d() -> Dict[int, int]: pass
[builtins fixtures/dict.pyi]
[out]
==
main:5: error: Argument 1 to "update" of "dict" has incompatible type "Dict[int, int]"; expected "SupportsKeysAndGetItem[int, str]"
main:5: error: Unpacked dict entry 1 has incompatible type "Dict[int, int]"; expected "SupportsKeysAndGetItem[int, str]"

[case testAwaitAndAsyncDef-only_when_nocache]
from a import g
Expand Down
2 changes: 1 addition & 1 deletion test-data/unit/pythoneval.test
Original file line number Diff line number Diff line change
Expand Up @@ -1350,7 +1350,7 @@ def f() -> Dict[int, str]:
def d() -> Dict[int, int]:
return {}
[out]
_testDictWithStarStarSpecialCase.py:4: error: Argument 1 to "update" of "MutableMapping" has incompatible type "Dict[int, int]"; expected "SupportsKeysAndGetItem[int, str]"
_testDictWithStarStarSpecialCase.py:4: error: Unpacked dict entry 1 has incompatible type "Dict[int, int]"; expected "SupportsKeysAndGetItem[int, str]"

[case testLoadsOfOverloads]
from typing import overload, Any, TypeVar, Iterable, List, Dict, Callable, Union
Expand Down

0 comments on commit 541639e

Please sign in to comment.