Some final touches for variadic types support (#16334)

I decided to go again over various parts of variadic types
implementation to double-check nothing is missing, checked interaction
with various "advanced" features (dataclasses, protocols, self-types,
match statement, etc.), added some more tests (including incremental),
and `grep`ed for potentially unhandled cases (and did found few
crashes). This mostly touches only variadic types but one thing goes
beyond, the fix for self-types upper bound, I think it is correct and
should be safe.

If there are no objections, next PR will flip the switch.

---------

Co-authored-by: Shantanu <[email protected]>

mypy/applytype.py

@@ -3,6 +3,7 @@
 from typing import Callable, Sequence
 
 import mypy.subtypes
+from mypy.erasetype import erase_typevars
 from mypy.expandtype import expand_type
 from mypy.nodes import Context
 from mypy.types import (
@@ -62,6 +63,11 @@ def get_target_type(
         report_incompatible_typevar_value(callable, type, tvar.name, context)
     else:
         upper_bound = tvar.upper_bound
+        if tvar.name == "Self":
+            # Internally constructed Self-types contain class type variables in upper bound,
+            # so we need to erase them to avoid false positives. This is safe because we do
+            # not support type variables in upper bounds of user defined types.
+            upper_bound = erase_typevars(upper_bound)
         if not mypy.subtypes.is_subtype(type, upper_bound):
             if skip_unsatisfied:
                 return None
@@ -121,6 +127,7 @@ def apply_generic_arguments(
     # Apply arguments to argument types.
     var_arg = callable.var_arg()
     if var_arg is not None and isinstance(var_arg.typ, UnpackType):
+        # Same as for ParamSpec, callable with variadic types needs to be expanded as a whole.
         callable = expand_type(callable, id_to_type)
         assert isinstance(callable, CallableType)
         return callable.copy_modified(variables=[tv for tv in tvars if tv.id not in id_to_type])

mypy/checker.py

@@ -1852,7 +1852,6 @@ def expand_typevars(
         if defn.info:
             # Class type variables
             tvars += defn.info.defn.type_vars or []
-        # TODO(PEP612): audit for paramspec
         for tvar in tvars:
             if isinstance(tvar, TypeVarType) and tvar.values:
                 subst.append([(tvar.id, value) for value in tvar.values])
@@ -2538,6 +2537,9 @@ def check_protocol_variance(self, defn: ClassDef) -> None:
         object_type = Instance(info.mro[-1], [])
         tvars = info.defn.type_vars
         for i, tvar in enumerate(tvars):
+            if not isinstance(tvar, TypeVarType):
+                # Variance of TypeVarTuple and ParamSpec is underspecified by PEPs.
+                continue
             up_args: list[Type] = [
                 object_type if i == j else AnyType(TypeOfAny.special_form)
                 for j, _ in enumerate(tvars)
@@ -2554,7 +2556,7 @@ def check_protocol_variance(self, defn: ClassDef) -> None:
                 expected = CONTRAVARIANT
             else:
                 expected = INVARIANT
-            if isinstance(tvar, TypeVarType) and expected != tvar.variance:
+            if expected != tvar.variance:
                 self.msg.bad_proto_variance(tvar.variance, tvar.name, expected, defn)
 
     def check_multiple_inheritance(self, typ: TypeInfo) -> None:
@@ -6695,19 +6697,6 @@ def check_possible_missing_await(
                 return
         self.msg.possible_missing_await(context, code)
 
-    def contains_none(self, t: Type) -> bool:
-        t = get_proper_type(t)
-        return (
-            isinstance(t, NoneType)
-            or (isinstance(t, UnionType) and any(self.contains_none(ut) for ut in t.items))
-            or (isinstance(t, TupleType) and any(self.contains_none(tt) for tt in t.items))
-            or (
-                isinstance(t, Instance)
-                and bool(t.args)
-                and any(self.contains_none(it) for it in t.args)
-            )
-        )
-
     def named_type(self, name: str) -> Instance:
         """Return an instance type with given name and implicit Any type args.
 
@@ -7471,10 +7460,22 @@ def builtin_item_type(tp: Type) -> Type | None:
                 return None
             if not isinstance(get_proper_type(tp.args[0]), AnyType):
                 return tp.args[0]
-    elif isinstance(tp, TupleType) and all(
-        not isinstance(it, AnyType) for it in get_proper_types(tp.items)
-    ):
-        return make_simplified_union(tp.items)  # this type is not externally visible
+    elif isinstance(tp, TupleType):
+        normalized_items = []
+        for it in tp.items:
+            # This use case is probably rare, but not handling unpacks here can cause crashes.
+            if isinstance(it, UnpackType):
+                unpacked = get_proper_type(it.type)
+                if isinstance(unpacked, TypeVarTupleType):
+                    unpacked = get_proper_type(unpacked.upper_bound)
+                assert (
+                    isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple"
+                )
+                normalized_items.append(unpacked.args[0])
+            else:
+                normalized_items.append(it)
+        if all(not isinstance(it, AnyType) for it in get_proper_types(normalized_items)):
+            return make_simplified_union(normalized_items)  # this type is not externally visible
     elif isinstance(tp, TypedDictType):
         # TypedDict always has non-optional string keys. Find the key type from the Mapping
         # base class.

mypy/checkexpr.py

@@ -410,7 +410,7 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type:
             result = self.alias_type_in_runtime_context(
                 node, ctx=e, alias_definition=e.is_alias_rvalue or lvalue
             )
-        elif isinstance(node, (TypeVarExpr, ParamSpecExpr)):
+        elif isinstance(node, (TypeVarExpr, ParamSpecExpr, TypeVarTupleExpr)):
             result = self.object_type()
         else:
             if isinstance(node, PlaceholderNode):
@@ -3316,6 +3316,7 @@ def infer_literal_expr_type(self, value: LiteralValue, fallback_name: str) -> Ty
 
     def concat_tuples(self, left: TupleType, right: TupleType) -> TupleType:
         """Concatenate two fixed length tuples."""
+        assert not (find_unpack_in_list(left.items) and find_unpack_in_list(right.items))
         return TupleType(
             items=left.items + right.items, fallback=self.named_type("builtins.tuple")
         )
@@ -6507,8 +6508,8 @@ def merge_typevars_in_callables_by_name(
             for tv in target.variables:
                 name = tv.fullname
                 if name not in unique_typevars:
-                    # TODO(PEP612): fix for ParamSpecType
-                    if isinstance(tv, ParamSpecType):
+                    # TODO: support ParamSpecType and TypeVarTuple.
+                    if isinstance(tv, (ParamSpecType, TypeVarTupleType)):
                         continue
                     assert isinstance(tv, TypeVarType)
                     unique_typevars[name] = tv

mypy/checkpattern.py

@@ -45,9 +45,13 @@
     Type,
     TypedDictType,
     TypeOfAny,
+    TypeVarTupleType,
     UninhabitedType,
     UnionType,
+    UnpackType,
+    find_unpack_in_list,
     get_proper_type,
+    split_with_prefix_and_suffix,
 )
 from mypy.typevars import fill_typevars
 from mypy.visitor import PatternVisitor
@@ -239,13 +243,29 @@ def visit_sequence_pattern(self, o: SequencePattern) -> PatternType:
         #
         # get inner types of original type
         #
+        unpack_index = None
         if isinstance(current_type, TupleType):
             inner_types = current_type.items
-            size_diff = len(inner_types) - required_patterns
-            if size_diff < 0:
-                return self.early_non_match()
-            elif size_diff > 0 and star_position is None:
-                return self.early_non_match()
+            unpack_index = find_unpack_in_list(inner_types)
+            if unpack_index is None:
+                size_diff = len(inner_types) - required_patterns
+                if size_diff < 0:
+                    return self.early_non_match()
+                elif size_diff > 0 and star_position is None:
+                    return self.early_non_match()
+            else:
+                normalized_inner_types = []
+                for it in inner_types:
+                    # Unfortunately, it is not possible to "split" the TypeVarTuple
+                    # into individual items, so we just use its upper bound for the whole
+                    # analysis instead.
+                    if isinstance(it, UnpackType) and isinstance(it.type, TypeVarTupleType):
+                        it = UnpackType(it.type.upper_bound)
+                    normalized_inner_types.append(it)
+                inner_types = normalized_inner_types
+                current_type = current_type.copy_modified(items=normalized_inner_types)
+                if len(inner_types) - 1 > required_patterns and star_position is None:
+                    return self.early_non_match()
         else:
             inner_type = self.get_sequence_type(current_type, o)
             if inner_type is None:
@@ -270,18 +290,18 @@ def visit_sequence_pattern(self, o: SequencePattern) -> PatternType:
             self.update_type_map(captures, type_map)
 
         new_inner_types = self.expand_starred_pattern_types(
-            contracted_new_inner_types, star_position, len(inner_types)
+            contracted_new_inner_types, star_position, len(inner_types), unpack_index is not None
         )
         rest_inner_types = self.expand_starred_pattern_types(
-            contracted_rest_inner_types, star_position, len(inner_types)
+            contracted_rest_inner_types, star_position, len(inner_types), unpack_index is not None
         )
 
         #
         # Calculate new type
         #
         new_type: Type
         rest_type: Type = current_type
-        if isinstance(current_type, TupleType):
+        if isinstance(current_type, TupleType) and unpack_index is None:
             narrowed_inner_types = []
             inner_rest_types = []
             for inner_type, new_inner_type in zip(inner_types, new_inner_types):
@@ -301,6 +321,14 @@ def visit_sequence_pattern(self, o: SequencePattern) -> PatternType:
             if all(is_uninhabited(typ) for typ in inner_rest_types):
                 # All subpatterns always match, so we can apply negative narrowing
                 rest_type = TupleType(rest_inner_types, current_type.partial_fallback)
+        elif isinstance(current_type, TupleType):
+            # For variadic tuples it is too tricky to match individual items like for fixed
+            # tuples, so we instead try to narrow the entire type.
+            # TODO: use more precise narrowing when possible (e.g. for identical shapes).
+            new_tuple_type = TupleType(new_inner_types, current_type.partial_fallback)
+            new_type, rest_type = self.chk.conditional_types_with_intersection(
+                new_tuple_type, [get_type_range(current_type)], o, default=new_tuple_type
+            )
         else:
             new_inner_type = UninhabitedType()
             for typ in new_inner_types:
@@ -345,17 +373,45 @@ def contract_starred_pattern_types(
 
         If star_pos in None the types are returned unchanged.
         """
-        if star_pos is None:
-            return types
-        new_types = types[:star_pos]
-        star_length = len(types) - num_patterns
-        new_types.append(make_simplified_union(types[star_pos : star_pos + star_length]))
-        new_types += types[star_pos + star_length :]
-
-        return new_types
+        unpack_index = find_unpack_in_list(types)
+        if unpack_index is not None:
+            # Variadic tuples require "re-shaping" to match the requested pattern.
+            unpack = types[unpack_index]
+            assert isinstance(unpack, UnpackType)
+            unpacked = get_proper_type(unpack.type)
+            # This should be guaranteed by the normalization in the caller.
+            assert isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple"
+            if star_pos is None:
+                missing = num_patterns - len(types) + 1
+                new_types = types[:unpack_index]
+                new_types += [unpacked.args[0]] * missing
+                new_types += types[unpack_index + 1 :]
+                return new_types
+            prefix, middle, suffix = split_with_prefix_and_suffix(
+                tuple([UnpackType(unpacked) if isinstance(t, UnpackType) else t for t in types]),
+                star_pos,
+                num_patterns - star_pos,
+            )
+            new_middle = []
+            for m in middle:
+                # The existing code expects the star item type, rather than the type of
+                # the whole tuple "slice".
+                if isinstance(m, UnpackType):
+                    new_middle.append(unpacked.args[0])
+                else:
+                    new_middle.append(m)
+            return list(prefix) + [make_simplified_union(new_middle)] + list(suffix)
+        else:
+            if star_pos is None:
+                return types
+            new_types = types[:star_pos]
+            star_length = len(types) - num_patterns
+            new_types.append(make_simplified_union(types[star_pos : star_pos + star_length]))
+            new_types += types[star_pos + star_length :]
+            return new_types
 
     def expand_starred_pattern_types(
-        self, types: list[Type], star_pos: int | None, num_types: int
+        self, types: list[Type], star_pos: int | None, num_types: int, original_unpack: bool
     ) -> list[Type]:
         """Undoes the contraction done by contract_starred_pattern_types.
 
@@ -364,6 +420,17 @@ def expand_starred_pattern_types(
         """
         if star_pos is None:
             return types
+        if original_unpack:
+            # In the case where original tuple type has an unpack item, it is not practical
+            # to coerce pattern type back to the original shape (and may not even be possible),
+            # so we only restore the type of the star item.
+            res = []
+            for i, t in enumerate(types):
+                if i != star_pos:
+                    res.append(t)
+                else:
+                    res.append(UnpackType(self.chk.named_generic_type("builtins.tuple", [t])))
+            return res
         new_types = types[:star_pos]
         star_length = num_types - len(types) + 1
         new_types += [types[star_pos]] * star_length
@@ -459,7 +526,15 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType:
             return self.early_non_match()
         if isinstance(type_info, TypeInfo):
             any_type = AnyType(TypeOfAny.implementation_artifact)
-            typ: Type = Instance(type_info, [any_type] * len(type_info.defn.type_vars))
+            args: list[Type] = []
+            for tv in type_info.defn.type_vars:
+                if isinstance(tv, TypeVarTupleType):
+                    args.append(
+                        UnpackType(self.chk.named_generic_type("builtins.tuple", [any_type]))
+                    )
+                else:
+                    args.append(any_type)
+            typ: Type = Instance(type_info, args)
         elif isinstance(type_info, TypeAlias):
             typ = type_info.target
         elif (

mypy/constraints.py

@@ -28,6 +28,7 @@
     Instance,
     LiteralType,
     NoneType,
+    NormalizedCallableType,
     Overloaded,
     Parameters,
     ParamSpecType,
@@ -1388,7 +1389,7 @@ def find_matching_overload_items(
     return res
 
 
-def get_tuple_fallback_from_unpack(unpack: UnpackType) -> TypeInfo | None:
+def get_tuple_fallback_from_unpack(unpack: UnpackType) -> TypeInfo:
     """Get builtins.tuple type from available types to construct homogeneous tuples."""
     tp = get_proper_type(unpack.type)
     if isinstance(tp, Instance) and tp.type.fullname == "builtins.tuple":
@@ -1399,10 +1400,10 @@ def get_tuple_fallback_from_unpack(unpack: UnpackType) -> TypeInfo | None:
         for base in tp.partial_fallback.type.mro:
             if base.fullname == "builtins.tuple":
                 return base
-    return None
+    assert False, "Invalid unpack type"
 
 
-def repack_callable_args(callable: CallableType, tuple_type: TypeInfo | None) -> list[Type]:
+def repack_callable_args(callable: CallableType, tuple_type: TypeInfo) -> list[Type]:
     """Present callable with star unpack in a normalized form.
 
     Since positional arguments cannot follow star argument, they are packed in a suffix,
@@ -1417,12 +1418,8 @@ def repack_callable_args(callable: CallableType, tuple_type: TypeInfo | None) ->
     star_type = callable.arg_types[star_index]
     suffix_types = []
     if not isinstance(star_type, UnpackType):
-        if tuple_type is not None:
-            # Re-normalize *args: X -> *args: *tuple[X, ...]
-            star_type = UnpackType(Instance(tuple_type, [star_type]))
-        else:
-            # This is unfortunate, something like tuple[Any, ...] would be better.
-            star_type = UnpackType(AnyType(TypeOfAny.from_error))
+        # Re-normalize *args: X -> *args: *tuple[X, ...]
+        star_type = UnpackType(Instance(tuple_type, [star_type]))
     else:
         tp = get_proper_type(star_type.type)
         if isinstance(tp, TupleType):
@@ -1544,7 +1541,9 @@ def infer_directed_arg_constraints(left: Type, right: Type, direction: int) -> l
 
 
 def infer_callable_arguments_constraints(
-    template: CallableType | Parameters, actual: CallableType | Parameters, direction: int
+    template: NormalizedCallableType | Parameters,
+    actual: NormalizedCallableType | Parameters,
+    direction: int,
 ) -> list[Constraint]:
     """Infer constraints between argument types of two callables.
 

mypy/erasetype.py

@@ -100,7 +100,9 @@ def visit_parameters(self, t: Parameters) -> ProperType:
         raise RuntimeError("Parameters should have been bound to a class")
 
     def visit_type_var_tuple(self, t: TypeVarTupleType) -> ProperType:
-        return AnyType(TypeOfAny.special_form)
+        # Likely, we can never get here because of aggressive erasure of types that
+        # can contain this, but better still return a valid replacement.
+        return t.tuple_fallback.copy_modified(args=[AnyType(TypeOfAny.special_form)])
 
     def visit_unpack_type(self, t: UnpackType) -> ProperType:
         return AnyType(TypeOfAny.special_form)