Update nesting level in typer state

compiler/src/dotty/tools/dotc/core/ConstraintHandling.scala

@@ -511,14 +511,8 @@ trait ConstraintHandling {
                 // `fixLevels`, this could lead to coarser types. But it has the potential
                 // to give a better approximation for the current type, since it avoids forming
                 // a Range in invariant position, which can lead to very coarse types further out.
-                // TODO: This widening is a side effect that is not undone if a typer state is aborted
-                // I don't think it's a soundness problem, since all that could happen is that
-                // the type variable causes earlier instantiations of other type variables
-                // down the line. But it could produce a hard-to-debug side effect that leads
-                // to worse types than expected. We should find a more robust way to do this.
-                // Maybe instantiating `tp` to another freshly created type at nesting level?
                 constr.println(i"widening nesting level of type variable $tp from ${tp.nestingLevel} to $maxLevel")
-                tp.nestingLevel = maxLevel
+                ctx.typerState.setNestingLevel(tp, maxLevel)
               true
             else false
           case _ =>

compiler/src/dotty/tools/dotc/core/TyperState.scala

@@ -10,7 +10,7 @@ import config.Config
 import config.Printers.constr
 import collection.mutable
 import java.lang.ref.WeakReference
-import util.Stats
+import util.{Stats, SimpleIdentityMap}
 import Decorators._
 
 import scala.annotation.internal.sharable
@@ -23,24 +23,28 @@ object TyperState {
       .setReporter(new ConsoleReporter())
       .setCommittable(true)
 
-  opaque type Snapshot = (Constraint, TypeVars, TypeVars)
+  type LevelMap = SimpleIdentityMap[TypeVar, Integer]
+
+  opaque type Snapshot = (Constraint, TypeVars, TypeVars, LevelMap)
 
   extension (ts: TyperState)
     def snapshot()(using Context): Snapshot =
       var previouslyInstantiated: TypeVars = SimpleIdentitySet.empty
       for tv <- ts.ownedVars do if tv.inst.exists then previouslyInstantiated += tv
-      (ts.constraint, ts.ownedVars, previouslyInstantiated)
+      (ts.constraint, ts.ownedVars, previouslyInstantiated, ts.upLevels)
 
     def resetTo(state: Snapshot)(using Context): Unit =
-      val (c, tvs, previouslyInstantiated) = state
+      val (c, tvs, previouslyInstantiated, upLevels) = state
       for tv <- tvs do
         if tv.inst.exists && !previouslyInstantiated.contains(tv) then
           tv.resetInst(ts)
       ts.ownedVars = tvs
       ts.constraint = c
+      ts.upLevels = upLevels
 }
 
 class TyperState() {
+  import TyperState.LevelMap
 
   private var myId: Int = _
   def id: Int = myId
@@ -89,6 +93,8 @@ class TyperState() {
   def ownedVars: TypeVars = myOwnedVars
   def ownedVars_=(vs: TypeVars): Unit = myOwnedVars = vs
 
+  private var upLevels: LevelMap = _
+
   /** Initializes all fields except reporter, isCommittable, which need to be
    *  set separately.
    */
@@ -99,20 +105,35 @@ class TyperState() {
     this.myConstraint = constraint
     this.previousConstraint = constraint
     this.myOwnedVars = SimpleIdentitySet.empty
+    this.upLevels = SimpleIdentityMap.empty
     this.isCommitted = false
     this
 
   /** A fresh typer state with the same constraint as this one. */
   def fresh(reporter: Reporter = StoreReporter(this.reporter, fromTyperState = true),
       committable: Boolean = this.isCommittable): TyperState =
     util.Stats.record("TyperState.fresh")
-    TyperState().init(this, this.constraint)
+    val ts = TyperState().init(this, this.constraint)
       .setReporter(reporter)
       .setCommittable(committable)
+    ts.upLevels = upLevels
+    ts
 
   /** The uninstantiated variables */
   def uninstVars: collection.Seq[TypeVar] = constraint.uninstVars
 
+  /** The nestingLevel of `tv` in this typer state */
+  def nestingLevel(tv: TypeVar): Int =
+    val own = upLevels(tv)
+    if own == null then tv.initNestingLevel else own.intValue()
+
+  /** Set the nestingLevel of `tv` in this typer state
+   *  @pre this level must be smaller than `tv.initNestingLevel`
+  */
+  def setNestingLevel(tv: TypeVar, level: Int) =
+    assert(level < tv.initNestingLevel)
+    upLevels = upLevels.updated(tv, level)
+
   /** The closest ancestor of this typer state (including possibly this typer state itself)
    *  which is not yet committed, or which does not have a parent.
    */
@@ -164,6 +185,12 @@ class TyperState() {
         if !ownedVars.isEmpty then ownedVars.foreach(targetState.includeVar)
       else
         targetState.mergeConstraintWith(this)
+
+    upLevels.foreachBinding { (tv, level) =>
+      if level < targetState.nestingLevel(tv) then
+        targetState.setNestingLevel(tv, level)
+    }
+
     targetState.gc()
     isCommitted = true
     ownedVars = SimpleIdentitySet.empty

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -4497,27 +4497,11 @@ object Types {
    *  is different from the variable's creation state (meaning unrolls are possible)
    *  in the current typer state.
    *
-   *  @param  origin        The parameter that's tracked by the type variable.
-   *  @param  creatorState  The typer state in which the variable was created.
-   *  @param  nestingLevel  Symbols with a nestingLevel strictly greater than this
-   *                        will not appear in the instantiation of this type variable.
-   *                        This is enforced in `ConstraintHandling`, dependig on the
-   *                        Config flags setting `checkLevelsOnConstraints` and
-   *                        `checkLevelsOnInstantiation`.
-   *
-   *                        Under `checkLevelsOnConstraints` we maintain the invariant that
-   *                        the `nonParamBounds` of a type variable never refer to a type with a
-   *                        greater `nestingLevel` (see `legalBound` for the reason why this
-   *                        cannot be delayed until instantiation). Then, on instantiation,
-   *                        we replace any param in the param bound with a level greater than
-   *                        nestingLevel (see `fullLowerBound`).
-   *
-   *                        Under `checkLevelsOnInstantiation`, we avoid incorrect levels only
-   *                        when a type variable is instantiated, see `ConstraintHandling$fixLevels`.
-   *                        Under this mode, the `nestingLevel` of a type variable can be made
-   *                        smaller when fixing the levels for some other type variable instance.
+   *  @param  origin           the parameter that's tracked by the type variable.
+   *  @param  creatorState     the typer state in which the variable was created.
+   *  @param  initNestingLevel the initial nesting level of the type variable. (c.f. nestingLevel)
    */
-  final class TypeVar private(initOrigin: TypeParamRef, creatorState: TyperState | Null, var nestingLevel: Int) extends CachedProxyType with ValueType {
+  final class TypeVar private(initOrigin: TypeParamRef, creatorState: TyperState | Null, val initNestingLevel: Int) extends CachedProxyType with ValueType {
     private var currentOrigin = initOrigin
 
     def origin: TypeParamRef = currentOrigin
@@ -4549,6 +4533,26 @@ object Types {
     private[core] var owningState: WeakReference[TyperState] | Null =
       if (creatorState == null) null else new WeakReference(creatorState)
 
+    /** The nesting level of this type variable in the current typer state. This is usually
+     *  the same as `initNestingLevel`, but can be decremented by calling `TyperState#setNestingLevel`.
+     *  Symbols with a nestingLevel strictly greater than this level will not appear in the
+     *  instantiation of this type variable. This is enforced in `ConstraintHandling`,
+     *  dependig on the Config flags setting `checkLevelsOnConstraints` and `checkLevelsOnInstantiation`.
+     *
+     *  Under `checkLevelsOnConstraints` we maintain the invariant that
+     *  the `nonParamBounds` of a type variable never refer to a type with a
+     *  greater `nestingLevel` (see `legalBound` for the reason why this
+     *  cannot be delayed until instantiation). Then, on instantiation,
+     *  we replace any param in the param bound with a level greater than
+     *  nestingLevel (see `fullLowerBound`).
+     *
+     *  Under `checkLevelsOnInstantiation`, we avoid incorrect levels only
+     *  when a type variable is instantiated, see `ConstraintHandling$fixLevels`.
+     *  Under this mode, the `nestingLevel` of a type variable can be made
+     *  smaller when fixing the levels for some other type variable instance.
+     */
+    def nestingLevel(using Context): Int = ctx.typerState.nestingLevel(this)
+
     /** The instance type of this variable, or NoType if the variable is currently
      *  uninstantiated
      */