Fix scala#8861: Avoid parameters when instantiating closure results

compiler/src/dotty/tools/dotc/typer/Namer.scala

@@ -1556,6 +1556,17 @@ class Namer { typer: Typer =>
     val termParamss = ctx.normalizeIfConstructor(vparamss.nestedMap(symbolOfTree), isConstructor)
     sym.setParamss(typeParams, termParamss)
     def wrapMethType(restpe: Type): Type = {
+      restpe match
+        case tv: TypeVar
+        if sym.isAnonymousFunction && ctx.typerState.constraint.contains(tv.origin) =>
+          // If the result type is a type variable R in an anonymous function, it comes from the
+          // environment of the function. In this case `instantiateDependent` risks
+          // instantiating R to a locally defined parameter, which violates avoidance rules.
+          // We instantiate instead to its avoided lower bound. See i8861.scala for a test.
+          val lowBound = ctx.typeComparer.fullLowerBound(tv.origin)
+          val lower = avoid(lowBound, termParamss.flatten)
+          if lower ne lowBound then tv.instantiateWith(lower)
+        case _ =>
       instantiateDependent(restpe, typeParams, termParamss)
       ctx.methodType(tparams map symbolOfTree, termParamss, restpe, isJava = ddef.mods.is(JavaDefined))
     }

tests/neg/i8861.scala

@@ -0,0 +1,31 @@
+object Test {
+  sealed trait Container { s =>
+    type A
+    def visit[R](int: IntV & s.type => R, str: StrV & s.type => R): R
+  }
+  final class IntV extends Container { s =>
+    type A = Int
+    val i: Int = 42
+    def visit[R](int: IntV & s.type => R, str: StrV & s.type => R): R = int(this)
+  }
+  final class StrV extends Container { s =>
+    type A = String
+    val t: String = "hello"
+    def visit[R](int: IntV & s.type => R, str: StrV & s.type => R): R = str(this)
+  }
+
+  def minimalOk[R](c: Container { type A = R }): R = c.visit[R](
+    int = vi => vi.i : vi.A,
+    str = vs => vs.t : vs.A
+  )
+  def minimalFail[M](c: Container { type A = M }): M = c.visit(
+    int = vi => vi.i : vi.A,
+    str = vs => vs.t : vs.A  // error
+  )
+
+  def main(args: Array[String]): Unit = {
+    val e: Container { type A = String } = new StrV
+    println(minimalOk(e)) // this one prints "hello"
+    println(minimalFail(e)) // this one fails with ClassCastException: class java.lang.String cannot be cast to class java.lang.Integer
+  }
+}